diff --git a/.changeset/flue-voice-safety.md b/.changeset/flue-voice-safety.md new file mode 100644 index 00000000000..5540dee7b70 --- /dev/null +++ b/.changeset/flue-voice-safety.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history. diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index 3b3a53c3142..a4d4ff018cf 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -9,13 +9,34 @@ import { join, resolve } from "node:path"; -import { defineConfig, loadConfigFromFile, mergeConfig } from "vite"; +import { + defineConfig, + loadConfigFromFile, + mergeConfig, + type UserConfig, +} from "vite"; import { defaultChatOrigin, petrinautLocalServer, } from "./src/http/local-origins.ts"; +interface PetrinautPanelConfigOptions { + readonly chatOrigin: string; + readonly loadedConfig: UserConfig; + readonly root: string; +} + +export const mergePetrinautPanelConfig = ({ + chatOrigin, + loadedConfig, + root, +}: PetrinautPanelConfigOptions): UserConfig => + mergeConfig(loadedConfig, { + root, + server: petrinautLocalServer(chatOrigin), + }); + export default defineConfig(async (environment) => { const websiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; if (!websiteRoot) { @@ -36,8 +57,9 @@ export default defineConfig(async (environment) => { throw new Error(`Could not load Petrinaut's Vite config from ${root}.`); const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin; - return mergeConfig(loaded.config, { + return mergePetrinautPanelConfig({ + chatOrigin, + loadedConfig: loaded.config, root, - server: petrinautLocalServer(chatOrigin), }); }); diff --git a/apps/brunch-agent/test/architecture/boundaries.integration.ts b/apps/brunch-agent/test/architecture/boundaries.integration.ts index f7b044eef51..8bd4e9aed0e 100644 --- a/apps/brunch-agent/test/architecture/boundaries.integration.ts +++ b/apps/brunch-agent/test/architecture/boundaries.integration.ts @@ -385,6 +385,7 @@ describe("core auxiliary subpaths stay in their assigned lanes", () => { ".", "./client-tools", "./flue", + "./question-marker", "./storage", "./workpiece", ]); @@ -424,6 +425,8 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12 * path enters here by review only. */ const SUBSTRATE_INTEGRATION_ENTRY_POINTS: Readonly> = { + "libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts": + "Types the Flue logger and calls the core marker tool with a mocked data-part writer and logger; no runtime boot, provider, key or socket.", "apps/brunch-agent/test/brunch-turn.test.ts": "Types Flue's client, admission, and conversation snapshot and constructs FlueExecutionError so the persona bridge can be unit-tested against a stubbed client — no provider key, no socket, no model call, no runtime boot.", "apps/brunch-agent/test/flue-transcript.test.ts": diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index ddf0d734ec3..8a20c937cae 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from "node:fs"; import { expect, test } from "vitest"; +import { mergePetrinautPanelConfig } from "../petrinaut-local.vite.config.ts"; import { defaultChatOrigin, localChatListen, @@ -21,7 +22,7 @@ test("one documented root command starts the Brunch server and Petrinaut panel", }; expect(rootPackage.scripts["dev:brunch"]).toBe( - "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", + "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", ); expect(rootPackage.scripts["dev:brunch:server"]).toBe( "yarn workspace @apps/brunch-agent dev", @@ -64,3 +65,19 @@ test("petrinaut:dev proxies the mounted Flue conversation route", () => { 'VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"', ); }); + +test("petrinaut:dev retains the website API handlers needed by Voice", () => { + const config = mergePetrinautPanelConfig({ + chatOrigin: defaultChatOrigin, + loadedConfig: { + plugins: [{ name: "petrinaut-api-dev" }], + }, + root: "/test/petrinaut-website", + }); + + expect(config.plugins).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "petrinaut-api-dev" }), + ]), + ); +}); diff --git a/apps/brunch-agent/test/petrinaut-chat-result.ts b/apps/brunch-agent/test/petrinaut-chat-result.ts index c7cc142689c..b1a70cb413c 100644 --- a/apps/brunch-agent/test/petrinaut-chat-result.ts +++ b/apps/brunch-agent/test/petrinaut-chat-result.ts @@ -21,6 +21,10 @@ export interface PetrinautChatResult { readonly resumedStatus: number; readonly resumedText: string; readonly resumedFinish: UIMessageChunk | undefined; + readonly questionMarkerLive: unknown; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleLive: boolean; + readonly questionToolVisibleHistory: boolean; readonly historyUserEntryCount: number; readonly historyClientToolResultCount: number; readonly historyGetStatus: number; @@ -51,5 +55,7 @@ export interface PetrinautChatResult { export interface PetrinautResumeResult { readonly historyGetStatus: number; readonly historyUserText: string; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleHistory: boolean; readonly transcript: string; } diff --git a/apps/brunch-agent/test/petrinaut-chat.integration.ts b/apps/brunch-agent/test/petrinaut-chat.integration.ts index 20750ca68a4..253fc8ff4d3 100644 --- a/apps/brunch-agent/test/petrinaut-chat.integration.ts +++ b/apps/brunch-agent/test/petrinaut-chat.integration.ts @@ -18,6 +18,10 @@ import { snapshotToUiMessages, } from "@hashintel/brunch-agent-transport-aisdk"; import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, +} from "@hashintel/brunch-agent/question-marker"; import { PING_TOOL_NAME } from "../src/agents/chat-agent/tools/ping.ts"; import { applyCaptureSweep } from "../src/capture/apply-sweep.ts"; @@ -43,6 +47,7 @@ const ACTIVATE_SKILL_TOOL_NAME = "activate_skill"; const CHAT_MODEL_ID = "claude-haiku-4-5"; const RUNBOOK_SKILL_NAME = "sdcpn-modelling"; const READ_SKILL_RESOURCE_TOOL_NAME = "read_skill_resource"; +const question = "Which documentation page should we inspect next?"; const principalKey = "principal-mission-1"; const conversationId = "conversation-mission-1"; @@ -79,6 +84,35 @@ const userTextFromHistory = ( .map((part) => part.text) .join(""); +const questionMarkerFromHistory = ( + messages: ReturnType, +): unknown => { + const marker = messages + .flatMap((message) => message.parts) + .find( + (part) => + part.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in part, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionMarkerFromChunks = ( + chunks: readonly UIMessageChunk[], +): unknown => { + const marker = chunks.find( + (chunk) => + chunk.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in chunk, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionToolVisibleInHistory = ( + messages: ReturnType, +): boolean => + messages + .flatMap((message) => message.parts) + .some((part) => part.type === `tool-${BRUNCH_QUESTION_TOOL_NAME}`); + const faux = fauxProvider({ provider: "anthropic", models: [{ id: CHAT_MODEL_ID, reasoning: true }], @@ -98,19 +132,24 @@ try { const panelTransport = createFlueChatTransport({ client: historyClient, clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); const projectHistory = ( snapshot: Awaited>, ) => snapshotToUiMessages(snapshot, { clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); if (process.env.BRUNCH_RESUME_PHASE === "1") { const snapshot = await historyClient.history(); + const historyMessages = projectHistory(snapshot); const result: PetrinautResumeResult = { historyGetStatus: 200, - historyUserText: userTextFromHistory(projectHistory(snapshot)), + historyUserText: userTextFromHistory(historyMessages), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), transcript: formatFlueTranscript(snapshot), }; process.stdout.write(`PETRINAUT_RESUME_RESULT ${JSON.stringify(result)}\n`); @@ -196,9 +235,19 @@ try { ], { stopReason: "toolUse" }, ), + fauxAssistantMessage( + [ + fauxToolCall( + BRUNCH_QUESTION_TOOL_NAME, + { question }, + { id: "tool-question-1" }, + ), + ], + { stopReason: "toolUse" }, + ), fauxAssistantMessage([ fauxText( - "The guide says the assistant can read its own documentation pages.", + `The guide says the assistant can read its own documentation pages. ${question}`, ), ]), fauxAssistantMessage([ @@ -400,6 +449,14 @@ try { .map((chunk) => chunk.delta) .join(""), resumedFinish: resumedChunks.at(-1), + questionMarkerLive: questionMarkerFromChunks(resumedChunks), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleLive: resumedChunks.some( + (chunk) => + chunk.type === "tool-input-available" && + chunk.toolName === BRUNCH_QUESTION_TOOL_NAME, + ), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), historyUserEntryCount: userEntryIds.length, historyClientToolResultCount: clientToolResultCount, historyGetStatus: 200, diff --git a/apps/brunch-agent/test/petrinaut-chat.test.ts b/apps/brunch-agent/test/petrinaut-chat.test.ts index daf97db27ea..2ee108a9ac0 100644 --- a/apps/brunch-agent/test/petrinaut-chat.test.ts +++ b/apps/brunch-agent/test/petrinaut-chat.test.ts @@ -72,6 +72,16 @@ test("the browser transport streams the mounted Flue agent through server and cl type: "finish", finishReason: "stop", }); + expect(result.questionMarkerLive).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleLive).toBe(false); + expect(result.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleHistory).toBe(false); expect(result.historyUserEntryCount).toBe(1); expect(result.historyClientToolResultCount).toBe(1); @@ -107,6 +117,7 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(result.interviewerToolNames).toContain("read_skill_resource"); expect(result.interviewerToolNames).toContain("ping"); expect(result.interviewerToolNames).toContain("readPetrinautDoc"); + expect(result.interviewerToolNames).toContain("brunch_mark_question"); expect(result.interviewerToolNames).not.toContain("brunch_ask"); expect(result.interviewerToolNames).not.toContain("sweep"); expect(result.interviewerToolNames).not.toContain("brunch_sweep"); @@ -151,10 +162,16 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(resumeResult.historyUserText).toContain( "Run the FE-1435 transport probe.", ); + expect(resumeResult.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(resumeResult.questionToolVisibleHistory).toBe(false); expect(resumeResult.transcript).toContain("tool ping"); expect(resumeResult.transcript).toContain("tool readPetrinautDoc"); expect(resumeResult.transcript).toContain("tool activate_skill"); expect(resumeResult.transcript).toContain("tool read_skill_resource"); + expect(resumeResult.transcript).toContain("tool brunch_mark_question"); } finally { await rm(dbDirectory, { recursive: true, force: true }); } diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 10a694237c6..ae357fc115d 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -97,7 +97,22 @@ disclosure before requesting microphone access. The disclosure also provides a microphone check and is remembered in browser storage only after Voice mode starts. -When Brunch is selected, typed and finalized spoken turns both enter the same mounted Flue conversation route. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Closing or speaking over Voice playback only stops local media; it does not alter canonical conversation history. Reopening the same net restores its observed Flue conversation without resubmitting a turn or replaying settled audio. +When Brunch is selected, typed turns and completed Voice transcripts both enter +the same mounted Flue conversation route. Each logical turn carries a stable +delivery key so a replayed request converges on the existing admission instead +of creating another turn. If admission cannot be confirmed, the UI reports the +ambiguity and does not retry automatically. **Stop** requests a durable Brunch +abort before the panel cancels its local response stream. Local playback +cancellation remains separate and does not alter canonical history. Canonical +Flue history is the source used when the same net is reopened. Automated +coverage guards a locally submitted turn from an older hydration snapshot and +does not resubmit turns or replay settled audio. The real hard-reload witness is +still pending, so reload parity is not yet claimed for this preview. +Voice-origin client-tool results retain their markers in Flue history. Direct +spoken user turns remain canonical text, but Flue 2.0.3 does not yet expose the +caller delivery metadata needed to restore their Voice chip after reopening. + +Browser execution and its continuation keep the shared composer busy; a local tool failure reaches Voice as an error rather than an apparently completed response. Durably aborted history entries retain their stopped label. If the Flue step has already completed, Stop can withhold not-yet-started browser work locally but cannot durably record that withholding: a reopen can recover those calls as pending. This cancellation/reopen limitation remains unresolved; the local guard is not a durable cancellation claim. An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a @@ -111,24 +126,43 @@ The text composer remains available. Sending typed text ends Voice mode first, then submits the draft exactly once through the same conversation; a failed handoff restores the draft. Closing the assistant pauses capture and speech before hiding it. Reopening preserves the mounted session in **Paused** state. -**Pause** and **End voice mode** live under **Voice mode actions**, while -**Resume** or **Reconnect** appears as the primary action when applicable. +The dock exposes **Your turn** while canonical audio owns the turn. That action +clears pending input and output, waits for the provider's matching +acknowledgements and response terminal event, and only then opens the +microphone for fresh capture. Its playback menu offers **Repeat question** and +**Read full response**. Full-response replay becomes available once the matching +response and audio output have both finished, enqueues all exact retained +canonical segments in order, and is disabled during capture, submission, +cancellation, pause, and errors. **Repeat question** has the same safety gates +and replays only exact question text carrying Brunch's non-interactive marker; +if the marker is missing, malformed, or does not match finalized prose, the +action stays disabled rather than guessing from the final segment. The browser sends its SDP offer to this app; the server initializes a trusted `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified -Realtime call endpoint. The provider key, model, instructions, tools, language, -and vocabulary policy stay server-side. The session uses semantic VAD with low -eagerness so natural thinking pauses are less likely to end an answer early. - -Realtime is the disposable media plane: it carries continuous microphone and remote audio, detects complete turns, and handles barge-in. Brunch remains the control plane and sole authority for questions, captures, state, completion, and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-question correlation. +Realtime call endpoint. The provider key, model, instructions, language, and +vocabulary policy stay server-side. Realtime exposes no tools, uses +`tool_choice: "none"`, and configures semantic VAD to detect an input boundary +without creating a model response. + +Realtime is the disposable media plane: it carries microphone and remote audio, +detects complete turns, and transcribes input. Brunch remains the control plane +and sole authority for questions, captures, state, completion, and durable +history. The bridge accepts only +`conversation.item.input_audio_transcription.completed` as an answer, ignores +model function arguments, and submits the normalized transcript through +Petrinaut's shared composer path. Connection epoch, item id, and content index +form its stable identity. Duplicate, empty, failed, unavailable, and over-limit +transcripts never submit; recoverable failures leave a not-heard or too-long +notice in the dock. Provisional transcription remains display-only. The bridge waits for the correlated Brunch turn before returning canonical -speech segments to Realtime. It then requests audio with tools disabled and -instructs Realtime to speak only those segments. Generated audio is not a -verbatim record: canonical Brunch text remains visible and authoritative. The -microphone stays active while the interviewer speaks and while Brunch is -working. Speaking over assistant audio interrupts playback automatically; -WebRTC truncates provider-side unheard audio without changing Brunch history. +speech segments to Realtime. It instructs Realtime to speak only those +segments. Generated audio is not a verbatim recording: canonical Brunch text +remains visible and authoritative. Voice is half-duplex: the physical +microphone is closed while the interviewer speaks, while Brunch is working, and +through cancellation. Audio captured before a **Your turn** handoff is +discarded and cannot become a later answer. The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts index 78d0abc3b90..2a77be95877 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts @@ -1,3 +1,4 @@ +import { FlueApiError } from "@flue/sdk"; import { expect, test, vi } from "vitest"; import { @@ -25,12 +26,18 @@ test("delegates one typed message to the supplied Flue conversation", async () = turnId: "turn-1", position: { batch: 1, index: 0 }, }); + await options?.onEvent?.({ + type: "message-completed", + conversationId: "conversation-stable", + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + }); await options?.onEvent?.({ type: "submission-settled", conversationId: "conversation-stable", submissionId: admission.submissionId, outcome: "completed", - position: { batch: 1, index: 1 }, + position: { batch: 1, index: 2 }, }); }); const client = { @@ -43,6 +50,10 @@ test("delegates one typed message to the supplied Flue conversation", async () = { kind: "user", messageId: "user-1" }, admissionListener, ); + const responseCompletedListener = vi.fn(); + tracker.subscribeToResponseMessageCompleted(responseCompletedListener); + const responseStartedListener = vi.fn(); + tracker.subscribeToResponseMessageStarted(responseStartedListener); const onAdmission = vi.fn(); const transport = createBrunchPanelTransport( Promise.resolve(client), @@ -80,6 +91,18 @@ test("delegates one typed message to the supplied Flue conversation", async () = expect(tracker.submissionsForResponse("assistant-1")).toEqual([ "submission-1", ]); + expect(responseStartedListener).toHaveBeenCalledOnce(); + expect(responseStartedListener).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }); + expect(responseCompletedListener).toHaveBeenCalledOnce(); + expect(responseCompletedListener).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + submissionId: "submission-1", + }); expect(onAdmission).toHaveBeenCalledOnce(); expect(onAdmission).toHaveBeenCalledWith(admission); }); @@ -141,15 +164,66 @@ test("matches client-tool admissions once and supports unsubscribe", () => { test("records every submission that wrote a resumed assistant message", () => { const tracker = new BrunchPanelConversationTracker(); - tracker.recordResponse("assistant-1", "submission-1"); - tracker.recordResponse("assistant-1", "submission-continuation"); - tracker.recordResponse("assistant-1", "submission-continuation"); + const responseStartedListener = vi.fn(); + tracker.subscribeToResponseMessageStarted(responseStartedListener); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }); expect(tracker.submissionsForResponse("assistant-1")).toEqual([ "submission-1", "submission-continuation", ]); expect(tracker.submissionsForResponse("assistant-2")).toBeUndefined(); + expect(responseStartedListener.mock.calls).toEqual([ + [ + { + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }, + ], + [ + { + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }, + ], + [ + { + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }, + ], + ]); +}); + +test("publishes Stop immediately and supports unsubscribe", () => { + const tracker = new BrunchPanelConversationTracker(); + const listener = vi.fn(); + const unsubscribedListener = vi.fn(); + tracker.subscribeToStopRequested(listener); + const unsubscribe = tracker.subscribeToStopRequested(unsubscribedListener); + unsubscribe(); + + tracker.recordStopRequested(); + + expect(listener).toHaveBeenCalledOnce(); + expect(unsubscribedListener).not.toHaveBeenCalled(); }); test("settles in-flight submissions before a durable abort can target them", async () => { @@ -254,3 +328,46 @@ test("refuses fixture traffic when the mounted Flue route is unavailable", async }), ).rejects.toThrow("Fixture route unavailable."); }); + +test("publishes a typed admission failure for the exact panel input", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(500, ""); + }); + const tracker = new BrunchPanelConversationTracker(); + const failureListener = vi.fn(); + tracker.subscribeToAdmissionFailure( + { kind: "user", messageId: "voice-realtime:1:item-1:0" }, + failureListener, + ); + const transport = createBrunchPanelTransport( + Promise.resolve({ send } as Pick as FlueClient), + tracker, + ); + + const submission = transport.sendMessages({ + trigger: "submit-message", + chatId: "conversation-stable", + messageId: undefined, + messages: [ + { + id: "voice-realtime:1:item-1:0", + role: "user", + parts: [{ type: "text", text: "One Voice turn." }], + }, + ], + abortSignal: undefined, + }); + + await expect(submission).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }); + expect(failureListener).toHaveBeenCalledOnce(); + expect(failureListener).toHaveBeenCalledWith( + expect.objectContaining({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }), + ); + expect(send).toHaveBeenCalledOnce(); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index 3e289ff7bcb..b8746caea34 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -1,8 +1,14 @@ -import { createFlueChatTransport } from "@hashintel/brunch-agent-transport-aisdk"; +import { + createFlueChatTransport, + FlueChatAdmissionError, +} from "@hashintel/brunch-agent-transport-aisdk"; import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import { sweepOutputSchema } from "../brunch-sweep-output"; -import { brunchClientToolNames } from "./brunch-client-tools"; + +const brunchClientToolNames = new Set([readPetrinautDocToolName]); import type { SweepCapture, @@ -10,7 +16,11 @@ import type { SweepCompletionReport, } from "../brunch-sweep-output"; import type { AgentSendResult, FlueClient } from "@flue/sdk"; -import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; +import type { + FlueChatResponseMessageCompletedEvent, + FlueChatResponseMessageStartedEvent, + FlueChatTransportOptions, +} from "@hashintel/brunch-agent-transport-aisdk"; import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; import type { UIMessageChunk } from "ai"; @@ -23,6 +33,10 @@ export type BrunchPanelAdmissionTarget = Pick< >; export class BrunchPanelConversationTracker { + readonly #admissionFailureSubscriptions = new Set<{ + readonly listener: (error: FlueChatAdmissionError) => void; + readonly target: BrunchPanelAdmissionTarget; + }>(); readonly #admissionSubscriptions = new Set<{ readonly listener: (admission: BrunchPanelAdmission) => void; readonly target: BrunchPanelAdmissionTarget; @@ -36,6 +50,13 @@ export class BrunchPanelConversationTracker { string, AgentSendResult["submissionId"][] >(); + readonly #responseMessageStartedListeners = new Set< + (event: FlueChatResponseMessageStartedEvent) => void + >(); + readonly #responseMessageCompletedListeners = new Set< + (event: FlueChatResponseMessageCompletedEvent) => void + >(); + readonly #stopRequestedListeners = new Set<() => void>(); public recordAdmission(admission: BrunchPanelAdmission): void { if (admission.kind === "user") { @@ -61,15 +82,29 @@ export class BrunchPanelConversationTracker { * all: Voice correlates a reply by membership, whichever side admitted the * continuation. */ - public recordResponse( - messageId: string, - submissionId: AgentSendResult["submissionId"], - ): void { - const recorded = this.#responseSubmissions.get(messageId); + public recordResponse(event: FlueChatResponseMessageStartedEvent): void { + const recorded = this.#responseSubmissions.get(event.messageId); if (recorded === undefined) { - this.#responseSubmissions.set(messageId, [submissionId]); - } else if (!recorded.includes(submissionId)) { - recorded.push(submissionId); + this.#responseSubmissions.set(event.messageId, [event.submissionId]); + } else if (!recorded.includes(event.submissionId)) { + recorded.push(event.submissionId); + } + for (const listener of this.#responseMessageStartedListeners) { + listener(event); + } + } + + public recordResponseMessageCompleted( + event: FlueChatResponseMessageCompletedEvent, + ): void { + for (const listener of this.#responseMessageCompletedListeners) { + listener(event); + } + } + + public recordStopRequested(): void { + for (const listener of this.#stopRequestedListeners) { + listener(); } } @@ -91,6 +126,21 @@ export class BrunchPanelConversationTracker { return submission; } + public recordAdmissionFailure( + target: BrunchPanelAdmissionTarget, + error: FlueChatAdmissionError, + ): void { + for (const subscription of this.#admissionFailureSubscriptions) { + if ( + subscription.target.kind === target.kind && + subscription.target.messageId === target.messageId + ) { + this.#admissionFailureSubscriptions.delete(subscription); + subscription.listener(error); + } + } + } + public submissionForInput( messageId: string, ): AgentSendResult["submissionId"] | undefined { @@ -111,6 +161,34 @@ export class BrunchPanelConversationTracker { this.#admissionSubscriptions.add(subscription); return () => this.#admissionSubscriptions.delete(subscription); } + + public subscribeToAdmissionFailure( + target: BrunchPanelAdmissionTarget, + listener: (error: FlueChatAdmissionError) => void, + ): () => void { + const subscription = { listener, target }; + this.#admissionFailureSubscriptions.add(subscription); + return () => this.#admissionFailureSubscriptions.delete(subscription); + } + + public subscribeToResponseMessageCompleted( + listener: (event: FlueChatResponseMessageCompletedEvent) => void, + ): () => void { + this.#responseMessageCompletedListeners.add(listener); + return () => this.#responseMessageCompletedListeners.delete(listener); + } + + public subscribeToResponseMessageStarted( + listener: (event: FlueChatResponseMessageStartedEvent) => void, + ): () => void { + this.#responseMessageStartedListeners.add(listener); + return () => this.#responseMessageStartedListeners.delete(listener); + } + + public subscribeToStopRequested(listener: () => void): () => void { + this.#stopRequestedListeners.add(listener); + return () => this.#stopRequestedListeners.delete(listener); + } } const formatFailure = (failure: SweepCompletionFailure): string => { @@ -254,14 +332,39 @@ export const createBrunchPanelTransport = ( ...(options?.mapClientToolInput === undefined ? {} : { mapClientToolInput: options.mapClientToolInput }), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), onAdmission: (event) => { tracker.recordAdmission(event); options?.onAdmission?.(event.admission); }, - onResponseMessage: ({ messageId, submissionId }) => - tracker.recordResponse(messageId, submissionId), + onResponseMessage: (event) => tracker.recordResponse(event), + onResponseMessageCompleted: (event) => + tracker.recordResponseMessageCompleted(event), }); - return decorateBrunchStream(await transport.sendMessages(sendOptions)); + try { + return decorateBrunchStream( + await transport.sendMessages(sendOptions), + ); + } catch (error) { + const messageId = + sendOptions.messageId ?? sendOptions.messages.at(-1)?.id; + if ( + error instanceof FlueChatAdmissionError && + messageId !== undefined + ) { + tracker.recordAdmissionFailure( + { + kind: + sendOptions.messageId === undefined + ? "user" + : "client-tool-result", + messageId, + }, + error, + ); + } + throw error; + } })(), ), }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index d61f4bd45bb..878bd25c03e 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -1,17 +1,17 @@ /** * @vitest-environment jsdom */ -import { act, cleanup, render } from "@testing-library/react"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/react"; +import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { brunchClientToolNames } from "./brunch-client-tools"; import { BrunchPanelConversationTracker } from "./brunch-panel-transport"; import { - brunchInteractiveTools, getBrunchVoiceMode, LocalStorageDemoApp, requestFlueStop, @@ -21,12 +21,30 @@ import { crewReservationFixtureId, } from "./prepared-crew-reservation-fixture"; -import type { FlueClient } from "@flue/sdk"; +import type { + AgentConversationObservationSnapshot, + FlueClient, +} from "@flue/sdk"; import type { PetrinautNavigationController } from "@hashintel/petrinaut/react"; +import type { PetrinautAiAssistant } from "@hashintel/petrinaut/ui"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, })); +const flueClientMock = vi.hoisted(() => ({ current: null as unknown })); +const renderedPetrinaut = vi.hoisted(() => ({ aiAssistant: null as unknown })); + +vi.mock("@flue/sdk", () => ({ + createFlueClient: () => flueClientMock.current, +})); + +const brunchPreviewConfig = vi.hoisted(() => ({ + chatEndpoint: "/agents/chat", + isBrunchConfigured: true, +})); +vi.mock("./brunch-preview-config", () => ({ + resolveBrunchPreviewConfig: () => brunchPreviewConfig, +})); const editorProps = vi.hoisted(() => ({ current: null as { @@ -51,6 +69,7 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ }, Petrinaut: (props: Record) => { editorProps.current = props; + renderedPetrinaut.aiAssistant = props.aiAssistant; return null; }, WalkthroughProvider: ({ children }: { children: ReactNode }) => children, @@ -64,44 +83,240 @@ describe("local storage demo Brunch voice integration", () => { test("installs the app-owned voice control for a configured Brunch transport", () => { const config = { available: true as const, connectionTimeoutMs: 15_000 }; - const voiceMode = getBrunchVoiceMode(config); - const control = voiceMode?.({ - canAcceptVoiceInput: true, - conversationId: "petrinaut-preview:net-1", - inputMode: "text", - isAiAssistantOpen: true, - messages: [], - registerVoiceModeControls: vi.fn(() => () => undefined), - reportVoiceSessionState: vi.fn(), - setInputMode: vi.fn(), - setVoiceActive: vi.fn(), - status: "ready", - stop: vi.fn(async () => undefined), - submitText: vi.fn(async () => ({ - kind: "message" as const, - messageId: "message-1", - })), - submitVoiceInput: vi.fn(async () => ({ - kind: "message" as const, - messageId: "voice-message-1", - })), - }); + const tracker = new BrunchPanelConversationTracker(); + const voiceMode = getBrunchVoiceMode(config, tracker); + const renderControl = () => + voiceMode?.({ + canAcceptVoiceInput: true, + conversationId: "petrinaut-preview:net-1", + inputMode: "text", + isAiAssistantOpen: true, + messages: [], + registerVoiceModeControls: vi.fn(() => () => undefined), + reportVoiceSessionState: vi.fn(), + setInputMode: vi.fn(), + setVoiceActive: vi.fn(), + status: "ready", + stop: vi.fn(async () => undefined), + submitText: vi.fn(async () => ({ + kind: "message" as const, + messageId: "message-1", + })), + submitVoiceInput: vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-message-1", + })), + }); + const control = renderControl(); expect(isValidElement(control)).toBe(true); if (!isValidElement(control)) { throw new Error("Expected the configured composer control to render."); } - expect(control).toMatchObject({ - props: { config }, - type: VoiceInterviewControl, + const failureListener = vi.fn(); + const responseCompletedListener = vi.fn(); + const responseStartedListener = vi.fn(); + const stopListener = vi.fn(); + const target = { kind: "user" as const, messageId: "voice-turn-1" }; + const controlProps = control.props as { + config: typeof config; + resolveInputSubmission: (messageId: string) => string | undefined; + resolveResponseSubmission: ( + messageId: string, + ) => readonly string[] | undefined; + subscribeToAdmission: ( + admissionTarget: typeof target, + listener: (submissionId: string) => void, + ) => () => void; + subscribeToAdmissionFailure: ( + admissionTarget: typeof target, + listener: (error: FlueChatAdmissionError) => void, + ) => () => void; + subscribeToResponseMessageCompleted: ( + listener: typeof responseCompletedListener, + ) => () => void; + subscribeToResponseMessageStarted: ( + listener: typeof responseStartedListener, + ) => () => void; + subscribeToStopRequested: (listener: () => void) => () => void; + }; + expect(control.type).toBe(VoiceInterviewControl); + expect(controlProps.config).toBe(config); + + const rerenderedControl = renderControl(); + expect(isValidElement(rerenderedControl)).toBe(true); + if (!isValidElement(rerenderedControl)) { + throw new Error("Expected the configured composer control to rerender."); + } + const rerenderedControlProps = + rerenderedControl.props as typeof controlProps; + expect(rerenderedControlProps.resolveInputSubmission).toBe( + controlProps.resolveInputSubmission, + ); + expect(rerenderedControlProps.resolveResponseSubmission).toBe( + controlProps.resolveResponseSubmission, + ); + expect(rerenderedControlProps.subscribeToAdmission).toBe( + controlProps.subscribeToAdmission, + ); + expect(rerenderedControlProps.subscribeToAdmissionFailure).toBe( + controlProps.subscribeToAdmissionFailure, + ); + expect(rerenderedControlProps.subscribeToResponseMessageCompleted).toBe( + controlProps.subscribeToResponseMessageCompleted, + ); + expect(rerenderedControlProps.subscribeToResponseMessageStarted).toBe( + controlProps.subscribeToResponseMessageStarted, + ); + expect(rerenderedControlProps.subscribeToStopRequested).toBe( + controlProps.subscribeToStopRequested, + ); + + const unsubscribe = controlProps.subscribeToAdmissionFailure( + target, + failureListener, + ); + const unsubscribeFromStop = + controlProps.subscribeToStopRequested(stopListener); + const unsubscribeFromResponseCompleted = + controlProps.subscribeToResponseMessageCompleted( + responseCompletedListener, + ); + const unsubscribeFromResponseStarted = + controlProps.subscribeToResponseMessageStarted(responseStartedListener); + const admissionError = new FlueChatAdmissionError({ kind: "ambiguous" }); + + tracker.recordAdmissionFailure(target, admissionError); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }); + tracker.recordResponseMessageCompleted({ + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + submissionId: "submission-1", }); + tracker.recordStopRequested(); + + expect(failureListener).toHaveBeenCalledWith(admissionError); + expect(responseStartedListener).toHaveBeenCalledOnce(); + expect(responseCompletedListener).toHaveBeenCalledOnce(); + expect(stopListener).toHaveBeenCalledOnce(); + unsubscribe(); + unsubscribeFromResponseCompleted(); + unsubscribeFromResponseStarted(); + unsubscribeFromStop(); }); - test("registers only interactive widgets that answer declared client tools", () => { - expect(brunchInteractiveTools.length).toBeGreaterThan(0); - for (const tool of brunchInteractiveTools) { - expect(brunchClientToolNames.has(tool.toolName)).toBe(true); - } + test("registers no brunch_ask tool in the production Brunch preview", async () => { + renderedPetrinaut.aiAssistant = null; + flueClientMock.current = { + observe: () => ({ + close: vi.fn(), + getSnapshot: () => ({ phase: "absent" }), + refresh: vi.fn(), + subscribe: () => () => undefined, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render( + {}} search={{}} />, + ); + await waitFor(() => expect(renderedPetrinaut.aiAssistant).not.toBeNull()); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + expect(aiAssistant.requestStop).toBeTypeOf("function"); + expect(aiAssistant.interactiveTools).toEqual([]); + expect( + aiAssistant.interactiveTools?.some( + ({ toolName }) => toolName === "brunch_ask", + ), + ).toBe(false); + + rendered.unmount(); + vi.unstubAllGlobals(); + }); + + test("keeps durable Flue Stop distinct from local playback cancellation", async () => { + renderedPetrinaut.aiAssistant = null; + let snapshot: AgentConversationObservationSnapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [], + messages: [], + }, + offset: "offset-before-stop", + phase: "live" as const, + error: undefined, + }; + const listeners = new Set<() => void>(); + const localPlaybackCancellation = vi.spyOn( + OpenAIRealtimeSession.prototype, + "cancelOutput", + ); + const abort = vi.fn(async () => { + snapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [ + { submissionId: "submission-stop", outcome: "aborted" as const }, + ], + messages: [], + }, + offset: "offset-after-stop", + phase: "live" as const, + error: undefined, + }; + for (const listener of listeners) listener(); + return { aborted: true }; + }); + flueClientMock.current = { + abort, + observe: () => ({ + close: vi.fn(), + getSnapshot: () => snapshot, + refresh: vi.fn(), + subscribe: (listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render( + {}} search={{}} />, + ); + await waitFor(() => + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant).requestStop, + ).toBeTypeOf("function"), + ); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + await expect(aiAssistant.requestStop?.()).resolves.toBe("stop-requested"); + expect(abort).toHaveBeenCalledOnce(); + expect(localPlaybackCancellation).not.toHaveBeenCalled(); + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant) + .renderComposerControl, + ).toBeUndefined(); + + rendered.unmount(); + localPlaybackCancellation.mockRestore(); + vi.unstubAllGlobals(); }); test("correlates the existing Brunch transport request", () => { @@ -137,6 +352,8 @@ describe("local storage demo Brunch voice integration", () => { const abort = vi.fn(async () => ({ aborted: true })); const client = { abort } as Pick as FlueClient; const tracker = new BrunchPanelConversationTracker(); + const stopListener = vi.fn(); + tracker.subscribeToStopRequested(stopListener); let admit: (() => void) | undefined; void tracker.trackSubmission( new Promise((resolve) => { @@ -145,6 +362,7 @@ describe("local storage demo Brunch voice integration", () => { ); const stop = requestFlueStop(Promise.resolve(client), tracker); + expect(stopListener).toHaveBeenCalledOnce(); await Promise.resolve(); await Promise.resolve(); expect(abort).not.toHaveBeenCalled(); @@ -346,11 +564,12 @@ describe("local storage demo prepared fixture", () => { afterEach(() => { cleanup(); editorProps.current = null; + brunchPreviewConfig.isBrunchConfigured = true; }); test("neither advertises nor opens the fixture while Brunch is unconfigured", () => { - // These tests run without `VITE_BRUNCH_CHAT_ENDPOINT`, so there is no Flue - // client to prepare the fixture conversation. Opening the fixture URL + brunchPreviewConfig.isBrunchConfigured = false; + // With Brunch disabled there is no Flue client to prepare the fixture conversation. Opening the fixture URL // anyway once left the banner on "preparing" forever with every send // unavailable; the URL now falls back to the ordinary per-net demo. seedStoredNet(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 3ed2a1dcec6..6f030d99c9e 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -26,7 +26,6 @@ import { import { DefaultChatTransport, Petrinaut, - type PetrinautAiInteractiveTool, type PetrinautAiMessage, type PetrinautAiStopResult, type PetrinautAiVoiceMode, @@ -46,10 +45,10 @@ import { type OpenAIVoiceConfig, VoiceInterviewControl, } from "../voice-interview/voice-interview-control"; -import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { getOrCreateBrunchConversationId } from "./brunch-conversation-id"; import { BrunchPanelConversationTracker, + type BrunchPanelAdmissionTarget, createBrunchPanelTransport, createUnavailableBrunchPanelTransport, } from "./brunch-panel-transport"; @@ -110,30 +109,43 @@ export const getBrunchVoiceMode = ( config: OpenAIVoiceConfig | null | undefined, tracker?: BrunchPanelConversationTracker, settlements?: readonly FlueConversationSettlement[], -): PetrinautAiVoiceMode | undefined => - config - ? (context: PetrinautAiVoiceModeContext) => ( - - tracker?.submissionForInput(messageId) - } - resolveResponseSubmission={(messageId) => - tracker?.submissionsForResponse(messageId) - } - subscribeToAdmission={ - tracker === undefined - ? undefined - : (target, listener) => - tracker.subscribeToAdmission(target, ({ admission }) => - listener(admission.submissionId), - ) - } - /> - ) - : undefined; +): PetrinautAiVoiceMode | undefined => { + if (!config) return undefined; + + const resolveInputSubmission = tracker?.submissionForInput.bind(tracker); + const resolveResponseSubmission = + tracker?.submissionsForResponse.bind(tracker); + const subscribeToResponseMessageCompleted = + tracker?.subscribeToResponseMessageCompleted.bind(tracker); + const subscribeToResponseMessageStarted = + tracker?.subscribeToResponseMessageStarted.bind(tracker); + const subscribeToStopRequested = + tracker?.subscribeToStopRequested.bind(tracker); + const subscribeToAdmission = + tracker === undefined + ? undefined + : (target: BrunchPanelAdmissionTarget, listener: (id: string) => void) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ); + const subscribeToAdmissionFailure = + tracker?.subscribeToAdmissionFailure.bind(tracker); + + return (context: PetrinautAiVoiceModeContext) => ( + + ); +}; const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => createJsonDocHandle({ @@ -144,11 +156,6 @@ const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => const brunchPrincipal = getOrCreateBrunchPrincipal(); -/** Every widget here must answer a tool named in `brunchClientToolNames`. */ -export const brunchInteractiveTools: readonly PetrinautAiInteractiveTool[] = [ - brunchAskInteractiveTool, -]; - const stockChatTransport = new DefaultChatTransport({ api: brunchPreviewConfig.chatEndpoint, headers: () => ({ @@ -180,6 +187,7 @@ export const requestFlueStop = async ( clientPromise: Promise>, tracker: BrunchPanelConversationTracker, ): Promise => { + tracker.recordStopRequested(); const client = await clientPromise; await tracker.settleInFlightSubmissions(); const result = await client.abort(); @@ -220,64 +228,6 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ fallbackNet: net, }); -type FlueChatHistory = ReturnType; - -const errorStatus = (error: Error | undefined): number | undefined => { - if ( - error !== undefined && - "status" in error && - typeof error.status === "number" - ) { - return error.status; - } - return undefined; -}; - -const BrunchConversationStatus = ({ - error, - latestSettlement, - phase, - refresh, -}: Pick< - FlueChatHistory, - "error" | "latestSettlement" | "phase" | "refresh" ->) => { - if (phase === undefined) return null; - - const label = - phase === "loading" - ? "Loading Brunch conversation…" - : phase === "connecting" - ? "Reconnecting to Brunch…" - : phase === "absent" - ? "New Brunch conversation" - : phase === "error" - ? errorStatus(error) === 401 || errorStatus(error) === 403 - ? "Brunch access was denied." - : "Brunch conversation unavailable." - : phase === "closed" - ? "Brunch conversation closed." - : latestSettlement?.outcome === "aborted" - ? "Last Brunch response stopped." - : latestSettlement?.outcome === "failed" - ? "Last Brunch response failed." - : "Brunch conversation ready."; - - return ( - - {label} - {phase === "error" && ( - <> - {" "} - - - )} - - ); -}; - /** * The demo's own palette command, registered beside Petrinaut's: picking it * in the palette starts a fresh net. @@ -658,7 +608,7 @@ export const LocalStorageDemoApp = ({ () => ({ ...(conversationId === null ? {} : { conversationId }), canClearMessages: flueClientPromise === null, - interactiveTools: brunchInteractiveTools, + interactiveTools: [], transport: petrinautAiChatTransport, ...(flueClientPromise === null ? {} @@ -666,18 +616,6 @@ export const LocalStorageDemoApp = ({ requestStop: () => requestFlueStop(flueClientPromise, conversationTracker), }), - ...(flueClientPromise === null - ? {} - : { - renderComposerControl: () => ( - - ), - }), messages: flueClientPromise === null ? currentNetId @@ -718,11 +656,7 @@ export const LocalStorageDemoApp = ({ conversationId, currentNetId, flueClientPromise, - flueHistory.error, - flueHistory.latestSettlement, flueHistory.messages, - flueHistory.phase, - flueHistory.refresh, petrinautAiChatTransport, setAiMessagesByNetId, ], diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.test.ts index 4c0c6f5e94a..a5ac7378027 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.test.ts @@ -66,6 +66,12 @@ describe("prepared crew-reservation fixture", () => { expect(preparedCrewReservationWorkpiece).toContain( "Exactly one dispatch crew", ); + expect(preparedCrewReservationWorkpiece).toContain( + "requires explicit true-user confirmation", + ); + expect(preparedCrewReservationWorkpiece).not.toContain( + "- Final inspection reserves the sole available dispatch crew.", + ); expect(preparedCrewReservationWorkpiece).toContain( "timing, failure modes, and recovery behavior remain unresolved", ); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.ts index b5c9f9f0630..50b5182536f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/prepared-crew-reservation-fixture.ts @@ -28,21 +28,21 @@ export const preparedCrewReservationWorkpiece = [ "# Final inspection and dispatch workpiece", "", "## Purpose and posture", - "Maintain the narrow batch path from final inspection to dispatch readiness and test one evidence-backed correction against the live Petrinaut document.", + "Maintain the narrow batch path from final inspection to dispatch readiness and test one evidence-backed decision against the live Petrinaut document.", "", "## Operational account", "- A batch that is ready enters final inspection.", - "- Final inspection reserves the sole available dispatch crew.", - "- Sign-off releases that crew and makes the batch ready for dispatch.", + "- The prepared topology returns the sole dispatch crew at sign-off.", + "- Whether final inspection reserves that crew is an unconfirmed hypothesis; changing the workpiece or net requires explicit true-user confirmation.", "", "## Quantity and resource policy", - "Exactly one dispatch crew is available in this fixture. Starting final inspection consumes that one available crew; sign-off returns it.", + "Exactly one dispatch crew is available in this fixture. Revision zero does not establish whether starting final inspection consumes it; the prepared topology currently returns it at sign-off.", "", "## Current Petrinaut correspondence", - "The prepared non-empty net contains the batch path and the crew return from sign-off. It deliberately lacks the standard weight-1 input arc from `Dispatch crew available` to `Start final inspection`.", + "The prepared non-empty net contains the batch path and the crew return from sign-off. The standard weight-1 input arc from `Dispatch crew available` to `Start final inspection` is absent while the reservation policy remains unconfirmed.", "", "## Explicit unknowns", - "Inspection and sign-off timing, failure modes, and recovery behavior remain unresolved.", + "Crew reservation awaits true-user confirmation. Inspection and sign-off timing, failure modes, and recovery behavior remain unresolved.", "", "## Claim boundary", "This prepared revision is test-authored diagnostic material. It is not model-produced evidence and does not establish capture provenance, behavioral execution, or broad projection quality.", diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts index 1b4d12a59ce..1ff5228fbdf 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts @@ -104,6 +104,90 @@ test("exposes the canonical settlement index for Voice correlation", async () => ]); }); +test("preserves every persisted Voice tool origin across hydration and reopen", async () => { + const harness = createObservationHarness({ + conversation: { + conversationId: "conversation-1", + settlements: [], + messages: [ + { + id: "assistant-voice-tools", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: "client-tool-result" }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }, + offset: "offset-voice", + phase: "live", + error: undefined, + }); + const firstOpen = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + + await waitFor(() => expect(firstOpen.result.current.ready).toBe(true)); + expect(firstOpen.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); + firstOpen.unmount(); + + const reopened = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + await waitFor(() => expect(reopened.result.current.ready).toBe(true)); + expect(reopened.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); +}); + test("asks nothing of the generic chat route, which keeps no history", () => { const { result } = renderHook(() => useFlueChatHistory(null, "conversation-1"), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts index c259215eab8..2701945d5b8 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts @@ -1,8 +1,8 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; - -import { brunchClientToolNames } from "./brunch-client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import type { AgentConversationObservation, @@ -15,6 +15,7 @@ import type { import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; const noSettlements: readonly FlueConversationSettlement[] = []; +const brunchClientToolNames = new Set([readPetrinautDocToolName]); /** * The observed canonical conversation together with the durable-stream offset @@ -40,6 +41,7 @@ const projectPetrinautMessages = ( snapshotToUiMessages(conversation, { clientToolNames, ...(mapClientToolInput === undefined ? {} : { mapClientToolInput }), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }) as PetrinautAiMessage[]; export const useFlueChatHistory = ( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index dc12cda21c6..50807c4ba97 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -1,9 +1,8 @@ import { describe, expect, test } from "vitest"; -import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; - import { hashCanonicalSpeechText, + selectCanonicalSpeech, selectCanonicalSpeechSegments, } from "./canonical-speech"; @@ -90,7 +89,7 @@ describe("canonical speech selection", () => { ]); }); - test("selects one exact validated brunch_ask question", () => { + test("does not treat structured tool input as canonical speech", () => { const messages = [ { id: "assistant-ask", @@ -99,21 +98,21 @@ describe("canonical speech selection", () => { { type: "dynamic-tool", toolCallId: "ask-1", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: "Which operator confirms the batch?" }, }, { type: "dynamic-tool", toolCallId: "ask-malformed", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: 42 }, }, { type: "dynamic-tool", toolCallId: "ask-submitted", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "output-available", input: { question: "Do not repeat an answered question." }, output: { answer: "Already answered." }, @@ -129,20 +128,128 @@ describe("canonical speech selection", () => { }, ] satisfies PetrinautAiMessage[]; - const selected = select(messages); - const contentHash = hashCanonicalSpeechText( - "Which operator confirms the batch?", - ); - expect(selected).toEqual([ + expect(select(messages)).toEqual([]); + }); + + test("selects an exact marked question separately from full-response text", () => { + const question = "Which operator confirms the batch?"; + const selection = selectCanonicalSpeech([ { - contentHash, - id: `canonical-speech:assistant-ask:ask-1:${contentHash}`, - messageId: "assistant-ask", - partId: "ask-1", - source: "brunch-ask", - text: "Which operator confirms the batch?", + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { + type: "text", + text: `The batch is ready. ${question} I can explain the choices.`, + state: "done", + }, + ], }, ]); + + expect(selection.segments.map(({ text }) => text)).toEqual([ + `The batch is ready. ${question} I can explain the choices.`, + ]); + expect(selection.questionSegment).toEqual({ + contentHash: hashCanonicalSpeechText(question), + id: `canonical-speech:assistant-question:question%3Atool-question-1:${hashCanonicalSpeechText(question)}`, + messageId: "assistant-question", + partId: "question:tool-question-1", + source: "assistant-question", + text: question, + }); + }); + + test.each([ + { + name: "missing exact finalized prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "A different question appears in the response.", + state: "done" as const, + }, + ], + }, + { + name: "only provisional prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "streaming" as const, + }, + ], + }, + { + name: "blank marker identity", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: " ", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "done" as const, + }, + ], + }, + ])("rejects a question marker with $name", ({ parts }) => { + expect( + selectCanonicalSpeech([ + { + id: "assistant-invalid-question", + role: "assistant", + parts, + }, + ]).questionSegment, + ).toBeUndefined(); + }); + + test("does not correlate a marker to text from another assistant message", () => { + const question = "Which operator confirms the batch?"; + + expect( + selectCanonicalSpeech([ + { + id: "assistant-marker", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + ], + }, + { + id: "assistant-text", + role: "assistant", + parts: [{ type: "text", text: question, state: "done" }], + }, + ]).questionSegment, + ).toBeUndefined(); }); test("uses stable source identity plus an exact-text fingerprint", () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index b99a0afdf30..fd466e1448d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -1,7 +1,7 @@ import { - ASK_TOOL_NAME, - parseBrunchAskInput, -} from "@hashintel/brunch-agent/client-tools"; + BRUNCH_QUESTION_DATA_NAME, + parseBrunchQuestionData, +} from "@hashintel/brunch-agent/question-marker"; import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; @@ -15,7 +15,7 @@ export interface CanonicalSpeechSegment { readonly id: string; readonly messageId: string; readonly partId: string; - readonly source: "assistant-text" | "brunch-ask"; + readonly source: "assistant-question" | "assistant-text"; /** * Every Flue submission that wrote to this segment's message: the one that * started it plus any client-tool continuation projected back onto it. @@ -46,16 +46,28 @@ const createSegment = ( }; }; -export const selectCanonicalSpeechSegments = ( +export interface CanonicalSpeechSelection { + readonly questionSegment?: CanonicalSpeechSegment; + readonly segments: CanonicalSpeechSegment[]; +} + +export const selectCanonicalSpeech = ( messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => { +): CanonicalSpeechSelection => { const segments: CanonicalSpeechSegment[] = []; + let questionSegment: CanonicalSpeechSegment | undefined; for (const message of messages) { if (message.role !== "assistant") { continue; } + const finalizedTexts = message.parts.flatMap((part) => + part.type === "text" && part.state !== "streaming" && part.text.trim() + ? [part.text] + : [], + ); + for (const [partIndex, part] of message.parts.entries()) { if ( part.type === "text" && @@ -70,32 +82,36 @@ export const selectCanonicalSpeechSegments = ( part.text, ), ); - continue; } + } - if ( - part.type !== "dynamic-tool" || - part.toolName !== ASK_TOOL_NAME || - part.state !== "input-available" - ) { - continue; + const questionMarkers = message.parts.flatMap((part) => { + if (part.type !== `data-${BRUNCH_QUESTION_DATA_NAME}`) { + return []; } - try { - const input = parseBrunchAskInput(part.input); - segments.push( - createSegment( - message.id, - part.toolCallId, - "brunch-ask", - input.question, - ), - ); - } catch { - // Malformed tool inputs remain visible as tool errors; they are not spoken. - } + const marker = parseBrunchQuestionData(part.data); + + return marker && + finalizedTexts.some((text) => text.includes(marker.question)) + ? [marker] + : []; + }); + const latestQuestionMarker = questionMarkers.at(-1); + + if (latestQuestionMarker) { + questionSegment = createSegment( + message.id, + `question:${latestQuestionMarker.toolCallId}`, + "assistant-question", + latestQuestionMarker.question, + ); } } - return segments; + return { questionSegment, segments }; }; + +export const selectCanonicalSpeechSegments = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 70f7bc4088f..245058d31d2 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -36,7 +36,7 @@ const canonicalSegment = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); @@ -221,7 +221,7 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); - test("keeps the microphone active through playback and reports automatic interruption", async () => { + test("keeps the microphone closed and rejects audio detected during playback", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); @@ -242,31 +242,325 @@ describe("OpenAIRealtimeSession", () => { item_id: "item-user", type: "input_audio_buffer.speech_started", }); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + channel.receive({ + content_index: 0, + item_id: "item-user", + transcript: "Assistant echo must not submit.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect(harness.events).not.toContainEqual( + expect.objectContaining({ itemId: "item-user", type: "completed" }), + ); + expect(harness.events).not.toContainEqual( + expect.objectContaining({ + itemId: "item-user", + type: "input-speech-started", + }), + ); + }); + + test("rejects an accepted input item whose transcript completes after output starts", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + channel.receive({ + audio_start_ms: 80, + item_id: "item-before-output", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + delta: "This started before output", + item_id: "item-before-output", + type: "conversation.item.input_audio_transcription.delta", + }); + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "This started before output", + type: "partial", + }); + + harness.session.speakCanonical([ + canonicalSegment("ask-1", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-canonical"); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + channel.receive({ + content_index: 0, + item_id: "item-before-output", + transcript: "This completed too late.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-output", + ), + ).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + }); + + test("invalidates accepted input before requesting canonical speech output", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + let microphoneEnabledWhenResponseRequested: boolean | undefined; + channel.send.mockImplementation((payload: string) => { + if (JSON.parse(payload).type === "response.create") { + microphoneEnabledWhenResponseRequested = + harness.localTracks[0]!.enabled; + } + }); + + channel.receive({ + audio_start_ms: 80, + item_id: "item-before-request", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + delta: "This started before canonical speech", + item_id: "item-before-request", + type: "conversation.item.input_audio_transcription.delta", + }); + + harness.session.speakCanonical([ + canonicalSegment("ask-request", "What happens next?"), + ]); + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "canonical-speech-requested" }), + ); + expect(microphoneEnabledWhenResponseRequested).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "This completed before output started.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-request", + ), + ).toBe(false); + + const handoff = harness.session.cancelOutput(); + let handoffSettled = false; + void handoff.then(() => { + handoffSettled = true; + }); + authorizeLatestSpeechResponse(channel, "response-before-output"); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response: { + id: "response-before-output", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await Promise.resolve(); + + expect(handoffSettled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response_id: "response-before-output", + type: "output_audio_buffer.cleared", + }); + await handoff; expect(harness.localTracks[0]!.enabled).toBe(true); - expect(harness.events).toEqual( - expect.arrayContaining([ - { - connectionEpoch: 1, - responseId: "response-canonical", - speechRequestId: "canonical-1-1", - type: "output-started", - }, - { - connectionEpoch: 1, - itemId: "item-user", - type: "input-speech-started", - }, - { + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ + audio_start_ms: 160, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This is fresh after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect( + harness.events.filter((event) => event.type === "completed"), + ).toEqual([ + { + key: { connectionEpoch: 1, - responseId: "response-canonical", - type: "output-interrupted", + contentIndex: 0, + itemId: "item-after-handoff", }, - ]), - ); + text: "This is fresh after the handoff.", + type: "completed", + }, + ]); }); - test("parses streamed tool arguments and the completed GA response output", async () => { + test("restores only the latest microphone preference after playback", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.session.speakCanonical([ + canonicalSegment("ask-1", "What happens next?"), + ]); + const channel = harness.channels[0]!; + authorizeLatestSpeechResponse(channel, "response-canonical"); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + harness.session.setMicrophoneEnabled(false); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + }); + + test("settles idle cancellation after input clear without response-scoped output", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + + const cancellation = harness.session.cancelOutput(); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + await Promise.resolve(); + + expect(settled).toBe(true); + expect(harness.localTracks[0]!.enabled).toBe(true); + expect(sentEvents(channel)).toEqual([ + { type: "input_audio_buffer.clear" }, + { type: "output_audio_buffer.clear" }, + ]); + }); + + test("waits for input, output, and response settlement before completing handoff", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + channel.receive({ + audio_start_ms: 40, + item_id: "item-before-handoff", + type: "input_audio_buffer.speech_started", + }); + harness.session.speakCanonical([ + canonicalSegment("ask-handoff", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-handoff"); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.started", + }); + + const cancellation = Promise.resolve(harness.session.cancelOutput()); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + expect(sentEvents(channel).slice(-3)).toEqual([ + { type: "input_audio_buffer.clear" }, + expect.objectContaining({ + response_id: "response-handoff", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); + channel.receive({ + content_index: 0, + item_id: "item-before-handoff", + transcript: "This began too early.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.cleared", + }); + await Promise.resolve(); + expect(settled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response: { + id: "response-handoff", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-handoff", + ), + ).toBe(false); + + channel.receive({ + audio_start_ms: 120, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This began after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "This began after the handoff.", + type: "completed", + }); + }); + + test("never exposes model function arguments as user input", async () => { const harness = createHarness(); await harness.session.connect(); const channel = harness.channels[0]!; @@ -287,6 +581,8 @@ describe("OpenAIRealtimeSession", () => { response_id: "response-tool", type: "response.function_call_arguments.delta", }); + expect(harness.events).toEqual([]); + channel.receive({ response: { id: "response-tool", @@ -305,76 +601,7 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ - { - callId: "call-1", - connectionEpoch: 1, - delta: '{"answer":"Approved"}', - itemId: "item-function", - responseId: "response-tool", - type: "tool-arguments-delta", - }, - { - arguments: '{"answer":"Approved"}', - callId: "call-1", - connectionEpoch: 1, - itemId: "item-function", - name: "continue_interview", - responseId: "response-tool", - type: "tool-arguments-done", - }, - { - connectionEpoch: 1, - responseId: "response-tool", - status: "completed", - type: "response-terminal", - }, - ]); - - harness.session.completeFunctionCall("call-1", [ - canonicalSegment("ask-2", "Who acts next?"), - ]); - const [functionOutput, responseCreate] = sentEvents(channel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ response_text: ["Who acts next?"] }), - }, - }); - expect(responseCreate).toMatchObject({ - type: "response.create", - response: { - instructions: - "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything.", - output_modalities: ["audio"], - parallel_tool_calls: false, - tool_choice: "none", - tools: [], - }, - }); - }); - - test("closes a stopped function call without requesting speech", async () => { - const harness = createHarness(); - await harness.session.connect(); - const channel = harness.channels[0]!; - const sentBefore = sentEvents(channel).length; - - harness.session.completeFunctionCallWithoutResponse( - "call-stopped", - "aborted", - ); - - expect(sentEvents(channel).slice(sentBefore)).toEqual([ - { - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-stopped", - output: JSON.stringify({ response_text: [], outcome: "aborted" }), - }, - }, + expect.objectContaining({ code: "invalid-response", type: "error" }), ]); }); @@ -383,18 +610,30 @@ describe("OpenAIRealtimeSession", () => { await harness.session.connect(); const channel = harness.channels[0]!; - harness.session.completeFunctionCall("call-exact", [ + harness.session.speakCanonical([ canonicalSegment("ask-exact", " Exact Brunch text.\n"), ]); - expect(sentEvents(channel)[0]).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-exact", - output: JSON.stringify({ - response_text: [" Exact Brunch text.\n"], - }), + expect(sentEvents(channel)[0]).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + input: [ + { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: JSON.stringify({ + response_text: [" Exact Brunch text.\n"], + }), + }, + ], + }, + ], + tool_choice: "none", + tools: [], }, }); const sentCount = sentEvents(channel).length; @@ -450,6 +689,166 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("keeps the microphone closed when an earlier stop follows a queued response request", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + harness.session.speakCanonical([ + canonicalSegment("early", "First canonical segment."), + ]); + authorizeLatestSpeechResponse(channel, "response-early"); + channel.receive({ + response_id: "response-early", + type: "output_audio_buffer.started", + }); + harness.session.speakCanonical([ + canonicalSegment("follow-on", "Second canonical segment."), + ]); + + channel.receive({ + response: { + id: "response-early", + output: [], + status: "completed", + }, + type: "response.done", + }); + expect(harness.events.at(-1)).toMatchObject({ + speechRequestId: "canonical-1-2", + type: "canonical-speech-requested", + }); + authorizeLatestSpeechResponse(channel, "response-follow-on"); + channel.receive({ + response: { + id: "response-follow-on", + output: [], + status: "completed", + }, + type: "response.done", + }); + + channel.receive({ + response_id: "response-early", + type: "output_audio_buffer.stopped", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response_id: "response-follow-on", + type: "output_audio_buffer.started", + }); + channel.receive({ + response_id: "response-follow-on", + type: "output_audio_buffer.stopped", + }); + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + responseId: "response-follow-on", + speechRequestId: "canonical-1-2", + status: "completed", + type: "response-terminal", + }); + expect(harness.localTracks[0]!.enabled).toBe(true); + }); + + test("releases active canonical ownership after acknowledged cancellation", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + harness.session.speakCanonical([ + canonicalSegment("cancelled", "Cancel this canonical segment."), + ]); + authorizeLatestSpeechResponse(channel, "response-cancelled"); + + const cancellation = harness.session.cancelOutput(); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response: { + id: "response-cancelled", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await Promise.resolve(); + + expect(settled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response_id: "response-cancelled", + type: "output_audio_buffer.cleared", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + }); + + test("waits for output clear when cancelling generated audio before playback", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + harness.session.speakCanonical([ + canonicalSegment("generated", "Generated canonical segment."), + ]); + authorizeLatestSpeechResponse(channel, "response-generated"); + channel.receive({ + response: { + id: "response-generated", + output: [], + status: "completed", + }, + type: "response.done", + }); + + const cancellation = harness.session.cancelOutput(); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + await Promise.resolve(); + + expect(settled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response_id: "response-generated", + type: "output_audio_buffer.cleared", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + }); + + test("force-settles cancellation when the provider fails before output clear", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const cancellation = harness.session.cancelOutput(); + + harness.channels[0]!.receive({ + error: { message: "private provider detail" }, + type: "error", + }); + + await expect(cancellation).resolves.toBeUndefined(); + expect(harness.localTracks[0]!.enabled).toBe(false); + expect(harness.localTracks[0]!.stop).toHaveBeenCalledOnce(); + expect(harness.events.at(-1)).toMatchObject({ + code: "invalid-response", + type: "error", + }); + }); + test("cancels canonical speech before the response starts", async () => { const harness = createHarness(); await harness.session.connect(); @@ -459,7 +858,7 @@ describe("OpenAIRealtimeSession", () => { ]); const responseCreate = sentEvents(channel)[0]!; - harness.session.cancelOutput(); + void harness.session.cancelOutput(); expect( sentEvents(channel).filter(({ type }) => type === "response.cancel"), @@ -503,6 +902,49 @@ describe("OpenAIRealtimeSession", () => { expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); }); + test("ignores leftover output after a cancelled response is already done", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + harness.session.speakCanonical([ + canonicalSegment("question", "Canonical question"), + ]); + const responseCreate = sentEvents(channel)[0]!; + + void harness.session.cancelOutput(); + + channel.receive({ + response: { + id: "response-canonical", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + channel.receive({ + response: { + id: "response-canonical", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + + expect(harness.events).not.toContainEqual( + expect.objectContaining({ type: "error" }), + ); + expect(harness.events).not.toContainEqual( + expect.objectContaining({ type: "output-started" }), + ); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + expect(sentEvents(channel).at(-1)).toEqual({ + type: "output_audio_buffer.clear", + }); + }); + test("retries a correlated canonical response after the active response ends", async () => { const harness = createHarness(); await harness.session.connect(); @@ -606,7 +1048,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -650,7 +1092,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -772,12 +1214,31 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); - test("treats transcripts as display-only and never closes capture", async () => { + test("requires a matching speech-start boundary before exposing transcripts", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); const channel = harness.channels[0]!; + channel.receive({ + content_index: 0, + delta: "Missing boundary", + item_id: "item-without-boundary", + type: "conversation.item.input_audio_transcription.delta", + }); + channel.receive({ + content_index: 0, + item_id: "item-without-boundary", + transcript: "This must stay rejected.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(harness.events).toEqual([]); + + channel.receive({ + audio_start_ms: 100, + item_id: "item-user", + type: "input_audio_buffer.speech_started", + }); channel.receive({ content_index: 0, delta: "The supervisor", @@ -792,6 +1253,11 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }, { key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-user" }, text: "The supervisor", @@ -806,11 +1272,67 @@ describe("OpenAIRealtimeSession", () => { expect(harness.localTracks[0]!.enabled).toBe(true); }); + test("does not retroactively accept a completion that precedes its speech boundary", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + + channel.receive({ + content_index: 0, + item_id: "item-reordered", + transcript: "This completed before its boundary.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ + audio_start_ms: 100, + item_id: "item-reordered", + type: "input_audio_buffer.speech_started", + }); + + expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-reordered", + type: "input-speech-started", + }, + ]); + }); + + test("does not reuse a speech boundary from a previous connection epoch", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.channels[0]!.receive({ + audio_start_ms: 100, + item_id: "reused-item", + type: "input_audio_buffer.speech_started", + }); + + await harness.session.disconnect(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.events.length = 0; + harness.channels[1]!.receive({ + content_index: 0, + item_id: "reused-item", + transcript: "This lacks a current-epoch boundary.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect(harness.events).toEqual([]); + }); + test("keeps the duplex session alive when optional input transcription fails", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); + harness.channels[0]!.receive({ + audio_start_ms: 100, + item_id: "item-user", + type: "input_audio_buffer.speech_started", + }); harness.channels[0]!.receive({ content_index: 0, error: { message: "private provider detail" }, @@ -819,6 +1341,11 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }, { key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-user" }, type: "transcription-failed", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 99b66aac151..0051a56fdb9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -19,13 +19,6 @@ export interface OpenAIRealtimeTranscriptKey { readonly itemId: string; } -interface RealtimeToolEventIdentity { - readonly callId: string; - readonly connectionEpoch: number; - readonly itemId: string; - readonly responseId: string; -} - export type OpenAIRealtimeSessionEvent = | { readonly key: OpenAIRealtimeTranscriptKey; @@ -71,18 +64,10 @@ export type OpenAIRealtimeSessionEvent = | { readonly connectionEpoch: number; readonly responseId: string; + readonly speechRequestId?: string; readonly status: "cancelled" | "completed" | "failed" | "incomplete"; readonly type: "response-terminal"; } - | (RealtimeToolEventIdentity & { - readonly delta: string; - readonly type: "tool-arguments-delta"; - }) - | (RealtimeToolEventIdentity & { - readonly arguments: string; - readonly name: string; - readonly type: "tool-arguments-done"; - }) | { readonly code: VoiceErrorCode; readonly message: string; @@ -135,6 +120,10 @@ type PendingClientEvent = }; type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; +type ResponseTerminalStatus = Extract< + OpenAIRealtimeSessionEvent, + { type: "response-terminal" } +>["status"]; const CANONICAL_RESPONSE_INSTRUCTIONS = "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything."; @@ -192,20 +181,25 @@ const waitForAbort = ( }; export class OpenAIRealtimeSession { + readonly #acceptedInputItemIds = new Set(); readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #activeResponseIds = new Set(); readonly #listeners = new Set(); readonly #authorizedResponseIds = new Set(); readonly #cancelledCanonicalResponseIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); + readonly #cancelOutputAwaitingRequestIds = new Set(); + readonly #cancelOutputAwaitingResponseIds = new Set(); readonly #canonicalResponseIds = new Set(); readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = []; readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); readonly #pendingSpeechRequests = new Map(); + readonly #playbackOverlappingInputItemIds = new Set(); readonly #remoteStreams = new Set(); readonly #speechRequestIds = new Map(); readonly #speechTimings = new Map(); + readonly #terminalCanonicalResponseIds = new Set(); readonly #transcriptionTimings = new Map(); #abortController: AbortController | null = null; #activeEpoch: number | null = null; @@ -214,6 +208,10 @@ export class OpenAIRealtimeSession { #connected = false; #connectedAt: number | null = null; #clientEventSequence = 0; + #cancelOutputAwaitingInputBufferClear = false; + #cancelOutputAwaitingOutputBufferClear = false; + #cancelOutputPromise: Promise | null = null; + #cancelOutputResolve: (() => void) | null = null; #connectionRequestId: string | null = null; #dataChannel: RTCDataChannel | null = null; #epoch = 0; @@ -223,6 +221,7 @@ export class OpenAIRealtimeSession { #meterHasSample = false; #meterLevel = 0; #meterSamples: Uint8Array | null = null; + #microphoneRequested = false; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; #remoteAudio: RemoteAudio | null = null; @@ -393,95 +392,83 @@ export class OpenAIRealtimeSession { } public setMicrophoneEnabled(enabled: boolean): void { - if (!this.#microphoneTrack) { - return; - } - const isEnabled = enabled && this.#connected; - this.#microphoneTrack.enabled = isEnabled; - if (isEnabled) { - this.#startMeter(); - } else { - this.#stopMeter(); - } + this.#microphoneRequested = enabled && this.#connected; + this.#syncMicrophoneTrack(); } public speakCanonical(segments: CanonicalSpeechSegment[]): void { this.#requestCanonicalSpeech(segments, true); } - public completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + public cancelOutput(): Promise { + if (!this.#connected || this.#dataChannel?.readyState !== "open") { + return Promise.resolve(); } - const responseText = this.#canonicalResponseText(segments); - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: responseText }), - }, - }); - this.#requestCanonicalSpeech(segments, false); - } - - /** - * Close a Realtime function call whose Brunch turn settled without a reply. - * The call output records the settlement so the model does not wait on it, - * and no speech is requested: a stopped turn has no canonical text to speak. - */ - public completeFunctionCallWithoutResponse( - callId: string, - outcome: "aborted" | "failed", - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + if (this.#cancelOutputPromise) { + return this.#cancelOutputPromise; } - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: [], outcome }), - }, + + const cancelOutputPromise = new Promise((resolve) => { + this.#cancelOutputResolve = resolve; }); - } + this.#cancelOutputPromise = cancelOutputPromise; + this.#cancelOutputAwaitingInputBufferClear = true; + this.#cancelOutputAwaitingOutputBufferClear = + this.#authorizedResponseIds.size > 0 || + this.#terminalCanonicalResponseIds.size > 0 || + this.#speakingResponseId !== null; + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); - public cancelOutput(): void { - if (!this.#connected || this.#dataChannel?.readyState !== "open") { - return; - } + try { + this.#send({ type: "input_audio_buffer.clear" }); - for (const request of this.#canonicalSpeechQueue.splice(0)) { - this.#cancelPendingSpeechRequest(request.speechRequestId); - } + for (const request of this.#canonicalSpeechQueue.splice(0)) { + this.#cancelPendingSpeechRequest(request.speechRequestId); + } - if (this.#responseCreateEventId !== null) { - const pendingEvent = this.#pendingClientEvents.get( - this.#responseCreateEventId, - ); - if (pendingEvent?.kind === "response-create") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, + if (this.#responseCreateEventId !== null) { + const pendingEvent = this.#pendingClientEvents.get( + this.#responseCreateEventId, ); + if (pendingEvent?.kind === "response-create") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + this.#cancelOutputAwaitingRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } } - } - for (const responseId of this.#canonicalResponseIds) { - if ( - this.#activeResponseIds.has(responseId) && - !this.#cancelledCanonicalResponseIds.has(responseId) - ) { - this.#cancelledCanonicalResponseIds.add(responseId); - this.#cancelOutputResponse(responseId); + for (const responseId of this.#canonicalResponseIds) { + if ( + this.#activeResponseIds.has(responseId) && + !this.#cancelledCanonicalResponseIds.has(responseId) + ) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + this.#cancelledCanonicalResponseIds.add(responseId); + this.#cancelResponse(responseId); + } } + + this.#send({ type: "output_audio_buffer.clear" }); + } catch { + this.#finishOutputCancellation(true); + this.#handleConnectionFailure("network", "speech"); } + + this.#finishOutputCancellation(); + return cancelOutputPromise; } #cancelOutputResponse(responseId: string): void { + if (this.#cancelOutputPromise) { + this.#cancelOutputAwaitingOutputBufferClear = true; + } this.#cancelResponse(responseId); this.#send({ type: "output_audio_buffer.clear" }); } @@ -599,6 +586,11 @@ export class OpenAIRealtimeSession { request, responseTerminalSequence: this.#responseTerminalSequence, }); + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); try { this.#send({ event_id: eventId, @@ -615,6 +607,7 @@ export class OpenAIRealtimeSession { } catch (error) { this.#responseCreateEventId = null; this.#pendingClientEvents.delete(eventId); + this.#syncMicrophoneTrack(); throw error; } } @@ -643,6 +636,12 @@ export class OpenAIRealtimeSession { this.#handleResponseDone(parsed, connectionEpoch); return; } + if (parsed.type === "input_audio_buffer.cleared") { + this.#acceptedInputItemIds.clear(); + this.#cancelOutputAwaitingInputBufferClear = false; + this.#finishOutputCancellation(); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -651,23 +650,22 @@ export class OpenAIRealtimeSession { if (parsed.type === "input_audio_buffer.speech_started") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_start_ms) === null) return; + if (this.#speakingResponseId || !this.#microphoneTrack?.enabled) { + this.#playbackOverlappingInputItemIds.add(itemId); + return; + } + this.#acceptedInputItemIds.add(itemId); this.#emit({ connectionEpoch, itemId, type: "input-speech-started", }); - if (this.#speakingResponseId) { - this.#emit({ - connectionEpoch, - responseId: this.#speakingResponseId, - type: "output-interrupted", - }); - } return; } if (parsed.type === "input_audio_buffer.speech_stopped") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_end_ms) === null) return; + if (this.#playbackOverlappingInputItemIds.has(itemId)) return; this.#emit({ connectionEpoch, itemId, @@ -677,15 +675,12 @@ export class OpenAIRealtimeSession { } if ( parsed.type === "output_audio_buffer.started" || - parsed.type === "output_audio_buffer.stopped" + parsed.type === "output_audio_buffer.stopped" || + parsed.type === "output_audio_buffer.cleared" ) { this.#handleOutputBufferEvent(parsed, connectionEpoch); return; } - if (parsed.type === "response.function_call_arguments.delta") { - this.#handleToolEvent(parsed, connectionEpoch); - return; - } if ( parsed.type === "conversation.item.input_audio_transcription.delta" || parsed.type === "conversation.item.input_audio_transcription.completed" || @@ -707,6 +702,9 @@ export class OpenAIRealtimeSession { if (metadata?.petrinaut_kind !== "canonical-speech" || !speechRequestId) { return; } + if (this.#cancelOutputAwaitingRequestIds.delete(speechRequestId)) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + } this.#completeResponseCreateEvent(speechRequestId); this.#canonicalResponseIds.add(responseId); if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) { @@ -762,6 +760,10 @@ export class OpenAIRealtimeSession { ) { this.#pendingClientEvents.delete(sourceEventId); this.#completedResponseCancelEventIds.delete(sourceEventId); + if (pendingEvent?.kind === "response-cancel") { + this.#cancelOutputAwaitingResponseIds.delete(pendingEvent.responseId); + this.#finishOutputCancellation(); + } return; } @@ -780,7 +782,11 @@ export class OpenAIRealtimeSession { pendingEvent.request.speechRequestId, ) ) { + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.speechRequestId, + ); this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); + this.#finishOutputCancellation(); return; } this.#canonicalSpeechQueue.unshift(pendingEvent.request); @@ -813,77 +819,54 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } + const terminalStatus = status as ResponseTerminalStatus; this.#responseTerminalSequence += 1; this.#activeResponseIds.delete(responseId); + this.#cancelOutputAwaitingResponseIds.delete(responseId); + this.#finishOutputCancellation(); this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; + const speechRequestId = this.#speechRequestIds.get(responseId); + const terminalEvent = { + connectionEpoch, + responseId, + ...(speechRequestId === undefined ? {} : { speechRequestId }), + status: terminalStatus, + type: "response-terminal" as const, + }; - if (this.#cancelledCanonicalResponseIds.delete(responseId)) { - this.#emit({ - connectionEpoch, - responseId, - status, - type: "response-terminal", - }); + if (this.#cancelledCanonicalResponseIds.has(responseId)) { + if (this.#speakingResponseId === responseId) { + this.#emit({ + connectionEpoch, + responseId, + type: "output-interrupted", + }); + } + this.#emit(terminalEvent); this.#finishSpeech(responseId, "request-aborted"); this.#resumeCanonicalSpeechQueue(); return; } - if (status === "completed") { + if (terminalStatus === "completed") { const output = response.output; if (!Array.isArray(output)) { this.#handleConnectionFailure("invalid-response", "connection"); return; } - const functionCalls = output - .map(asRecord) - .filter( - (item): item is Record => - item?.type === "function_call", - ); - if ( - functionCalls.length > 1 || - (functionCalls.length > 0 && this.#canonicalResponseIds.has(responseId)) - ) { + if (output.some((item) => asRecord(item)?.type === "function_call")) { this.#handleConnectionFailure("invalid-response", "connection"); return; } - for (const item of functionCalls) { - const argumentsJson = nonEmptyString(item.arguments); - const callId = nonEmptyString(item.call_id); - const itemId = nonEmptyString(item.id); - const name = nonEmptyString(item.name); - if ( - !argumentsJson || - !callId || - !itemId || - !name || - (item.status !== undefined && item.status !== "completed") - ) { - this.#handleConnectionFailure("invalid-response", "connection"); - return; - } - this.#emit({ - arguments: argumentsJson, - callId, - connectionEpoch, - itemId, - name, - responseId, - type: "tool-arguments-done", - }); + if (this.#authorizedResponseIds.has(responseId)) { + this.#terminalCanonicalResponseIds.add(responseId); } - this.#emit({ - connectionEpoch, - responseId, - status, - type: "response-terminal", - }); + this.#emit(terminalEvent); this.#resumeCanonicalSpeechQueue(); return; } - if (status === "cancelled") { + if (terminalStatus === "cancelled") { if (this.#speakingResponseId === responseId) { this.#emit({ connectionEpoch, @@ -891,22 +874,12 @@ export class OpenAIRealtimeSession { type: "output-interrupted", }); } - this.#emit({ - connectionEpoch, - responseId, - status, - type: "response-terminal", - }); + this.#emit(terminalEvent); this.#finishSpeech(responseId, "request-aborted"); this.#resumeCanonicalSpeechQueue(); return; } - this.#emit({ - connectionEpoch, - responseId, - status, - type: "response-terminal", - }); + this.#emit(terminalEvent); if (this.#authorizedResponseIds.has(responseId)) { this.#finishSpeech(responseId, "invalid-response"); } @@ -941,6 +914,9 @@ export class OpenAIRealtimeSession { if (!responseId) return; if (event.type === "output_audio_buffer.started") { if (this.#cancelledCanonicalResponseIds.has(responseId)) { + if (this.#cancelOutputPromise) { + this.#cancelOutputAwaitingOutputBufferClear = true; + } this.#send({ type: "output_audio_buffer.clear" }); return; } @@ -949,7 +925,12 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); this.#speakingResponseId = responseId; + this.#syncMicrophoneTrack(); const speechRequestId = this.#speechRequestIds.get(responseId); if (!speechRequestId) { this.#handleConnectionFailure("invalid-response", "connection"); @@ -964,35 +945,29 @@ export class OpenAIRealtimeSession { return; } const wasSpeaking = this.#speakingResponseId === responseId; + const wasCleared = event.type === "output_audio_buffer.cleared"; this.#finishSpeech( responseId, - this.#cancelledCanonicalResponseIds.has(responseId) + wasCleared || this.#cancelledCanonicalResponseIds.has(responseId) ? "request-aborted" : undefined, ); + if (wasCleared && this.#cancelOutputAwaitingOutputBufferClear) { + for (const terminalResponseId of this.#terminalCanonicalResponseIds) { + this.#finishSpeech(terminalResponseId, "request-aborted"); + } + } if (wasSpeaking) { - this.#emit({ connectionEpoch, responseId, type: "output-stopped" }); + this.#emit({ + connectionEpoch, + responseId, + type: wasCleared ? "output-interrupted" : "output-stopped", + }); + } + if (wasCleared && this.#cancelOutputAwaitingOutputBufferClear) { + this.#cancelOutputAwaitingOutputBufferClear = false; + this.#finishOutputCancellation(); } - } - - #handleToolEvent( - event: Record, - connectionEpoch: number, - ): void { - const callId = nonEmptyString(event.call_id); - const itemId = nonEmptyString(event.item_id); - const responseId = nonEmptyString(event.response_id); - const outputIndex = nonNegativeInteger(event.output_index); - if (!callId || !itemId || !responseId || outputIndex === null) return; - if (typeof event.delta !== "string") return; - this.#emit({ - callId, - connectionEpoch, - delta: event.delta, - itemId, - responseId, - type: "tool-arguments-delta", - }); } #handleTranscriptEvent( @@ -1003,9 +978,29 @@ export class OpenAIRealtimeSession { const contentIndex = nonNegativeInteger(event.content_index); if (!itemId || contentIndex === null) return; const key = { connectionEpoch, contentIndex, itemId }; + const overlapsPlayback = + this.#playbackOverlappingInputItemIds.has(itemId) || + !this.#acceptedInputItemIds.has(itemId); + if (overlapsPlayback) { + if ( + event.type === + "conversation.item.input_audio_transcription.completed" || + event.type === "conversation.item.input_audio_transcription.failed" + ) { + this.#finishTranscription( + itemId, + event.type === "conversation.item.input_audio_transcription.failed" + ? "invalid-response" + : undefined, + ); + this.#acceptedInputItemIds.delete(itemId); + } + return; + } this.#startTranscription(itemId); if (event.type === "conversation.item.input_audio_transcription.failed") { this.#finishTranscription(itemId, "invalid-response"); + this.#acceptedInputItemIds.delete(itemId); this.#emit({ key, type: "transcription-failed" }); return; } @@ -1018,6 +1013,7 @@ export class OpenAIRealtimeSession { event.type === "conversation.item.input_audio_transcription.completed" ) { this.#finishTranscription(itemId); + this.#acceptedInputItemIds.delete(itemId); } this.#emit({ key, @@ -1056,9 +1052,34 @@ export class OpenAIRealtimeSession { } this.#speechRequestIds.delete(responseId); this.#authorizedResponseIds.delete(responseId); + this.#terminalCanonicalResponseIds.delete(responseId); if (this.#speakingResponseId === responseId) { this.#speakingResponseId = null; } + this.#syncMicrophoneTrack(); + } + + #finishOutputCancellation(force = false): void { + if ( + !this.#cancelOutputPromise || + (!force && + (this.#cancelOutputAwaitingInputBufferClear || + this.#cancelOutputAwaitingOutputBufferClear || + this.#cancelOutputAwaitingRequestIds.size > 0 || + this.#cancelOutputAwaitingResponseIds.size > 0)) + ) { + return; + } + + const resolve = this.#cancelOutputResolve; + this.#cancelOutputPromise = null; + this.#cancelOutputResolve = null; + this.#cancelOutputAwaitingInputBufferClear = false; + this.#cancelOutputAwaitingOutputBufferClear = false; + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); + this.#syncMicrophoneTrack(); + resolve?.(); } #emit(event: OpenAIRealtimeSessionEvent): void { @@ -1239,6 +1260,26 @@ export class OpenAIRealtimeSession { this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); } + #syncMicrophoneTrack(): void { + if (!this.#microphoneTrack) { + return; + } + const enabled = + this.#microphoneRequested && + this.#connected && + this.#cancelOutputPromise === null && + this.#authorizedResponseIds.size === 0 && + this.#canonicalSpeechQueue.length === 0 && + this.#responseCreateEventId === null && + this.#speakingResponseId === null; + this.#microphoneTrack.enabled = enabled; + if (enabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } + } + #stopMeter(): void { if (this.#meterFrame === null) return; this.#dependencies.cancelAnimationFrame(this.#meterFrame); @@ -1328,20 +1369,26 @@ export class OpenAIRealtimeSession { ); } this.#transcriptionTimings.clear(); + this.#acceptedInputItemIds.clear(); this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); this.#cancelledSpeechRequestIds.clear(); + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); this.#canonicalSpeechQueue.length = 0; this.#completedResponseCancelEventIds.clear(); this.#pendingClientEvents.clear(); this.#pendingSpeechRequests.clear(); + this.#playbackOverlappingInputItemIds.clear(); this.#speechTimings.clear(); this.#speechRequestIds.clear(); + this.#terminalCanonicalResponseIds.clear(); this.#authorizedResponseIds.clear(); this.#canonicalResponseIds.clear(); this.#responseCreateEventId = null; this.#responseTerminalSequence = 0; this.#speakingResponseId = null; + this.#microphoneRequested = false; this.#waitingForResponseTerminal = false; this.#activeEpoch = null; this.#connected = false; @@ -1388,6 +1435,7 @@ export class OpenAIRealtimeSession { this.#mediaStream = null; } this.#microphoneTrack = null; + this.#finishOutputCancellation(true); } #waitForDataChannelOpen( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index b65fc510f9c..3f2596e316d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import { createRealtimeSubmissionId, RealtimeBrunchBridge, @@ -7,26 +9,63 @@ import { } from "./realtime-brunch-bridge"; import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; const segment = ( id: string, text: string, - source: CanonicalSpeechSegment["source"] = "brunch-ask", + submissionId?: string, ): CanonicalSpeechSegment => ({ contentHash: "fnv1a32:12345678", id, messageId: `message-${id}`, partId: id, - source, + source: "assistant-text", + ...(submissionId === undefined ? {} : { submissionIds: [submissionId] }), + text, +}); + +const transcriptKey = ( + connectionEpoch: number, + itemId = "user-item-1", + contentIndex = 0, +): OpenAIRealtimeTranscriptKey => ({ connectionEpoch, contentIndex, itemId }); + +const completedTranscript = ( + connectionEpoch: number, + text = "The supervisor approves it.", + itemId = "user-item-1", + contentIndex = 0, +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId, contentIndex), text, + type: "completed", +}); + +const failedTranscript = ( + connectionEpoch: number, + itemId = "user-item-1", +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId), + type: "transcription-failed", +}); + +const completedResponseMessage = ( + messageId: string, + submissionId: string, + index: number, +) => ({ + messageId, + position: { batch: 1, index }, + submissionId, }); const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { - completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; @@ -39,10 +78,14 @@ const createHarness = () => { ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"] - >(async () => ({ - kind: "interactive-tool", - toolCallId: "ask-current", - })); + >(async (input) => { + input.onAdmission("submission-voice-1"); + return { + kind: "message", + messageId: input.id, + submissionId: "submission-voice-1", + }; + }); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -59,797 +102,808 @@ const createHarness = () => { }; }; -const toolDelta = ( - connectionEpoch: number, - delta: string, -): Extract => ({ - callId: "call-1", - connectionEpoch, - delta, - itemId: "function-item-1", - responseId: "response-1", - type: "tool-arguments-delta", -}); - -const toolDone = ( - connectionEpoch: number, - argumentsJson = '{"answer":"The supervisor approves it."}', -): Extract => ({ - arguments: argumentsJson, - callId: "call-1", - connectionEpoch, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done", -}); - -const responseTerminal = ( - connectionEpoch: number, - status: "cancelled" | "completed" | "failed" | "incomplete", - responseId = "response-1", -): Extract => ({ - connectionEpoch, - responseId, - status, - type: "response-terminal", -}); +const startReady = ( + harness: ReturnType, + connectionEpoch = 3, +): void => { + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + harness.bridge.start(connectionEpoch); +}; describe("RealtimeBrunchBridge", () => { - test("speaks the current canonical turn without replaying history", () => { + test("rehydrates settled canonical speech without submission or playback", () => { const harness = createHarness(); - const historical = segment( - "history", - "Do not replay this.", - "assistant-text", - ); - const preamble = { - ...segment("preamble", "Thanks. One more question.", "assistant-text"), - messageId: "message-current-turn", - }; - const question = { - ...segment("ask-current", "What happens after approval?"), - messageId: "message-current-turn", - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [historical, preamble, question], + canonicalSegments: [ + segment("settled", "Already delivered.", "submission-settled"), + ], status: "ready", }); - harness.bridge.start(4); + harness.bridge.start(9); - expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([ - preamble, - question, - ]); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events).toEqual([]); }); - test("rehydrates the settled Voice turn without resubmission or playback", () => { + test("does not dispatch canonical updates that arrive during output cancellation", () => { const harness = createHarness(); - const settledResponse = { - ...segment( - "settled-response", - "This canonical response was already delivered.", - "assistant-text", - ), - submissionIds: ["submission-settled"], - }; + startReady(harness); + const cancelledSegment = segment( + "cancelled-update", + "Do not speak this cancelled update.", + ); + harness.bridge.cancelPendingSpeech(); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [settledResponse], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [cancelledSegment], + status: "streaming", }); - harness.bridge.start(9); - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - expect(harness.events).toEqual([]); - }); - test("streams and validates one tool call, preserves ask correlation, and waits for canonical Brunch output", async () => { - const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); + harness.bridge.completeTurnHandoff(); + const laterSegment = segment("later-update", "Speak this later update."); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], + canonicalSegments: [cancelledSegment, laterSegment], status: "ready", }); - harness.bridge.start(7); - harness.session.speakCanonical.mockClear(); - harness.emit(toolDelta(7, '{"answer":"The supervisor')); - harness.emit(toolDelta(7, ' approves it."}')); - harness.emit(toolDone(7)); + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([laterSegment]); + }); + + test("submits only a completed transcript through the user admission target", async () => { + const harness = createHarness(); + startReady(harness, 7); + const key = transcriptKey(7); + + harness.emit({ key, text: "The supervisor", type: "partial" }); + harness.emit({ + arguments: '{"answer":"Fabricated answer"}', + callId: "legacy-call", + connectionEpoch: 7, + itemId: "legacy-item", + name: "continue_interview", + responseId: "legacy-response", + type: "tool-arguments-done", + } as unknown as OpenAIRealtimeSessionEvent); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + + harness.emit(completedTranscript(7, " The supervisor\napproves it. ")); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); + const deliveryId = createRealtimeSubmissionId(key); expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(7, "call-1"), + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, text: "The supervisor approves it.", }), ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + answer: "The supervisor approves it.", + deliveryId, + type: "submission-started", + }); + expect(JSON.stringify(harness.events)).not.toContain("Fabricated answer"); + }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", + test("rejects unfinished input invalidated by output and accepts fresh input", async () => { + const harness = createHarness(); + startReady(harness); + + harness.emit({ + connectionEpoch: 3, + itemId: "item-before-output", + type: "input-speech-started", }); - const acknowledgement = segment( - "acknowledgement", - "Thanks. I have recorded that.", - "assistant-text", - ); - const nextQuestion = segment( - "ask-next", - "Who is informed next?", - "brunch-ask", + harness.emit({ + connectionEpoch: 3, + responseId: "response-output", + speechRequestId: "speech-output", + type: "output-started", + }); + harness.emit( + completedTranscript(3, "This completed too late.", "item-before-output"), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, acknowledgement, nextQuestion], - status: "ready", + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", }); + harness.emit({ + connectionEpoch: 3, + responseId: "response-output", + type: "output-stopped", + }); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-output", + type: "input-speech-started", + }); + harness.emit(completedTranscript(3, "This is fresh.", "item-after-output")); await vi.waitFor(() => - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [acknowledgement, nextQuestion], - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), + ); + + harness.emit(completedTranscript(3, "Stale replay.", "item-before-output")); + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); }); - test("emits the real admission before composer submission completes", async () => { + test("rejects unfinished input as soon as canonical speech is requested", async () => { const harness = createHarness(); - let finishSubmission: (() => void) | undefined; - harness.submitInterviewAnswer.mockImplementationOnce(async () => { - await new Promise((resolve) => { - finishSubmission = resolve; - }); - return { - kind: "interactive-tool", - toolCallId: "ask-current", - }; + startReady(harness); + + harness.emit({ + connectionEpoch: 3, + itemId: "item-before-request", + type: "input-speech-started", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + harness.emit( + completedTranscript( + 3, + "This completed before output started.", + "item-before-request", + ), + ); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + harness.emit( + completedTranscript( + 3, + "The stale item cannot recover authority.", + "item-before-request", + ), ); - const submission = harness.submitInterviewAnswer.mock.calls[0]?.[0]; - expect(submission).toBeDefined(); - submission?.onAdmission("submission-early"); - submission?.onAdmission("submission-early"); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "submission-started" }), - { - callId: "call-1", - submissionId: "submission-early", - type: "submission-admitted", - }, - ]); + harness.bridge.completeTurnHandoff(); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emit( + completedTranscript(3, "This is fresh.", "item-after-handoff"), + ); - finishSubmission?.(); await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - const unrelated = segment("unrelated", "Do not select this."); - const correlated = { - ...segment("correlated", "Select this response."), - submissionIds: ["submission-early"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated, correlated], - status: "ready", - }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [correlated], + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), ); - harness.bridge.stop(); - submission?.onAdmission("submission-stale"); - expect( - harness.events.filter(({ type }) => type === "submission-admitted"), - ).toHaveLength(1); }); - test("admits one finalized Realtime answer through Flue once", async () => { + test("retains follow-on output ownership across an earlier response stop", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "message", - messageId: "message-kickoff", - submissionId: "submission-voice-1", + startReady(harness); + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-early", + type: "canonical-speech-requested", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", + harness.emit({ + connectionEpoch: 3, + responseId: "response-early", + speechRequestId: "speech-early", + type: "output-started", }); - harness.bridge.start(7); - - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); - - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "user", - messageId: createRealtimeSubmissionId(7, "call-1"), - }, - id: createRealtimeSubmissionId(7, "call-1"), - text: "Battery charger workflow", - }), - ), - ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", + harness.emit({ + connectionEpoch: 3, + responseId: "response-early", + status: "completed", + type: "response-terminal", }); - const unrelated = { - ...segment("unrelated", "Do not speak this response."), - submissionIds: ["submission-other"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated], - status: "ready", + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-follow-on", + type: "canonical-speech-requested", }); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - const firstQuestion = { - ...segment("ask-first", "What starts the battery charger workflow?"), - submissionIds: ["submission-voice-1"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated, firstQuestion], - status: "ready", + harness.emit({ + connectionEpoch: 3, + responseId: "response-follow-on", + speechRequestId: "speech-follow-on", + status: "completed", + type: "response-terminal", + }); + harness.emit({ + connectionEpoch: 3, + responseId: "response-early", + type: "output-stopped", }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstQuestion], + harness.emit({ + connectionEpoch: 3, + itemId: "item-during-follow-on", + type: "input-speech-started", + }); + harness.emit( + completedTranscript( + 3, + "This overlaps pending follow-on output.", + "item-during-follow-on", + ), ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); - }); - test("rejects a submission result that disagrees with the transport admission", async () => { - const harness = createHarness(); - harness.submitInterviewAnswer.mockImplementationOnce(async (input) => { - input.onAdmission("submission-early"); - return { - kind: "message", - messageId: input.id, - submissionId: "submission-other", - }; + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", + + harness.bridge.completeTurnHandoff(); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-handoff", + type: "input-speech-started", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + harness.emit( + completedTranscript(3, "This is fresh.", "item-after-handoff"), + ); await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), ); }); - test("requires a correlated Brunch busy cycle before accepting new canonical segments", async () => { + test("releases pending output ownership when cancellation settles before playback", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", + startReady(harness); + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-cancelled", + type: "canonical-speech-requested", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + + harness.emit({ + connectionEpoch: 3, + responseId: "response-cancelled", + speechRequestId: "speech-cancelled", + status: "cancelled", + type: "response-terminal", + } as OpenAIRealtimeSessionEvent); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-cancellation", + type: "input-speech-started", + }); + harness.emit( + completedTranscript( + 3, + "This follows acknowledged cancellation.", + "item-after-cancellation", + ), + ); + await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - const unrelated = segment( - "unrelated", - "An unrelated canonical update.", - "assistant-text", + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + text: "This follows acknowledged cancellation.", + }), ); + }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", - }); + test("derives stable delivery identity from epoch, item, and content index", () => { + expect( + createRealtimeSubmissionId(transcriptKey(12, "item/with spaces", 4)), + ).toBe("voice-realtime:12:item%2Fwith%20spaces:4"); + }); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + test("submits duplicate completed transcript events exactly once", async () => { + const harness = createHarness(); + startReady(harness); + const transcript = completedTranscript(3); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question, unrelated], - status: "submitted", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", - }); + harness.emit(transcript); + harness.emit(transcript); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [unrelated], + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); + expect(harness.events).toContainEqual({ + reason: "duplicate", + type: "transcript-rejected", + }); }); - test("rejects streamed arguments whose response or item identity changes", async () => { + test.each([ + ["", "empty"], + [" \n\t ", "empty"], + ["a".repeat(32_001), "over-limit"], + ] as const)( + "rejects an invalid completed transcript as %s", + (text, reason) => { + const harness = createHarness(); + startReady(harness); + + harness.emit(completedTranscript(3, text)); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toEqual([{ reason, type: "transcript-rejected" }]); + }, + ); + + test("rejects a failed transcript and accepts the next keyed turn", async () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Answer"}')); - - harness.emit({ - ...toolDone(3, '{"answer":"Answer"}'), - responseId: "response-2", - }); - await Promise.resolve(); + startReady(harness); - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + harness.emit(failedTranscript(3, "failed-item")); expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + { reason: "failed", type: "transcript-rejected" }, ]); + + harness.emit(completedTranscript(3, "Retried answer.", "retry-item")); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "Retried answer." }), + ), + ); }); - test("rejects concurrent argument streams before either can submit", async () => { + test("rejects completed transcripts while the shared submission path is unavailable", () => { const harness = createHarness(); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", }); harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"First"}')); - harness.emit({ - ...toolDelta(3, '{"answer":"Second"}'), - callId: "call-2", - itemId: "function-item-2", - }); + harness.emit(completedTranscript(3)); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), + { reason: "unavailable", type: "transcript-rejected" }, ]); }); - test("discards a cancelled argument stream without poisoning the next answer", async () => { + test("ignores transcripts from an inactive connection epoch", () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Cancelled"}')); - harness.emit(responseTerminal(3, "cancelled")); - harness.emit(toolDone(3, '{"answer":"Cancelled"}')); + startReady(harness, 2); - harness.emit({ - ...toolDelta(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); - harness.emit({ - ...toolDone(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); + harness.emit(completedTranscript(1, "Stale answer")); + harness.emit(failedTranscript(1, "stale-failed")); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toEqual([]); + }); + + test("correlates the admitted submission with exact canonical response segments", async () => { + const harness = createHarness(); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(3, "call-2"), - text: "Accepted", - }), + const input = harness.submitInterviewAnswer.mock.calls[0]?.[0]; + expect(input).toBeDefined(); + + input?.onAdmission("submission-voice-1"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "submitted", + }); + const unrelated = segment( + "unrelated", + "Do not speak this.", + "submission-other", ); - expect(harness.events).not.toContainEqual( - expect.objectContaining({ type: "error" }), + const correlated = segment( + "correlated", + "Speak this canonical response.", + "submission-voice-1", ); - }); - - test("rejects an unfinished argument stream from a completed response", () => { - const harness = createHarness(); + const correlatedQuestion: CanonicalSpeechSegment = { + ...segment( + "correlated-question", + "Which operator confirms the batch?", + "submission-voice-1", + ), + messageId: correlated.messageId, + source: "assistant-question", + }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], + canonicalSegments: [unrelated, correlated], + questionSegment: correlatedQuestion, status: "ready", }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Incomplete')); - - harness.emit(responseTerminal(3, "completed")); - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.events.map(({ type }) => type)).toEqual([ + "submission-started", + "submission-admitted", + "submission-accepted", + "canonical-text-ready", + "submission-settled", + "canonical-response-ready", ]); + expect(harness.events.at(-1)).toEqual({ + deliveryId, + questionSegment: correlatedQuestion, + segments: [correlated], + type: "canonical-response-ready", + }); }); - test("rejects duplicate, stale, overlapping, and malformed calls without another Brunch submission", async () => { + test("speaks a completed canonical segment while chat remains streaming and settles separately", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(2); - - harness.emit(toolDone(1)); - harness.emit(toolDelta(2, '{"answer":"The supervisor approves it."}')); - harness.emit(toolDone(2)); - harness.emit(toolDone(2)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - - harness.emit({ - ...toolDone(2, '{"answer":"Overlapping"}'), - callId: "call-2", - itemId: "function-item-2", + const correlated = segment( + "correlated", + "Speak this committed response.", + "submission-voice-1", + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [correlated], + status: "streaming", }); - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); - expect(harness.events.at(-1)).toMatchObject({ type: "error" }); - }); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(correlated.messageId, "submission-voice-1", 1), + ); + + expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.events.map(({ type }) => type)).not.toContain( + "submission-settled", + ); + expect(harness.events.map(({ type }) => type)).not.toContain( + "canonical-response-ready", + ); - test.each([ - ["wrong tool", { ...toolDone(3), name: "invent_question" }], - ["invalid JSON", toolDone(3, "not-json")], - ["extra property", toolDone(3, '{"answer":"Valid","extra":true}')], - ["empty answer", toolDone(3, '{"answer":" "}')], - ])("rejects %s arguments", async (_label, event) => { - const harness = createHarness(); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], + canonicalSegments: [correlated], status: "ready", }); - harness.bridge.start(3); - harness.emit(event); - await Promise.resolve(); - - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.events.slice(-2).map(({ type }) => type)).toEqual([ + "submission-settled", + "canonical-response-ready", ]); }); - test("rejects a composer result that does not match the pending brunch_ask", async () => { + test("does not let a completed reasoning-only or tool-only step authorize later text", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "interactive-tool", - toolCallId: "another-ask", + startReady(harness, 7); + harness.emit(completedTranscript(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", }); + + const messageId = "reasoning-or-tool-message"; + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(messageId, "submission-voice-1", 1), + ); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + harness.bridge.notifyResponseMessageStarted({ + messageId, + position: { batch: 1, index: 2 }, + submissionId: "submission-voice-1", + }); + const laterText = { + ...segment( + "not-yet-completed", + "Do not let the earlier completion authorize this text.", + ), + messageId, + submissionIds: ["submission-voice-1"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [laterText], + status: "streaming", }); - harness.bridge.start(5); - harness.emit(toolDone(5)); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - await vi.waitFor(() => - expect(harness.events.at(-1)).toMatchObject({ type: "error" }), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(messageId, "submission-voice-1", 3), ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([laterText]); }); - test("records first canonical text before the turn settles", async () => { + test("speaks later continuation segments once and in canonical order", async () => { const harness = createHarness(); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-text", - ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + const first = { + ...segment("first", "First committed segment."), + messageId: "assistant-response", + submissionIds: ["submission-voice-1"], + }; + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [first], status: "streaming", }); - const firstText = { - ...segment("first", "First completed block.", "assistant-text"), - submissionIds: ["submission-text"], + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-voice-1", 1), + ); + + const second = { + ...segment("second", "Second committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], + }; + const third = { + ...segment("third", "Third committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], }; harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question, firstText], + canonicalSegments: [first, second, third], status: "streaming", }); + expect(harness.session.speakCanonical).toHaveBeenCalledTimes(1); - const typesWhileStreaming = harness.events.map(({ type }) => type); - expect(typesWhileStreaming).toContain("canonical-text-ready"); - expect(typesWhileStreaming).not.toContain("submission-settled"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-continuation", 2), + ); + const fourth = { + ...segment("fourth", "Fourth committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], + }; harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, firstText], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [first, second, third, fourth], + status: "streaming", }); - expect( - harness.events.filter(({ type }) => type === "canonical-text-ready"), - ).toHaveLength(1); - expect(harness.events.map(({ type }) => type)).toContain( - "submission-settled", - ); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstText], + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-continuation", 3), ); + + expect(harness.session.speakCanonical.mock.calls).toEqual([ + [[first]], + [[second, third]], + [[fourth]], + ]); }); - test("closes a durably stopped Voice turn without speaking", async () => { + test("does not start speech cancelled while its correlated response is pending", async () => { const harness = createHarness(); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-stopped", + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + + harness.bridge.cancelPendingSpeech(); + + const correlated = segment( + "correlated", + "Retain this without speaking it.", + "submission-voice-1", ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(correlated.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [correlated], status: "streaming", }); - - // A completed step with no text is not a stop: the panel may still be - // sending the client-tool follow-up that carries the reply. harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [ - { outcome: "completed", submissionId: "submission-stopped" }, - ], + canonicalSegments: [correlated], status: "ready", }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).not.toHaveBeenCalled(); - expect(harness.events.map(({ type }) => type)).not.toContain( - "submission-settled", - ); - const speechRequestsBeforeStop = - harness.session.speakCanonical.mock.calls.length; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [{ outcome: "aborted", submissionId: "submission-stopped" }], - status: "ready", + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toMatchObject({ + segments: [correlated], + speechCancelled: true, + type: "canonical-response-ready", }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).toHaveBeenCalledWith("call-1", "aborted"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - expect(harness.session.speakCanonical).toHaveBeenCalledTimes( - speechRequestsBeforeStop, - ); - expect(harness.events.map(({ type }) => type)).toEqual( - expect.arrayContaining(["submission-settled", "submission-stopped"]), - ); - expect(harness.events.some(({ type }) => type === "error")).toBe(false); }); - test("speaks the folded continuation that answers a Voice brunch_ask follow-up", async () => { + test("does not speak a completed segment from an aborted submission", async () => { const harness = createHarness(); - const question = { - ...segment("ask-current", "Question"), - submissionIds: ["submission-question"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-answer", + const aborted = segment( + "aborted", + "Never speak an aborted response.", + "submission-voice-1", ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(aborted.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [aborted], + settlements: [{ outcome: "aborted", submissionId: "submission-voice-1" }], status: "streaming", }); - - // The continuation is projected onto the message that asked, so that - // message is now written by both submissions. - const nextQuestion = { - ...segment("ask-next", "Next question"), - messageId: question.messageId, - submissionIds: ["submission-question", "submission-answer"], - }; - const askedAgain = { - ...question, - submissionIds: nextQuestion.submissionIds, - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [askedAgain, nextQuestion], + canonicalSegments: [aborted], + settlements: [{ outcome: "aborted", submissionId: "submission-voice-1" }], status: "ready", }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [nextQuestion], - ); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toEqual({ + deliveryId: createRealtimeSubmissionId(transcriptKey(7)), + outcome: "aborted", + type: "submission-stopped", + }); }); - test("speaks a reply that arrives through a client-tool follow-up", async () => { + test("rejects a path-B result that does not preserve the delivery identity", async () => { const harness = createHarness(); harness.submitInterviewAnswer.mockResolvedValueOnce({ kind: "message", - messageId: "message-kickoff", + messageId: "different-message", submissionId: "submission-voice-1", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7, '{"answer":"Read the guide first."}')); + startReady(harness); + + harness.emit(completedTranscript(3)); + await vi.waitFor(() => expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), + expect.objectContaining({ + code: "interview-correlation", + type: "error", + }), ), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", - }); + }); - // Brunch read a doc mid-turn; the panel's follow-up submission finished - // the same assistant message. - const reply = { - ...segment("reply", "The guide says hello.", "assistant-text"), - submissionIds: ["submission-voice-1", "submission-doc-follow-up"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [reply], - status: "ready", - }); + test.each([ + { + code: "admission-rejected", + failure: { kind: "rejected", status: 403 } as const, + message: "Brunch rejected the message before admission (HTTP 403).", + }, + { + code: "admission-conflict", + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + } as const, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + }, + { + code: "admission-ambiguous", + failure: { kind: "ambiguous" } as const, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + }, + { + code: "admission-aborted", + failure: { kind: "aborted" } as const, + message: "The local chat submission was cancelled.", + }, + ])( + "preserves a $failure.kind admission outcome", + async ({ code, failure, message }) => { + const harness = createHarness(); + harness.submitInterviewAnswer.mockRejectedValueOnce( + new FlueChatAdmissionError(failure), + ); + startReady(harness); + + harness.emit(completedTranscript(3)); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code, + failure, + message, + type: "error", + }), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); + }, + ); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [reply], + test("requires a shared chat busy cycle before accepting new canonical text", async () => { + const harness = createHarness(); + startReady(harness); + harness.emit(completedTranscript(3)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + const response = segment( + "response", + "Canonical response.", + "submission-voice-1", ); - }); - test("speaks new canonical text turns without creating a Realtime tool result", () => { - const harness = createHarness(); - const question = segment("ask-current", "Question"); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], + canonicalSegments: [response], status: "ready", }); - harness.bridge.start(8); - harness.session.speakCanonical.mockClear(); - const response = segment( - "typed-response", - "Canonical response", - "assistant-text", - ); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [response], + status: "streaming", + }); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question, response], + canonicalSegments: [response], status: "ready", }); expect(harness.session.speakCanonical).toHaveBeenCalledWith([response]); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index a7b0014cc59..3172b93b15d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,7 +1,17 @@ +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; import type { AgentSendResult, FlueConversationSettlement } from "@flue/sdk"; -import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; +import type { + FlueChatAdmissionFailure, + FlueChatResponseMessageCompletedEvent, + FlueChatResponseMessageStartedEvent, + FlueChatTransportOptions, +} from "@hashintel/brunch-agent-transport-aisdk"; import type { PetrinautAiComposerSubmitTextResult, PetrinautAiVoiceModeContext, @@ -15,20 +25,15 @@ export type VoiceSubmissionSettlement = Pick< interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; - /** Flue's settlement index: the only witness that a turn ended short of a reply. */ + readonly questionSegment?: CanonicalSpeechSegment; + /** Local logical termination when the panel withheld a continuation. */ + readonly stopped?: boolean; + /** Flue's settlement index remains the durable outcome authority. */ readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly status: PetrinautAiVoiceModeContext["status"]; } interface RealtimeBridgeSession { - completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void; - completeFunctionCallWithoutResponse( - callId: string, - outcome: Exclude, - ): void; speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -64,69 +69,94 @@ interface RealtimeBrunchBridgeDependencies { ) => Promise; } +interface CompletedResponseMessage extends FlueChatResponseMessageCompletedEvent { + consumed: boolean; +} + interface ActiveSubmission { readonly abortController: AbortController; readonly baselineSegmentIds: ReadonlySet; - readonly callId: string; - readonly epoch: number; - readonly pendingQuestionId: string | null; - readonly pendingQuestionMessageId: string | null; + readonly completedResponseMessages: CompletedResponseMessage[]; + readonly deliveryId: string; correlated: boolean; firstTextEmitted: boolean; sawBusyChatStatus: boolean; + speechCancelled: boolean; submissionId: AgentSendResult["submissionId"] | null; } -interface ArgumentStream { - readonly chunks: string[]; - readonly itemId: string; - readonly responseId: string; -} +type RealtimeAdmissionErrorCode = + | "admission-aborted" + | "admission-ambiguous" + | "admission-conflict" + | "admission-rejected"; -export type RealtimeBridgeErrorCode = +type RealtimeInterviewErrorCode = | "interview-correlation" | "interview-response" | "interview-submission"; +export type RealtimeBridgeErrorCode = + | RealtimeAdmissionErrorCode + | RealtimeInterviewErrorCode; + +export type RealtimeTranscriptRejectionReason = + | "duplicate" + | "empty" + | "failed" + | "over-limit" + | "unavailable"; + export type RealtimeBrunchBridgeEvent = | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-started"; } | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-accepted"; } | { - readonly callId: string; + readonly deliveryId: string; readonly submissionId: AgentSendResult["submissionId"]; readonly type: "submission-admitted"; } | { - readonly callId: string; + readonly deliveryId: string; + readonly questionSegment?: CanonicalSpeechSegment; readonly segments: CanonicalSpeechSegment[]; + readonly speechCancelled?: true; readonly type: "canonical-response-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "canonical-text-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-settled"; } | { - readonly callId: string; - readonly outcome: Exclude< - VoiceSubmissionSettlement["outcome"], - "completed" - >; + readonly deliveryId: string; + readonly outcome: + | Exclude + | "withheld"; readonly type: "submission-stopped"; } | { - readonly code: RealtimeBridgeErrorCode; + readonly reason: RealtimeTranscriptRejectionReason; + readonly type: "transcript-rejected"; + } + | { + readonly code: RealtimeInterviewErrorCode; + readonly message: string; + readonly type: "error"; + } + | { + readonly code: RealtimeAdmissionErrorCode; + readonly failure: FlueChatAdmissionFailure; readonly message: string; readonly type: "error"; }; @@ -137,45 +167,53 @@ const INVALID_BRIDGE_EVENT = "The voice response could not be matched to the interview. Reconnect voice or use text instead."; const ANSWER_LIMIT = 32_000; -export const createRealtimeSubmissionId = ( - connectionEpoch: number, - callId: string, -): string => `voice-realtime:${connectionEpoch}:${encodeURIComponent(callId)}`; - -const latestPendingQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); - -const parseContinueInterviewArguments = ( - argumentsJson: string, -): string | null => { - try { - const value: unknown = JSON.parse(argumentsJson); - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; - } - const record = value as Record; - if (Object.keys(record).length !== 1 || typeof record.answer !== "string") { - return null; - } - const answer = record.answer.trim(); - return answer && Array.from(answer).length <= ANSWER_LIMIT ? answer : null; - } catch { - return null; +export const createRealtimeSubmissionId = ({ + connectionEpoch, + contentIndex, + itemId, +}: OpenAIRealtimeTranscriptKey): string => + `voice-realtime:${connectionEpoch}:${encodeURIComponent(itemId)}:${contentIndex}`; + +const transcriptKeyId = (key: OpenAIRealtimeTranscriptKey): string => + createRealtimeSubmissionId(key); + +const normalizeTranscript = (transcript: string): string => + transcript.trim().replace(/\s+/gu, " "); + +const positionPrecedes = ( + first: FlueChatResponseMessageCompletedEvent["position"], + second: FlueChatResponseMessageStartedEvent["position"], +): boolean => + first.batch < second.batch || + (first.batch === second.batch && first.index < second.index); + +const admissionErrorCode = ( + failure: FlueChatAdmissionFailure, +): RealtimeAdmissionErrorCode => { + switch (failure.kind) { + case "aborted": + return "admission-aborted"; + case "ambiguous": + return "admission-ambiguous"; + case "rejected": + return "admission-rejected"; + case "submission-conflict": + return "admission-conflict"; } }; export class RealtimeBrunchBridge { - readonly #argumentDeltas = new Map(); + readonly #acceptedInputItemIds = new Set(); + readonly #activeOutputResponseIds = new Set(); readonly #listeners = new Set(); - readonly #processedCalls = new Set(); + readonly #pendingSpeechRequestIds = new Set(); + readonly #playbackOverlappingInputItemIds = new Set(); + readonly #processedTranscripts = new Set(); readonly #session: RealtimeBridgeSession; readonly #submitInterviewAnswer: ( input: SubmitInterviewAnswerInput, ) => Promise; readonly #seenSegmentIds = new Set(); - readonly #terminalResponseIds = new Set(); #activeEpoch: number | null = null; #activeSubmission: ActiveSubmission | null = null; #chat: ChatUpdate = { @@ -184,6 +222,7 @@ export class RealtimeBrunchBridge { status: "ready", }; #generation = 0; + #outputCancellationPending = false; public constructor({ session, @@ -199,27 +238,73 @@ export class RealtimeBrunchBridge { return () => this.#listeners.delete(listener); } + public cancelPendingSpeech(): void { + this.#outputCancellationPending = true; + if (this.#activeSubmission) { + this.#activeSubmission.speechCancelled = true; + } + } + + public completeTurnHandoff(): void { + this.#activeOutputResponseIds.clear(); + this.#outputCancellationPending = false; + this.#pendingSpeechRequestIds.clear(); + } + + public notifyResponseMessageCompleted( + event: FlueChatResponseMessageCompletedEvent, + ): void { + const active = this.#activeSubmission; + if ( + active === null || + active.completedResponseMessages.some( + ({ position }) => + position.batch === event.position.batch && + position.index === event.position.index, + ) + ) { + return; + } + active.completedResponseMessages.push({ + ...event, + consumed: false, + }); + this.#completeCorrelatedSubmission(); + } + + public notifyResponseMessageStarted( + event: FlueChatResponseMessageStartedEvent, + ): void { + const active = this.#activeSubmission; + if (active === null) { + return; + } + for (const completion of active.completedResponseMessages) { + if ( + !completion.consumed && + completion.messageId === event.messageId && + positionPrecedes(completion.position, event.position) + ) { + completion.consumed = true; + } + } + } + public start(connectionEpoch: number): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#processedCalls.clear(); + this.#acceptedInputItemIds.clear(); + this.#playbackOverlappingInputItemIds.clear(); + this.#processedTranscripts.clear(); + this.#activeOutputResponseIds.clear(); + this.#outputCancellationPending = false; + this.#pendingSpeechRequestIds.clear(); this.#seenSegmentIds.clear(); - this.#terminalResponseIds.clear(); for (const segment of this.#chat.canonicalSegments) { this.#seenSegmentIds.add(segment.id); } - - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if (question) { - this.#session.speakCanonical( - this.#chat.canonicalSegments.filter( - ({ messageId }) => messageId === question.messageId, - ), - ); - } } public stop(): void { @@ -227,8 +312,12 @@ export class RealtimeBrunchBridge { this.#activeSubmission?.abortController.abort(); this.#activeEpoch = null; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#terminalResponseIds.clear(); + this.#acceptedInputItemIds.clear(); + this.#playbackOverlappingInputItemIds.clear(); + this.#processedTranscripts.clear(); + this.#activeOutputResponseIds.clear(); + this.#outputCancellationPending = false; + this.#pendingSpeechRequestIds.clear(); } public updateChat(update: ChatUpdate): void { @@ -250,6 +339,12 @@ export class RealtimeBrunchBridge { this.#completeCorrelatedSubmission(); return; } + if (this.#outputCancellationPending || update.stopped) { + for (const segment of update.canonicalSegments) { + this.#seenSegmentIds.add(segment.id); + } + return; + } if (update.status !== "ready") { return; } @@ -276,167 +371,163 @@ export class RealtimeBrunchBridge { } } + #rejectTranscript(reason: RealtimeTranscriptRejectionReason): void { + this.#emit({ reason, type: "transcript-rejected" }); + } + #fail( message: string, - code: RealtimeBridgeErrorCode = "interview-correlation", + code: RealtimeInterviewErrorCode = "interview-correlation", ): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeSubmission = null; - this.#argumentDeltas.clear(); this.#emit({ code, message, type: "error" }); } + #failAdmission(error: FlueChatAdmissionError): void { + ++this.#generation; + this.#activeSubmission?.abortController.abort(); + this.#activeSubmission = null; + this.#emit({ + code: admissionErrorCode(error.failure), + failure: error.failure, + message: error.message, + type: "error", + }); + } + #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { if ( - !("connectionEpoch" in event) || + "connectionEpoch" in event && event.connectionEpoch !== this.#activeEpoch ) { return; } - if (event.type === "response-terminal") { - this.#handleResponseTerminal(event); + if (event.type === "input-speech-started") { + if (this.#ownsOutputTurn()) { + this.#playbackOverlappingInputItemIds.add(event.itemId); + } else { + this.#acceptedInputItemIds.add(event.itemId); + } + return; + } + if (event.type === "canonical-speech-requested") { + this.#pendingSpeechRequestIds.add(event.speechRequestId); + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + return; + } + if (event.type === "output-started") { + this.#pendingSpeechRequestIds.delete(event.speechRequestId); + this.#activeOutputResponseIds.add(event.responseId); + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); return; } if ( - event.type !== "tool-arguments-delta" && - event.type !== "tool-arguments-done" + event.type === "output-stopped" || + event.type === "output-interrupted" ) { + this.#activeOutputResponseIds.delete(event.responseId); return; } - - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - if (this.#terminalResponseIds.has(responseKey)) { + if (event.type === "response-terminal") { + if (event.status !== "completed" && event.speechRequestId !== undefined) { + this.#pendingSpeechRequestIds.delete(event.speechRequestId); + } return; } - const callKey = `${event.connectionEpoch}:${event.callId}`; - if (this.#processedCalls.has(callKey)) { + if (event.type !== "completed" && event.type !== "transcription-failed") { return; } - if (event.type === "tool-arguments-delta") { - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if ( - stream && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId) - ) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if (stream) { - stream.chunks.push(event.delta); - } else { - this.#argumentDeltas.set(callKey, { - chunks: [event.delta], - itemId: event.itemId, - responseId: event.responseId, - }); - } + if (event.key.connectionEpoch !== this.#activeEpoch) { return; } - this.#processedCalls.add(callKey); - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#fail(INVALID_BRIDGE_EVENT); + const keyId = transcriptKeyId(event.key); + if (this.#processedTranscripts.has(keyId)) { + this.#rejectTranscript("duplicate"); return; } - this.#argumentDeltas.delete(callKey); - if ( - this.#activeSubmission || - event.name !== "continue_interview" || - (stream !== undefined && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId || - stream.chunks.join("") !== event.arguments)) - ) { - this.#fail(INVALID_BRIDGE_EVENT); + this.#processedTranscripts.add(keyId); + this.#acceptedInputItemIds.delete(event.key.itemId); + + if (this.#playbackOverlappingInputItemIds.has(event.key.itemId)) { + this.#rejectTranscript("unavailable"); return; } - const answer = parseContinueInterviewArguments(event.arguments); - const question = latestPendingQuestion(this.#chat.canonicalSegments); + if (event.type === "transcription-failed") { + this.#rejectTranscript("failed"); + return; + } if ( - !answer || + this.#activeSubmission || !this.#chat.canAcceptInterviewAnswer || - (!question && this.#chat.status !== "ready") + this.#chat.status !== "ready" ) { - this.#fail(INVALID_BRIDGE_EVENT); + this.#rejectTranscript("unavailable"); return; } + const answer = normalizeTranscript(event.text); + if (answer.length === 0) { + this.#rejectTranscript("empty"); + return; + } + if (Array.from(answer).length > ANSWER_LIMIT) { + this.#rejectTranscript("over-limit"); + return; + } + + const deliveryId = createRealtimeSubmissionId(event.key); const generation = this.#generation; this.#activeSubmission = { abortController: new AbortController(), baselineSegmentIds: new Set( this.#chat.canonicalSegments.map(({ id }) => id), ), - callId: event.callId, + completedResponseMessages: [], correlated: false, - epoch: event.connectionEpoch, + deliveryId, firstTextEmitted: false, - pendingQuestionId: question?.partId ?? null, - pendingQuestionMessageId: question?.messageId ?? null, sawBusyChatStatus: false, + speechCancelled: false, submissionId: null, }; - this.#emit({ answer, callId: event.callId, type: "submission-started" }); - void this.#submit(event, answer, generation); + this.#emit({ answer, deliveryId, type: "submission-started" }); + void this.#submit(answer, deliveryId, generation); } - #handleResponseTerminal( - event: Extract, - ): void { - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - const matchingStreams = [...this.#argumentDeltas].filter( - ([, stream]) => stream.responseId === event.responseId, + #ownsOutputTurn(): boolean { + return ( + this.#activeOutputResponseIds.size > 0 || + this.#pendingSpeechRequestIds.size > 0 ); - if (event.status === "completed" && matchingStreams.length > 0) { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - - for (const [callKey] of matchingStreams) { - this.#argumentDeltas.delete(callKey); - this.#processedCalls.add(callKey); - } - this.#terminalResponseIds.add(responseKey); } async #submit( - event: Extract, answer: string, + deliveryId: string, generation: number, ): Promise { try { const activeAtSubmission = this.#activeSubmission; if (!activeAtSubmission) return; - const voiceMessageId = createRealtimeSubmissionId( - event.connectionEpoch, - event.callId, - ); const result = await this.#submitInterviewAnswer({ - admissionTarget: - activeAtSubmission.pendingQuestionMessageId === null - ? { kind: "user", messageId: voiceMessageId } - : { - kind: "client-tool-result", - messageId: activeAtSubmission.pendingQuestionMessageId, - }, - id: voiceMessageId, + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, onAdmission: (submissionId) => { const active = this.#activeSubmission; if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } @@ -448,7 +539,7 @@ export class RealtimeBrunchBridge { } active.submissionId = submissionId; this.#emit({ - callId: event.callId, + deliveryId, submissionId, type: "submission-admitted", }); @@ -460,22 +551,15 @@ export class RealtimeBrunchBridge { if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } - const resultMatchesSubmission = - active.pendingQuestionId === null - ? result.kind === "message" - : result.kind === "interactive-tool" && - result.toolCallId === active.pendingQuestionId; - if (!resultMatchesSubmission) { + if (result.kind !== "message" || result.messageId !== deliveryId) { this.#fail(INVALID_BRIDGE_EVENT); return; } - const resultSubmissionId = - result.kind === "message" ? (result.submissionId ?? null) : null; + const resultSubmissionId = result.submissionId ?? null; if ( active.submissionId !== null && resultSubmissionId !== null && @@ -486,18 +570,18 @@ export class RealtimeBrunchBridge { } active.submissionId ??= resultSubmissionId; active.correlated = true; - this.#emit({ - answer, - callId: event.callId, - type: "submission-accepted", - }); + this.#emit({ answer, deliveryId, type: "submission-accepted" }); this.#completeCorrelatedSubmission(); - } catch { + } catch (error) { if (generation === this.#generation) { - this.#fail( - "The interview could not accept that answer. Use the composer to retry.", - "interview-submission", - ); + if (error instanceof FlueChatAdmissionError) { + this.#failAdmission(error); + } else { + this.#fail( + "The interview could not accept that answer. Use the composer to retry.", + "interview-submission", + ); + } } } } @@ -507,6 +591,27 @@ export class RealtimeBrunchBridge { if (!active?.correlated || !active.sawBusyChatStatus) { return; } + if (this.#chat.stopped && this.#chat.status === "ready") { + // Cancellation can finish before this step commits its final prose. + // Retire it now so a later render cannot restart the withheld speech. + for (const segment of this.#chat.canonicalSegments) { + this.#seenSegmentIds.add(segment.id); + } + const settlement = this.#chat.settlements?.find( + ({ submissionId }) => submissionId === active.submissionId, + ); + this.#emit({ deliveryId: active.deliveryId, type: "submission-settled" }); + this.#activeSubmission = null; + this.#emit({ + deliveryId: active.deliveryId, + outcome: + settlement && settlement.outcome !== "completed" + ? settlement.outcome + : "withheld", + type: "submission-stopped", + }); + return; + } // A reply may be written by the admitted submission itself or by a // client-tool continuation projected onto the same message, and an ask // follow-up writes into the message that asked; so match membership and @@ -521,7 +626,61 @@ export class RealtimeBrunchBridge { // Completed canonical text can land while the turn is still streaming; // record that instant separately from settlement. active.firstTextEmitted = true; - this.#emit({ callId: active.callId, type: "canonical-text-ready" }); + this.#emit({ + deliveryId: active.deliveryId, + type: "canonical-text-ready", + }); + } + const stoppedSettlement = + active.submissionId === null + ? undefined + : this.#chat.settlements?.find( + ({ submissionId }) => submissionId === active.submissionId, + ); + if (stoppedSettlement && stoppedSettlement.outcome !== "completed") { + if (this.#chat.status === "ready") { + this.#completeStoppedSubmission(active); + } + return; + } + const completionMatchesSegment = ( + completion: CompletedResponseMessage, + segment: CanonicalSpeechSegment, + ): boolean => + completion.messageId === segment.messageId && + (segment.submissionIds?.includes(completion.submissionId) ?? false); + const pendingCompletions = active.completedResponseMessages.filter( + ({ consumed }) => !consumed, + ); + const eligibleCompletions = pendingCompletions.filter((completion) => + responseSegments.some( + (segment) => + !this.#seenSegmentIds.has(segment.id) && + completionMatchesSegment(completion, segment), + ), + ); + const completedSegments = responseSegments.filter( + (segment) => + !this.#seenSegmentIds.has(segment.id) && + eligibleCompletions.some((completion) => + completionMatchesSegment(completion, segment), + ), + ); + if (!active.speechCancelled) { + if (completedSegments.length > 0) { + try { + this.#session.speakCanonical(completedSegments); + for (const segment of completedSegments) { + this.#seenSegmentIds.add(segment.id); + } + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } + } + for (const completion of eligibleCompletions) { + completion.consumed = true; } if (this.#chat.status !== "ready") { return; @@ -531,20 +690,42 @@ export class RealtimeBrunchBridge { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCall(active.callId, responseSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); + if (!active.speechCancelled) { + const unscheduledSegments = responseSegments.filter( + ({ id }) => !this.#seenSegmentIds.has(id), + ); + if (unscheduledSegments.length > 0) { + try { + this.#session.speakCanonical(unscheduledSegments); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } } for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } + const questionSegment = this.#chat.questionSegment; + const correlatedQuestion = + questionSegment && + responseSegments.some( + ({ messageId }) => messageId === questionSegment.messageId, + ) && + (active.submissionId === null || + (questionSegment.submissionIds?.includes(active.submissionId) ?? false)) + ? questionSegment + : undefined; this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, + ...(correlatedQuestion ? { questionSegment: correlatedQuestion } : {}), segments: responseSegments, + ...(active.speechCancelled ? { speechCancelled: true as const } : {}), type: "canonical-response-ready", }); } @@ -563,19 +744,13 @@ export class RealtimeBrunchBridge { if (settlement === undefined || settlement.outcome === "completed") { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCallWithoutResponse( - active.callId, - settlement.outcome, - ); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, outcome: settlement.outcome, type: "submission-stopped", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx new file mode 100644 index 00000000000..08f33472342 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx @@ -0,0 +1,305 @@ +/** @vitest-environment jsdom */ +import { act, cleanup, render, waitFor } from "@testing-library/react"; +import { useLayoutEffect } from "react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { createJsonDocHandle } from "@hashintel/petrinaut-core"; +import { Petrinaut } from "@hashintel/petrinaut/ui"; + +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "../local-storage-demo/brunch-panel-transport"; +import { selectCanonicalSpeech } from "./canonical-speech"; +import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; +import { submitVoiceInputWithAdmission } from "./voice-interview-control"; + +import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; +import type { AgentSendResult, FlueClient } from "@flue/sdk"; +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +vi.hoisted(() => { + window.matchMedia = (media) => ({ + media, + matches: false, + onchange: null, + addListener() {}, + removeListener() {}, + addEventListener() {}, + removeEventListener() {}, + dispatchEvent: () => true, + }); +}); + +const VoiceObserver = ({ + current, + onUpdate, +}: { + current: PetrinautAiVoiceModeContext; + onUpdate: (context: PetrinautAiVoiceModeContext) => void; +}) => { + useLayoutEffect(() => onUpdate(current), [current, onUpdate]); + return null; +}; +const inertWorker = () => ({ + postMessage() {}, + addEventListener() {}, + removeEventListener() {}, + terminate() {}, +}); +const hosts: Array<() => void> = []; +afterEach(() => { + cleanup(); + for (const close of hosts.splice(0)) close(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +test.each([ + { preamble: true, outcome: "completed" }, + { preamble: false, outcome: "completed" }, + { preamble: false, outcome: "invalid-input" }, + { preamble: false, outcome: "withheld" }, + { preamble: true, outcome: "withheld" }, +])( + "settles the real panel/Voice browser-tool path ($outcome, preamble: $preamble)", + async ({ preamble, outcome }) => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + observe() {} + unobserve() {} + disconnect() {} + }, + ); + const tracker = new BrunchPanelConversationTracker(); + let context: PetrinautAiVoiceModeContext | undefined; + let emitInput: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; + let finishContinuation: (() => void) | undefined; + let finishStoppedStep: (() => void) | undefined; + const events: RealtimeBrunchBridgeEvent[] = []; + const speakCanonical = + vi.fn<(segments: CanonicalSpeechSegment[]) => void>(); + const send = vi.fn( + async (): Promise => ({ + submissionId: `submission-${send.mock.calls.length}`, + uid: "uid", + offset: "0", + streamUrl: "http://local.test/agents/chat/test/stream", + }), + ); + const wait = vi.fn(async (admission, options) => { + const submissionId = (admission as AgentSendResult).submissionId; + const continuation = submissionId === "submission-2"; + if (continuation) + await new Promise((resolve) => { + finishContinuation = resolve; + }); + if (!continuation && outcome === "withheld") + await new Promise((resolve) => { + finishStoppedStep = resolve; + }); + const messageId = continuation ? "continuation" : "assistant"; + let ordinal = 0; + const position = () => ({ + batch: continuation ? 2 : 1, + index: ordinal++, + }); + await options?.onEvent?.({ + type: "message-started", + conversationId: "test", + submissionId, + messageId, + turnId: messageId, + position: position(), + }); + if (preamble || continuation) + await options?.onEvent?.({ + type: "message-delta", + conversationId: "test", + messageId, + kind: "text", + delta: continuation + ? "The guide is available." + : "Checking the guide.", + position: position(), + }); + if (!continuation) + await options?.onEvent?.({ + type: "tool-input", + conversationId: "test", + messageId, + toolCallId: "read-guide", + toolName: "readPetrinautDoc", + input: { + doc: outcome === "invalid-input" ? "missing-page" : "ai-assistant", + }, + position: position(), + }); + await options?.onEvent?.({ + type: "message-completed", + conversationId: "test", + messageId, + position: position(), + }); + await options?.onEvent?.({ + type: "submission-settled", + conversationId: "test", + submissionId, + outcome: "completed", + position: position(), + }); + }); + const client = { send, wait } as Pick< + FlueClient, + "send" | "wait" + > as FlueClient; + const bridge = new RealtimeBrunchBridge({ + session: { + speakCanonical, + subscribe: (listener) => { + emitInput = listener; + return () => {}; + }, + }, + submitInterviewAnswer: async (input) => { + if (!context) throw new Error("Panel did not mount"); + return submitVoiceInputWithAdmission({ + input, + submitVoiceInput: context.submitVoiceInput, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }); + }, + }); + hosts.push(() => bridge.stop()); + bridge.subscribe((event) => events.push(event)); + tracker.subscribeToResponseMessageCompleted((event) => + bridge.notifyResponseMessageCompleted(event), + ); + tracker.subscribeToResponseMessageStarted((event) => + bridge.notifyResponseMessageStarted(event), + ); + const updateVoice = (current: PetrinautAiVoiceModeContext) => { + context = current; + bridge.updateChat({ + canAcceptInterviewAnswer: current.canAcceptVoiceInput, + status: current.status, + stopped: current.stopped, + canonicalSegments: selectCanonicalSpeech(current.messages).segments.map( + (segment) => ({ + ...segment, + submissionIds: tracker.submissionsForResponse(segment.messageId), + }), + ), + }); + }; + const handle = createJsonDocHandle({ + id: "voice-browser-test", + initial: { + places: [], + transitions: [], + types: [], + parameters: [], + differentialEquations: [], + }, + }); + render( + { + tracker.recordStopRequested(); + bridge.cancelPendingSpeech(); + bridge.completeTurnHandoff(); + finishStoppedStep?.(); + return "already-settled"; + }, + transport: createBrunchPanelTransport( + Promise.resolve(client), + tracker, + ), + renderVoiceMode: (current) => ( + + ), + }} + />, + ); + await waitFor(() => expect(context).toBeDefined()); + await act(async () => { + bridge.start(1); + emitInput?.({ + type: "completed", + key: { connectionEpoch: 1, contentIndex: 0, itemId: "spoken-input" }, + text: "Read the guide.", + }); + }); + if (outcome === "invalid-input") { + await waitFor(() => expect(context?.status).toBe("error")); + expect(events).toContainEqual( + expect.objectContaining({ type: "error", code: "interview-response" }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(speakCanonical).not.toHaveBeenCalled(); + return; + } + if (outcome === "withheld") { + await waitFor(() => expect(finishStoppedStep).toBeDefined()); + await act(async () => { + await context?.stop(); + }); + await waitFor(() => + expect(events).toContainEqual( + expect.objectContaining({ + type: "submission-stopped", + outcome: "withheld", + }), + ), + ); + // A later render must not resurrect prose committed after cancellation. + if (context) updateVoice(context); + expect(send).toHaveBeenCalledOnce(); + expect(speakCanonical).not.toHaveBeenCalled(); + return; + } + await waitFor(() => expect(finishContinuation).toBeDefined()); + expect(send).toHaveBeenCalledTimes(2); + expect(context?.status).not.toBe("ready"); + expect( + events.some((event) => event.type === "canonical-response-ready"), + ).toBe(false); + expect(send.mock.calls[1]?.[0].message).toMatchObject({ + kind: "signal", + attributes: { toolCallIds: "read-guide" }, + }); + await act(async () => { + finishContinuation?.(); + }); + await waitFor(() => + expect(events).toContainEqual( + expect.objectContaining({ type: "canonical-response-ready" }), + ), + ); + expect(context?.status).toBe("ready"); + expect( + speakCanonical.mock.calls + .flatMap(([segments]) => segments) + .map((segment) => segment.text), + ).toEqual( + preamble + ? ["Checking the guide.", "The guide is available."] + : ["The guide is available."], + ); + }, +); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 6e94c159fe0..632a4332269 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -12,6 +12,8 @@ import { import { StrictMode, useState } from "react"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { acknowledgeVoiceInterviewDisclosure, @@ -265,10 +267,55 @@ describe("voice interview control", () => { abortController.abort(); - await expect(resultPromise).rejects.toMatchObject({ name: "AbortError" }); + await expect(resultPromise).rejects.toMatchObject({ + failure: { kind: "aborted" }, + name: "FlueChatAdmissionError", + }); expect(unsubscribe).toHaveBeenCalledOnce(); }); + test("preserves a typed failure reported after the panel submission resolves", async () => { + const admissionError = new FlueChatAdmissionError({ kind: "ambiguous" }); + let reportFailure: ((error: FlueChatAdmissionError) => void) | undefined; + const unsubscribeFromAdmission = vi.fn(); + const unsubscribeFromFailure = vi.fn(); + const resultPromise = submitVoiceInputWithAdmission({ + input: { + admissionTarget: { kind: "user", messageId: "voice-turn-1" }, + id: "voice-turn-1", + onAdmission: vi.fn(), + signal: new AbortController().signal, + text: "One Voice turn.", + }, + submitVoiceInput: async () => ({ + kind: "message", + messageId: "voice-turn-1", + }), + subscribeToAdmission: () => unsubscribeFromAdmission, + subscribeToAdmissionFailure: (_target, listener) => { + reportFailure = listener; + return unsubscribeFromFailure; + }, + }); + let settled = false; + void resultPromise.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await Promise.resolve(); + expect(settled).toBe(false); + + reportFailure?.(admissionError); + + await expect(resultPromise).rejects.toBe(admissionError); + expect(unsubscribeFromAdmission).toHaveBeenCalledOnce(); + expect(unsubscribeFromFailure).toHaveBeenCalledOnce(); + }); + test("stores and reads the versioned disclosure acknowledgement", () => { const values = new Map(); const storage = { @@ -324,7 +371,7 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("keeps the first-use disclosure inline without a text-handoff action", () => { + test("keeps the first-use disclosure inline without a text-handoff action", async () => { render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); @@ -332,16 +379,33 @@ describe("voice interview control", () => { const disclosure = screen.getByRole("region", { name: "Voice mode consent", }); - expect(disclosure).not.toBeNull(); - expect(within(disclosure).getByText("Voice mode")).not.toBeNull(); expect( - screen.getByText("OpenAI processes live audio", { exact: false }), + within(disclosure).getByText("Start a voice conversation"), ).not.toBeNull(); expect( - screen - .getByRole("button", { name: "Start voice mode" }) - .hasAttribute("disabled"), - ).toBe(true); + within(disclosure).getByText( + "OpenAI processes live audio and speaks the interviewer’s words. Petrinaut saves finalized answers—not audio.", + ), + ).not.toBeNull(); + + const consent = within(disclosure).getByRole("checkbox", { + name: "I understand how voice data is handled.", + }); + const start = within(disclosure).getByRole("button", { + name: "Start voice", + }); + expect(start.hasAttribute("disabled")).toBe(true); + fireEvent.click(consent); + await waitFor(() => + expect( + within(disclosure) + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + expect( + within(disclosure).getByRole("button", { name: "Test microphone" }), + ).not.toBeNull(); expect( screen.queryByRole("button", { name: "Use text instead" }), ).toBeNull(); @@ -361,7 +425,14 @@ describe("voice interview control", () => { fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect(await screen.findByText("Session: error")).not.toBeNull(); expect(screen.getByText("Voice active")).not.toBeNull(); @@ -377,6 +448,75 @@ describe("voice interview control", () => { expect(screen.getByText("Session: error")).not.toBeNull(); }); + test("keeps one microphone check pending and reports its result", async () => { + let resolveCheck: ((stream: MediaStream) => void) | undefined; + const getUserMedia = vi.fn( + () => + new Promise((resolve) => { + resolveCheck = resolve; + }), + ); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const check = screen.getByRole("button", { name: "Test microphone" }); + fireEvent.click(check); + fireEvent.click(check); + + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(check.getAttribute("aria-busy")).toBe("true"); + + resolveCheck?.({ getTracks: () => [] } as unknown as MediaStream); + + expect(await screen.findByText("Microphone ready.")).not.toBeNull(); + await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); + }); + + test.each([ + { + failure: "media devices are missing", + stubMedia: () => vi.stubGlobal("navigator", {}), + }, + { + failure: "getUserMedia throws synchronously", + stubMedia: () => + vi.stubGlobal("navigator", { + mediaDevices: { + getUserMedia: () => { + throw new DOMException("Unavailable", "NotSupportedError"); + }, + }, + }), + }, + ])( + "reports an accessible microphone failure when $failure", + async ({ stubMedia }) => { + stubMedia(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const disclosure = screen.getByRole("region", { + name: "Voice mode consent", + }); + const check = within(disclosure).getByRole("button", { + name: "Test microphone", + }); + + fireEvent.click(check); + + const status = await within(disclosure).findByText( + "Microphone access was not available.", + ); + expect(status.getAttribute("aria-live")).toBe("polite"); + expect(status.getAttribute("aria-atomic")).toBe("true"); + expect(check.getAttribute("aria-describedby")).toBe(status.id); + await waitFor(() => + expect(check.getAttribute("aria-busy")).toBe("false"), + ); + }, + ); + test("starts directly after acknowledgement and ends through the registered control", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, @@ -401,6 +541,18 @@ describe("voice interview control", () => { expect(screen.getByText("Voice inactive")).not.toBeNull(); }); + test("registers replay controls that remain snapshot-gated", async () => { + render(); + + await waitFor(() => expect(registeredVoiceModeControls).toBeDefined()); + + expect(registeredVoiceModeControls?.readFullResponse).toBeTypeOf( + "function", + ); + expect(registeredVoiceModeControls?.takeTurn).toBeTypeOf("function"); + expect(registeredVoiceModeControls?.repeatQuestion).toBeTypeOf("function"); + }); + test("restarts when Voice is reselected before teardown completes", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, @@ -494,18 +646,25 @@ describe("voice interview control", () => { expect(screen.getByText("Panel closed")).not.toBeNull(); }); - test("records acknowledgement only when the interview starts", () => { + test("records acknowledgement only when the interview starts", async () => { stubUnavailableMicrophone(); render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); - fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); + fireEvent.click(screen.getByRole("button", { name: "Test microphone" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBeNull(); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBe("acknowledged"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 5206ac7d8c1..7094a09f9aa 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -6,11 +6,16 @@ import { useSyncExternalStore, } from "react"; -import { Button } from "@hashintel/ds-components"; +import { + FlueChatAdmissionError, + type FlueChatResponseMessageCompletedEvent, + type FlueChatResponseMessageStartedEvent, +} from "@hashintel/brunch-agent-transport-aisdk"; +import { Button, Checkbox } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, @@ -24,6 +29,7 @@ import { type VoiceTurnSnapshot, } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; @@ -37,6 +43,17 @@ type SubscribeToAdmission = ( target: RealtimeBrunchAdmissionTarget, listener: (submissionId: AgentSendResult["submissionId"]) => void, ) => () => void; +type SubscribeToAdmissionFailure = ( + target: RealtimeBrunchAdmissionTarget, + listener: (error: FlueChatAdmissionError) => void, +) => () => void; +type SubscribeToResponseMessageCompleted = ( + listener: (event: FlueChatResponseMessageCompletedEvent) => void, +) => () => void; +type SubscribeToResponseMessageStarted = ( + listener: (event: FlueChatResponseMessageStartedEvent) => void, +) => () => void; +type SubscribeToStopRequested = (listener: () => void) => () => void; type SubmitInterviewAnswer = ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"]; @@ -48,17 +65,20 @@ export const submitVoiceInputWithAdmission = async ({ resolveInputSubmission, submitVoiceInput, subscribeToAdmission, + subscribeToAdmissionFailure, }: { readonly input: SubmitInterviewAnswerInput; readonly resolveInputSubmission?: ResolveSubmission; readonly submitVoiceInput: PetrinautAiVoiceModeContext["submitVoiceInput"]; readonly subscribeToAdmission?: SubscribeToAdmission; + readonly subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; }): Promise => { let unsubscribe = () => {}; + let unsubscribeFromFailure = () => {}; let removeAbortListener = () => {}; const cancelled = new Promise((_resolve, reject) => { const rejectForAbort = () => - reject(new DOMException("Voice admission cancelled", "AbortError")); + reject(new FlueChatAdmissionError({ kind: "aborted" })); if (input.signal.aborted) { rejectForAbort(); return; @@ -68,16 +88,25 @@ export const submitVoiceInputWithAdmission = async ({ input.signal.removeEventListener("abort", rejectForAbort); }); const admissionObserved = - subscribeToAdmission === undefined + subscribeToAdmission === undefined && + subscribeToAdmissionFailure === undefined ? Promise.resolve() - : new Promise((resolve) => { - unsubscribe = subscribeToAdmission( - input.admissionTarget, - (submissionId) => { - input.onAdmission(submissionId); - resolve(); - }, - ); + : new Promise((resolve, reject) => { + if (subscribeToAdmission !== undefined) { + unsubscribe = subscribeToAdmission( + input.admissionTarget, + (submissionId) => { + input.onAdmission(submissionId); + resolve(); + }, + ); + } + if (subscribeToAdmissionFailure !== undefined) { + unsubscribeFromFailure = subscribeToAdmissionFailure( + input.admissionTarget, + reject, + ); + } }); try { const [result] = await Promise.race([ @@ -96,6 +125,7 @@ export const submitVoiceInputWithAdmission = async ({ } finally { removeAbortListener(); unsubscribe(); + unsubscribeFromFailure(); } }; @@ -186,31 +216,82 @@ export const loadOpenAIVoiceConfig = async ( } }; -const disclosureStyle = css({ - display: "flex", +const VoiceModeIcon = () => ( + +); + +const disclosureFrameStyle = css({ width: "full", - flexDirection: "column", - gap: "2", - paddingX: "2", - paddingY: "2", + padding: "2", borderTopWidth: "thin", borderTopStyle: "solid", borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", color: "neutral.s100", + _focus: { outline: "none" }, +}); + +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); + +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", }); const disclosureTitleStyle = css({ display: "flex", + minWidth: "[0]", flexDirection: "column", - gap: "1", + gap: "0.5", +}); + +const disclosureHeadingStyle = css({ fontSize: "sm", fontWeight: "semibold", + lineHeight: "tight", }); const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs", - fontWeight: "normal", }); const disclosureCopyStyle = css({ @@ -219,6 +300,14 @@ const disclosureCopyStyle = css({ lineHeight: "relaxed", }); +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); + const disclosureActionsStyle = css({ display: "flex", flexWrap: "wrap", @@ -226,13 +315,22 @@ const disclosureActionsStyle = css({ gap: "2", }); +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + const VoiceInterviewDisclosure = ({ + checkingMicrophone, consented, microphoneCheck, onCheckMicrophone, onConsentChange, onStart, }: { + readonly checkingMicrophone: boolean; readonly consented: boolean; readonly microphoneCheck: string; readonly onCheckMicrophone: () => void; @@ -248,40 +346,65 @@ const VoiceInterviewDisclosure = ({ return (
-
- Voice mode - - Talk through your process with AI - -
-

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut keeps finalized answers in this conversation, not the audio. -

- - {microphoneCheck && ( -

- {microphoneCheck} +

+
+ + + +
+ + Start a voice conversation + + + Talk through your process with AI + +
+
+

+ OpenAI processes live audio and speaks the interviewer’s words. + Petrinaut saves finalized answers—not audio.

- )} -
- - + +
+ + +
+
+ {microphoneCheck} +
); @@ -306,6 +429,10 @@ const AvailableVoiceInterviewControl = ({ resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, + subscribeToResponseMessageCompleted, + subscribeToResponseMessageStarted, + subscribeToStopRequested, }: { config: OpenAIVoiceConfig; context: PetrinautAiVoiceModeContext; @@ -313,6 +440,10 @@ const AvailableVoiceInterviewControl = ({ resolveResponseSubmission?: ResolveSubmissions; settlements?: readonly VoiceSubmissionSettlement[]; subscribeToAdmission?: SubscribeToAdmission; + subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; + subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; + subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; + subscribeToStopRequested?: SubscribeToStopRequested; }) => { "use no memo"; @@ -323,6 +454,7 @@ const AvailableVoiceInterviewControl = ({ let latestSubmitVoiceInput = context.submitVoiceInput; let latestResolveInputSubmission = resolveInputSubmission; let latestSubscribeToAdmission = subscribeToAdmission; + let latestSubscribeToAdmissionFailure = subscribeToAdmissionFailure; const session = new OpenAIRealtimeSession({ cancelAnimationFrame: (handle) => globalThis.cancelAnimationFrame(handle), connectionTimeoutMs: config.connectionTimeoutMs, @@ -344,6 +476,7 @@ const AvailableVoiceInterviewControl = ({ resolveInputSubmission: latestResolveInputSubmission, submitVoiceInput: latestSubmitVoiceInput, subscribeToAdmission: latestSubscribeToAdmission, + subscribeToAdmissionFailure: latestSubscribeToAdmissionFailure, }), }); const controller = new VoiceTurnController({ @@ -353,6 +486,7 @@ const AvailableVoiceInterviewControl = ({ submitText: (input) => latestSubmitVoiceInput(input), }); return { + bridge, controller, getSnapshot: () => controller.getSnapshot(), subscribe: (listener: (snapshot: VoiceTurnSnapshot) => void) => @@ -363,10 +497,14 @@ const AvailableVoiceInterviewControl = ({ | ((messageId: string) => string | undefined) | undefined, nextSubscribeToAdmission: SubscribeToAdmission | undefined, + nextSubscribeToAdmissionFailure: + | SubscribeToAdmissionFailure + | undefined, ) => { latestSubmitVoiceInput = nextSubmitVoiceInput; latestResolveInputSubmission = nextResolveInputSubmission; latestSubscribeToAdmission = nextSubscribeToAdmission; + latestSubscribeToAdmissionFailure = nextSubscribeToAdmissionFailure; }, }; }); @@ -378,6 +516,7 @@ const AvailableVoiceInterviewControl = ({ const [showDisclosure, setShowDisclosure] = useState(false); const [consented, setConsented] = useState(false); const [microphoneCheck, setMicrophoneCheck] = useState(""); + const [checkingMicrophone, setCheckingMicrophone] = useState(false); const handledVoiceSelectionRef = useRef(false); const { inputMode, @@ -387,34 +526,61 @@ const AvailableVoiceInterviewControl = ({ setVoiceActive, } = context; + useEffect( + () => + subscribeToResponseMessageCompleted?.((event) => + store.bridge.notifyResponseMessageCompleted(event), + ), + [store, subscribeToResponseMessageCompleted], + ); + useEffect( + () => + subscribeToResponseMessageStarted?.((event) => + store.bridge.notifyResponseMessageStarted(event), + ), + [store, subscribeToResponseMessageStarted], + ); + useEffect( + () => + subscribeToStopRequested?.(() => store.controller.cancelPendingSpeech()), + [store, subscribeToStopRequested], + ); + useLayoutEffect(() => { store.updateSubmissionContext( context.submitVoiceInput, resolveInputSubmission, subscribeToAdmission, + subscribeToAdmissionFailure, ); + const canonicalSpeech = selectCanonicalSpeech(context.messages); + const correlateSegment = (segment: CanonicalSpeechSegment) => { + const submissionIds = resolveResponseSubmission?.(segment.messageId); + return submissionIds === undefined || submissionIds.length === 0 + ? segment + : { ...segment, submissionIds }; + }; store.controller.updateChat({ canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages).map( - (segment) => { - const submissionIds = resolveResponseSubmission?.(segment.messageId); - return submissionIds === undefined || submissionIds.length === 0 - ? segment - : { ...segment, submissionIds }; - }, - ), + canonicalSegments: canonicalSpeech.segments.map(correlateSegment), + ...(canonicalSpeech.questionSegment + ? { questionSegment: correlateSegment(canonicalSpeech.questionSegment) } + : {}), settlements, + stopped: context.stopped, status: context.status, }); }, [ context.canAcceptVoiceInput, context.messages, context.status, + context.stopped, context.submitVoiceInput, resolveInputSubmission, resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, store, ]); @@ -425,12 +591,17 @@ const AvailableVoiceInterviewControl = ({ registerVoiceModeControls({ end: () => store.controller.end(), pause: () => store.controller.pause(), + readFullResponse: () => store.controller.readFullResponse(), reconnect: () => { void store.controller.reconnect(); }, - resume: () => store.controller.resume(), + repeatQuestion: () => store.controller.repeatQuestion(), + resume: () => { + void store.controller.resume(); + }, setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), + takeTurn: () => store.controller.takeTurn(), }), [registerVoiceModeControls, store], ); @@ -497,19 +668,38 @@ const AvailableVoiceInterviewControl = ({ return ( { - setMicrophoneCheck("Checking microphone…"); - void navigator.mediaDevices.getUserMedia({ audio: true }).then( - (stream) => { + if (checkingMicrophone) { + return; + } + setCheckingMicrophone(true); + setMicrophoneCheck(""); + let microphoneCheckPromise: Promise; + try { + const { mediaDevices } = navigator as { + readonly mediaDevices?: MediaDevices; + }; + microphoneCheckPromise = + mediaDevices === undefined + ? Promise.reject(new Error("Microphone access is unavailable.")) + : mediaDevices.getUserMedia({ audio: true }); + } catch (error) { + microphoneCheckPromise = Promise.reject(error); + } + void microphoneCheckPromise + .then((stream) => { for (const track of stream.getTracks()) { track.stop(); } setMicrophoneCheck("Microphone ready."); - }, - () => setMicrophoneCheck("Microphone access was not available."), - ); + }) + .catch(() => + setMicrophoneCheck("Microphone access was not available."), + ) + .finally(() => setCheckingMicrophone(false)); }} onConsentChange={setConsented} onStart={() => { @@ -531,6 +721,10 @@ export const VoiceInterviewControl = ({ resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, + subscribeToResponseMessageCompleted, + subscribeToResponseMessageStarted, + subscribeToStopRequested, ...context }: PetrinautAiVoiceModeContext & { readonly config: OpenAIVoiceConfig; @@ -538,6 +732,10 @@ export const VoiceInterviewControl = ({ readonly resolveResponseSubmission?: ResolveSubmissions; readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly subscribeToAdmission?: SubscribeToAdmission; + readonly subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; + readonly subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; + readonly subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; + readonly subscribeToStopRequested?: SubscribeToStopRequested; }) => ( ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index ec153c77af7..6f81ef30b4b 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -1,3 +1,4 @@ +import { FlueApiError } from "@flue/sdk"; import { describe, expect, test, vi } from "vitest"; import { createOpenAIRealtimeCallHandler } from "../../../server/voice/openai-realtime-call"; @@ -9,11 +10,13 @@ import { BrunchPanelConversationTracker, createBrunchPanelTransport, } from "../local-storage-demo/brunch-panel-transport"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; +import { submitVoiceInputWithAdmission } from "./voice-interview-control"; import { VoiceTurnController } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; import type { AgentSendResult, FlueClient } from "@flue/sdk"; @@ -73,11 +76,16 @@ const initialMessages = [ id: "initial-question-message", parts: [ { - input: { question: "What happens after approval?" }, - state: "input-available", - toolCallId: "ask-current", - toolName: "brunch_ask", - type: "dynamic-tool", + data: { + question: "What happens after approval?", + toolCallId: "tool-initial-question", + }, + type: "data-brunch-question", + }, + { + state: "done", + text: "What happens after approval?", + type: "text", }, ], role: "assistant", @@ -95,19 +103,100 @@ const responseMessages = [ id: "next-question-message", parts: [ { - input: { question: canonicalQuestion }, - state: "input-available", - toolCallId: "ask-next", - toolName: "brunch_ask", - type: "dynamic-tool", + data: { + question: canonicalQuestion, + toolCallId: "tool-next-question", + }, + type: "data-brunch-question", + }, + { + state: "done", + text: canonicalQuestion, + type: "text", }, ], role: "assistant", }, ] satisfies PetrinautAiMessage[]; +const createAdmissionOutcomeHarness = ( + client: Pick, +) => { + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve(client as FlueClient), + tracker, + ); + let realtimeListener: + | ((event: OpenAIRealtimeSessionEvent) => void) + | undefined; + const bridge = new RealtimeBrunchBridge({ + session: { + speakCanonical: vi.fn(), + subscribe: (listener) => { + realtimeListener = listener; + return () => { + realtimeListener = undefined; + }; + }, + }, + submitInterviewAnswer: (input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + void transport + .sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }) + .catch(() => undefined); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), + }); + const events: RealtimeBrunchBridgeEvent[] = []; + bridge.subscribe((event) => events.push(event)); + bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + bridge.start(1); + + return { + emitCompletedTranscript: (itemId: string) => + realtimeListener?.({ + key: { connectionEpoch: 1, contentIndex: 0, itemId }, + text: spokenAnswer, + type: "completed", + }), + events, + }; +}; + describe("controlled voice preview", () => { - test("bridges one Realtime tool call through Brunch and back to canonical duplex audio", async () => { + test("bridges one completed transcript through Brunch and back to canonical half-duplex audio", async () => { const diagnostics: VoiceDiagnosticEvent[] = []; const reportDiagnostic = (event: VoiceDiagnosticEvent) => diagnostics.push(event); @@ -211,10 +300,59 @@ describe("controlled voice preview", () => { reportDiagnostic, requestAnimationFrame: vi.fn(() => 1), }); - const submitInterviewAnswer = vi.fn(async () => ({ - kind: "interactive-tool" as const, - toolCallId: "ask-current", - })); + const admission: AgentSendResult = { + offset: "offset-voice-1", + streamUrl: "https://petrinaut.test/agents/chat/instance-1", + submissionId: "submission-voice-1", + uid: "uid-voice-1", + }; + const send = vi.fn(async () => admission); + const wait = vi.fn(async () => undefined); + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve({ send, wait } as Pick< + FlueClient, + "send" | "wait" + > as FlueClient), + tracker, + ); + type SubmitInterviewAnswer = ConstructorParameters< + typeof RealtimeBrunchBridge + >[0]["submitInterviewAnswer"]; + const submitInterviewAnswer = vi.fn((input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + const stream = await transport.sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }); + void stream.pipeTo(new WritableStream()); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission: admitted }) => + listener(admitted.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), + ); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -222,32 +360,90 @@ describe("controlled voice preview", () => { const controller = new VoiceTurnController({ bridge, session, - submitText: submitInterviewAnswer, + submitText: vi.fn(async () => ({ kind: "message" as const })), + }); + await controller.start(); + dataChannel.receive({ + audio_start_ms: 200, + item_id: "pre-output-item", + type: "input_audio_buffer.speech_started", }); + dataChannel.receive({ + content_index: 0, + delta: "Speech started before output", + item_id: "pre-output-item", + type: "conversation.item.input_audio_transcription.delta", + }); + expect(controller.getSnapshot().partialText).toBe( + "Speech started before output", + ); + const initialSelection = selectCanonicalSpeech(initialMessages); + const initialSegments = initialSelection.segments; controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, + questionSegment: initialSelection.questionSegment, status: "ready", }); + dataChannel.receive({ + content_index: 0, + item_id: "pre-output-item", + transcript: "This completed before output started.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + microphoneEnabled: true, + partialText: "", + }); + expect(track.enabled).toBe(false); + expect(submitInterviewAnswer).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + + dataChannel.receive({ + content_index: 0, + item_id: "pre-output-item", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(send).not.toHaveBeenCalled(); - await controller.start(); authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); dataChannel.receive({ response_id: "response-initial-question", type: "output_audio_buffer.started", }); + expect(controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + output: "speaking", + }); + + const handoff = controller.takeTurn(); dataChannel.receive({ audio_start_ms: 300, - item_id: "user-item", + item_id: "playback-overlap", type: "input_audio_buffer.speech_started", }); + dataChannel.receive({ + content_index: 0, + item_id: "playback-overlap", + transcript: "Playback must not become input.", + type: "conversation.item.input_audio_transcription.completed", + }); + dataChannel.receive({ type: "input_audio_buffer.cleared" }); dataChannel.receive({ response: { id: "response-initial-question", + output: [], status: "cancelled", }, type: "response.done", }); + dataChannel.receive({ + response_id: "response-initial-question", + type: "output_audio_buffer.cleared", + }); + await handoff; expect(controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -255,75 +451,93 @@ describe("controlled voice preview", () => { }); dataChannel.receive({ - call_id: "call-1", - delta: `{"answer":"${spokenAnswer}"}`, - item_id: "function-item-1", - output_index: 0, - response_id: "response-tool-1", - type: "response.function_call_arguments.delta", + audio_start_ms: 500, + item_id: "user-item", + type: "input_audio_buffer.speech_started", }); dataChannel.receive({ - response: { - id: "response-tool-1", - output: [ - { - arguments: `{"answer":"${spokenAnswer}"}`, - call_id: "call-1", - id: "function-item-1", - name: "continue_interview", - status: "completed", - type: "function_call", - }, - ], - status: "completed", - }, - type: "response.done", + content_index: 0, + delta: "The supervisor", + item_id: "user-item", + type: "conversation.item.input_audio_transcription.delta", + }); + dataChannel.receive({ + content_index: 0, + item_id: "user-item", + transcript: spokenAnswer, + type: "conversation.item.input_audio_transcription.completed", }); await vi.waitFor(() => expect(submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ admissionTarget: { - kind: "client-tool-result", - messageId: "initial-question-message", + kind: "user", + messageId: "voice-realtime:1:user-item:0", }, - id: "voice-realtime:1:call-1", + id: "voice-realtime:1:user-item:0", text: spokenAnswer, }), ), ); - expect(controller.getSnapshot()).toMatchObject({ - input: "submitting", - lastAnswerDelivery: "delivered", - microphoneEnabled: true, - output: "waiting-for-tool", - }); + await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:user:voice-realtime:1:user-item:0", + message: { body: spokenAnswer, kind: "user" }, + }), + ); + await vi.waitFor(() => + expect(controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastAnswerDelivery: "delivered", + microphoneEnabled: true, + output: "waiting-for-tool", + }), + ); controller.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, status: "streaming", }); + const initialSegmentIds = new Set(initialSegments.map(({ id }) => id)); + const responseSelection = selectCanonicalSpeech(responseMessages); + const correlateResponse = (segment: CanonicalSpeechSegment) => + initialSegmentIds.has(segment.id) + ? segment + : { ...segment, submissionIds: [admission.submissionId] }; + const correlatedSegments = + responseSelection.segments.map(correlateResponse); controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(responseMessages), + canonicalSegments: correlatedSegments, + questionSegment: responseSelection.questionSegment + ? correlateResponse(responseSelection.questionSegment) + : undefined, status: "ready", }); - const [functionOutput, responseCreate] = sentEvents(dataChannel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], - }), - }, - }); + const responseCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); expect(responseCreate).toMatchObject({ type: "response.create", response: { + input: [ + { + content: [ + { + text: JSON.stringify({ + response_text: [canonicalReply, canonicalQuestion], + }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], output_modalities: ["audio"], tool_choice: "none", tools: [], @@ -341,6 +555,48 @@ describe("controlled voice preview", () => { microphoneEnabled: true, output: "speaking", }); + expect(track.enabled).toBe(false); + + dataChannel.receive({ + response_id: "response-canonical-reply", + type: "output_audio_buffer.stopped", + }); + dataChannel.receive({ + response: { + id: "response-canonical-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + expect(controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + output: "idle", + }); + + controller.repeatQuestion(); + + const replayCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); + expect(replayCreate).toMatchObject({ + response: { + input: [ + { + content: [ + { + text: JSON.stringify({ response_text: [canonicalQuestion] }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], + }, + type: "response.create", + }); const remoteTrack = { kind: "audio", stop: vi.fn() }; const remoteStream = { @@ -373,11 +629,13 @@ describe("controlled voice preview", () => { turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, + tool_choice: "none", + tools: [], }); expect(diagnostics).toEqual( expect.arrayContaining([ @@ -443,8 +701,6 @@ describe("controlled voice preview", () => { | undefined; const bridge = new RealtimeBrunchBridge({ session: { - completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: (listener) => { realtimeListener = listener; @@ -457,6 +713,7 @@ describe("controlled voice preview", () => { admissionTarget, id, onAdmission, + signal, text, }) => { const unsubscribe = tracker.subscribeToAdmission( @@ -475,7 +732,7 @@ describe("controlled voice preview", () => { parts: [{ type: "text", text }], }, ], - abortSignal: undefined, + abortSignal: signal, }); try { await stream.pipeTo(new WritableStream()); @@ -498,14 +755,14 @@ describe("controlled voice preview", () => { }); bridge.start(1); - const finalized = { - arguments: '{"answer":"The supervisor approves it."}', - callId: "call-1", - connectionEpoch: 1, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done" as const, + const finalized: OpenAIRealtimeSessionEvent = { + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "input-item-1", + }, + text: spokenAnswer, + type: "completed", }; realtimeListener?.(finalized); realtimeListener?.(finalized); @@ -513,7 +770,7 @@ describe("controlled voice preview", () => { await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(bridgeEvents).toContainEqual({ - callId: "call-1", + deliveryId: "voice-realtime:1:input-item-1:0", submissionId: admission.submissionId, type: "submission-admitted", }), @@ -521,11 +778,12 @@ describe("controlled voice preview", () => { expect(bridgeEvents).not.toContainEqual( expect.objectContaining({ type: "submission-accepted" }), ); - expect(send).toHaveBeenCalledWith({ - idempotencyKey: "ai-sdk:user:voice-realtime:1:call-1", - message: { kind: "user", body: "The supervisor approves it." }, - signal: undefined, + const sendInput = send.mock.calls[0]?.[0]; + expect(sendInput).toMatchObject({ + idempotencyKey: "ai-sdk:user:voice-realtime:1:input-item-1:0", + message: { kind: "user", body: spokenAnswer }, }); + expect(sendInput?.signal).toBeInstanceOf(AbortSignal); expect(admission.streamUrl).toContain("/agents/chat/"); settleSubmission?.(); @@ -535,4 +793,80 @@ describe("controlled voice preview", () => { ), ); }); + + test("surfaces an ambiguous Flue admission through the panel observer without retrying", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(500, ""); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-ambiguous"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-ambiguous", + failure: { kind: "ambiguous" }, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("preserves a conflicting submission through the production admission path", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-conflict"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-conflict", + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + }, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("keeps local admission abort distinct from durable Flue abort", async () => { + const send = vi.fn(async () => { + throw new DOMException("Local admission cancelled", "AbortError"); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-aborted"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-aborted", + failure: { kind: "aborted" }, + message: "The local chat submission was cancelled.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index d637a762607..912282facab 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -5,6 +5,9 @@ import { toVoiceSessionState } from "./voice-session-state"; import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -12,6 +15,7 @@ const listeningSnapshot = { errorMessage: "", errorRequestId: "", input: "listening", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: true, @@ -30,13 +34,40 @@ describe("toVoiceSessionState", () => { test("reports a listening turn with its microphone level", () => { expect(mapSnapshot()).toEqual({ + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, + notice: null, phase: "listening", }); }); + test("publishes safe handoff and canonical playback availability", () => { + expect( + mapSnapshot({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }), + ).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }); + }); + + test("describes recoverable transcript rejections", () => { + expect(mapSnapshot({ inputNotice: "not-heard" })?.notice).toBe( + "We didn't catch that. Please try again.", + ); + expect(mapSnapshot({ inputNotice: "too-long" })?.notice).toBe( + "That answer is too long. Please try a shorter response.", + ); + }); + test("hands the turn to the assistant while it speaks", () => { expect(mapSnapshot({ output: "speaking", partialText: "" })).toMatchObject({ phase: "speaking", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index e1a06464eb0..5d8bfce2219 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -65,6 +65,7 @@ const phaseOf = ( return "speaking"; } if ( + snapshot.output === "cancelling" || snapshot.output === "waiting-for-tool" || snapshot.input === "submitting" ) { @@ -93,6 +94,9 @@ export const toVoiceSessionState = ({ } return { + canReadFullResponse: snapshot.canReadFullResponse, + canRepeatQuestion: snapshot.canRepeatQuestion, + canTakeTurn: snapshot.canTakeTurn, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: @@ -100,6 +104,12 @@ export const toVoiceSessionState = ({ snapshot.input !== "paused" && !snapshot.microphoneEnabled, microphoneLevel: snapshot.microphoneLevel, + notice: + snapshot.inputNotice === "not-heard" + ? "We didn't catch that. Please try again." + : snapshot.inputNotice === "too-long" + ? "That answer is too long. Please try a shorter response." + : null, phase: phaseOf(snapshot), }; }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 08dbffb2e7a..7152451dcd0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -16,10 +16,11 @@ const createHarness = () => { | undefined; let bridgeListener: ((event: RealtimeBrunchBridgeEvent) => void) | undefined; const session = { - cancelOutput: vi.fn(), + cancelOutput: vi.fn<() => Promise>(async () => undefined), connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), + speakCanonical: vi.fn(), subscribe: vi.fn( (listener: (event: OpenAIRealtimeSessionEvent) => void) => { sessionListener = listener; @@ -30,6 +31,8 @@ const createHarness = () => { ), }; const bridge = { + cancelPendingSpeech: vi.fn(), + completeTurnHandoff: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -73,230 +76,1279 @@ const question = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); +const markedQuestion = ( + id: string, + text = "What happens after approval?", +): CanonicalSpeechSegment => ({ + ...question(id, text), + source: "assistant-question", +}); + describe("VoiceTurnController", () => { test("records the content-free Voice lifecycle once in causal order", async () => { const harness = createHarness(); await harness.controller.start(); - + + harness.emitBridge({ + answer: "Private finalized answer", + deliveryId: "call-opaque", + type: "submission-started", + }); + harness.advanceTime(10); + harness.emitBridge({ + deliveryId: "call-opaque", + submissionId: "submission-opaque", + type: "submission-admitted", + }); + harness.emitBridge({ + deliveryId: "call-opaque", + submissionId: "submission-opaque", + type: "submission-admitted", + }); + harness.advanceTime(10); + harness.emitBridge({ + answer: "Private finalized answer", + deliveryId: "call-opaque", + type: "submission-accepted", + }); + harness.emitBridge({ + deliveryId: "call-opaque", + type: "canonical-text-ready", + }); + harness.advanceTime(10); + harness.emitBridge({ + deliveryId: "call-opaque", + type: "submission-settled", + }); + harness.advanceTime(10); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-opaque", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-duplicate", + type: "canonical-speech-requested", + }); + harness.advanceTime(10); + const outputStarted: OpenAIRealtimeSessionEvent = { + connectionEpoch: 1, + responseId: "response-opaque", + speechRequestId: "speech-opaque", + type: "output-started", + }; + harness.emitSession(outputStarted); + harness.emitSession(outputStarted); + + expect(harness.latencyEvents).toEqual([ + { + correlationId: "call-opaque", + elapsedMs: 10, + name: "submission-admitted", + }, + { + correlationId: "call-opaque", + elapsedMs: 20, + name: "first-canonical-text", + }, + { + correlationId: "call-opaque", + elapsedMs: 30, + name: "submission-settled", + }, + { + correlationId: "call-opaque", + elapsedMs: 40, + name: "first-tts-request", + }, + { + correlationId: "call-opaque", + elapsedMs: 50, + name: "first-tts-audio", + }, + ]); + expect(JSON.stringify(harness.latencyEvents)).not.toContain( + "Private finalized answer", + ); + + await harness.controller.end(); + harness.emitBridge({ + deliveryId: "call-opaque", + type: "submission-settled", + }); + harness.emitSession(outputStarted); + expect(harness.latencyEvents).toHaveLength(5); + }); + + test("opens a continuous microphone before starting canonical question speech", async () => { + const harness = createHarness(); + const order: string[] = []; + harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { + if (enabled) order.push("microphone-on"); + }); + harness.bridge.start.mockImplementation(() => order.push("bridge-start")); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), + status: "ready", + }); + + await harness.controller.start(); + + expect(order).toEqual(["microphone-on", "bridge-start"]); + expect(harness.controller.getSnapshot()).toMatchObject({ + connection: "connected", + currentQuestion: "What happens after approval?", + input: "listening", + microphoneEnabled: true, + output: "idle", + }); + }); + + test("tracks assistant playback without admitting automatic barge-in", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-1", + speechRequestId: "speech-1", + type: "output-started", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "speaking", + }); + + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + microphoneEnabled: true, + output: "speaking", + }); + expect(harness.session.cancelOutput).not.toHaveBeenCalled(); + }); + + test("clears pre-output capture and only commits fresh post-handoff input", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-late-transcript")], + questionSegment: markedQuestion("ask-late-transcript"), + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-before-output", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "Pre-output partial", + type: "partial", + }); + expect(harness.controller.getSnapshot().partialText).toBe( + "Pre-output partial", + ); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-output", + speechRequestId: "speech-output", + type: "output-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "This completed too late.", + type: "completed", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + partialText: "", + }); + + await harness.controller.takeTurn(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "Fresh post-handoff answer.", + type: "completed", + }); + harness.emitBridge({ + answer: "Fresh post-handoff answer.", + deliveryId: "fresh-delivery", + type: "submission-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastCommittedText: "Fresh post-handoff answer.", + partialText: "", + }); + }); + + test("clears capture when canonical speech is requested before output starts", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-request")], + questionSegment: markedQuestion("ask-request"), + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-request", + segments: [question("ask-request")], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-before-request", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "Provisional pre-request words", + type: "partial", + }); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + lastCommittedText: "", + partialText: "", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "This completed before output started.", + type: "completed", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + + await harness.controller.takeTurn(); + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "Fresh post-handoff answer.", + type: "completed", + }); + harness.emitBridge({ + answer: "Fresh post-handoff answer.", + deliveryId: "fresh-delivery", + type: "submission-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastCommittedText: "Fresh post-handoff answer.", + partialText: "", + }); + }); + + test("offers handoff for canonical output without a question marker", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-without-question", + type: "canonical-speech-requested", + }); + + expect(harness.controller.getSnapshot().canTakeTurn).toBe(true); + await harness.controller.takeTurn(); + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); + }); + + test("offers handoff when canonical output follows an answered question", async () => { + const harness = createHarness(); + const answeredQuestion = markedQuestion("ask-answered"); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [answeredQuestion], + questionSegment: answeredQuestion, + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-answered", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-answered", + type: "submission-accepted", + }); + harness.emitBridge({ + deliveryId: "call-answered", + type: "submission-settled", + }); + harness.emitBridge({ + deliveryId: "call-answered", + segments: [question("follow-on", "Here is the follow-on detail.")], + type: "canonical-response-ready", + }); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-after-answer", + type: "canonical-speech-requested", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + input: "listening", + output: "waiting-for-tool", + }); + }); + + test("keeps handoff unavailable while disconnected or paused", async () => { + const harness = createHarness(); + expect(harness.controller.getSnapshot().canTakeTurn).toBe(false); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-before-pause", + type: "canonical-speech-requested", + }); + + harness.controller.pause(); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + input: "paused", + }); + }); + + test("hands off an active response once and applies the latest mute preference after cancellation", async () => { + const harness = createHarness(); + let finishCancellation: (() => void) | undefined; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), + status: "ready", + }); + await harness.controller.start(); + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + harness.session.cancelOutput.mockClear(); + + expect(harness.controller.getSnapshot().canTakeTurn).toBe(true); + const handoff = harness.controller.takeTurn(); + const repeatedHandoff = harness.controller.takeTurn(); + + expect(repeatedHandoff).toBe(handoff); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + output: "cancelling", + }); + + harness.session.setMicrophoneEnabled.mockClear(); + harness.controller.setMicrophoneMuted(true); + harness.controller.setMicrophoneMuted(false); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot().microphoneEnabled).toBe(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + type: "output-interrupted", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + status: "cancelled", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().output).toBe("cancelling"); + + finishCancellation?.(); + await handoff; + + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("reopens the microphone only after cancellation and Brunch settlement", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("answered-question")], + questionSegment: markedQuestion("answered-question"), + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + answer: "The approved answer.", + deliveryId: "voice-request", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The approved answer.", + deliveryId: "voice-request", + type: "submission-accepted", + }); + harness.controller.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question("next-question")], + questionSegment: markedQuestion("next-question"), + status: "streaming", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + const handoff = harness.controller.takeTurn(); + let handoffFinished = false; + void handoff.then(() => { + handoffFinished = true; + }); + await Promise.resolve(); + + expect(handoffFinished).toBe(false); + expect(harness.bridge.completeTurnHandoff).not.toHaveBeenCalled(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(false); + + harness.emitBridge({ + deliveryId: "voice-request", + type: "submission-settled", + }); + await handoff; + + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("cancels queued and later speech when the host stops a response", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.controller.cancelPendingSpeech(); + + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + }); + + test("releases bridge ownership after pending output cancellation is acknowledged", async () => { + const harness = createHarness(); + let finishCancellation: (() => void) | undefined; + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-pending-cancellation", + type: "canonical-speech-requested", + }); + + harness.controller.cancelPendingSpeech(); + expect(harness.bridge.completeTurnHandoff).not.toHaveBeenCalled(); + + finishCancellation?.(); + await vi.waitFor(() => + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(), + ); + }); + + test("keeps the user turn when cancelled pending speech settles later", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + + await harness.controller.takeTurn(); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [question("ask-late", "Retained late response")], + speechCancelled: true, + type: "canonical-response-ready", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "interrupted", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("restores capture when cancelled settlement arrives after interrupted early speech", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-interrupted-early", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-interrupted-early", + type: "submission-accepted", + }); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-interrupted-early", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-interrupted-early", + speechRequestId: "speech-interrupted-early", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-interrupted-early", + type: "output-interrupted", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-interrupted-early", + status: "cancelled", + type: "response-terminal", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitBridge({ + deliveryId: "call-interrupted-early", + segments: [], + speechCancelled: true, + type: "canonical-response-ready", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "interrupted", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("replays exact canonical response segments but does not infer a question from the final segment", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const nextQuestion = question("ask-replay", "Who approves release?"); + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + segments: [context, nextQuestion], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "unrelated-response", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().canRepeatQuestion).toBe(false); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: false, + }); + + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + context, + nextQuestion, + ]); + }); + + test("repeats only the exact Brunch-marked question after replay settles", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const finalProse = question( + "response-prose", + "The approver is recorded. I can explain the escalation path.", + ); + const exactQuestion: CanonicalSpeechSegment = { + ...question("marked-question", "Who approves release?"), + messageId: finalProse.messageId, + source: "assistant-question", + }; + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + questionSegment: exactQuestion, + segments: [context, finalProse], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + + harness.controller.repeatQuestion(); + + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + exactQuestion, + ]); + }); + + test.each(["resolved", "rejected"] as const)( + "keeps replay disabled until generic cancellation is %s", + async (cancellationOutcome) => { + const harness = createHarness(); + const replayQuestion = markedQuestion( + "replay-after-cancellation", + "Who approves release?", + ); + let finishCancellation: (() => void) | undefined; + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve, reject) => { + finishCancellation = () => { + if (cancellationOutcome === "resolved") { + resolve(); + } else { + reject(new Error("Cancellation failed.")); + } + }; + }), + ); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-replay", + questionSegment: replayQuestion, + segments: [replayQuestion], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + speechRequestId: "speech-replay", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + input: "listening", + output: "idle", + }); + + harness.controller.cancelPendingSpeech(); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.readFullResponse(); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + finishCancellation?.(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + input: "listening", + output: "idle", + }), + ); + }, + ); + + test("disables replay while the user is capturing input", async () => { + const harness = createHarness(); + const segment = question("ask-capture"); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [segment], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: false, + }); + + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + }); + + test("keeps capture closed from submission until canonical output settles", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-1", + type: "submission-started", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastAnswerDelivery: "pending", + lastCommittedText: "The supervisor approves it.", + microphoneEnabled: true, + output: "waiting-for-tool", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-1", + type: "submission-accepted", + }); + harness.emitBridge({ + deliveryId: "call-1", + segments: [question("ask-2", "Who acts next?")], + type: "canonical-response-ready", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + lastAnswerDelivery: "delivered", + microphoneEnabled: true, + output: "waiting-for-tool", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-next", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + speechRequestId: "speech-next", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + status: "completed", + type: "response-terminal", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + type: "output-stopped", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("returns to listening after a durably stopped turn without speaking", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitBridge({ + answer: "Stop this one.", + deliveryId: "voice-1", + type: "submission-started", + }); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ + answer: "Stop this one.", + deliveryId: "voice-1", + type: "submission-accepted", + }); + harness.advanceTime(40); + harness.emitBridge({ deliveryId: "voice-1", type: "submission-settled" }); + harness.emitBridge({ + deliveryId: "voice-1", + outcome: "aborted", + type: "submission-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + lastAnswerDelivery: "delivered", + output: "idle", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(harness.latencyEvents).toContainEqual({ + correlationId: "voice-1", + elapsedMs: 40, + name: "submission-settled", + }); + }); + + test("keeps finished early speech idle and restores capture after canonical settlement", async () => { + const harness = createHarness(); + const nextQuestion = markedQuestion("ask-early", "Who acts next?"); + await harness.controller.start(); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-early", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "call-early", + type: "submission-accepted", + }); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-early", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + speechRequestId: "speech-early", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + type: "output-stopped", + }); + + harness.emitBridge({ + deliveryId: "call-early", + questionSegment: nextQuestion, + segments: [nextQuestion], + type: "canonical-response-ready", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + input: "listening", + output: "idle", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("keeps capture closed when more canonical speech starts at settlement", async () => { + const harness = createHarness(); + const finalSegment = markedQuestion("ask-final", "Who acts next?"); + await harness.controller.start(); harness.emitBridge({ - answer: "Private finalized answer", - callId: "call-opaque", + answer: "The supervisor approves it.", + deliveryId: "call-queued", type: "submission-started", }); - harness.advanceTime(10); - harness.emitBridge({ - callId: "call-opaque", - submissionId: "submission-opaque", - type: "submission-admitted", + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-early", + type: "canonical-speech-requested", }); - harness.emitBridge({ - callId: "call-opaque", - submissionId: "submission-opaque", - type: "submission-admitted", + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + speechRequestId: "speech-early", + type: "output-started", }); - harness.advanceTime(10); - harness.emitBridge({ - answer: "Private finalized answer", - callId: "call-opaque", - type: "submission-accepted", + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + status: "completed", + type: "response-terminal", }); - harness.emitBridge({ - callId: "call-opaque", - type: "canonical-text-ready", + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + type: "output-stopped", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-final", + type: "canonical-speech-requested", }); - harness.advanceTime(10); harness.emitBridge({ - callId: "call-opaque", - type: "submission-settled", + deliveryId: "call-queued", + questionSegment: finalSegment, + segments: [finalSegment], + type: "canonical-response-ready", }); - harness.advanceTime(10); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + input: "listening", + output: "waiting-for-tool", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + harness.emitSession({ connectionEpoch: 1, - speechRequestId: "speech-opaque", - type: "canonical-speech-requested", + responseId: "response-final", + speechRequestId: "speech-final", + type: "output-started", }); harness.emitSession({ connectionEpoch: 1, - speechRequestId: "speech-duplicate", - type: "canonical-speech-requested", + responseId: "response-final", + status: "completed", + type: "response-terminal", }); - harness.advanceTime(10); - const outputStarted: OpenAIRealtimeSessionEvent = { + harness.emitSession({ connectionEpoch: 1, - responseId: "response-opaque", - speechRequestId: "speech-opaque", - type: "output-started", - }; - harness.emitSession(outputStarted); - harness.emitSession(outputStarted); - - expect(harness.latencyEvents).toEqual([ - { - correlationId: "call-opaque", - elapsedMs: 10, - name: "submission-admitted", - }, - { - correlationId: "call-opaque", - elapsedMs: 20, - name: "first-canonical-text", - }, - { - correlationId: "call-opaque", - elapsedMs: 30, - name: "submission-settled", - }, - { - correlationId: "call-opaque", - elapsedMs: 40, - name: "first-tts-request", - }, - { - correlationId: "call-opaque", - elapsedMs: 50, - name: "first-tts-audio", - }, - ]); - expect(JSON.stringify(harness.latencyEvents)).not.toContain( - "Private finalized answer", - ); + responseId: "response-final", + type: "output-stopped", + }); - await harness.controller.end(); - harness.emitBridge({ - callId: "call-opaque", - type: "submission-settled", + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + output: "idle", }); - harness.emitSession(outputStarted); - expect(harness.latencyEvents).toHaveLength(5); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); - test("opens a continuous microphone before starting canonical question speech", async () => { + test("keeps follow-on speech pending when the earlier output stop arrives late", async () => { const harness = createHarness(); - const order: string[] = []; - harness.session.setMicrophoneEnabled.mockImplementation((enabled) => { - if (enabled) order.push("microphone-on"); - }); - harness.bridge.start.mockImplementation(() => order.push("bridge-start")); + const followOnQuestion = markedQuestion("ask-follow-on", "Who acts next?"); harness.controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question("ask-1")], + canonicalSegments: [followOnQuestion], + questionSegment: followOnQuestion, status: "ready", }); - await harness.controller.start(); - - expect(order).toEqual(["microphone-on", "bridge-start"]); - expect(harness.controller.getSnapshot()).toMatchObject({ - connection: "connected", - currentQuestion: "What happens after approval?", - input: "listening", - microphoneEnabled: true, - output: "idle", + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-early", + type: "canonical-speech-requested", }); - }); - - test("keeps capture active while the interviewer speaks and interrupts automatically", async () => { - const harness = createHarness(); - await harness.controller.start(); - harness.emitSession({ connectionEpoch: 1, - responseId: "response-1", - speechRequestId: "speech-1", + responseId: "response-early", + speechRequestId: "speech-early", type: "output-started", }); - expect(harness.controller.getSnapshot()).toMatchObject({ - input: "listening", - microphoneEnabled: true, - output: "speaking", + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-early", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-follow-on", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-follow-on", + speechRequestId: "speech-follow-on", + status: "completed", + type: "response-terminal", }); + harness.session.setMicrophoneEnabled.mockClear(); harness.emitSession({ connectionEpoch: 1, - itemId: "item-user", - type: "input-speech-started", + responseId: "response-early", + type: "output-stopped", }); + expect(harness.controller.getSnapshot()).toMatchObject({ - microphoneEnabled: true, - output: "interrupted", + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: true, + output: "waiting-for-tool", }); - expect(harness.session.cancelOutput).not.toHaveBeenCalled(); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-follow-on", + speechRequestId: "speech-follow-on", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-follow-on", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot().output).toBe("idle"); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); - test("represents submitting and output independently without closing capture", async () => { + test("preserves speaking output when canonical settlement arrives during playback", async () => { const harness = createHarness(); + const nextQuestion = markedQuestion("ask-playing", "Who acts next?"); await harness.controller.start(); - harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-playing", type: "submission-started", }); - expect(harness.controller.getSnapshot()).toMatchObject({ - input: "submitting", - lastAnswerDelivery: "pending", - lastCommittedText: "The supervisor approves it.", - microphoneEnabled: true, - output: "waiting-for-tool", + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-playing", + type: "canonical-speech-requested", }); - harness.emitBridge({ - answer: "The supervisor approves it.", - callId: "call-1", - type: "submission-accepted", + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + speechRequestId: "speech-playing", + type: "output-started", }); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ - callId: "call-1", - segments: [question("ask-2", "Who acts next?")], + deliveryId: "call-playing", + questionSegment: nextQuestion, + segments: [nextQuestion], type: "canonical-response-ready", }); expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, input: "listening", - lastAnswerDelivery: "delivered", - microphoneEnabled: true, - output: "waiting-for-tool", + output: "speaking", }); - expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith( - false, - ); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + output: "idle", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); - test("returns to listening after a durably stopped turn without speaking", async () => { + test("restores capture after a cancelled reply finishes provider cancellation", async () => { const harness = createHarness(); + let finishCancellation: (() => void) | undefined; await harness.controller.start(); - harness.emitBridge({ - answer: "Stop this one.", - callId: "call-1", + answer: "Cancel this reply.", + deliveryId: "call-cancelled", type: "submission-started", }); + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + harness.controller.cancelPendingSpeech(); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ - answer: "Stop this one.", - callId: "call-1", - type: "submission-accepted", - }); - harness.advanceTime(40); - harness.emitBridge({ callId: "call-1", type: "submission-settled" }); - harness.emitBridge({ - callId: "call-1", - outcome: "aborted", - type: "submission-stopped", + deliveryId: "call-cancelled", + segments: [], + speechCancelled: true, + type: "canonical-response-ready", }); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", - lastAnswerDelivery: "delivered", - output: "idle", - }); - expect(harness.latencyEvents).toContainEqual({ - correlationId: "call-1", - elapsedMs: 40, - name: "submission-settled", + output: "interrupted", }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + finishCancellation?.(); + await Promise.resolve(); + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); test("restores submission state when resumed before Brunch releases the turn", async () => { @@ -304,13 +1356,14 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); - harness.controller.resume(); + await harness.controller.resume(); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "submitting", lastAnswerDelivery: "pending", @@ -319,11 +1372,11 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -339,27 +1392,29 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), status: "ready", }); await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-2", "Who acts next?")], + questionSegment: markedQuestion("ask-2", "Who acts next?"), status: "ready", }); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -381,13 +1436,13 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -397,9 +1452,9 @@ describe("VoiceTurnController", () => { microphoneEnabled: false, output: "interrupted", }); - expect(harness.session.cancelOutput).toHaveBeenCalledTimes(2); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -434,6 +1489,33 @@ describe("VoiceTurnController", () => { expect(harness.submitText).not.toHaveBeenCalled(); }); + test.each(["empty", "failed"] as const)( + "reports a recoverable not-heard notice for a %s transcript", + async (reason) => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-1", + type: "input-speech-started", + }); + harness.emitSession({ + key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-1" }, + text: "Provisional words", + type: "partial", + }); + + harness.emitBridge({ reason, type: "transcript-rejected" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + inputNotice: "not-heard", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + }, + ); + test("keeps completed display transcripts until submission and rejects late events", async () => { const harness = createHarness(); await harness.controller.start(); @@ -449,11 +1531,11 @@ describe("VoiceTurnController", () => { }); harness.emitBridge({ answer: "First answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -539,13 +1621,44 @@ describe("VoiceTurnController", () => { output: "interrupted", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, }); }); + test("reuses pending output cancellation across paused chat updates", async () => { + const harness = createHarness(); + let finishCancellation: (() => void) | undefined; + harness.session.cancelOutput.mockImplementation( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + await harness.controller.start(); + harness.controller.pause(); + const listener = vi.fn(); + harness.controller.subscribe(listener); + const update = { + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready" as const, + }; + + harness.controller.updateChat(update); + harness.controller.updateChat(update); + + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(listener).not.toHaveBeenCalled(); + + finishCancellation?.(); + await vi.waitFor(() => + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(), + ); + }); + test("mutes capture without interrupting what the interviewer is saying", async () => { const harness = createHarness(); await harness.controller.start(); @@ -617,7 +1730,7 @@ describe("VoiceTurnController", () => { output: "idle", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.bridge.start).toHaveBeenCalledWith(1); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", @@ -625,7 +1738,7 @@ describe("VoiceTurnController", () => { }); }); - test("cancels output that starts while paused without exposing speaking", async () => { + test("keeps pending cancellation when output starts while paused", async () => { const harness = createHarness(); await harness.controller.start(); harness.controller.pause(); @@ -640,7 +1753,7 @@ describe("VoiceTurnController", () => { type: "output-started", }); - expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).not.toHaveBeenCalled(); expect(observedOutputs).not.toContain("speaking"); expect(harness.controller.getSnapshot()).toMatchObject({ input: "paused", @@ -735,6 +1848,10 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-reconnect", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-reconnect", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); @@ -761,12 +1878,16 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-failed-delivery", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-failed-delivery", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ @@ -784,6 +1905,52 @@ describe("VoiceTurnController", () => { expect(harness.session.connect).toHaveBeenCalledTimes(2); }); + test.each([ + { + code: "admission-rejected" as const, + failure: { kind: "rejected", status: 403 } as const, + message: "Brunch rejected the message before admission (HTTP 403).", + }, + { + code: "admission-conflict" as const, + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + } as const, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + }, + { + code: "admission-ambiguous" as const, + failure: { kind: "ambiguous" } as const, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + }, + { + code: "admission-aborted" as const, + failure: { kind: "aborted" } as const, + message: "The local chat submission was cancelled.", + }, + ])("surfaces $failure.kind admission safely", async (admissionFailure) => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "voice-turn-1", + type: "submission-started", + }); + + harness.emitBridge({ ...admissionFailure, type: "error" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + connection: "error", + errorCode: admissionFailure.code, + errorMessage: admissionFailure.message, + lastAnswerDelivery: "failed", + }); + }); + test("clears a provisional transcript when the interview fails", async () => { const harness = createHarness(); await harness.controller.start(); @@ -835,7 +2002,7 @@ describe("VoiceTurnController", () => { await bridgeFailure.controller.start(); bridgeFailure.emitBridge({ answer: "Pending answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); bridgeFailure.emitBridge({ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 0f4e42cbac6..4c8c5950c3a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -19,13 +19,18 @@ export type VoiceConnectionState = | "error"; export type VoiceInputState = "listening" | "paused" | "submitting"; export type VoiceOutputState = + | "cancelling" | "idle" | "waiting-for-tool" | "speaking" | "interrupted"; export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed"; +export type VoiceInputNotice = "none" | "not-heard" | "too-long"; export interface VoiceTurnSnapshot { + readonly canReadFullResponse: boolean; + readonly canRepeatQuestion: boolean; + readonly canTakeTurn: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -33,6 +38,7 @@ export interface VoiceTurnSnapshot { readonly errorMessage: string; readonly errorRequestId: string; readonly input: VoiceInputState; + readonly inputNotice: VoiceInputNotice; readonly lastAnswerDelivery: VoiceAnswerDelivery; readonly lastCommittedText: string; readonly microphoneEnabled: boolean; @@ -57,14 +63,17 @@ export interface VoiceLatencyEvent { } interface RealtimeSession { - cancelOutput(): void; + cancelOutput(): Promise; connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; + speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } interface RealtimeBridge { + cancelPendingSpeech(): void; + completeTurnHandoff(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -90,13 +99,24 @@ interface VoiceTurnControllerDependencies { interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly questionSegment?: CanonicalSpeechSegment; readonly settlements?: readonly VoiceSubmissionSettlement[]; + readonly stopped?: boolean; readonly status: PetrinautAiVoiceModeContext["status"]; } +interface PendingSubmissionSettlement { + readonly deliveryId: string; + readonly promise: Promise; + readonly resolve: () => void; +} + type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -104,6 +124,7 @@ const initialSnapshot: VoiceTurnSnapshot = { errorMessage: "", errorRequestId: "", input: "paused", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: false, @@ -112,11 +133,6 @@ const initialSnapshot: VoiceTurnSnapshot = { partialText: "", }; -const latestQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); - export class VoiceTurnController { readonly #bridge: RealtimeBridge; readonly #listeners = new Set(); @@ -125,18 +141,29 @@ export class VoiceTurnController { readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; #activeEpoch: number | null = null; + #activeSpeechOutputEnded = false; + #activeSpeechResponseId: string | null = null; + #activeSpeechResponseTerminal = false; #answerFinalizedAt: number | null = null; #answeredQuestionId: string | null = null; #bridgeStarted = false; #currentQuestionId: string | null = null; #generation = 0; #inputStateOnResume: Exclude | null = null; + #inputTurnPending = false; #latencyCorrelationId: string | null = null; + #lastResponseQuestion: CanonicalSpeechSegment | null = null; + #lastResponseSegments: CanonicalSpeechSegment[] = []; + #outputCancellationPromise: Promise | null = null; #pauseRequested = false; + readonly #pendingSpeechRequestIds = new Set(); + #pendingSubmissionSettlement: PendingSubmissionSettlement | null = null; readonly #recordedLatencyEvents = new Set(); #snapshot = initialSnapshot; #submittingQuestionId: string | null = null; + #takingTurnPromise: Promise | null = null; #teardownPromise: Promise | null = null; + readonly #terminalSpeechRequestIds = new Set(); #transcriptItemId: string | null = null; #transcriptKey: string | null = null; #ttsSpeechRequestId: string | null = null; @@ -195,8 +222,16 @@ export class VoiceTurnController { } this.#inputStateOnResume = null; + this.#inputTurnPending = false; + this.#outputCancellationPromise = null; this.#pauseRequested = false; + this.#pendingSpeechRequestIds.clear(); + this.#terminalSpeechRequestIds.clear(); + this.#completeSubmissionSettlement(); this.#bridgeStarted = false; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#update({ connection: "connecting", errorCode: null, @@ -234,14 +269,25 @@ export class VoiceTurnController { public async end(): Promise { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#answerFinalizedAt = null; this.#answeredQuestionId = null; this.#bridgeStarted = false; this.#currentQuestionId = null; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#lastResponseQuestion = null; + this.#lastResponseSegments = []; + this.#outputCancellationPromise = null; + this.#pendingSpeechRequestIds.clear(); this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = null; + this.#takingTurnPromise = null; + this.#terminalSpeechRequestIds.clear(); + this.#completeSubmissionSettlement(); this.#pauseRequested = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -292,7 +338,7 @@ export class VoiceTurnController { this.#inputStateOnResume = this.#snapshot.input; this.#pauseRequested = true; const output = this.#snapshot.output === "idle" ? "idle" : "interrupted"; - this.#session.cancelOutput(); + this.cancelPendingSpeech(); this.#session.setMicrophoneEnabled(false); this.#update({ input: "paused", @@ -315,17 +361,51 @@ export class VoiceTurnController { ) { return; } - this.#session.setMicrophoneEnabled(!muted); + if ( + this.#takingTurnPromise === null && + this.#outputCancellationPromise === null && + this.#activeSpeechResponseId === null && + (this.#snapshot.output === "idle" || + this.#snapshot.output === "interrupted") + ) { + this.#session.setMicrophoneEnabled(!muted); + } this.#update({ microphoneEnabled: !muted, microphoneLevel: 0 }); } - public resume(): void { + public async resume(): Promise { if ( this.#snapshot.connection !== "connected" || this.#snapshot.input !== "paused" ) { return; } + const generation = this.#generation; + while (this.#outputCancellationPromise || this.#takingTurnPromise) { + try { + await (this.#outputCancellationPromise ?? this.#takingTurnPromise); + } catch (error) { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + return; + } + const snapshotAfterCancellation = this.getSnapshot(); + if ( + generation !== this.#generation || + snapshotAfterCancellation.connection !== "connected" || + snapshotAfterCancellation.input !== "paused" + ) { + return; + } + } const input = this.#inputStateOnResume ?? "listening"; this.#inputStateOnResume = null; this.#pauseRequested = false; @@ -350,6 +430,11 @@ export class VoiceTurnController { this.#update({ input, microphoneEnabled: true }); } + public cancelPendingSpeech(): void { + this.#bridge.cancelPendingSpeech(); + void this.#cancelOutput(); + } + public async submitCorrection(correction: string): Promise { const correctedText = correction.trim(); const previousText = this.#snapshot.lastCommittedText; @@ -382,39 +467,123 @@ export class VoiceTurnController { } } + public readFullResponse(): void { + if (!this.#snapshot.canReadFullResponse) return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([...this.#lastResponseSegments]); + } + + public repeatQuestion(): void { + if (!this.#snapshot.canRepeatQuestion || !this.#lastResponseQuestion) + return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([this.#lastResponseQuestion]); + } + + /** + * Hands the turn to the user only after provider cancellation has cleared + * input and output and the active response has reached a terminal state. + */ + public takeTurn(): Promise { + if (this.#takingTurnPromise) return this.#takingTurnPromise; + if (!this.#snapshot.canTakeTurn) return Promise.resolve(); + + const generation = this.#generation; + this.#bridge.cancelPendingSpeech(); + this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ output: "cancelling", partialText: "" }); + + const submissionSettlement = + this.#pendingSubmissionSettlement?.promise ?? Promise.resolve(); + const takingTurnPromise = Promise.all([ + this.#session.cancelOutput(), + submissionSettlement, + ]) + .then(() => { + if ( + generation !== this.#generation || + this.#snapshot.connection !== "connected" || + this.#snapshot.input === "paused" + ) { + return; + } + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + this.#pendingSpeechRequestIds.clear(); + this.#terminalSpeechRequestIds.clear(); + this.#bridge.completeTurnHandoff(); + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + this.#update({ output: "interrupted" }); + }) + .catch((error: unknown) => { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + }) + .finally(() => { + if (this.#takingTurnPromise === takingTurnPromise) { + this.#takingTurnPromise = null; + this.#update({}); + } + }); + this.#takingTurnPromise = takingTurnPromise; + this.#update({}); + return takingTurnPromise; + } + public updateChat(update: ChatUpdate): void { - const question = latestQuestion(update.canonicalSegments); + const question = update.questionSegment; if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); this.#recordLatency("question-visible", question.id); } this.#bridge.updateChat(update); - if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + if ( + this.#snapshot.input === "paused" && + (this.#activeSpeechResponseId !== null || + this.#pendingSpeechRequestIds.size > 0) + ) { + void this.#cancelOutput(); } } #handleBridgeEvent(event: RealtimeBrunchBridgeEvent): void { if (this.#snapshot.connection !== "connected") return; if (event.type === "error") { + this.#completeSubmissionSettlement(); this.#setError(event.message, event.code); return; } if (event.type === "submission-started") { + this.#beginSubmissionSettlement(event.deliveryId); const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "submitting"; } + this.#inputTurnPending = false; this.#answerFinalizedAt = this.#now(); - this.#latencyCorrelationId = event.callId; + this.#latencyCorrelationId = event.deliveryId; this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = this.#currentQuestionId; this.#transcriptItemId = null; this.#transcriptKey = null; this.#ttsSpeechRequestId = null; + this.#session.setMicrophoneEnabled(false); this.#update({ input: paused ? "paused" : "submitting", + inputNotice: "none", lastAnswerDelivery: "pending", lastCommittedText: event.answer, output: "waiting-for-tool", @@ -422,6 +591,18 @@ export class VoiceTurnController { }); return; } + if (event.type === "transcript-rejected") { + if (event.reason === "duplicate" || event.reason === "unavailable") { + return; + } + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ + inputNotice: event.reason === "over-limit" ? "too-long" : "not-heard", + partialText: "", + }); + return; + } if (event.type === "submission-accepted") { this.#answeredQuestionId = this.#submittingQuestionId; this.#submittingQuestionId = null; @@ -429,18 +610,20 @@ export class VoiceTurnController { return; } if (event.type === "submission-admitted") { - this.#recordLatency("submission-admitted", event.callId); + this.#recordLatency("submission-admitted", event.deliveryId); return; } if (event.type === "canonical-text-ready") { - this.#recordLatency("first-canonical-text", event.callId); + this.#recordLatency("first-canonical-text", event.deliveryId); return; } if (event.type === "submission-settled") { - this.#recordLatency("submission-settled", event.callId); + this.#completeSubmissionSettlement(event.deliveryId); + this.#recordLatency("submission-settled", event.deliveryId); return; } if (event.type === "submission-stopped") { + this.#completeSubmissionSettlement(event.deliveryId); // Brunch was stopped before it replied: nothing to speak, and the // interviewer is free to listen again. const pausedWhileStopped = this.#snapshot.input === "paused"; @@ -451,21 +634,46 @@ export class VoiceTurnController { input: pausedWhileStopped ? "paused" : "listening", output: "idle", }); + this.#restoreMicrophoneIfCaptureAvailable(); + return; + } + this.#completeSubmissionSettlement(event.deliveryId); + this.#lastResponseQuestion = event.questionSegment ?? null; + this.#lastResponseSegments = [...event.segments]; + const responseEnd = event.segments.at(-1); + if (event.speechCancelled) { + const paused = this.#snapshot.input === "paused"; + if (paused) { + this.#inputStateOnResume = "listening"; + } + this.#update({ + input: paused ? "paused" : "listening", + output: "interrupted", + }); + this.#restoreMicrophoneIfCaptureAvailable(); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); return; } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; - this.#session.cancelOutput(); + void this.#cancelOutput(); } + const preserveSettledOutput = + this.#snapshot.output === "speaking" || + this.#snapshot.output === "waiting-for-tool" || + (this.#snapshot.output === "idle" && + this.#snapshot.input === "submitting"); this.#update({ input: paused ? "paused" : "listening", - output: paused ? "interrupted" : "waiting-for-tool", + output: paused + ? "interrupted" + : preserveSettledOutput + ? this.#snapshot.output + : "waiting-for-tool", }); - const question = event.segments.findLast( - ({ source }) => source === "brunch-ask", - ); - if (question) this.#recordLatency("answer-ready", question.id); + this.#restoreMicrophoneIfCaptureAvailable(); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { @@ -486,6 +694,12 @@ export class VoiceTurnController { return; } if (event.type === "canonical-speech-requested") { + this.#pendingSpeechRequestIds.add(event.speechRequestId); + this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ output: "waiting-for-tool", partialText: "" }); if ( this.#latencyCorrelationId !== null && this.#ttsSpeechRequestId === null @@ -496,12 +710,20 @@ export class VoiceTurnController { return; } if (event.type === "output-started") { + this.#pendingSpeechRequestIds.delete(event.speechRequestId); + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = event.responseId; + this.#activeSpeechResponseTerminal = + this.#terminalSpeechRequestIds.delete(event.speechRequestId); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); - this.#update({ output: "interrupted" }); + void this.#cancelOutput(); + this.#update({ output: "interrupted", partialText: "" }); return; } - this.#update({ output: "speaking" }); + this.#update({ output: "speaking", partialText: "" }); if ( this.#latencyCorrelationId !== null && event.speechRequestId === this.#ttsSpeechRequestId @@ -514,32 +736,69 @@ export class VoiceTurnController { return; } if (event.type === "output-stopped") { - this.#update({ output: "idle" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#outputAfterPlaybackEnds("idle"), + }); + this.#restoreMicrophoneIfCaptureAvailable(); if (this.#currentQuestionId) { this.#recordLatency("question-spoken", this.#currentQuestionId); } return; } if (event.type === "output-interrupted") { - this.#update({ output: "interrupted" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#outputAfterPlaybackEnds("interrupted"), + }); + this.#restoreMicrophoneIfCaptureAvailable(); return; } if (event.type === "input-speech-started") { + if ( + this.#takingTurnPromise || + this.#snapshot.output === "speaking" || + this.#snapshot.output === "cancelling" + ) { + return; + } + this.#inputTurnPending = true; this.#transcriptItemId = event.itemId; this.#transcriptKey = null; - if (this.#snapshot.output === "speaking") { - this.#update({ output: "interrupted", partialText: "" }); - } else { - this.#update({ partialText: "" }); + this.#update({ inputNotice: "none", partialText: "" }); + return; + } + if (event.type === "response-terminal") { + if (event.responseId === this.#activeSpeechResponseId) { + if (this.#activeSpeechOutputEnded) { + this.#clearSettledSpeech(); + } else { + this.#activeSpeechResponseTerminal = true; + } + this.#update({}); + this.#restoreMicrophoneIfCaptureAvailable(); + } else if ( + event.speechRequestId !== undefined && + this.#pendingSpeechRequestIds.has(event.speechRequestId) + ) { + if (event.status === "completed") { + this.#terminalSpeechRequestIds.add(event.speechRequestId); + } else { + this.#pendingSpeechRequestIds.delete(event.speechRequestId); + this.#terminalSpeechRequestIds.delete(event.speechRequestId); + } } return; } - if ( - event.type === "input-speech-stopped" || - event.type === "response-terminal" || - event.type === "tool-arguments-delta" || - event.type === "tool-arguments-done" - ) { + if (event.type === "input-speech-stopped") { return; } @@ -547,6 +806,7 @@ export class VoiceTurnController { if (event.key.connectionEpoch !== this.#activeEpoch) return; if (event.key.itemId !== this.#transcriptItemId) return; if (event.type === "transcription-failed") { + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ partialText: "" }); @@ -560,6 +820,7 @@ export class VoiceTurnController { }); return; } + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ @@ -574,9 +835,18 @@ export class VoiceTurnController { ): void { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#outputCancellationPromise = null; + this.#pendingSpeechRequestIds.clear(); this.#recordedLatencyEvents.clear(); + this.#takingTurnPromise = null; + this.#terminalSpeechRequestIds.clear(); + this.#completeSubmissionSettlement(); this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -601,6 +871,99 @@ export class VoiceTurnController { }); } + #cancelOutput(): Promise { + if (this.#outputCancellationPromise) { + return this.#outputCancellationPromise; + } + const cancellationPromise = this.#session.cancelOutput(); + this.#outputCancellationPromise = cancellationPromise; + this.#update({}); + void cancellationPromise.then( + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + this.#pendingSpeechRequestIds.clear(); + this.#terminalSpeechRequestIds.clear(); + this.#clearSettledSpeech(); + this.#bridge.completeTurnHandoff(); + const output = + this.#snapshot.output === "waiting-for-tool" || + this.#snapshot.output === "speaking" + ? "interrupted" + : this.#snapshot.output; + this.#update({ output }); + this.#restoreMicrophoneIfCaptureAvailable(); + } + }, + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + this.#update({}); + this.#restoreMicrophoneIfCaptureAvailable(); + } + }, + ); + return cancellationPromise; + } + + #beginSubmissionSettlement(deliveryId: string): void { + this.#completeSubmissionSettlement(); + let resolve = () => {}; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + this.#pendingSubmissionSettlement = { deliveryId, promise, resolve }; + } + + #completeSubmissionSettlement(deliveryId?: string): void { + const pending = this.#pendingSubmissionSettlement; + if ( + pending === null || + (deliveryId !== undefined && deliveryId !== pending.deliveryId) + ) { + return; + } + this.#pendingSubmissionSettlement = null; + pending.resolve(); + } + + #clearSettledSpeech(): void { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + } + + #outputAfterPlaybackEnds( + settledOutput: Extract, + ): VoiceOutputState { + if (this.#takingTurnPromise) { + return "cancelling"; + } + if ( + this.#pendingSpeechRequestIds.size > 0 && + this.#outputCancellationPromise === null && + this.#snapshot.input !== "paused" + ) { + return "waiting-for-tool"; + } + return settledOutput; + } + + #restoreMicrophoneIfCaptureAvailable(): void { + if ( + this.#snapshot.connection === "connected" && + this.#snapshot.input === "listening" && + this.#activeSpeechResponseId === null && + this.#pendingSpeechRequestIds.size === 0 && + this.#takingTurnPromise === null && + this.#outputCancellationPromise === null && + (this.#snapshot.output === "idle" || + this.#snapshot.output === "interrupted") + ) { + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + } + } + #recordLatency(name: VoiceLatencyEvent["name"], correlationId: string): void { if (this.#answerFinalizedAt === null) return; const eventKey = `${correlationId}:${name}`; @@ -623,14 +986,41 @@ export class VoiceTurnController { ); } + #canReplay(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input === "listening" && + !this.#inputTurnPending && + this.#activeSpeechResponseId === null && + this.#pendingSpeechRequestIds.size === 0 && + this.#outputCancellationPromise === null && + this.#takingTurnPromise === null && + (snapshot.output === "idle" || snapshot.output === "interrupted") + ); + } + + #canTakeTurn(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input !== "paused" && + (snapshot.output === "waiting-for-tool" || + snapshot.output === "speaking") && + this.#takingTurnPromise === null + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } #update(update: Partial): void { const snapshot = { ...this.#snapshot, ...update }; + const canReplay = this.#canReplay(snapshot); this.#snapshot = { ...snapshot, + canReadFullResponse: canReplay && this.#lastResponseSegments.length > 0, + canRepeatQuestion: canReplay && this.#lastResponseQuestion !== null, + canTakeTurn: this.#canTakeTurn(snapshot), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts index 38188a178c3..c7f4f94c284 100644 --- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts @@ -86,7 +86,7 @@ describe("OpenAI Realtime call handler", () => { expect(fetch).not.toHaveBeenCalled(); }); - test("forwards only the SDP and server-owned duplex Realtime policy", async () => { + test("forwards only the SDP and server-owned half-duplex Realtime policy", async () => { const reportDiagnostic = vi.fn(); const fetch = vi.fn( async () => @@ -131,16 +131,16 @@ describe("OpenAI Realtime call handler", () => { type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], - tool_choice: "required", - tools: [{ name: "continue_interview", type: "function" }], + tool_choice: "none", + tools: [], audio: { input: { transcription: { model: "gpt-4o-transcribe", language: "en" }, turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts index 1cdd3f0fead..523b8c85f17 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts @@ -46,18 +46,18 @@ describe("OpenAI voice policy", () => { ).toEqual({ available: true, connectionTimeoutMs: 15_000 }); }); - test("owns the trusted GPT-Realtime-2 duplex session policy", () => { - expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v1"); + test("owns the trusted GPT-Realtime-2 half-duplex session policy", () => { + expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v3"); expect(createOpenAIRealtimeSession()).toEqual({ type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], reasoning: { effort: "low" }, parallel_tool_calls: false, - tool_choice: "required", + tool_choice: "none", instructions: `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -65,29 +65,16 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`, - tools: [ - { - type: "function", - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object", - additionalProperties: false, - properties: { answer: { type: "string" } }, - required: ["answer"], - }, - }, - ], +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`, + tools: [], audio: { input: { noise_reduction: { type: "far_field" }, @@ -100,8 +87,8 @@ After the tool result arrives, speak only its response_text strings, in array or turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" }, @@ -109,12 +96,17 @@ After the tool result arrives, speak only its response_text strings, in array or }); }); - test("allows no provider-owned interview decisions or unrestricted tools", () => { - const serializedPolicy = JSON.stringify(createOpenAIRealtimeSession()); + test("lets Realtime neither answer for the user nor call tools between turns", () => { + const policy = createOpenAIRealtimeSession(); + const serializedPolicy = JSON.stringify(policy); expect(serializedPolicy).not.toContain("response.create"); expect(serializedPolicy).not.toContain("gpt-realtime-1.5"); + expect(serializedPolicy).not.toContain("continue_interview"); expect(serializedPolicy).not.toContain('"tool_choice":"auto"'); - expect(createOpenAIRealtimeSession().tools).toHaveLength(1); + expect(serializedPolicy).not.toContain('"tool_choice":"required"'); + expect(policy.tools).toHaveLength(0); + expect(policy.audio.input.turn_detection.create_response).toBe(false); + expect(policy.audio.input.transcription.model).toBe("gpt-4o-transcribe"); }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts index fc401fc0c7b..653b622dc3a 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts @@ -1,5 +1,5 @@ export const OPENAI_REALTIME_CONNECTION_TIMEOUT_MS = 15_000; -export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v1"; +export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v3"; interface VoiceEnvironment { readonly NODE_ENV?: string; @@ -24,7 +24,7 @@ export const getOpenAIVoiceAvailability = (environment: VoiceEnvironment) => ({ const REALTIME_INSTRUCTIONS = `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -32,38 +32,31 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`; +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`; +/** + * The completed input transcription is the only source of the user's answer. + * Semantic VAD therefore commits audio without creating a response or + * interrupting playback, and the Realtime model has no tools with which to + * manufacture an answer. + */ export const createOpenAIRealtimeSession = () => ({ type: "realtime" as const, model: "gpt-realtime-2", output_modalities: ["audio"] as const, reasoning: { effort: "low" as const }, parallel_tool_calls: false, - tool_choice: "required" as const, + tool_choice: "none" as const, instructions: REALTIME_INSTRUCTIONS, - tools: [ - { - type: "function" as const, - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object" as const, - additionalProperties: false, - properties: { answer: { type: "string" as const } }, - required: ["answer"] as const, - }, - }, - ], + tools: [] as const, audio: { input: { noise_reduction: { type: "far_field" as const }, @@ -76,8 +69,8 @@ export const createOpenAIRealtimeSession = () => ({ turn_detection: { type: "semantic_vad" as const, eagerness: "low" as const, - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" as const }, diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index f6249fc36b4..b4a2d1638c1 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,162 +1,86 @@ -# Mission 6 — resume one prepared workpiece and Petrinaut document +# Mission 6b — Reconcile Voice with resumable browser work ## Status -**Closed on `ln/fe-1575-resumable-workpiece-petrinaut` by owner decision on 2026-09-04.** [FE-1575](https://linear.app/hash/issue/FE-1575/resume-one-brunch-workpiece-and-petrinaut-document-across-tabs) delivered the implementation, outer mechanical witness, cold-reader adjudication, and the product manager's fresh two-tab conversation/workpiece/document demo; see the [retained implementation and witness evidence](docs/evidence/implementations/fe-1575-resumable-workpiece-petrinaut.md). The owner explicitly waived re-running the Voice-origin and aborted-assistant presentation clauses in the fresh product-manager conversation and closed the mission anyway: those records were absent from that run, their behavior is mechanically covered by the outer witness, and their future scenario obligations are carried in [`MISSION.next.md`](MISSION.next.md#voice-after-the-live-transport-cut). This is a closure exception, not evidence that the skipped human checks passed. Earlier on 2026-09-04 the owner amended only the Deferred section, to point at the recut future planning record and carry two admissions from this mission's evidence; the imperative, throughline, proof, constraints, fog-line, and stop conditions otherwise remain the historical execution contract. +**Accepted by Lu on 2026-09-07 with explicit limitations**, on `ln/fe-1580-reconcile-voice-resumable-workpiece`, [PR #9564](https://github.com/hashintel/hash/pull/9564), above Mission 6 and Mission 5. The accepted [owner witness](docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/witness.md) proved the local Voice → causal browser mutation → coherent resume → active-submission Stop/reopen path after repairing cross-step client-result accumulation and the fixture's non-causal prepared answer. Direct spoken-user Voice attribution after hydration, durable recovery of locally withheld post-settlement browser work, and comparative audible latency are explicitly deferred with narrowed claims; the full pre-registered telemetry bundle was not retained and is not inferred. -## Imperative - -Determine whether one canonical Brunch conversation can maintain a useful Markdown workpiece and drive a meaningful change to a real Petrinaut document through the browser without reviving a comprehensive typed domain IR. - -Mission 3 separately showed a recoverable Markdown workpiece and hermetic canonical Petrinaut callbacks, but its paid model could not carry a nested construction schema and no product path joined the two results. Mission 4 accepted the independent core `elicitation` capability and SDCPN job-skill composition but produced no full-run candidate. This mission must retire the join and resume uncertainty honestly with one deliberately prepared fixture rather than treating either historical result as an integrated product. +KA's branch and [PR #9531](https://github.com/hashintel/hash/pull/9531) remain untouched. The replacement imports the contribution `58f75840804766a84ce85b9daab5b5194f3875ec..be56a18ff0244c5750a8702e9c7f45c0b607dc06` with attribution, never the distant merge-base delta. Its live `MISSION.md` is historical source, not imported authority. This is the explicit exception to one new issue per mission; FE-1580 was referenced without rewriting its issue. No Linear write or KA-record change is part of acceptance. -### Visible product advance +**Accepted implementation and evidence:** the pre-witness restacked candidate passed 39 uncached scoped build/test/type/lint tasks (1,318 tests). Commits `1e238f498e` and `48e2b66666` repair causal client-result delivery and require explicit true-user fixture evidence. Focused post-repair checks and the sanitized canonical record are listed in the [owner witness](docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/witness.md); the earlier [verification](docs/evidence/implementations/voice-resumable-reconciliation/verification.md) retains the broader local suite and the exact accepted dispositions. Mission 7 may consume this narrowed accepted foundation after restack; its own integrated witnesses remain necessary. -**Release note:** Brunch edits the Petrinaut net you are looking at from the conversation, and your work survives closing the tab. - -**Demo script (no engineer present):** with the local Brunch/Petrinaut development stack running, open the stable demo fixture selector for the labelled prepared crew-reservation case. Its canonical Brunch conversation, current Markdown workpiece, and non-empty Petrinaut net come back together. The fixture visibly states that it is test-authored and prepared, and what it does not claim. Tell Brunch the one new realistic fact the fixture is prepared for: final inspection uses the single dispatch crew, and sign-off releases it. Watch the workpiece update and a new arc appear in the live net from `Dispatch crew available` to `Start final inspection`. Wait until the fixture reports that the conversation, workpiece, and automatically mirrored document are settled. Open the same fixture in a second tab, confirm that messages retain their typed/Voice provenance and that an aborted assistant entry still presents as stopped rather than as an ordinary truncated answer, submit one follow-up message, and receive Brunch's response in the same conversation without duplicate submission or identity drift. +## Imperative -**Previously impossible:** Brunch only produced off-canvas net JSON for manual load; nothing it did touched the live document or survived a reload. +Make KA's completed-transcript, half-duplex Voice experience work safely over Mission 6's resumable browser mutations and coherent workpiece/document recovery. Preserve both capabilities instead of replacing either. Distinguish committed prose, submission settlement, pending browser work/continuation, coherent document settlement and terminal provider output at the actual shared boundaries. Start from the parent's new busy/follow-up/Stop behavior rather than adding a parallel coordinator. -**Deployment posture:** the demo runs against the locally run Petrinaut website and Brunch agent (`yarn dev:brunch`). Mission 8 stopped before remote deployment, so no product-manager-noticeable claim here depends on remote infrastructure; remote durability stays with Mission 8. +**Release note:** speak to Brunch, let it change the prepared net, interrupt or stop safely, and reopen the same work without replaying speech or duplicating the change. Transcript, tool failures and stopped entries remain understandable. Direct spoken-user attribution on reopen is explicitly unsupported; Stop is durable for active Flue submissions, while browser work withheld after a settled tool-call step may reappear as pending after reopen. -**Completion:** the mission is done when a product manager can run that demo script end to end for this fixture and every readiness-gate obligation in [Proof](#proof) is closed. The first green pass through the two-tab path is an internal milestone inside the mission, not its completion. +**Demo:** run `yarn dev:brunch`, open the honestly labelled crew-reservation fixture, make a typed turn followed by a spoken confirmation, and watch the single crew-reservation arc and coherent bundle settle. During another response use **Your turn**, wait for safe fresh capture, and speak again. Separately Stop before completion. Reopen in Tab B and inspect the conversation and net, then continue without duplicate preparation, mutation or autoplay. Inspect compact/expanded Voice, exact full-response and question replay, and a visible tool failure. This local demo and its acceptance gates, not merely a clean merge or green unit tests, define the visible advance. Vestera construction/explanation remain Mission 7. ## Throughline -### Observed departure point and first unproved boundary - -The production browser already has most local pieces: - -- `apps/petrinaut-website`'s local-storage demo owns an editable `PetrinautDocHandle`, automatically writes handle changes to `petrinaut-sdcpn`, and maps each net to a persistent Brunch conversation id; -- Petrinaut's stock AI panel already validates and executes canonical read, mutation, and command tools against the active browser document and returns the original tool-call id; -- live Mission 5 is replacing the server-side `GET`/`POST /api/chat` adapter with one browser `FlueClient` plus host-supplied AI SDK `ChatTransport` over the mounted `/agents/chat/:instanceId` route; that route carries typed turns, `history()` hydration, and correlated client-tool-result signals, and Mission 6 must consume rather than duplicate or reverse that transport; and -- the SDCPN skill already emits a full recoverable `runbook-ir` block and requires construction to consume that workpiece rather than transcript archaeology. - -The first unproved boundary is ordinary Brunch conversation over Mission 5's browser Flue transport → mounted canonical document read / least mutation → browser execution → correlated Flue continuation. Today ordinary SDCPN conversations mount only the Petrinaut documentation reader as a browser tool; the validated construction subset is headless-only, and no stable fixture selector or coherent cross-tab witness joins the conversation, workpiece, and document lifecycles. Mission 6 may develop independent fixture/workpiece/document logic while Mission 5 proceeds, but it must consume Mission 5's landed browser transport before integrated or outer proof and must not add another conversation route in the interim. - -### Accepted fixture and boundary crossings - -Prepare the existing final-inspection / dispatch-crew case as an explicitly test-authored fixture. Its starting workpiece and non-empty net preserve this narrow account: one crew is reserved during final inspection, sign-off releases it, the batch then becomes dispatch-ready, and timing plus failure/recovery remain unresolved. Its prepared material must identify its authorship and must not be presented as a Mission 4 candidate or model-produced evidence. - -Deliver the prepared starting workpiece exactly once through Flue's public dispatch surface as a tagged structured signal. Its canonical record must remain `role: system`, `purpose: dispatch`, carry the fixture id, `test-authored` authorship, and non-claims as signal attributes, and preserve the exact Markdown body. This record is prepared revision zero. Later full `runbook-ir` blocks emitted in genuine assistant responses are model-produced revisions; the workpiece resolver selects the latest eligible revision without rewriting Flue's append-only history. This is analogous to last-one-wins selection of extension-contributed artifacts in a Pi raw session log, not permission to overwrite either log. - -The disposable production-route probe established the carrier facts: the tagged signal retained its exact body and attributes, an exact idempotent retry converged on the original submission without adding messages, and the snapshot survived process reopen unchanged. The current `ChatAgent` rejected fixture authorship supplied as `initialData` with `400` and created no history; `initialData` is not a substitute for the public workpiece source. A user delivery would impersonate the person, while faux-provider output, hand-authored assistant records, private canonical record types, direct database writes, and a second history store are not preparation routes. The probe's configured Anthropic credential was rejected with `401`, so its separately classified faux assistant response established no model-behavior claim. - -Prepare the net with `Batch ready`, `Under final inspection`, `Ready for dispatch`, and `Dispatch crew available` places plus `Start final inspection` and `Sign-off` transitions. Preserve the batch-flow arcs and the return of the crew from sign-off, but deliberately omit the standard input arc that reserves the sole crew when final inspection starts. Use one realistic confirming answer: final inspection consumes the sole available dispatch crew, sign-off returns it, and timing plus recovery remain unknown. The least candidate mutation is one canonical weight-1 standard input arc from `Dispatch crew available` to `Start final inspection`. The exact before/after edge makes the semantic oracle discriminating while avoiding Mission 3's deeply nested schema failure. If that shallow mutation still cannot cross Flue faithfully, stop with the carrier blocker rather than weakening the claim. - ```text -stable prepared-fixture selector -→ resolve distinct fixture, Petrinaut document, and Flue conversation identities -→ open the prepared non-empty browser document and use the browser Flue client to idempotently deliver or recover the tagged revision-zero signal -→ hydrate canonical Flue history through `history()` on the mounted route -→ recover prepared revision zero from the tagged dispatch record, or the latest eligible assistant revision, by source message id plus content hash -→ submit the realistic crew-reservation confirmation through Mission 5's production browser Flue transport -→ Brunch emits an inspectable full workpiece revision without erasing prior meaning or the remaining unknown -→ SDCPN construction reads that current workpiece and the live browser document -→ Brunch requests the least canonical meaningful mutation -→ Petrinaut validates and executes it against the bound document -→ the original tool-call id and result return as one correlated signal through the same browser Flue transport and resume the same conversation -→ inspect the canonical non-empty document and advance the runtime settled manifest only after conversation/workpiece/document state is observable -→ a second tab opens the same fixture selector, resolves the same identities and settled hashes, submits one follow-up, and receives Brunch's response +completed current-turn microphone transcript +→ shared panel submitVoiceInputWithAdmission/useChat admission +→ browser ChatTransport over the memoized FlueClient +→ same-origin /agents/chat/:instanceId and mounted Brunch ChatAgent +→ committed canonical prose, hidden server question marker, browser-tool requests +→ existing canonical browser validation and effects on the bound document +→ original call-id outputs resume the same conversation +→ canonical speech queue and acknowledged cancellation +→ coherent workpiece/document settlement +→ canonical history reopen and another real turn ``` -The settled fixture manifest is runtime local product state, not merely retained evaluation evidence: the stable selector uses it to choose the latest coherent observed bundle of distinct fixture, Flue conversation, workpiece source/hash, and Petrinaut document/hash or revision identities across reopen. It is a small viability pointer, not a new event log, independent workpiece store, or distributed transaction. A failed history load, workpiece recovery, rejected/no-op mutation, or missing result correlation must leave the prior settled bundle selected while partial state and failure remain visible for diagnosis. Retained witness artifacts copy and inspect this runtime state but do not select the product bundle. The existing automatic localStorage mirror is the only document-save mechanism unless a real failure proves it insufficient; this mission adds no explicit Save affordance. +### Departure and protected sources -### Expected touched paths +- Mission 5 `b1295ad454` holds composer status busy across automatic follow-up and permits Stop to withhold it. Mission 6 `976bb1c67c` repairs fixture routing, docs-reader catalogue retention, workpiece numbering, coherent persistence and mutation no-op honesty. These committed repairs satisfy the earlier wait-for-parent handoff. Recheck the combined deferred static-tool path rather than assuming that either source closes it. +- Read KA's pinned `MISSION.md` and the imported `docs/evidence/implementations/mission-5-voice-safety-parity/{donor-behavior-matrix,provenance-blocker,witness-blocker}.md` plus `docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md`. Import their historical evidence without relabelling its tests or witnesses as this candidate's proof. Retain the latest repeated-output-cancellation regression from `db8184b2e6`. +- Mission 6's accepted authority is [archived](docs/mission-archive/6-resumable-workpiece-petrinaut.md). Its [implementation record](docs/evidence/implementations/fe-1575-resumable-workpiece-petrinaut.md) and `fe-1575-outer-browser-witness-2026-09-04{,-r2}` raw bundles establish the prepared document/workpiece path, not Voice/stopped-entry presentation: the inspected bundles have completed settlements and no recorded Voice origins. Preserve the historical owner close and raw records while correcting current interpretation. +- Trace `packages/transport-aisdk/src/{index,transcript,ui-stream,client-tool-history}.ts`, website `local-storage-demo/{brunch-panel-transport,use-flue-chat-history,use-crew-reservation-fixture-session,crew-reservation-settled-manifest}.ts`, Petrinaut `ai-assistant-panel.tsx` and mutation helper, and website `voice-interview/{openai-realtime-session,realtime-brunch-bridge,voice-turn-controller,canonical-speech,voice-interview-control}.ts*`. Matching source tests, installed SDK 2.0.3 types and [Flue routing](docs/reference/architecture/flue-routing.md) guide the smallest repair. -This manifest is provisional and may shrink or move when the first real probe exposes the deeper existing boundary: +### Import and reconciliation boundary -```text -libs/@hashintel/brunch-agent/ -├── MISSION.md ~ live authority and eventual close evidence -├── MISSION.next.md ~ future joins and carried flags only -├── packages/plugin-sdcpn/ ~ mount only the read/mutation capability earned by this tracer -└── docs/evidence/ + prepared fixture manifest and browser witness -apps/brunch-agent/ -├── src/agents/chat-agent/ and src/conversation/ ? only if fixture-scoped mounting or workpiece recovery belongs outside the landed browser transport -└── test/ + real Flue/client-tool fixture integration -apps/petrinaut-website/ -└── src/main/app/local-storage-demo/ ~ consume Mission 5 transport; fixture selection, prepared signal, runtime settled manifest, and cross-tab continuation -libs/@hashintel/brunch-agent/packages/transport-aisdk/ ? consume the landed Mission 5 public surface; do not duplicate its implementation here -libs/@hashintel/petrinaut-core/ or libs/@hashintel/petrinaut/ ? only for an observed canonical contract or browser-host defect -``` +Commit this authority separately, then a credited squashed source import with necessary conflict resolutions recorded, followed by focused reconciliation commits and verification evidence. Retain the existing launcher repair and fixture configuration. Reconcile the hidden question marker with the scoped browser catalogue and identical live/history normalization; a browser mutation cannot become server-executed through a missing catalogue entry. Reconcile deterministic user/tool keys with stable payload ordering, bounded keys, causal per-step result batches and admission outcomes; prefix selection alone is insufficient. Carry source output-insertion failure handling through the actual deferred automatic-tool path and preserve fixture refusal/coherent-bundle feedback in the new Voice presentation. ## Proof -The visible advance is the demo script in the imperative, run by a product manager against the named local posture. The evidence that backs the claim is one stable local demo URL or fixture selector plus its labelled prepared manifest, exact before/after Flue snapshots, recovered Markdown workpiece revisions, canonical Petrinaut document states, and two-tab witness; those are oracles for the builder and adjudicator, not the advance itself. Together they establish single-fixture browser-backed viability. They do **not** establish automatic full-net projection, capture-backed or selected-pair provenance, behavioral execution, broad scenario coverage, remote replacement durability, concurrent editing, Mission 3/4 quality superiority, or a promoted reusable product seed. - -### Internal milestone: first green throughline - -The first internal milestone is one pass through the throughline for the prepared fixture: a cold reader can reconstruct the spine and distinguish supplied evidence, inference, and the explicit unknown in the workpiece; one realistic turn produces an inspectable workpiece revision without erasing the unknown; Brunch reads the live document and applies the one supported arc through the real browser client-tool boundary; the canonical net is non-empty and visibly corresponds to the confirmed meaning; and a second tab observes the same settled conversation, workpiece, and document revision and continues without duplicate submission or identity drift. Reaching this milestone authorizes the readiness work below; it does not close the mission. +The first milestone is a spoken fixture turn whose browser mutation returns through the shared route and produces canonical audio without duplication. Readiness additionally requires the following discriminators. Existing test locations are relative to their packages; scenario names describe required assertions, not pre-existing test claims. Evidence lives under `docs/evidence/implementations/voice-resumable-reconciliation/`, pinned to the final implementation, source and parent commits. -### Readiness gate: completion bar +1. **Canonical input and explicit half-duplex handoff.** Website `voice-interview/{openai-realtime-session,realtime-brunch-bridge,voice-turn-controller,voice-interview-control}.test.ts*` retain completed keyed transcripts, speech-request-before-audio invalidation, stale/duplicate/boundaryless rejection, queued-output ownership, latest mute preference, acknowledged cancellation and in-flight/repeated-cancel reuse. `voice-preview.integration.test.ts` proves actual shared panel/transport admission once, with model function arguments unable to submit. +2. **Browser continuations and Stop.** A test mounting the real `AiAssistantPanel` with the Voice bridge holds browser execution/output insertion and continuation at intermediate `ready`, both with preceding canonical prose and without it. Capture and replay must not become available prematurely. Stop before tool execution, during output insertion and before scheduled continuation prevents later work that has not been admitted; already-applied mutations stay inspectable without a rollback claim. Your turn cancels audio without durably aborting admitted Brunch work. Parent regression tests remain green. +3. **Tools and canonical projection.** Website `local-storage-demo/{brunch-panel-transport,use-flue-chat-history}.test.ts` and transport `test/{ui-stream,transcript}.test.ts` preserve fixture browser tools while hiding only the server marker; normalize the same client input live and from history; and fold continuations without losing surviving Voice origins. `canonical-speech.test.ts` and bridge/controller tests allow exact canonical segments only, seed history without autoplay, gate exact replay until all terminal conditions, and leave question replay disabled for absent/unmatched markers. +4. **Admission identity and failure.** Transport `test/chat-transport.test.ts` covers exact user retry, cumulative/reordered logical tool-result retries, changed-payload conflict retaining the original submission ID, bounded identity, ambiguous admission without automatic retry and local abort without durable abort. App `test/petrinaut-chat.test.ts`/its built-runtime integration verify deduplicated receipts. Petrinaut `ai-assistant-panel.test.tsx` covers matching per-tool output errors; combined panel/Voice tests cover textless browser-continuation failure. Distinguish input rejection, effect failure/no-op, output insertion rejection and continuation rejection. Partial failure cannot advance the prior coherent bundle or strand ownership. +5. **Supported reopen.** Transport/history tests reconstruct surviving client-tool Voice origins and each aborted assistant entry from canonical data without browser origin storage. Retain before/after/Tab-B snapshots and rendered stopped-entry evidence, including a later completed response so a global latest-status banner is not mistaken for per-message state. Direct spoken-user attribution has its own gate below. +6. **Real product/stock coexistence.** Human/browser witness of the demo retains `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical snapshots, settlements and commit/hash manifest. Verify original call/result IDs, one target arc, coherent bundle identity, fresh Tab-B continuation, no duplicate mutation/autoplay and same-origin routes. Panel/contents tests and rendered inspection cover compact/expanded Voice, persistent/copyable errors and stock behavior when Brunch is absent/unselected. Actual microphone/audible behavior cannot be claimed from simulation. +7. **Comparative latency.** Keep KA's gate: ten comparable real-audio donor #9496 trials at `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final candidate, same machine/browser/input/model and warm/cold policy, finalized speech to first audible canonical TTS. Candidate median must not regress and p95 regression must be below 20%. Retain raw sanitized samples, method, environment and pins. Earlier diagnostic turns with nearly zero text-to-settlement delay prove no improvement. No paid trials are authorized by this cut; Lu must first approve caller/model, bounded trials, ceiling and accounting owner. Mission 7's budget is unavailable here. +8. **Repository verification and docs.** Run root Yarn/Turbo `build test:unit lint:tsc lint:eslint` for `@hashintel/brunch-agent`, binding-flue, plugin-sdcpn, transport-aisdk, `@apps/brunch-agent`, `@hashintel/petrinaut`, and `@apps/petrinaut-website`; use narrow package tests first to discriminate failures. Check changed-file formatting, `git diff --check` and `yarn workspace @local/petrinaut-arch-docs lint:arch-docs`. User docs describe exact supported behavior and limits; exactly one source patch changeset covers this PR's published Petrinaut behavior. Report screenshot updates if needed. Prior counts are not a final run. -The mission completes only when the demo script works for this fixture and these obligations are closed: stale fixture/workpiece/document revision refusal, duplicate tool delivery, read/write failure visibility, unsupported meaning, no-op mutation honesty, partial-save behavior, second-tab rehydration, separate identity integrity, and one negative mutation case. Do not close every consequential-element provenance link, remote task replacement, broad scenario coverage, or repeated automatic projection here; those become Mission 7 or Mission 9 obligations only after this tracer exposes a finite peer set and load-bearing seams. +**Direct-user provenance gate:** SDK 2.0.3's canonical user messages do not expose caller Voice metadata or idempotency keys. Supported signal/tool-result origin reconstruction is not direct-user provenance. The owner witness observed both live Voice chips disappear after Tab-B hydration. Lu explicitly deferred this chip with truthful presentation on 2026-09-07: canonical spoken text survives, but direct spoken-user origin is not claimed after reopen. No local Flue patch, sidecar/signal admission or text encoding is authorized. -Every final leaf has a discriminating oracle: - -1. **The prepared fixture is honest and sufficient for this narrow test.** The committed fixture manifest, raw Flue snapshot, and a cold-reader adjudication identify the prepared workpiece's tagged system/dispatch source, exact test-authored Markdown, process spine, constrained crew, release policy, quantity context, explicit unknown, prepared net meaning, and non-claims. The same inspection distinguishes every later assistant revision as model-produced and must not require transcript archaeology. -2. **One evidence turn maintains the Markdown workpiece.** A production-agent fixture integration mechanically recovers prepared revision zero from the tagged dispatch record, then selects the latest eligible assistant `runbook-ir` block after the confirming turn, retaining each source message id and SHA-256. Before/after adjudication must find the supplied contextual quantity, retained crew/release meaning, retained unsupported context, and no invented fact or hardened unknown. -3. **The real browser executes a correlated Petrinaut read and write.** Focused plugin/transport tests prove that `getLatestNetDefinition` and the selected `addArc` schema come mechanically from Petrinaut's canonical contracts, fixture mode advertises only the selected operations, duplicate result delivery does not apply the mutation twice, rejected input remains visible, and a mutation that would change nothing is reported as a no-op rather than as a change. The browser witness must retain tool name, call id, parsed input, execution output, resumed signal, and resulting canonical definition; a headless callback alone does not pass. -4. **The document change is meaningful rather than merely accepted.** A structural comparison proves there was no standard input arc from `Dispatch crew available` to `Start final inspection` before the turn and exactly one weight-1 arc afterward, while `Sign-off` still returns the crew and the prepared net remains non-empty. The changed workpiece retains the reservation/release meaning and unresolved timing/recovery. Parser/schema acceptance or a disconnected convenience element fails. -5. **The runtime settled manifest cannot bless partial state.** A focused failure test injects history/workpiece-recovery failure, rejected `addArc`, or missing/duplicate result correlation and shows that the prior coherent runtime bundle remains selected while the failure and any partial state are inspectable. A retained evidence manifest alone does not pass this leaf, and no localStorage failure interface is invented solely to satisfy it. -6. **A second tab resumes and continues the same fixture.** With `yarn dev:brunch` running, the recorded browser protocol opens the stable selector in Tab A, performs and settles the turn, then opens it in Tab B. The witness compares fixture id, document id and canonical definition hash, Flue conversation id and history, latest workpiece source/hash, runtime settled-manifest identity, per-message typed/Voice provenance, stopped-turn presentation, and absence of duplicate submission. An aborted entry rendered as ordinary truncated content fails this check even if a global latest-settlement banner still says stopped. Tab B must then submit one follow-up message and receive its correlated Brunch response in that same conversation. A read alone does not pass. Tab B opened against a stale or mismatched revision must refuse visibly rather than silently select older artifacts. -7. **The cut has not smuggled in the later architecture.** Public-schema and dependency inspection finds only fixture identity/revision links, Markdown recovery metadata, and canonical Petrinaut payloads—no closed process ontology, typed capture-to-workpiece reducer, graph database, second conversation log, or general projection engine. - -Verification proceeds inside-out but closure requires the outer boundary: - -- **Inner:** fixture parsing and prepared-label checks; identity separation; workpiece recovery/hash; canonical `addArc` schema and exact before/after edge assertion; idempotent client-tool result handling; runtime-manifest refusal. -- **Middle:** the built production `ChatAgent` and Mission 5 browser `FlueClient`/`ChatTransport` path at `/agents/chat/:instanceId` hydrate the prepared conversation, accept the evidence turn, recover the revised workpiece, and carry actual browser-tool calls/results. No `GET` or `POST /api/chat` evidence passes. Run the focused workspaces through root Turbo (`test:unit`, `lint:tsc`, `lint:eslint`, and `build` where changed). -- **Outer:** the two-tab `yarn dev:brunch` witness above, with retained before/after artifacts, one Voice-origin message, and one durably stopped assistant turn. A content-only transcript match does not establish faithful resume. -- **Semantic:** a cold human accepts fixture/workpiece honesty and the workpiece-to-document correspondence. The oracle may falsify those claims; it may not rewrite the interaction or architecture policy. -- **Product:** a product manager who did not watch the work runs the demo script from the imperative without an engineer and notices the advance. This is the last check before close, after the readiness gate; it is not a substitute for the oracles above. +**Close:** Lu accepted the narrowed mission claim on 2026-09-07 after the owner witness. The real path and automated evidence passed as recorded; the three deferred claims and evidence-bundle limitation remain visible rather than being counted as proof. ## Constraints -- Keep fixture id, Flue conversation id, latest workpiece source/revision, and Petrinaut document id/revision distinct and explicitly linked. One id must not impersonate all lifecycles. -- Flue history remains the canonical conversation log. Browser message caches and fixture artifacts are projections or evidence, never a second authority. -- Consume Mission 5's browser `FlueClient` plus host-supplied AI SDK `ChatTransport`; typed turns, prepared signals, history hydration, and client-tool results all cross `/agents/chat/:instanceId`. Do not keep, restore, or add another product conversation route. -- The tagged prepared signal is the only test-authored workpiece source admitted by this fixture. It remains a diagnostic system/dispatch record; latest-revision selection may supersede it with a genuine assistant workpiece but may not mutate, relabel, or hide its authorship. -- Markdown remains the semantic workpiece. Recover its full latest version; do not introduce a comprehensive typed domain IR to make fixture lookup convenient. -- Projection consumes the current workpiece. The transcript may establish provenance and help recover that artifact but may not become the primary construction IR. -- Petrinaut owns canonical schemas, browser validation, mutations, and document state. Brunch imports or mechanically derives those contracts and does not hand-copy their field shapes. -- Client tools execute against the active bound browser document and return the original tool-call id. Stale, duplicate, cross-document, malformed, failed, and no-op outcomes fail visibly. -- Advance the runtime settled manifest only after the claimed Flue snapshot, workpiece revision, and document state can all be inspected. It selects the coherent local bundle but does not make the browser and Flue stores transactional. Automatic localStorage mirroring remains the only save behavior; do not invent an explicit Save affordance or cross-store transaction machinery without an observed recovery failure requiring it. -- Preserve the accepted Mission 4 `useBrunchAgent()` plus `useSdcpnPlugin()` architecture. The app composes; the plugin owns SDCPN operation semantics; the transport carries results; the UI executes them. -- Keep construction tools unavailable to unrelated ordinary conversations unless the real path proves the smallest safe selection can be scoped to this fixture/mode. Stock-assistant behavior must remain unchanged when Brunch is absent or unselected. -- The fixture is local and deliberately prepared. Make no remote durability, capture provenance, automatic projection, behavioral execution, or concurrent collaboration claim. -- No HASH Graph, Temporal, Redis, new database, observer, workflow engine, second agent, second event log, or closed workpiece schema. -- Update the affected Petrinaut user guide in the same change if the selector, save/resume behavior, or panel behavior becomes user-facing; add one Petrinaut changeset only if a published Petrinaut package changes. -- Repair typed/Voice provenance and stopped-turn presentation at the canonical history-to-Petrinaut projection boundary; do not add a second transcript store. Account for the observed discoverability strain around **Show transcript**, **Exit voice mode**, and the chat composer's durable **Stop** without conflating local Voice exit with Flue abort. +- One conversation/log, memoized Flue client, shared `useChat` path-B admission and mounted route. No direct Voice send, separate mutable transcript, simplifier, live `brunch_ask`, or interactive question path. The core marker annotates exact existing prose without accepting answers. +- Realtime has no tools, `tool_choice: none`, and semantic VAD with `create_response: false`. Normalize completed transcript once in the bridge (trim/Unicode whitespace collapse), then enforce 32,000 code points. Generic panel validation must not mutate that normalized payload. +- Microphone closes from canonical speech request through queued/playing output, cancellation, pause, error and submission; invalidate unfinished input before sending `response.create`. Fresh capture needs explicit handoff, acknowledged provider cancellation and settled correlated conversation work. Automatic duplex remains rejected because playback can become authoritative user input. +- Only new durably completed, submission-correlated canonical segments may speak before settlement. Never deltas, unfinished text, reasoning, tool payloads, inferred prose, hydrated history or failed/aborted continuation segments. Exact full-response and marked-question replay remain gated by conversation/output/input terminal conditions; cancellation suppresses queued and later continuation speech. +- Keep local playback, observation, HTTP cancellation and durable conversation Stop distinct. Stable logical delivery identity plus stable payload ordering yields at most one admission; ambiguous outcomes never auto-retry. Preserve each surviving tool Voice origin independently. +- Preserve repaired fixture/conversation/document/workpiece identity, canonical browser schemas/callbacks, scoped catalogue, recovery, no-op honesty, prior-coherent-bundle refusal and automatic document persistence. Transient UI/audio state cannot bless durability. No cross-store atomicity or concurrency claim. +- Preserve KA's authorship and source records. Existing source policy excluding Mission 6 mutation work is superseded only for this explicit combined-path reconciliation; unrelated donor and stakeholder PRs remain untouched. Import source evidence as history, not candidate acceptance. ## Fog-line -- Whether the selected shallow `addArc` schema survives the provider-visible Flue carrier and results in exactly one browser mutation without reopening the broader nested-schema problem. -- The least safe way to expose canonical `getLatestNetDefinition` plus `addArc` in a fixture conversation while retaining the headless-only guard for broader construction. -- Whether the latest `runbook-ir` message id and hash are sufficient workpiece revision identity or the two-tab consumer exposes a need for a separate persisted workpiece artifact. -- Whether Mantine/localStorage synchronization plus the active `PetrinautDocHandle` is sufficient for the same-browser two-tab witness, and which document hash/revision signal best distinguishes settled from stale state. -- Whether the known provider-visible nested-schema failure is absent for the selected flat mutation. Do not generalize one success to nested construction classes. -- Which of history recovery, invalid `addArc`, or duplicate result delivery is the cheapest discriminating failure for the settled-witness rule after the first real path reveals the ordering. +The source-grounded intermediate-ready hazard may already be reduced by the parent fix; the deferred static-tool path must decide what remains. Output insertion rejection, textless continuation failure, retained idempotency compatibility and cancellation ordering need discriminators before mechanisms. Prefer existing SDK and local mechanisms; no parallel scheduler or generalized state machine merely to name a boundary. Source green suites and a textual merge do not prove these joins. -Resolve these at the named production/browser boundaries. Clarifying prose alone does not clear them. If a choice changes the accepted interaction policy, architectural ownership, or proof claim, return it to the owner and amend this authority before implementation continues. +Question-marker compliance remains a model limitation: missing/unmatched markers disable replay, never justify inference. The real microphone witness passed. Direct-user attribution and comparative latency were explicitly deferred with no corresponding claim. No unobserved evidence may be inferred from owner acceptance. ## Stop or reorient -Stop and surface evidence if: - -- fixture preparation requires pretending a Mission 4 candidate exists, placing prepared text in a user or assistant record, accepting an untagged preparation signal, or otherwise hiding test-authored/model-authored boundaries; -- the path conflates fixture, conversation, workpiece, and document identities or creates a second canonical conversation history; -- typed traffic, prepared signals, history, or client-tool results cross a product route other than Mission 5's mounted browser Flue route; -- the agent rereads transcript prose as its primary projection input because the current Markdown workpiece cannot carry the needed meaning; -- parser/schema acceptance, document non-emptiness, or a disconnected convenience element is offered as semantic correspondence; -- client-tool results lose their original call id, can target the wrong document, or duplicate execution on retry/reload; -- a partial or failed write advances the runtime settled manifest, second-tab reopening silently selects stale/mismatched artifacts, or Tab B proves only a read without a real continuation; -- exposing one browser mutation requires mounting an unrestricted construction surface for every ordinary conversation; -- the selected provider/Flue schema cannot faithfully carry the least meaningful mutation—record the crisp blocker rather than hand-copying Petrinaut schemas or widening into Mission 9; -- the tracer needs a closed ontology, typed claim ledger, general projection engine, distributed transaction, or new durable service before a concrete failure demonstrates that need; or -- work widens into capture-backed why/provenance, automatic projection breadth, remote deployment durability, concurrent collaboration, or broad scenario readiness. +Stop if source/parent pins move without inspection, another checkout's work would be disturbed, or the join requires another conversation route/authority, ambiguous automatic retry, rewritten speech, new batch/termination policy, local Flue patch or provenance store. Reorient if half-duplex cannot ensure fresh post-barrier capture, provider acknowledgement cannot bound cancellation, mutations duplicate, Stop allows withheld work to execute, failures disappear, or coherent settlement is falsely reported. Do not manufacture human/latency evidence or hide an unresolved gate to call the base verified. ## Deferred -On 2026-09-04 the future planning record was recut around provenance by lineage with declared basis; see the [2026-09-04 migration disposition](MISSION.next.md#2026-09-04-provenance-replanning-migration-disposition). Mission 7 now owns construction and explanation of one real net region from a genuine conversation: settled workpiece revisions as `update_workpiece` tool calls, constructor-declared basis on each mutation, verifiable transition records, the why operation with its safety and utility gates, schema-carrier repair, scenario-selected tool admission, and retirement of the orphaned `ask` and `sweep` client handling. Mission 9 owns repeatable projection breadth: unchanged repeat, changed input, retirement, concurrent change, cross-conversation document access, and the schema classes an extended region adds. Two facts from this mission carry into that record and its close report: the prepared fixture's "Current Petrinaut correspondence" section was fixture-authored rather than produced by any skill directive, so this fixture is a viability proof and is not promoted into the provenance pair; and the fenced `runbook-ir` block plus message-id-and-hash selection is a Mission 6 contract that Mission 7 replaces for model-produced revisions, keeping the tagged prepared signal for test-authored material only. This mission's constraint that construction tools stay out of ordinary conversations is amended by the Mission 7 cut, not here. Remote replacement durability and release infrastructure remain in the historical Mission 8 handoff, and a Mission 8 successor must be scheduled before any remote claim. Multi-tab concurrent editing, a durable cross-store commit protocol, explicit localStorage failure injection and refusal of a concurrent write from a tab holding an older revision (distinct from the readiness-gate refusal to reopen onto stale or mismatched artifacts), and promotion of this prepared fixture into a reusable product seed re-enter only if the automatic mirror loses or overwrites state, a later consumer requires atomic bundle identity, or this mission otherwise exposes concrete strain; their current planning home and re-entry conditions remain in [`MISSION.next.md`](MISSION.next.md) and the linked Mission 7/9 drafts. +Mission 7 consumes this accepted local reconciliation, not a new Vestera implementation. Amend its departure base and preserve the hidden/server marker versus browser-tool distinction, canonical identity, speech exclusions, causal per-step client results, continuation and cancellation contracts in A2/A3; re-pin the prompt/tool baseline before instrument freeze or paid runs. Its Step B genuine typed/Voice/stopped-entry witness remains necessary over new revision/basis semantics and cannot inherit Mission 6b's scenario evidence as its own. + +The [future spine](MISSION.next.md) retains construction/explanation, declared basis, workpiece revision tools, broad projection, orphan-code retirement, concurrent editing, remote durability/deployment and further UX policy changes with their existing owners. Direct-user Voice attribution, post-settlement durable withholding and comparative latency re-enter only under the conditions in the owner witness. The observed verbose negative-control answer and Stop discoverability strain are future UX inputs, not silent passes. Retirement of KA's original PR requires separate authorization. No Linear write is part of this close. diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md index af1f9c4d7d3..c65e005691c 100644 --- a/libs/@hashintel/brunch-agent/MISSION.next.md +++ b/libs/@hashintel/brunch-agent/MISSION.next.md @@ -1,6 +1,6 @@ # Brunch future mission spine -> Canonical future-planning spine, shared frame, and backlog index only. This file is not execution authority and authorizes no implementation. [`MISSION.md`](MISSION.md) is the closed Mission 6 execution record on this branch, which is stacked on the Mission 5 FE-1574 branch. Detailed provisional clusters are context repositories, not missions; re-evaluate and convert one into `MISSION.md` on its own branch before acting. +> Canonical future-planning spine, shared frame, and backlog index only. This file is not execution authority and authorizes no implementation. [`MISSION.md`](MISSION.md) is accepted Mission 6b, the owner-witnessed Voice reconciliation above repaired Mission 6. Mission 6 is [archived](docs/mission-archive/6-resumable-workpiece-petrinaut.md). Mission 7's Step A branch is restacked above this accepted narrowed foundation; its own scenario evidence remains required. Detailed provisional clusters are context repositories, not missions; re-evaluate and convert one into `MISSION.md` on its own branch before acting. This spine and its four linked drafts form one future-planning record. Keep each consequential meaning in one authoritative planning home: shared contracts and unallocated concerns live here; mission-specific detail lives in its draft. A spine pointer is not a second contract. Material omitted from a future cut returns to this record at full fidelity, and the consumed draft is removed. @@ -12,7 +12,7 @@ Mission 4 closed on this branch by owner adjudication on 2026-09-03. The accepte A future Mission 4 close-out addendum requires its own issue, branch, PR, and mission authority. It may stack on this closed branch and own broader reliability/hardening if warranted, browser parity, fixture/seed promotion contracts, topology-neutral case allocation, contract/readiness sweeps, archive subtraction, and Mission 8 preparation. It also owns the observed S4 report-versus-immediate-ask decision unless a later numbered mission first makes it load-bearing: re-enter only when a real review must continue immediately or repeated gap-only reports create visible friction; preserve S3 restraint while testing S4 activation and asking under a fresh instrument. Its exact issue/name and minimum scope remain owner decisions; do not create another Mission 4 draft. -Mission 6 closed on the FE-1575 branch under root [`MISSION.md`](MISSION.md): one deliberately prepared, honestly labelled fixture joined canonical conversation, session history, Markdown workpiece, and Petrinaut document through a browser-backed read/write change and cross-tab resume. Its consumed draft remains removed; its product-manager litmus, demo script, proof, and explicit owner waiver remain in the closed authority. The owner closed despite not re-running Voice-origin provenance and aborted-assistant presentation in the fresh product-manager conversation; those future scenario obligations live under [Voice after the live transport cut](#voice-after-the-live-transport-cut). Mission 5 owns the direct Voice/Flue transport cut on the FE-1574 branch directly beneath this one; its full contract lives only in that branch's root `MISSION.md`. Neither tracer requires a Mission 4 full-run candidate. The two were cut as independent siblings, but Mission 5's recut made the browser Flue `ChatTransport` the only door into a Brunch conversation and removed the `/api/chat` path Mission 6 had named as its departure point; the owner therefore corrected Mission 6 to consume Mission 5's landed transport, and this branch stacks on Mission 5's committed typed-panel transport tracer. +Mission 6 closed on the FE-1575 branch under its [archived authority](docs/mission-archive/6-resumable-workpiece-petrinaut.md): one deliberately prepared, honestly labelled fixture joined canonical conversation, session history, Markdown workpiece, and Petrinaut document through a browser-backed read/write change and cross-tab resume. Its consumed draft remains removed; its product-manager litmus, demo script, proof, and explicit owner waiver remain in the closed authority. The owner closed despite not re-running Voice-origin provenance and aborted-assistant presentation in the fresh product-manager conversation; those future scenario obligations live under [Voice after the live transport cut](#voice-after-the-live-transport-cut). Mission 5 owns the direct Voice/Flue transport cut on the FE-1574 branch directly beneath this one; its full contract lives only in that branch's root `MISSION.md`. Neither tracer requires a Mission 4 full-run candidate. The two were cut as independent siblings, but Mission 5's recut made the browser Flue `ChatTransport` the only door into a Brunch conversation and removed the `/api/chat` path Mission 6 had named as its departure point; the owner therefore corrected Mission 6 to consume Mission 5's landed transport, and this branch stacks on Mission 5's committed typed-panel transport tracer. On 2026-09-04, while Mission 6 was closing, the owner and an agent reviewed the provenance design that Missions 7, 9, and 10 had assumed, and two independent adversarial reviews tested the result. The outcome, recorded in the [decision log](docs/evidence/design/provenance-and-tooling-decision-log-2026-09-04.md), [mini spec](docs/evidence/design/provenance-by-lineage-mini-spec-2026-09-04.md), [independent review](docs/evidence/design/provenance-by-lineage-independent-review-2026-09-04.md), and [follow-up review](docs/evidence/design/provenance-by-lineage-follow-up-review-2026-09-04.md), changed the spine in four ways. Provenance is no longer a capture-envelope and hand-authored derivation seam over a prepared pair; it is recovered lineage in the canonical Flue log (workpiece revisions and net mutations as tool calls) plus a constructor-declared basis carried on each mutation request, with passage evidence, element origin, current state, attempt history, and recorded roles kept as distinct relations. Construction and explanation are consolidated into Mission 7 on a genuine conversation, because lineage exists only when the model actually constructs and because the owner chose fully connected parts over thin tracers; Mission 7 closes the readiness of its own claim and hands only breadth to Mission 9. The prepared Mission 6 fixture is a viability proof and is not promoted; real fixtures come from persona interviews run to construction. Tool admission ends its deferral: the inherited six-tool subset is retired in favour of scenario-selected operations with canonically derived schemas over a repaired provider carrier. These are owner decisions expressed in conversation; they become authority only when the Mission 7 draft is cut. @@ -21,6 +21,7 @@ M4 closed — core/plugin elicitation pattern accepted; S4 transition and full M4+ optional successor — broader hardening or source promotion only under separate authority M5 live on FE-1574, beneath this branch — direct Voice/Flue turn, canonical streamed reply, cancellation, and reopen M6 closed on FE-1575 — conversation → Markdown workpiece → Petrinaut read/write → cross-tab resume proved; two fresh-human Voice/stopped checks waived and carried +M6b live reconciliation — KA's Voice behavior over repaired M6; human/latency and direct-user attribution gates remain explicit in root authority M7 construct and explain — one genuine conversation builds and explains one real net region; two-step authority; closes its own readiness M8 deployment handoff — historical branch stopped after local application proof, before infrastructure deployment; a successor must be scheduled before any remote claim M9 repeatable projection breadth — unchanged repeat, changed input, retirement, concurrent change, schema classes over the M7 seam @@ -28,7 +29,7 @@ M10 revision — ship bounded authorized reviewer revision and a scoped patch o M11 optimisation — ship an accepted optimisation handoff after its consumer contract exists; early non-binding consumer discovery before M9's region ``` -Every numbered product mission after the proof-of-life exception must pass the **product-manager litmus**: a product manager who did not watch the work must be able to notice that the product materially moved forward. Each mission therefore states, in its draft's visible-product-advance section and then in its cut `MISSION.md` imperative, a release-note sentence, a demo script a product manager can run without an engineer, and the thing that was impossible before. Snapshots, manifests, event ledgers, and negative controls are oracles that belong in the evidence sections; they are not the visible advance. A mission is complete at its readiness gate, when the demo script works for the named scenario, not at the first green throughline tracer, which is an internal milestone inside the mission. Mission 5 names the Petrinaut Brunch panel's typed and Voice surface over one Flue route, with its litmus stated in the FE-1574 branch's `MISSION.md`; closed Mission 6 names the stable fixture and browser Petrinaut document, with its litmus retained in this branch's [`MISSION.md`](MISSION.md#visible-product-advance); Missions 7, 9, and 10 name the Petrinaut Brunch panel. Because Mission 8 stopped before remote deployment, those panel missions must name the deployment posture available at cut time, and a locally run panel is acceptable for the demo; a product-manager-noticeable claim must never depend on infrastructure that does not exist, while remote durability obligations stay in their readiness gates. Architecture, schema repair, fixtures, evaluation, rehearsal, and spikes may support the advance but cannot be the sole outcome. Parallel work means separate issue, branch, PR, worktree, and mission authority; it never means multiple live missions here. +Every numbered product mission after the proof-of-life exception must pass the **product-manager litmus**: a product manager who did not watch the work must be able to notice that the product materially moved forward. Each mission therefore states, in its draft's visible-product-advance section and then in its cut `MISSION.md` imperative, a release-note sentence, a demo script a product manager can run without an engineer, and the thing that was impossible before. Snapshots, manifests, event ledgers, and negative controls are oracles that belong in the evidence sections; they are not the visible advance. A mission is complete at its readiness gate, when the demo script works for the named scenario, not at the first green throughline tracer, which is an internal milestone inside the mission. Mission 5 names the Petrinaut Brunch panel's typed and Voice surface over one Flue route, with its litmus stated in the FE-1574 branch's `MISSION.md`; closed Mission 6 names the stable fixture and browser Petrinaut document, with its litmus retained in the [archive](docs/mission-archive/6-resumable-workpiece-petrinaut.md#visible-product-advance); Missions 7, 9, and 10 name the Petrinaut Brunch panel. Because Mission 8 stopped before remote deployment, those panel missions must name the deployment posture available at cut time, and a locally run panel is acceptable for the demo; a product-manager-noticeable claim must never depend on infrastructure that does not exist, while remote durability obligations stay in their readiness gates. Architecture, schema repair, fixtures, evaluation, rehearsal, and spikes may support the advance but cannot be the sole outcome. Parallel work means separate issue, branch, PR, worktree, and mission authority; it never means multiple live missions here. ## Successor mission précis @@ -368,7 +369,7 @@ Before claiming long-running provenance, prove panel/transcript/workpiece recove ### Voice after the live transport cut -The Mission 5 contract, recut on 2026-09-03, owns the single-route consolidation: the typed panel's browser `ChatTransport` over `@flue/sdk`, removal of the server-side `/api/chat` door, repurposing `transport-aisdk` as the browser-side adapter, direct Voice/Flue reconciliation, its selected external-PR evidence, and the bounded local tracer. Its 2026-09-04 human witness passed typed and Voice admission, spoken playback, barge-in, and durable Stop, then failed faithful reopen: per-message typed/Voice provenance disappeared and the stopped entry returned as ordinary truncated content. Mission 6 repaired those two projection defects and its outer witness exercised them, but its fresh product-manager conversation contained neither record. On 2026-09-04 the owner explicitly waived that fresh-human re-check and closed Mission 6; the waiver is not a pass. +The Mission 5 contract, recut on 2026-09-03, owns the single-route consolidation: the typed panel's browser `ChatTransport` over `@flue/sdk`, removal of the server-side `/api/chat` door, repurposing `transport-aisdk` as the browser-side adapter, direct Voice/Flue reconciliation, its selected external-PR evidence, and the bounded local tracer. Its 2026-09-04 human witness passed typed and Voice admission, spoken playback, barge-in, and durable Stop, then failed faithful reopen: per-message typed/Voice provenance disappeared and the stopped entry returned as ordinary truncated content. On 2026-09-04 the owner explicitly waived the fresh-human re-check and closed Mission 6; its fresh product-manager conversation contained neither record. A subsequent source/artifact audit could not substantiate the earlier mechanical-coverage claim: both retained outer-witness bundles contain only completed settlements and no recorded Voice origins, and the analyzed history projector did not reconstruct either per-message property. Preserve the historical close and immutable records, but neither the waiver nor those bundles establish a presentation pass. Mission 6b's root authority owns the combined foundation check and distinguishes supported client-tool attribution from blocked direct-user attribution. A later mission that exercises Voice, exact conversation resume, or pre-release scenario breadth must include one reproducible scenario containing at least one typed-origin message, one Voice-origin message, and one durably aborted assistant entry. After closing and reopening in a second tab, the oracle must verify per-message typed/Voice provenance, render the aborted entry as stopped rather than ordinary truncated content, and distinguish local **Exit voice mode** from durable composer **Stop**. Fold this scenario into that mission's named test portfolio before closure; do not treat Mission 6's prepared fixture or mechanical witness as a permanent substitute for the skipped human check. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md new file mode 100644 index 00000000000..f664d2175f4 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md @@ -0,0 +1,97 @@ +# Mission 5 question replay and direct-user provenance decision + +## Decision + +The owner approved two changes to the live FE-1580 authority on 2026-09-04: + +1. Brunch may expose a non-interactive, model-facing question-marker tool. The + marker identifies exact assistant-authored question text for Voice replay, + but it never suspends a response, renders an answer affordance, accepts an + answer, or changes Voice path B. +2. Direct-user Voice provenance must wait for an upstream Flue contract that + durably projects caller metadata on a canonical `kind: "user"` message. This + branch must not patch Flue locally or approximate provenance with a second + signal admission, browser storage, or encoded user text. + +## Exact question marker + +Brunch core owns a `brunch_mark_question` server tool and a +`data-brunch-question` client marker. Before asking the user a direct question, +the model calls the tool with the exact question text. The tool writes a durable +data part containing that text and its Flue `toolCallId`, then returns a small +acknowledgement. It does not terminate the response. Brunch instructions require +the same exact text to appear in ordinary assistant prose after the tool call. + +The browser transport hides the marker tool's implementation call while +retaining the data part. This keeps an internal annotation out of Petrinaut's +tool-activity UI without creating another conversation representation. Both the +live stream and canonical snapshot projection apply the same hidden-tool rule. + +Canonical speech accepts a question marker only when all of these facts hold: + +- the marker has a non-empty string question and non-empty `toolCallId`; +- it belongs to an assistant message; +- the same assistant message contains the exact marked string in finalized + ordinary text; and +- the marker data part is complete and canonical, not provisional Voice state. + +Malformed, unmatched, stale, or absent markers do not enable **Repeat +question**. The final text segment and punctuation are never used as fallback +question authority. The selected question segment derives stable identity from +the assistant message id, marker tool-call id, and exact-text hash. Full-response +speech remains the ordered ordinary text segments and is not rewritten or +duplicated by the marker. + +The Voice controller carries the selected question separately from the full +response. **Repeat question** reuses the existing exact canonical queue and the +same settlement, output-completion, idle-input, submission, cancellation, +capture, pause, and error gates as **Read full response**. The control remains +disabled when the settled response has no matching marker. + +## Production proof + +Tests are written and observed failing before implementation. Closing evidence +must cover: + +- Brunch's built Flue agent mounting `brunch_mark_question` while continuing to + omit `brunch_ask`; +- a real server-tool call writing a durable `data-brunch-question` part; +- live transport and snapshot projection hiding the implementation tool while + retaining the marker; +- canonical selection rejecting malformed and unmatched markers and preserving + exact text and stable identity for a valid marker; +- the production Voice host registering `repeatQuestion` and the panel invoking + it only when `canRepeatQuestion` is true; and +- controller and preview integration proving exact question-only replay after + correlated Brunch settlement and matching Realtime output completion, with + every existing replay exclusion still enforced. + +## Direct-user Voice provenance + +Flue 2.0.3 and current upstream `main` accept only `body` and image +`attachments` on `kind: "user"`. The caller's idempotency key is irreversibly +hashed into `submissionId`; canonical snapshots do not expose that key or +caller-authored user metadata. Agent-authored response metadata cannot annotate +the canonical user message. + +The accepted route is an upstream Flue extension that admits caller metadata on +the user delivery, persists it atomically with the canonical user record, and +projects it on live and historical user messages. FE-1580 can adopt that seam +only after a released dependency is available and the branch is explicitly +authorized to upgrade. The closing oracle is a snapshot-only fresh-process test +that restores the Voice marker without browser correlation state. + +Rejected alternatives: + +- a local Yarn patch to Flue, because it forks substrate persistence and wire + projection inside this product PR; +- a correlated provenance signal, because it is a second, non-atomic admission + that can independently fail or wake the agent; +- browser or application sidecar storage, because it becomes a second durable + authority; and +- hidden transcript, attachment, or visible-text encoding, because it changes + the canonical user representation or smuggles metadata through content. + +Until the upstream contract is released and adopted, direct spoken user text +remains canonically durable but its Voice chip after reopen remains blocked and +must not be reported as complete. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md new file mode 100644 index 00000000000..0afe5e39b2b --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -0,0 +1,113 @@ +# FE-1580 donor-behavior matrix + +## Decision frame + +This record pins the semantic disposition of the Voice donor branches for the live [FE-1580 mission](../../../../MISSION.md). The parent and donors are read-only source evidence at these exact heads: + +| Source | Pinned head | Role | +| --- | --- | --- | +| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `58f75840804766a84ce85b9daab5b5194f3875ec` | Unified Flue route and path-B departure base | +| Donor PR [#9496](https://github.com/hashintel/hash/pull/9496) | `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` | Canonical TTS queue and replay mechanics | +| Donor PR [#9500](https://github.com/hashintel/hash/pull/9500) | `935aa9f02a5ac635a50eb8bc130edb3e258af8e4` | Completed-transcript authority | +| Donor PR [#9507](https://github.com/hashintel/hash/pull/9507) | `252b9dbb0c77fae8cee45a506f09cac3e20c381c` | Temporary `brunch_ask` shim, excluded | +| Donor PR [#9512](https://github.com/hashintel/hash/pull/9512) | `d13535d1077b3a78d6a1411031b7d0a0a78e3144` | Half-duplex cancellation, replay UX, and provenance | + +No source is merged, cherry-picked, rebased, retargeted, rewritten, or closed by the implementation. Tests are transplanted first and adapted to the one Flue submission route; production behavior is reimplemented semantically. + +The owner selected half-duplex turn ownership on 2026-09-03: assistant output owns the audio turn until **Your turn** completes an acknowledged cancellation barrier. Automatic duplex is not an admissible fallback. + +## Behavior disposition + +| Source | Behavior | Disposition | Reason | Outstanding adaptation or proof | +| --- | --- | --- | --- | --- | +| #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | +| #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | +| #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Implemented with transport convergence/conflict tests and typed Voice admission outcomes. | +| #9528 | Canonical hydration guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation | **Adopt through restack** | These mechanisms remain parent-owned and must enter the successor through the stack rather than copied fixes. | Restacked onto `58f758408047`; hydration no longer blocks the real witness. Further defects in these mechanisms remain parent scope. | +| #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | +| #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | +| #9496 | `canRepeatQuestion`, `repeatQuestion()`, and playback-menu action | **Adopt UX; reject final-segment inference** | The final segment may be ordinary prose and is not authority for question identity. The approved `brunch_mark_question` data marker now supplies deterministic identity without accepting an answer. | Implemented by replaying only exact marked text found in finalized prose from the same assistant message; a missing or unmatched marker leaves the action disabled. | +| #9496 | Realtime-generated concise response preparation or any fallback that rewrites canonical text | **Reject** | Response simplification is a non-goal and violates exact canonical speech. | Tests compare retained segment ids and exact queued strings; no preparation API remains on this path. | +| #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Implemented in policy, session, and production-preview integration tests. | +| #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Implemented with current-turn speech-boundary, stale, reordered, and late-output rejection tests. | +| #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Implemented through path B; the panel preserves the bridge-normalized payload unchanged. | +| #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Implemented with reason-specific bridge/controller UI coverage. | +| #9500 | Silently settling ownership by discarding every playback-overlapping utterance without an explicit handoff | **Supersede** | It avoids echo but leaves users without a deliberate way to take the turn. | Use #9512 half-duplex `canTakeTurn`/`takeTurn()` and reject all speech captured before the completed handoff. | +| #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response facts; exact question replay uses the non-interactive marker instead. | +| #9507 | Temporary `brunch_ask` registration, widget, correlated spoken ask answer, transcript formatting | **Reject entire shim** | The current transport only admits the supported follow-up set; a spoken ask can otherwise wait forever. Structured questions are a separate product decision. | Remove or gate dormant `brunchAskInteractiveTool` and `"brunch-ask"` canonical-speech recognition only if still present after restack. | +| #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Implemented through the public Voice store and production panel registration path. | +| #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Implemented with acknowledgement/race tests, latest-mute behavior, and fresh post-handoff capture. | +| #9512 | Replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Implemented against parent segment/submission correlation for exact full-response and marked-question replay. | +| #9512 | Voice answer icon/provenance before interactive answers | **Partially adopt; blocked for direct user turns** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. Flue's client-tool result signal can durably carry those origins. Its direct-user delivery and snapshot types expose no caller metadata or idempotency key, so a direct spoken user message cannot be identified after reopen without a forbidden second store or text encoding. | Keep `voiceToolCallIds`, preserve successful siblings on partial failure, and reconstruct supported tool-result origins from Flue signals. Re-enter direct-user attribution only when Flue provides a supported durable correlation seam. | +| #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | + +## Adopted-behavior replacement coverage + +| Adopted behavior | Replacement implementation | Regression test | Production integration proof | Status | +| --- | --- | --- | --- | --- | +| One path-B Flue admission route | `local-storage-demo-app.tsx`, `brunch-panel-transport.ts`, transport `src/index.ts` | `brunch-panel-transport.test.ts`, `chat-transport.test.ts` | `voice-preview.integration.test.ts` crosses completed transcript → panel submission → Flue transport → canonical speech | **Implemented**; parent defects remain downstack | +| Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview carries 409 conflict, 500 ambiguity, and local abort through transport → tracker → `submitVoiceInputWithAdmission` → bridge; each observes one `send()`, and local abort never invokes durable `FlueClient.abort()` | **Implemented** | +| Exact canonical TTS queue and full-response replay | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Petrinaut playback menu | session queue/cancellation cases; controller exact-segment and terminal-gating cases; panel action tests | real host registration exposes `readFullResponse`; panel forwards it through `voiceSessionStore` | **Implemented** | +| Exact question replay | core `brunch_mark_question` tool/data contract; transport hidden-tool projection; `canonical-speech.ts`; bridge/controller; Voice host callback | core marker tests; live/snapshot transport projection tests; canonical selector malformed/unmatched/cross-message cases; controller final-segment negative and exact-marker replay cases | real Flue integration persists and reopens `data-brunch-question` while hiding the marker tool; controlled Voice preview carries the marker through response correlation and queues only the exact question; panel host forwards the action | **Implemented**; missing or unmatched markers fail closed | +| Disabled Realtime generation/tools | `openai-voice-policy.ts`, `openai-realtime-session.ts` | policy/session tests reject tools and function arguments | controlled production preview negotiates the server policy and emits only canonical speech | **Implemented** | +| Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, canonical-request-before-output, and late-output cases | controlled production preview proves a pre-request item cannot call Flue `send()` before output starts and only fresh post-handoff input submits through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | +| Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | canonical-request invalidation, input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves the microphone closes before `response.create` and fresh post-handoff capture submits once | **Implemented** | +| Durable Stop distinct from local cancellation | app `requestFlueStop`, panel `stopComposer`, session `cancelOutput` | panel durable-before-local Stop, controller/session local-cancel cases, app host Stop case | configured Brunch app invokes `FlueClient.abort()`, observes an aborted settlement, and does not invoke local playback cancellation | **Implemented**; parent-owned Stop races excluded | +| Multi-origin Voice client-tool provenance | panel `addMappedToolOutput`, transport client-tool-result signal/projection, `useFlueChatHistory` | sibling partial-failure, persisted-signal projection, hydration/reopen cases | configured app consumes the public Flue observation and restores every `voiceToolCallId` | **Implemented for client-tool results**; direct-user marker **blocked** | +| No live `brunch_ask` | Brunch app registers `interactiveTools: []`; canonical speech selector ignores the ask name | canonical-speech negative case and configured-app registration negative case | captured production Brunch `PetrinautAiAssistant` has no ask tool while retaining Flue Stop wiring | **Implemented exclusion** | + +## Outstanding acceptance ledger + +| Area | Required closing evidence | Current state | +| --- | --- | --- | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, canonical-speech-request and late-output invalidation, provisional UI clearing, exact bridge normalization, and unchanged panel payload are covered. | +| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity; production-path integration preserves the original conflict `submissionId` and keeps local admission abort distinct from durable abort. | +| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races cover the barrier and mute preference; panel registration and configured-app Stop cases cover the production host seams. | +| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay preserves every exact segment. **Repeat question** uses only a durable non-interactive Brunch marker that exactly matches finalized prose in the same assistant message; final-segment inference remains rejected. Both actions share terminal/output/input gating. | +| Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | +| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | +| Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Parent hydration blocker resolved by restack; human browser/microphone run and retained artifacts remain outstanding. | +| Comparative latency | Ten pinned #9496 trials and ten final-candidate trials retain raw finalized-speech-to-first-audible-canonical-TTS samples and show no median regression with p95 regression below 20%. | Donor isolated worktree is prepared and its five focused Voice suites pass 108/108 after dependency build. Twenty comparable human audible trials and statistics remain outstanding. | +| Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | + +## Corrective verification + +Fresh local checks on 2026-09-07 cover the 72-file successor diff against the +verified #9528 head `58f75840804766a84ce85b9daab5b5194f3875ec`. +The 62-commit replay required semantic resolutions in the mission authority, +the already-equivalent launcher comment, and the Voice transcript panel. The +combined panel keeps the parent's editor-owned width together with the +successor's always-live transcript and complete-error behavior. The full gate +then caught one unused parent import left by that merge; removing it returned +the complete verification set to green. The verified code head before this +evidence-only update is `b66869f393`: + +| Command | Result | +| --- | --- | +| `NODE_OPTIONS=--no-experimental-webstorage mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks and 1,216/1,216 tests passed: 16/16 Brunch app files with 80/80 tests, 10/10 Brunch core files with 86/86 tests, 3/3 transport files with 32/32 tests, 5/5 binding files with 18/18 tests, 2/2 plugin files with 8/8 tests, 83/83 Petrinaut files with 673/673 tests, and 32/32 website files with 319/319 tests. | +| `mise exec -- yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 70 layers, 356 edges, 725 files, 71 generated pages, and 38 authored pages. | +| `mise exec -- yarn lint:format` | Exit 0; all 5,586 matched repository files use the correct format. | +| `git diff --check` | Exit 0. | + +### Earlier focused evidence + +These focused checks were established on the earlier 2026-09-04 candidate. +Their complete files were rerun inside the 2026-09-07 seven-workspace gate: + +| Command | Result | +| --- | --- | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts -t 'invalidates accepted input before requesting canonical speech output\|rejects unfinished input as soon as canonical speech is requested\|clears capture when canonical speech is requested before output starts\|bridges one completed transcript through Brunch and back to canonical half-duplex audio'` | Exit 0; 4/4 selected tests passed and 92 unrelated tests were filtered across four files. This covers the request-before-output race at session, bridge, controller, and production integration layers. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | +| `mise exec -- yarn workspace @hashintel/brunch-agent test:unit test/question-marker.test.ts` | Exit 0; 9/9 exact question-marker tests passed. | +| `mise exec -- yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` | Exit 0; 32/32 transport tests passed, including live and snapshot marker projection plus bounded Flue-error serialization. | +| `mise exec -- yarn workspace @apps/brunch-agent test:unit test/petrinaut-chat.test.ts` | Exit 0; 1/1 real-Flue integration test passed, including exact marker persistence through fresh-process reopen while marker tools remain hidden. | +| `mise exec -- yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 46/46 production host-registration and panel tests passed. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts` | Exit 0; 79/79 exact replay, queue, terminal-gating, and turn-controller tests passed. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @hashintel/brunch-agent` | Exit 0; 5/5 tasks passed, including 10/10 test files and 86/86 tests; the four question-marker mock lint failures are resolved with production-interface signatures. | +| In isolated detached worktree `/Users/kostandin/Projects/hashdev/worktrees/fe-1580-latency-baseline-9496`: `mise exec -- yarn exec turbo run build --filter '@apps/petrinaut-website^...'`, then `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; dependency build passed 14/14 tasks, then all 5/5 donor Voice files and 108/108 tests passed at pinned #9496 head. The isolated donor and candidate panels return HTTP 200 on ports 4916 and 4915 respectively; real audible samples remain uncollected. | + +No production Voice source under `apps/petrinaut-website/src/main/app/voice-interview` +calls `FlueClient.send()`; its only `.send()` is the OpenAI Realtime data +channel. Production Brunch registration supplies `interactiveTools: []`, and +canonical speech has no `brunch_ask` recognition. The dormant ask source remains +unmounted. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md new file mode 100644 index 00000000000..b3e9e897b07 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -0,0 +1,46 @@ +# FE-1580 direct-user Voice provenance blocker + +## Observed boundary + +Flue 2.0.3 can durably preserve Voice provenance for client-tool results: the +existing client-tool result signal carries each Voice-origin tool-call id, and +canonical snapshot projection can reconstruct every surviving sibling origin. +Regression coverage preserves successful siblings after a partial failure, +projects both origins from the persisted signal, and restores them through the +production observation hook after unmount and reopen. + +The corresponding direct-user seam does not exist in the installed public +contract: + +- `DeliveredMessage` user input accepts only `body` and image `attachments`; +- the caller's `idempotencyKey` is accepted for admission but is not projected + into `FlueConversationMessage` or `FlueConversationSettlement`; +- materialized user messages expose the generated `submissionId`, but no Voice + source metadata; and +- snapshot `metadata` is agent-authored response metadata, not caller-authored + user-message metadata. + +The discarded implementation persisted Voice `submissionId` values in browser +storage and correlated them after hydration. That would create a second durable +store, which the mission explicitly names as a stop condition. Encoding the +origin in visible user text is also prohibited. Replacing the canonical direct +user message with a hidden Flue signal would change the delivery semantics and +require a synthetic second transcript projection, so it is not a transparent +representation of the existing path-B turn. + +## Current disposition + +Direct spoken user turns still render with a Voice chip while their AI SDK +message metadata is live. Their canonical text and submission survive Flue +hydration, but the Voice chip cannot be reconstructed after reopen. This portion +of proof item 5 is blocked rather than reported as complete. + +Re-enter only when Flue projects caller metadata or the caller idempotency key +onto the canonical direct-user message, or when the product owner explicitly +authorizes a different durable representation. The oracle is a snapshot-only +test that reconstructs the Voice marker after a fresh process with no browser +correlation state. + +The restacked branch still installs `@flue/sdk` 2.0.3 with this same public +shape. No supported projection seam or owner-approved deferral has been +recorded, so direct-user reopen attribution remains blocked. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md new file mode 100644 index 00000000000..420b908da58 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -0,0 +1,61 @@ +# FE-1580 human-evidence gate + +## Current disposition + +The real Voice witness has **not** been run and no witness bundle is claimed. +Completed-transcript authority, admission idempotency, half-duplex handoff, +acknowledged cancellation, exact full-response replay, durable Stop, dormant-ask +exclusion, exact Brunch-marked question replay, and the supported client-tool +portion of Voice provenance have focused automated coverage. Automated coverage +cannot replace the microphone, handoff, unsettled Stop, hard-reload, and +network-route witness required for mission acceptance. Direct-user Voice +attribution has a separate [Flue projection blocker](provenance-blocker.md). + +The successor is restacked onto [PR #9528](https://github.com/hashintel/hash/pull/9528) +head `58f75840804766a84ce85b9daab5b5194f3875ec`. That parent now guards its +once-per-conversation hydration from replacing a locally visible assistant +response with an older canonical snapshot, so hydration no longer blocks this +witness. The remaining gate is the required human browser and microphone run. + +An owner-directed PR #9531 side quest also removed a local launcher blocker +found at the real boundary on 2026-09-04. The Brunch-specific Vite config had +removed Petrinaut's entire `petrinaut-api-dev` plugin, so +`/api/voice/config` returned transformed module source instead of the handler's +JSON. The launcher now retains the website API plugin while continuing to +proxy only `/agents/chat/*` to Brunch. A config-level regression test loads the +real merged config, and an isolated `yarn dev:brunch` panel process with an +enabled non-secret test environment returned +`{"available":true,"connectionTimeoutMs":15000}`. This proves local Voice API +wiring only; it does not satisfy the human witness below. + +## Re-entry gate + +Using the final source/build commit: + +1. submit one typed turn; +2. run one real microphone turn and confirm exactly one matching user message; +3. confirm visible text and synthesized speech use the same canonical response; +4. use **Your turn** during output and retain cancellation acknowledgements; +5. confirm pre-handoff audio cannot submit and fresh post-handoff speech can; +6. durably stop an unsettled turn and retain its stopped settlement; +7. replay the exact full response and exact marked question; +8. hard-reload the settled conversation and confirm no resubmission or + automatic replay; +9. retain the canonical Flue snapshot and settlement index; +10. retain a network route summary proving the absolute Flue `streamUrl` + remains on the same-origin proxy; and +11. record the exact source/build and evidence commits plus hashes for every + retained artifact. + +The comparative latency proof also requires ten audible trials at pinned donor +#9496 head `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final +candidate. Both sets use the same machine, browser, microphone/input phrase, +model configuration, warm/cold-start policy, and finalized-speech-to-first- +audible-canonical-TTS boundary. Raw sanitized samples, the calculation method, +environment, commit identities, median, and p95 must be retained; the candidate +median may not regress and p95 regression must remain below 20%. + +Until then, `witness.md`, `voice-events.jsonl`, `network-routes.json`, +`flue-snapshot.json`, and `settlements.json` are intentionally absent rather +than populated with simulated evidence. Latency samples and statistics are also +intentionally absent until the comparable human trials run. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/import.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/import.md new file mode 100644 index 00000000000..d47fada46a6 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/import.md @@ -0,0 +1,30 @@ +# Mission 6b source import + +## Provenance and scope + +Lu authorized Mission 6b implementation and subsequent Mission 7 restack on 2026-09-07. No Linear writes or PR submission are authorized; the later replacement PR will reference FE-1580 without rewriting it. KA's original branch and PR #9531 remain untouched. + +- Source contribution: `58f75840804766a84ce85b9daab5b5194f3875ec..be56a18ff0244c5750a8702e9c7f45c0b607dc06`, authored by Kostandin Angjellari and retained with attribution in the import commit. +- Destination: Mission 6 `01649899eb65ab8d7a8fec9407dc3ea613128264` over Mission 5 `7538264feeb1487aa494e991831bed0338ae76df`. +- Authority-only cut: `ecee802ce3`. Original preparation: Mission 7 commit `86e3755` and its reconciliation draft. The old `eecbe99e20..b53b1006fb` analysis range is not the import source. +- Import method: three-way application of the exact source contribution, excluding only its root `libs/@hashintel/brunch-agent/MISSION.md`. Source evidence remains immutable history; the replacement authority supersedes source prohibitions on integrating Mission 6 only within the accepted combined-path scope. + +## Necessary join resolutions + +- Retain Mission 6's configurable browser-tool catalogue and input mapper alongside KA's hidden non-interactive question marker in transport and history. Default remains the docs reader; fixture-specific tools and their canonical input normalization remain available. The source deleted an imported default-catalogue constant, so preserve a stable default set locally rather than losing fixture configurability. +- Retain the established Mission 6 `ai-sdk:user:` / `ai-sdk:client-tools:` delivery namespaces and sorted tool-call key identity, adding KA's bounded-key validation, typed rejected/conflict/ambiguous/aborted outcomes and canonical completed-transcript Voice identities. Update source test expectations to that retained namespace; do not invent a new prefix to bypass previous admission receipts. +- Combine snapshot input normalization with KA's persisted per-tool origin records and hidden marker projection. Preserve parent continuation folding. The later reconciliation must test origins contributed by folded continuation messages; mechanically joining the two maps alone is not proof. +- Keep all independent tests added at the same insertion points: fixture transport/refusal and input mapping/pending-tool step tests from the parent; admission failure and rich stream error tests from KA. The Voice route test uses the keyed completed-transcript identity and a real request AbortSignal, not the old provider function-call identity. +- Preserve the parent's composite composer busy status and Stop-withheld follow-up behavior; no new Voice scheduler is introduced by the import. The automatic browser-tool output path still requires its own combined lifecycle/failure discriminator. +- KA's host test mocked Brunch permanently configured; the repaired parent's unconfigured-fixture test consequently failed. Make the mock explicitly configurable for that test rather than undoing the parent's fallback behavior. +- Extend the reviewed architecture inventory for the core question-marker export and its hermetic logger/tool-run test. It invokes the tool with mocked writer/logger and no runtime, key, socket or model. This is the source feature crossing the newer parent inventory, not permission to loosen the inventory check. + +## Import verification, not acceptance + +The first checks caught retained-key test expectations, the conflicting configured-host mock and the new architecture inventory entries. These were corrected at the join. The complete seven-workspace command then passed **39/39 tasks** (23 cached) before focused reconciliation: + +```sh +yarn exec turbo run build test:unit lint:tsc lint:eslint --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk --filter @apps/brunch-agent --filter @hashintel/petrinaut --filter @apps/petrinaut-website --continue=always --output-logs errors-only +``` + +This establishes that the joined source builds and passes the existing package gates. It does not establish safe pending static-tool execution across Stop, deterministic reordered result payloads, combined Voice failure release, faithful stopped-entry reopen, a real microphone/browser witness or comparative audible latency. The active authority owns those remaining discriminators and owner-held gates. No paid provider call or human acceptance is claimed. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/main-restack.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/main-restack.md new file mode 100644 index 00000000000..da53d471f1d --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/main-restack.md @@ -0,0 +1,31 @@ +# Full-stack restack onto main + +Lu explicitly requested `gt restack --no-interactive` on 2026-09-07 after the review-only Mission 7 move. That operation restacked Missions 5, 6, 6b and 7 onto `main` at `b1d3ffcfd1077546276a8698f45cee8f7144c6dc`. No push, PR submission, Linear write, provider call or acceptance occurred. Mission 6b remains unaccepted and Mission 7's shared/paid foundation gate remains closed. + +## Resolution and pins + +Only `apps/brunch-agent/src/app.ts` and `apps/brunch-agent/src/http/routes.ts` conflicted, while replaying Mission 5's removal of the legacy chat route. Main's newly added liveness route is retained alongside the ownership-guarded Flue conversation route. `/api/chat`, its handler and its route constant remain removed. The rest of the stack replayed without source conflicts. + +| Role | Rebased pin | +| --- | --- | +| Mission 5 | `b1295ad454ba7548a927e7d9771c5bb211e04827` | +| Mission 6 | `976bb1c67cc6673c06356b855a667584b662f8c9` | +| Mission 6b code candidate | `8ed08e1eba50ea972a96481227540461e03bba39` | +| Mission 6b before this pointer refresh | `ccf93d5bb5b33c4cc54c5b349c469bc632df221b` | +| Mission 7 at the verification run | `11bfa2a18da1c782ae0ed191f12f44a68a11e2c2` | + +The runtime tree differs from the previous review candidate by main's container/liveness changes, not new Voice reconciliation behavior. The imported KA contribution remains pinned at `be56a18ff0244c5750a8702e9c7f45c0b607dc06`. KA's frozen branch was observed at the later `9415e1b0075d7cb8c5b7fe19e0512b8bc917c97f` and was left untouched; that newer contribution is not imported by this restack. Original source, witness and pre-restack evidence pins remain historical records, not rewritten results. + +## Verification + +The same seven-package command in [the reconciliation verification](verification.md#verification-run), with `--force`, passed **39/39 tasks with zero cached tasks**: builds, unit tests, TypeScript and ESLint. Scoped suites passed **1,318 tests in 167 files**. The increase from the earlier run is main's new health unit test. The architecture-doc check passed with 70 layers, 356 edges, 736 files, 71 generated pages and 38 authored pages. Whitespace and conflict-marker checks passed. + +A separate Node probe loaded the real built application through `loadBuiltBrunchApplication()`, with a fresh temporary `BRUNCH_DEV_DB_PATH` and `OTEL_SDK_DISABLED=true`. It used the production application's `fetch`, not a test-only Hono route, and shut it down afterward. Assertions verified: + +```json +{"health":{"status":200,"body":{"status":"pass"}},"legacy":404,"guardedFlue":401} +``` + +The health response also had `cache-control: no-store` and `application/health+json` content type. Requests were `/health`, `/api/chat` and `/agents/chat/missing-identity`; none admitted a conversation or contacted a model. This proves that the conflict resolution retained liveness and the single guarded conversation door in the emitted application. It is not a container-runtime, deployment, microphone, reload or latency witness. + +Subsequent commits refresh the live Mission 6b/7 dependency pointers only; they do not change this tested runtime tree or clear any acceptance gate. The existing [acceptance dispositions](verification.md#acceptance-disposition--still-open) remain open. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/canonical-summary.json b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/canonical-summary.json new file mode 100644 index 00000000000..c78a7b36f29 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/canonical-summary.json @@ -0,0 +1,168 @@ +{ + "schemaVersion": 1, + "capturedAt": "2026-09-07", + "branch": "ln/fe-1580-reconcile-voice-resumable-workpiece", + "implementationHead": "48e2b66666df05034777f9410024c2e1228c86be", + "implementationCommits": ["1e238f498e", "48e2b66666"], + "canonicalConversationId": "conv_01M1Y4SPKHEKPVFVAMQG9QNH4Y", + "streamPath": "agents/brunch-chat-agent/7b484f39802178a2222836a7ee4c0c5e3ef123c457d507d75b3faf5ac98b3e32", + "submissions": [ + { + "sequence": 2, + "submissionId": "sub_ik_53770e9263c875a6a3ac812ed2ae291e", + "kind": "signal", + "status": "settled", + "outcome": "completed", + "fixtureId": "crew-reservation-v1" + }, + { + "sequence": 3, + "submissionId": "sub_ik_9942648a081a4a1d257dc20b41e0d9ea", + "kind": "user", + "status": "settled", + "outcome": "completed", + "body": "SDCPN" + }, + { + "sequence": 4, + "submissionId": "sub_ik_40c1b6c5489dc05c9bf9c49e243db5ed", + "kind": "signal", + "status": "settled", + "outcome": "completed", + "results": [ + { + "toolCallId": "toolu_01Ff2VXcErUd5pLxPiVFHPgc", + "toolName": "getLatestNetDefinition", + "applied": null + } + ] + }, + { + "sequence": 5, + "submissionId": "sub_ik_b15d9c3a910649fef2d5a559db61a6ee", + "kind": "user", + "status": "settled", + "outcome": "completed", + "body": "Starting final inspection reserves the single dispatch crew immediately. Sign-off releases it. The timing, failure, and recovery behavior are still unknown." + }, + { + "sequence": 6, + "submissionId": "sub_ik_f7f80b5cd8b6e3360d6a8d1bfca86f86", + "kind": "signal", + "status": "settled", + "outcome": "completed", + "results": [ + { + "toolCallId": "toolu_01XoXQED2JUy5Xk6MiH3axDs", + "toolName": "addArc", + "applied": true + } + ] + }, + { + "sequence": 7, + "submissionId": "sub_ik_785634ce1e717631217d8fc0484a660d", + "kind": "signal", + "status": "settled", + "outcome": "completed", + "results": [ + { + "toolCallId": "toolu_01MaeDyPhW4iuULiJvWkpKeE", + "toolName": "getLatestNetDefinition", + "applied": null + } + ] + }, + { + "sequence": 8, + "submissionId": "sub_ik_8eb1e09c5ebb8b651ef29965140ecd40", + "kind": "user", + "status": "settled", + "outcome": "completed", + "body": "What remains unresolved in this workpiece? Do not mutate the net." + }, + { + "sequence": 9, + "submissionId": "sub_ik_336c2a985788889d677697626db20ba1", + "kind": "user", + "status": "settled", + "outcome": "aborted", + "body": "Please give a detailed explanation of every unresolved timing, failure, and recovery question in this workpiece without changing the net.", + "abortRequested": true + }, + { + "sequence": 10, + "submissionId": "sub_ik_bcdfbccbd9856a83982a7aa2a49e1972", + "kind": "user", + "status": "settled", + "outcome": "completed", + "body": "Just give me a very brief overview of what is less than optimal in the current model." + }, + { + "sequence": 11, + "submissionId": "sub_ik_3926c3230c824a5a3c7a6a13089a6729", + "kind": "user", + "status": "settled", + "outcome": "completed", + "body": "For testing purposes only, ask me a question please." + } + ], + "toolCalls": [ + { + "sequence": 32, + "submissionId": "sub_ik_9942648a081a4a1d257dc20b41e0d9ea", + "messageId": "entry_01M1Y4T7VKYRMJPBQ87QNY9HHF", + "toolCallId": "toolu_01Ff2VXcErUd5pLxPiVFHPgc", + "toolName": "getLatestNetDefinition", + "arguments": {} + }, + { + "sequence": 115, + "submissionId": "sub_ik_b15d9c3a910649fef2d5a559db61a6ee", + "messageId": "entry_01M1Y511RH9XABSYGBYAJE430A", + "toolCallId": "toolu_01XoXQED2JUy5Xk6MiH3axDs", + "toolName": "addArc", + "arguments": { + "transitionId": "start-final-inspection", + "arcDirection": "input", + "placeId": "dispatch-crew-available", + "weight": "1", + "type": "standard" + } + }, + { + "sequence": 126, + "submissionId": "sub_ik_f7f80b5cd8b6e3360d6a8d1bfca86f86", + "messageId": "entry_01M1Y51F3JYQVJW5N5CKC9PWE6", + "toolCallId": "toolu_01MaeDyPhW4iuULiJvWkpKeE", + "toolName": "getLatestNetDefinition", + "arguments": {} + } + ], + "modelProducedWorkpieceMessages": [ + { + "messageId": "entry_01M1Y511RH9XABSYGBYAJE430A", + "submissionId": "sub_ik_b15d9c3a910649fef2d5a559db61a6ee", + "sequence": 98, + "runbookBlockCount": 1 + }, + { + "messageId": "entry_01M1Y51QS8WTR61G9Z1V1YNFY3", + "submissionId": "sub_ik_785634ce1e717631217d8fc0484a660d", + "sequence": 148, + "runbookBlockCount": 1 + } + ], + "assertions": { + "targetAddArcCallCount": 1, + "clientResultSignalsAreSingleStep": true, + "durableAbortCount": 1, + "directSpokenUserOriginPresentAfterHydration": false, + "browserObservedSettledRevision": 2, + "browserObservedTargetArc": "present", + "browserObservedAutoplayOnHydration": false, + "browserObservedResumedWorkAfterAbort": false, + "browserObservedStoppedLabel": true, + "browserObservedPlaybackControlsPassed": true + } +} diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/failed-cumulative-results.png b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/failed-cumulative-results.png new file mode 100644 index 00000000000..fc935e7a845 Binary files /dev/null and b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/failed-cumulative-results.png differ diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/passing-negative-control.png b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/passing-negative-control.png new file mode 100644 index 00000000000..a3150ba3975 Binary files /dev/null and b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/passing-negative-control.png differ diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/witness.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/witness.md new file mode 100644 index 00000000000..a753b931f24 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/owner-witness-2026-09-07/witness.md @@ -0,0 +1,74 @@ +# Mission 6b owner witness — 2026-09-07 + +## Verdict + +Lu Nelson accepted Mission 6b on 2026-09-07 with the explicit claim and limitations below. The accepted local product path is: completed Voice transcript → canonical Brunch conversation → explicit half-duplex handoff → one browser mutation and verification → coherent revision 2 → Tab-B continuation → durable active-submission Stop → Tab-C stopped-entry recovery without autoplay or resumed work. + +This witness does not establish comparative latency, direct spoken-user Voice attribution after hydration, or durable recovery of browser work withheld locally after its Flue tool-call step has already settled. Those claims were explicitly deferred rather than passed. + +## Environment and pins + +- Branch: `ln/fe-1580-reconcile-voice-resumable-workpiece`. +- Tested implementation head after the two witness repairs: `48e2b66666df05034777f9410024c2e1228c86be`. +- Causal client-result repair: `1e238f498e`. +- Explicit-evidence fixture repair: `48e2b66666`. +- Local entrypoint: `yarn dev:brunch`. +- Browser fixture: `http://127.0.0.1:4915/?brunch-fixture=crew-reservation-v1` in a fresh private browsing context after origin storage was cleared. +- Canonical conversation: `conv_01M1Y4SPKHEKPVFVAMQG9QNH4Y`. +- Model reported by the canonical stream: `claude-haiku-4-5`. +- Secrets, authorization headers, SDP, audio, provider payloads, and browser principal are not retained. + +## Discovery run and repairs + +The first attempted spoken confirmation was transcribed canonically as only `SDCPN`. The pre-repair agent nevertheless inferred the intended fixture correction from revision zero, issued two browser reads across separate model steps, applied `addArc`, and then received a cumulative client-result signal containing the new mutation result plus both stale reads. Tool results were sorted by call id rather than causal order, so the continuation misread the old definitions as post-mutation verification and reported an anomaly. The browser showed the arc while the coherent-bundle guard correctly refused settlement and retained revision zero. + +Database inspection established that the duplicate-looking reads were not duplicate execution of one canonical tool call: they were distinct calls from successive assistant steps, accumulated into later AI SDK message state. `completedClientToolResults` scanned the entire folded assistant message on every automatic continuation. The first repair added a red public-seam test at `createFlueChatTransport().sendMessages()` and changed collection to the most recent assistant step containing client-tool output. A second real attempt exposed a mixed server/browser batch: `activate_skill` continued on the server while `getLatestNetDefinition` completed in the browser, leaving a later server-only step after the pending browser result. The regression was extended to that exact topology before the collector was corrected. The passing negative and positive runs each admitted one client result per signal, with no stale cumulative results. + +Revision zero also contradicted the model-facing fixture instruction: it said to wait for confirmed true-user evidence while the prepared workpiece already stated crew reservation as fact and described the missing arc as an approved correction. A red fixture test preceded the repair. Revision zero now labels crew reservation as an unconfirmed hypothesis, and the plugin instruction says fragments, topic labels, inspect/explain requests, and unrelated messages cannot authorize mutation. + +The failed databases remain outside the repository at `/tmp/brunch-agent-failed-witness-20260907T134055Z` and `/tmp/brunch-agent-negative-control-error-20260907T143604Z` for the life of this machine session. They are diagnostic inputs, not accepted evidence. + +## Accepted run + +1. The prepared fixture settled at revision zero with the target arc absent and its Markdown workpiece available. +2. Voice connected and the microphone check responded. +3. Lu supplied the negative control `SDCPN`. Canonical submission `sub_ik_9942648a081a4a1d257dc20b41e0d9ea` completed. Brunch inspected the net once, did not call `addArc`, kept revision zero settled, and asked for explicit confirmation. The response was much more verbose than necessary; this is interaction strain, not a correctness failure. +4. Lu used **Your turn**, waited for fresh listening, and said: `Starting final inspection reserves the single dispatch crew immediately. Sign-off releases it. The timing, failure, and recovery behavior are still unknown.` Canonical submission `sub_ik_b15d9c3a910649fef2d5a559db61a6ee` completed. +5. The assistant emitted one model-produced workpiece before construction, then issued `addArc` call `toolu_01XoXQED2JUy5Xk6MiH3axDs`. Result submission `sub_ik_f7f80b5cd8b6e3360d6a8d1bfca86f86` contained only that result and reported `applied: true`. +6. A later, separate `getLatestNetDefinition` call `toolu_01MaeDyPhW4iuULiJvWkpKeE` verified the changed document. Its result submission contained only that read. The assistant emitted the final full workpiece in another message, so revision 2 correctly represents distinct pre-mutation and post-verification model-produced workpieces rather than two user turns. +7. The browser showed exactly one standard weight-1 input arc from `Dispatch crew available` to `Start final inspection`, a settled revision-2 bundle, one visible canonical reply, and one audible rendering. +8. Tab B reopened revision 2 with the target arc, conversation, and workpiece intact. No audio autoplayed and no work or mutation duplicated. The typed follow-up `What remains unresolved in this workpiece? Do not mutate the net.` completed without another `addArc`. +9. Lu started another Voice turn asking for a detailed account of unresolved timing, failure, and recovery, exited Voice mode, and pressed durable Stop while the response was active. Submission `sub_ik_336c2a985788889d677697626db20ba1` has `abort_requested_at` and canonical outcome `aborted` with `submission_aborted` error. +10. Tab C retained the streamed partial prose as formatted headings/list items with a message-level **Response stopped** label. The final phrase remained honestly truncated. Revision 2 and the target arc stayed coherent; no audio autoplayed and no tool work resumed. +11. **Read full response**, **Repeat question**, stopped-response gating, and compact/expanded Voice controls behaved as specified. Two additional non-mutating test turns used for those controls remain visible in `canonical-summary.json`. + +## Owner dispositions + +- **Direct spoken-user Voice attribution after hydration — deferred truthfully.** Live Voice chips were visible, but both disappeared after Tab-B snapshot hydration. Flue/AI SDK 2.0.3 does not retain caller Voice metadata. The accepted claim is that spoken text is canonical and durable and client-tool Voice origins survive; direct spoken-user origin is not shown after reopen until the upstream SDK exposes durable caller metadata. No local sidecar, text encoding, or Flue patch is authorized. +- **Post-settlement local withholding — deferred with a narrowed Stop claim.** Durable Stop is accepted for active Flue submissions, as witnessed. If Flue has already settled a tool-call step, browser work withheld locally in the current process has no canonical withholding record and may reappear as pending after reopen. Already-applied mutations are not rolled back. Re-enter when the platform provides a durable canonical withholding/cancellation operation or a product consumer requires this race to close. +- **Comparative audible latency — deferred with no latency claim.** The required 10 donor + 10 candidate campaign did not run. Re-enter if latency becomes a release criterion, measured complaint, or performance regression investigation. +- **Interaction strain — accepted, not erased.** The negative-control response recited excessive net detail before asking the necessary question. Durable Stop was poorly discoverable while Voice was active: Lu had to exit Voice mode before using the streaming Stop action. These are future UX inputs, not evidence that the accepted control path failed. +- **Evidence bundle limitation — accepted explicitly.** The run retains canonical submissions, settlements, tool ids, workpiece-message ids, owner observations, and two screenshots. It does not retain the pre-registered full `voice-events.jsonl`, browser network-route export, raw canonical snapshot, or audible latency samples. No missing artifact is inferred or manufactured. + +## Artifacts + +- [`canonical-summary.json`](canonical-summary.json) — sanitized SQLite-derived submissions, tool calls, workpiece sources, and owner-observed browser assertions. +- [`failed-cumulative-results.png`](failed-cumulative-results.png) — first-run UI showing the out-of-order reasoning/tool chain and eventual incoherent state before repair. +- [`passing-negative-control.png`](passing-negative-control.png) — repaired negative control showing one net read, no mutation, and an explicit confirmation question. + +## Automated verification + +The red/green transport command was `yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit chat-transport.test.ts`. Before repair it dispatched `mutation-latest,read-before-1,read-before-2`; the mixed-batch refinement then reproduced `The client-tool follow-up has no completed result.` After repair it passes 19 tests. + +Final focused checks observed during the witness: + +- `yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` — 42 passed. +- `yarn workspace @hashintel/brunch-agent-transport-aisdk lint:tsc` — passed. +- `yarn workspace @hashintel/brunch-agent-transport-aisdk lint:eslint` — no errors; two pre-existing sequential retry-test warnings. +- `yarn workspace @hashintel/brunch-agent-plugin-sdcpn test:unit` — 11 passed. +- `yarn workspace @hashintel/brunch-agent-plugin-sdcpn lint:tsc` and `lint:eslint` — passed without warnings. +- Focused prepared fixture and settlement tests — 14 passed. +- `@hashintel/petrinaut` `ai-assistant-panel.test.tsx` — 56 passed; existing React Compiler warnings only. +- Focused website transport, Voice preview, browser-tool integration, and local-storage app tests — 32 passed. + +These focused checks and the owner witness establish the accepted local claim. They do not replace the repository-wide final check or create a remote deployment claim. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/verification.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/verification.md new file mode 100644 index 00000000000..d1d033a268c --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/voice-resumable-reconciliation/verification.md @@ -0,0 +1,83 @@ +# Mission 6b — Local reconciliation verification + +## State and pins + +Local implementation candidate, **not mission acceptance or a real microphone/browser witness**. Recorded on 2026-09-07. Implementation approval did not waive provenance, human acceptance, latency or newly observed cancellation/reopen limits. No paid calls, Linear writes, PR submission or changes to KA's original branch/PR were made. + +- Candidate code: `c649eec3ba5d31b27294f6a870a0a5d676b79fd8`. +- Authority: `ecee802ce3`; source import: `66ac62693f`; import test joins/provenance: `221f3e53a0`. +- Mission 6 parent: `01649899eb65ab8d7a8fec9407dc3ea613128264`; Mission 5: `7538264feeb1487aa494e991831bed0338ae76df`. +- KA source contribution: `58f75840804766a84ce85b9daab5b5194f3875ec..be56a18ff0244c5750a8702e9c7f45c0b607dc06`, still unchanged at inspection. See [import provenance](import.md). +- Mission 7 remains at `86e37556e363c06bdd5700b67ba58991363ba5a3` when this record is first written; it has not yet moved above this candidate. + +## Demonstrated repairs + +The repaired parent already holds ordinary SDK automatic follow-up busy. The additional code addresses observed defects at deferred browser execution and canonical history, not a replacement scheduler. + +| Discriminator | Before repair | Candidate evidence | +| --- | --- | --- | +| Reordered cumulative client-tool results | Same idempotency key, different serialized payload order | `transport-aisdk/test/chat-transport.test.ts`: identical payload bytes and key after reordering | +| Folded continuation origins | Voice call IDs on a continuation disappeared when it folded into the root message | `transport-aisdk/test/transcript.test.ts`: surviving origins are merged | +| Reopened aborted entry followed by a completed reply | No per-message stopped metadata | Canonical settlements identify only the aborted entry; transcript and contents tests retain its label without a global Stop banner | +| Stop before deferred browser execution | Follow-up withheld, but the deferred mutation still ran | Panel regression asserts the not-yet-started mutation does not run | +| Textless automatic-tool failure | Hosts saw `ready` and Voice could remain owned | Panel records matching `output-error`, retains visible detail, exposes terminal error; combined test releases Voice through the error path | +| Durable aborted history with pending tool input | Reopening executed the stopped tool | Panel skips runnable parts of canonically stopped messages | +| StrictMode setup/cleanup/setup | Cleanup cancelled the execution timer but retained its claim; recovered work stayed busy | Pending timer claims are released on cleanup; initially hydrated StrictMode test executes exactly one continuation | +| Stop in conversation A, then switch to B | A's terminated generation suppressed B's recovered work | Conversation changes invalidate the generation and reset local turn presentation; B executes once | +| Async command from A completes after switch to B | A's output scheduled an unsolicited continuation in populated B | Generation and conversation ownership are checked before insertion and continuation; deferred-layout test observes no send to B | +| Preamble commits after cancellation finishes | A later ready render spoke the previously withheld prose | Bridge retires stopped segments; combined preamble/Stop test observes no speech after a repeated update | + +A read-only independent review identified the last four discriminators (three findings, with two conversation-identity cases). They were reproduced before repair and pass afterward. An initial conversation test fixture incorrectly returned an endless empty finish and used the wrong composer-control prop; it was corrected before adjudicating the identity cases. The resulting red tests, not that harness failure, support the findings. + +## Combined production-component test + +`apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx` mounts the published `Petrinaut` component, its actual panel and `useChat`, production `createBrunchPanelTransport`, admission tracker, `submitVoiceInputWithAdmission`, canonical speech selection and `RealtimeBrunchBridge`. + +Five cases cover browser continuation with and without preamble, textless invalid browser input, and local withholding with and without preamble. They assert busy ownership while the continuation is held, original tool-call identity in the delivered signal, exact canonical speech after continuation, visible terminal failure, and no speech or continuation after local Stop. The preamble Stop case repeats the final update to detect speech resurrection. + +Flue send/wait events, media input/output and cancellation acknowledgement are controlled by the test. The browser tool reads documentation; it does not prove the real fixture mutation, microphone/VAD timing, audible cancellation, network route or fresh-tab persistence. The panel mutation test separately checks Stop-before-execution. These distinctions prevent a component integration pass from being presented as the required product witness. + +## Verification run + +At the candidate code state, all **39/39** tasks passed, with **0 cached** tasks: + +```bash +yarn exec turbo run build test:unit lint:tsc lint:eslint --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk --filter @apps/brunch-agent --filter @hashintel/petrinaut --filter @apps/petrinaut-website --force --continue=always --output-logs errors-only +``` + +The scoped unit suites passed **1,317 tests in 166 files**: + +| Package | Files | Tests | +| --- | ---: | ---: | +| Brunch core | 11 | 93 | +| Flue binding | 5 | 18 | +| SDCPN plugin | 2 | 11 | +| AI SDK transport | 4 | 41 | +| Brunch application | 19 | 109 | +| Petrinaut | 84 | 687 | +| Petrinaut website | 41 | 358 | + +Additional checks: `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` passed (70 layers, 356 edges, 736 files, 71 generated pages, 38 authored pages); changed TypeScript formatting, the three changed publishable/user Markdown files and `git diff --check` passed. Commit hooks passed formatting and Markdown lint. Existing non-blocking React Compiler and Node configuration warnings are not repaired here. Brunch Markdown is explicitly excluded from the repository formatter and Markdown lint, so those tools are not claimed as checks of this record. Full Local CI/GitHub CI and live screenshot/audio evidence were not run; no push occurred. + +## Acceptance disposition — accepted with explicit limitations on 2026-09-07 + +Lu Nelson accepted the narrowed Mission 6b claim after the real owner witness in [`owner-witness-2026-09-07/witness.md`](owner-witness-2026-09-07/witness.md). That witness exposed and repaired cumulative cross-step client results and the fixture's non-causal prepared answer, then passed the negative control, explicit spoken mutation, Your turn, coherent revision-2 settlement, Tab-B continuation, active-submission durable Stop, Tab-C stopped-entry recovery, and playback controls. The full pre-registered telemetry bundle was not retained; the owner accepted that evidence limitation explicitly. + +| Obligation | Disposition | +| --- | --- | +| Source preservation, scoped catalogue, canonical normalization, deterministic admission and inherited automated contracts | Imported with provenance; scoped suites pass | +| Deferred execution, termination and history joins above | Discriminated and repaired locally; the owner witness additionally proved one result per causal step after the cross-step accumulation repair | +| Exact Stop timing during held output insertion and insertion rejection through the full combined host | Not separately demonstrated by the owner witness; source/component cases remain bounded automated evidence rather than a claim that every race was witnessed | +| Real fixture spoken mutation, Your turn, Stop, coherent bundle, Tab-B continuation and compact/expanded inspection | Passed by the owner witness, including one causal mutation, no duplicate/autoplay, canonical active-submission abort and stopped-entry recovery | +| Direct spoken-user Voice chip after snapshot-only reopen | Observed missing and explicitly deferred by Lu; canonical text survives, but no direct-user Voice-origin claim is made after hydration | +| Reload-safe cancellation of a locally withheld tool continuation | Explicitly deferred with the narrowed Stop claim below; no invented durable marker | +| Comparative audible latency | Explicitly deferred; Mission 6b makes no comparative latency or no-regression claim | +| Human acceptance and original PR retirement | Narrowed mission claim accepted by Lu; KA's PR remains untouched and requires separate retirement authorization | + +### Local withholding is not a durable stopped record + +When a Flue tool-call step has already completed, the parent Stop adapter can return `already-settled`. The panel can withhold pending browser execution and its follow-up, and Voice can release that logical turn without speaking its late prose. The bridge labels this outcome `withheld`, not a fabricated Flue abortion. Canonical history still records the original step as completed with pending tool input. + +A fresh process cannot infer the local withholding from that snapshot. Pending completed-step tools remain recoverable work, whereas genuinely aborted submissions now project `metadata.stopped` and are not executed. The successful aborted-entry tests do **not** solve this local-withholding/reopen case. User docs explicitly warn that reopening can recover the locally withheld tool as pending work. + +Resolving that distinction durably requires a supported recording/termination boundary. Do not add a browser sidecar, forge aborted settlements, admit an extra hidden turn, or silently disable ordinary pending-tool recovery. Lu accepted the narrower behavior on 2026-09-07: Stop is durable while the Flue submission is active; after a tool-call step settles, locally withheld browser work may reappear as pending after reopen, and already-applied mutations are not rolled back. Re-enter when the platform supplies a durable canonical withholding/cancellation operation or a product consumer makes this race load-bearing. diff --git a/libs/@hashintel/brunch-agent/docs/mission-archive/6-resumable-workpiece-petrinaut.md b/libs/@hashintel/brunch-agent/docs/mission-archive/6-resumable-workpiece-petrinaut.md new file mode 100644 index 00000000000..18f9c2b79b0 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/mission-archive/6-resumable-workpiece-petrinaut.md @@ -0,0 +1,162 @@ +# Mission 6 — resume one prepared workpiece and Petrinaut document + +## Status + +**Closed on `ln/fe-1575-resumable-workpiece-petrinaut` by owner decision on 2026-09-04.** [FE-1575](https://linear.app/hash/issue/FE-1575/resume-one-brunch-workpiece-and-petrinaut-document-across-tabs) delivered the implementation, outer mechanical witness, cold-reader adjudication, and the product manager's fresh two-tab conversation/workpiece/document demo; see the [retained implementation and witness evidence](../evidence/implementations/fe-1575-resumable-workpiece-petrinaut.md). The owner explicitly waived re-running the Voice-origin and aborted-assistant presentation clauses in the fresh product-manager conversation and closed the mission anyway: those records were absent from that run, their behavior is mechanically covered by the outer witness, and their future scenario obligations are carried in [`MISSION.next.md`](../../MISSION.next.md#voice-after-the-live-transport-cut). This is a closure exception, not evidence that the skipped human checks passed. Earlier on 2026-09-04 the owner amended only the Deferred section, to point at the recut future planning record and carry two admissions from this mission's evidence; the imperative, throughline, proof, constraints, fog-line, and stop conditions otherwise remain the historical execution contract. + +## Imperative + +Determine whether one canonical Brunch conversation can maintain a useful Markdown workpiece and drive a meaningful change to a real Petrinaut document through the browser without reviving a comprehensive typed domain IR. + +Mission 3 separately showed a recoverable Markdown workpiece and hermetic canonical Petrinaut callbacks, but its paid model could not carry a nested construction schema and no product path joined the two results. Mission 4 accepted the independent core `elicitation` capability and SDCPN job-skill composition but produced no full-run candidate. This mission must retire the join and resume uncertainty honestly with one deliberately prepared fixture rather than treating either historical result as an integrated product. + +### Visible product advance + +**Release note:** Brunch edits the Petrinaut net you are looking at from the conversation, and your work survives closing the tab. + +**Demo script (no engineer present):** with the local Brunch/Petrinaut development stack running, open the stable demo fixture selector for the labelled prepared crew-reservation case. Its canonical Brunch conversation, current Markdown workpiece, and non-empty Petrinaut net come back together. The fixture visibly states that it is test-authored and prepared, and what it does not claim. Tell Brunch the one new realistic fact the fixture is prepared for: final inspection uses the single dispatch crew, and sign-off releases it. Watch the workpiece update and a new arc appear in the live net from `Dispatch crew available` to `Start final inspection`. Wait until the fixture reports that the conversation, workpiece, and automatically mirrored document are settled. Open the same fixture in a second tab, confirm that messages retain their typed/Voice provenance and that an aborted assistant entry still presents as stopped rather than as an ordinary truncated answer, submit one follow-up message, and receive Brunch's response in the same conversation without duplicate submission or identity drift. + +**Previously impossible:** Brunch only produced off-canvas net JSON for manual load; nothing it did touched the live document or survived a reload. + +**Deployment posture:** the demo runs against the locally run Petrinaut website and Brunch agent (`yarn dev:brunch`). Mission 8 stopped before remote deployment, so no product-manager-noticeable claim here depends on remote infrastructure; remote durability stays with Mission 8. + +**Completion:** the mission is done when a product manager can run that demo script end to end for this fixture and every readiness-gate obligation in [Proof](#proof) is closed. The first green pass through the two-tab path is an internal milestone inside the mission, not its completion. + +## Throughline + +### Observed departure point and first unproved boundary + +The production browser already has most local pieces: + +- `apps/petrinaut-website`'s local-storage demo owns an editable `PetrinautDocHandle`, automatically writes handle changes to `petrinaut-sdcpn`, and maps each net to a persistent Brunch conversation id; +- Petrinaut's stock AI panel already validates and executes canonical read, mutation, and command tools against the active browser document and returns the original tool-call id; +- live Mission 5 is replacing the server-side `GET`/`POST /api/chat` adapter with one browser `FlueClient` plus host-supplied AI SDK `ChatTransport` over the mounted `/agents/chat/:instanceId` route; that route carries typed turns, `history()` hydration, and correlated client-tool-result signals, and Mission 6 must consume rather than duplicate or reverse that transport; and +- the SDCPN skill already emits a full recoverable `runbook-ir` block and requires construction to consume that workpiece rather than transcript archaeology. + +The first unproved boundary is ordinary Brunch conversation over Mission 5's browser Flue transport → mounted canonical document read / least mutation → browser execution → correlated Flue continuation. Today ordinary SDCPN conversations mount only the Petrinaut documentation reader as a browser tool; the validated construction subset is headless-only, and no stable fixture selector or coherent cross-tab witness joins the conversation, workpiece, and document lifecycles. Mission 6 may develop independent fixture/workpiece/document logic while Mission 5 proceeds, but it must consume Mission 5's landed browser transport before integrated or outer proof and must not add another conversation route in the interim. + +### Accepted fixture and boundary crossings + +Prepare the existing final-inspection / dispatch-crew case as an explicitly test-authored fixture. Its starting workpiece and non-empty net preserve this narrow account: one crew is reserved during final inspection, sign-off releases it, the batch then becomes dispatch-ready, and timing plus failure/recovery remain unresolved. Its prepared material must identify its authorship and must not be presented as a Mission 4 candidate or model-produced evidence. + +Deliver the prepared starting workpiece exactly once through Flue's public dispatch surface as a tagged structured signal. Its canonical record must remain `role: system`, `purpose: dispatch`, carry the fixture id, `test-authored` authorship, and non-claims as signal attributes, and preserve the exact Markdown body. This record is prepared revision zero. Later full `runbook-ir` blocks emitted in genuine assistant responses are model-produced revisions; the workpiece resolver selects the latest eligible revision without rewriting Flue's append-only history. This is analogous to last-one-wins selection of extension-contributed artifacts in a Pi raw session log, not permission to overwrite either log. + +The disposable production-route probe established the carrier facts: the tagged signal retained its exact body and attributes, an exact idempotent retry converged on the original submission without adding messages, and the snapshot survived process reopen unchanged. The current `ChatAgent` rejected fixture authorship supplied as `initialData` with `400` and created no history; `initialData` is not a substitute for the public workpiece source. A user delivery would impersonate the person, while faux-provider output, hand-authored assistant records, private canonical record types, direct database writes, and a second history store are not preparation routes. The probe's configured Anthropic credential was rejected with `401`, so its separately classified faux assistant response established no model-behavior claim. + +Prepare the net with `Batch ready`, `Under final inspection`, `Ready for dispatch`, and `Dispatch crew available` places plus `Start final inspection` and `Sign-off` transitions. Preserve the batch-flow arcs and the return of the crew from sign-off, but deliberately omit the standard input arc that reserves the sole crew when final inspection starts. Use one realistic confirming answer: final inspection consumes the sole available dispatch crew, sign-off returns it, and timing plus recovery remain unknown. The least candidate mutation is one canonical weight-1 standard input arc from `Dispatch crew available` to `Start final inspection`. The exact before/after edge makes the semantic oracle discriminating while avoiding Mission 3's deeply nested schema failure. If that shallow mutation still cannot cross Flue faithfully, stop with the carrier blocker rather than weakening the claim. + +```text +stable prepared-fixture selector +→ resolve distinct fixture, Petrinaut document, and Flue conversation identities +→ open the prepared non-empty browser document and use the browser Flue client to idempotently deliver or recover the tagged revision-zero signal +→ hydrate canonical Flue history through `history()` on the mounted route +→ recover prepared revision zero from the tagged dispatch record, or the latest eligible assistant revision, by source message id plus content hash +→ submit the realistic crew-reservation confirmation through Mission 5's production browser Flue transport +→ Brunch emits an inspectable full workpiece revision without erasing prior meaning or the remaining unknown +→ SDCPN construction reads that current workpiece and the live browser document +→ Brunch requests the least canonical meaningful mutation +→ Petrinaut validates and executes it against the bound document +→ the original tool-call id and result return as one correlated signal through the same browser Flue transport and resume the same conversation +→ inspect the canonical non-empty document and advance the runtime settled manifest only after conversation/workpiece/document state is observable +→ a second tab opens the same fixture selector, resolves the same identities and settled hashes, submits one follow-up, and receives Brunch's response +``` + +The settled fixture manifest is runtime local product state, not merely retained evaluation evidence: the stable selector uses it to choose the latest coherent observed bundle of distinct fixture, Flue conversation, workpiece source/hash, and Petrinaut document/hash or revision identities across reopen. It is a small viability pointer, not a new event log, independent workpiece store, or distributed transaction. A failed history load, workpiece recovery, rejected/no-op mutation, or missing result correlation must leave the prior settled bundle selected while partial state and failure remain visible for diagnosis. Retained witness artifacts copy and inspect this runtime state but do not select the product bundle. The existing automatic localStorage mirror is the only document-save mechanism unless a real failure proves it insufficient; this mission adds no explicit Save affordance. + +### Expected touched paths + +This manifest is provisional and may shrink or move when the first real probe exposes the deeper existing boundary: + +```text +libs/@hashintel/brunch-agent/ +├── MISSION.md ~ live authority and eventual close evidence +├── MISSION.next.md ~ future joins and carried flags only +├── packages/plugin-sdcpn/ ~ mount only the read/mutation capability earned by this tracer +└── docs/evidence/ + prepared fixture manifest and browser witness +apps/brunch-agent/ +├── src/agents/chat-agent/ and src/conversation/ ? only if fixture-scoped mounting or workpiece recovery belongs outside the landed browser transport +└── test/ + real Flue/client-tool fixture integration +apps/petrinaut-website/ +└── src/main/app/local-storage-demo/ ~ consume Mission 5 transport; fixture selection, prepared signal, runtime settled manifest, and cross-tab continuation +libs/@hashintel/brunch-agent/packages/transport-aisdk/ ? consume the landed Mission 5 public surface; do not duplicate its implementation here +libs/@hashintel/petrinaut-core/ or libs/@hashintel/petrinaut/ ? only for an observed canonical contract or browser-host defect +``` + +## Proof + +The visible advance is the demo script in the imperative, run by a product manager against the named local posture. The evidence that backs the claim is one stable local demo URL or fixture selector plus its labelled prepared manifest, exact before/after Flue snapshots, recovered Markdown workpiece revisions, canonical Petrinaut document states, and two-tab witness; those are oracles for the builder and adjudicator, not the advance itself. Together they establish single-fixture browser-backed viability. They do **not** establish automatic full-net projection, capture-backed or selected-pair provenance, behavioral execution, broad scenario coverage, remote replacement durability, concurrent editing, Mission 3/4 quality superiority, or a promoted reusable product seed. + +### Internal milestone: first green throughline + +The first internal milestone is one pass through the throughline for the prepared fixture: a cold reader can reconstruct the spine and distinguish supplied evidence, inference, and the explicit unknown in the workpiece; one realistic turn produces an inspectable workpiece revision without erasing the unknown; Brunch reads the live document and applies the one supported arc through the real browser client-tool boundary; the canonical net is non-empty and visibly corresponds to the confirmed meaning; and a second tab observes the same settled conversation, workpiece, and document revision and continues without duplicate submission or identity drift. Reaching this milestone authorizes the readiness work below; it does not close the mission. + +### Readiness gate: completion bar + +The mission completes only when the demo script works for this fixture and these obligations are closed: stale fixture/workpiece/document revision refusal, duplicate tool delivery, read/write failure visibility, unsupported meaning, no-op mutation honesty, partial-save behavior, second-tab rehydration, separate identity integrity, and one negative mutation case. Do not close every consequential-element provenance link, remote task replacement, broad scenario coverage, or repeated automatic projection here; those become Mission 7 or Mission 9 obligations only after this tracer exposes a finite peer set and load-bearing seams. + +Every final leaf has a discriminating oracle: + +1. **The prepared fixture is honest and sufficient for this narrow test.** The committed fixture manifest, raw Flue snapshot, and a cold-reader adjudication identify the prepared workpiece's tagged system/dispatch source, exact test-authored Markdown, process spine, constrained crew, release policy, quantity context, explicit unknown, prepared net meaning, and non-claims. The same inspection distinguishes every later assistant revision as model-produced and must not require transcript archaeology. +2. **One evidence turn maintains the Markdown workpiece.** A production-agent fixture integration mechanically recovers prepared revision zero from the tagged dispatch record, then selects the latest eligible assistant `runbook-ir` block after the confirming turn, retaining each source message id and SHA-256. Before/after adjudication must find the supplied contextual quantity, retained crew/release meaning, retained unsupported context, and no invented fact or hardened unknown. +3. **The real browser executes a correlated Petrinaut read and write.** Focused plugin/transport tests prove that `getLatestNetDefinition` and the selected `addArc` schema come mechanically from Petrinaut's canonical contracts, fixture mode advertises only the selected operations, duplicate result delivery does not apply the mutation twice, rejected input remains visible, and a mutation that would change nothing is reported as a no-op rather than as a change. The browser witness must retain tool name, call id, parsed input, execution output, resumed signal, and resulting canonical definition; a headless callback alone does not pass. +4. **The document change is meaningful rather than merely accepted.** A structural comparison proves there was no standard input arc from `Dispatch crew available` to `Start final inspection` before the turn and exactly one weight-1 arc afterward, while `Sign-off` still returns the crew and the prepared net remains non-empty. The changed workpiece retains the reservation/release meaning and unresolved timing/recovery. Parser/schema acceptance or a disconnected convenience element fails. +5. **The runtime settled manifest cannot bless partial state.** A focused failure test injects history/workpiece-recovery failure, rejected `addArc`, or missing/duplicate result correlation and shows that the prior coherent runtime bundle remains selected while the failure and any partial state are inspectable. A retained evidence manifest alone does not pass this leaf, and no localStorage failure interface is invented solely to satisfy it. +6. **A second tab resumes and continues the same fixture.** With `yarn dev:brunch` running, the recorded browser protocol opens the stable selector in Tab A, performs and settles the turn, then opens it in Tab B. The witness compares fixture id, document id and canonical definition hash, Flue conversation id and history, latest workpiece source/hash, runtime settled-manifest identity, per-message typed/Voice provenance, stopped-turn presentation, and absence of duplicate submission. An aborted entry rendered as ordinary truncated content fails this check even if a global latest-settlement banner still says stopped. Tab B must then submit one follow-up message and receive its correlated Brunch response in that same conversation. A read alone does not pass. Tab B opened against a stale or mismatched revision must refuse visibly rather than silently select older artifacts. +7. **The cut has not smuggled in the later architecture.** Public-schema and dependency inspection finds only fixture identity/revision links, Markdown recovery metadata, and canonical Petrinaut payloads—no closed process ontology, typed capture-to-workpiece reducer, graph database, second conversation log, or general projection engine. + +Verification proceeds inside-out but closure requires the outer boundary: + +- **Inner:** fixture parsing and prepared-label checks; identity separation; workpiece recovery/hash; canonical `addArc` schema and exact before/after edge assertion; idempotent client-tool result handling; runtime-manifest refusal. +- **Middle:** the built production `ChatAgent` and Mission 5 browser `FlueClient`/`ChatTransport` path at `/agents/chat/:instanceId` hydrate the prepared conversation, accept the evidence turn, recover the revised workpiece, and carry actual browser-tool calls/results. No `GET` or `POST /api/chat` evidence passes. Run the focused workspaces through root Turbo (`test:unit`, `lint:tsc`, `lint:eslint`, and `build` where changed). +- **Outer:** the two-tab `yarn dev:brunch` witness above, with retained before/after artifacts, one Voice-origin message, and one durably stopped assistant turn. A content-only transcript match does not establish faithful resume. +- **Semantic:** a cold human accepts fixture/workpiece honesty and the workpiece-to-document correspondence. The oracle may falsify those claims; it may not rewrite the interaction or architecture policy. +- **Product:** a product manager who did not watch the work runs the demo script from the imperative without an engineer and notices the advance. This is the last check before close, after the readiness gate; it is not a substitute for the oracles above. + +## Constraints + +- Keep fixture id, Flue conversation id, latest workpiece source/revision, and Petrinaut document id/revision distinct and explicitly linked. One id must not impersonate all lifecycles. +- Flue history remains the canonical conversation log. Browser message caches and fixture artifacts are projections or evidence, never a second authority. +- Consume Mission 5's browser `FlueClient` plus host-supplied AI SDK `ChatTransport`; typed turns, prepared signals, history hydration, and client-tool results all cross `/agents/chat/:instanceId`. Do not keep, restore, or add another product conversation route. +- The tagged prepared signal is the only test-authored workpiece source admitted by this fixture. It remains a diagnostic system/dispatch record; latest-revision selection may supersede it with a genuine assistant workpiece but may not mutate, relabel, or hide its authorship. +- Markdown remains the semantic workpiece. Recover its full latest version; do not introduce a comprehensive typed domain IR to make fixture lookup convenient. +- Projection consumes the current workpiece. The transcript may establish provenance and help recover that artifact but may not become the primary construction IR. +- Petrinaut owns canonical schemas, browser validation, mutations, and document state. Brunch imports or mechanically derives those contracts and does not hand-copy their field shapes. +- Client tools execute against the active bound browser document and return the original tool-call id. Stale, duplicate, cross-document, malformed, failed, and no-op outcomes fail visibly. +- Advance the runtime settled manifest only after the claimed Flue snapshot, workpiece revision, and document state can all be inspected. It selects the coherent local bundle but does not make the browser and Flue stores transactional. Automatic localStorage mirroring remains the only save behavior; do not invent an explicit Save affordance or cross-store transaction machinery without an observed recovery failure requiring it. +- Preserve the accepted Mission 4 `useBrunchAgent()` plus `useSdcpnPlugin()` architecture. The app composes; the plugin owns SDCPN operation semantics; the transport carries results; the UI executes them. +- Keep construction tools unavailable to unrelated ordinary conversations unless the real path proves the smallest safe selection can be scoped to this fixture/mode. Stock-assistant behavior must remain unchanged when Brunch is absent or unselected. +- The fixture is local and deliberately prepared. Make no remote durability, capture provenance, automatic projection, behavioral execution, or concurrent collaboration claim. +- No HASH Graph, Temporal, Redis, new database, observer, workflow engine, second agent, second event log, or closed workpiece schema. +- Update the affected Petrinaut user guide in the same change if the selector, save/resume behavior, or panel behavior becomes user-facing; add one Petrinaut changeset only if a published Petrinaut package changes. +- Repair typed/Voice provenance and stopped-turn presentation at the canonical history-to-Petrinaut projection boundary; do not add a second transcript store. Account for the observed discoverability strain around **Show transcript**, **Exit voice mode**, and the chat composer's durable **Stop** without conflating local Voice exit with Flue abort. + +## Fog-line + +- Whether the selected shallow `addArc` schema survives the provider-visible Flue carrier and results in exactly one browser mutation without reopening the broader nested-schema problem. +- The least safe way to expose canonical `getLatestNetDefinition` plus `addArc` in a fixture conversation while retaining the headless-only guard for broader construction. +- Whether the latest `runbook-ir` message id and hash are sufficient workpiece revision identity or the two-tab consumer exposes a need for a separate persisted workpiece artifact. +- Whether Mantine/localStorage synchronization plus the active `PetrinautDocHandle` is sufficient for the same-browser two-tab witness, and which document hash/revision signal best distinguishes settled from stale state. +- Whether the known provider-visible nested-schema failure is absent for the selected flat mutation. Do not generalize one success to nested construction classes. +- Which of history recovery, invalid `addArc`, or duplicate result delivery is the cheapest discriminating failure for the settled-witness rule after the first real path reveals the ordering. + +Resolve these at the named production/browser boundaries. Clarifying prose alone does not clear them. If a choice changes the accepted interaction policy, architectural ownership, or proof claim, return it to the owner and amend this authority before implementation continues. + +## Stop or reorient + +Stop and surface evidence if: + +- fixture preparation requires pretending a Mission 4 candidate exists, placing prepared text in a user or assistant record, accepting an untagged preparation signal, or otherwise hiding test-authored/model-authored boundaries; +- the path conflates fixture, conversation, workpiece, and document identities or creates a second canonical conversation history; +- typed traffic, prepared signals, history, or client-tool results cross a product route other than Mission 5's mounted browser Flue route; +- the agent rereads transcript prose as its primary projection input because the current Markdown workpiece cannot carry the needed meaning; +- parser/schema acceptance, document non-emptiness, or a disconnected convenience element is offered as semantic correspondence; +- client-tool results lose their original call id, can target the wrong document, or duplicate execution on retry/reload; +- a partial or failed write advances the runtime settled manifest, second-tab reopening silently selects stale/mismatched artifacts, or Tab B proves only a read without a real continuation; +- exposing one browser mutation requires mounting an unrestricted construction surface for every ordinary conversation; +- the selected provider/Flue schema cannot faithfully carry the least meaningful mutation—record the crisp blocker rather than hand-copying Petrinaut schemas or widening into Mission 9; +- the tracer needs a closed ontology, typed claim ledger, general projection engine, distributed transaction, or new durable service before a concrete failure demonstrates that need; or +- work widens into capture-backed why/provenance, automatic projection breadth, remote deployment durability, concurrent collaboration, or broad scenario readiness. + +## Deferred + +On 2026-09-04 the future planning record was recut around provenance by lineage with declared basis; see the [2026-09-04 migration disposition](../../MISSION.next.md#2026-09-04-provenance-replanning-migration-disposition). Mission 7 now owns construction and explanation of one real net region from a genuine conversation: settled workpiece revisions as `update_workpiece` tool calls, constructor-declared basis on each mutation, verifiable transition records, the why operation with its safety and utility gates, schema-carrier repair, scenario-selected tool admission, and retirement of the orphaned `ask` and `sweep` client handling. Mission 9 owns repeatable projection breadth: unchanged repeat, changed input, retirement, concurrent change, cross-conversation document access, and the schema classes an extended region adds. Two facts from this mission carry into that record and its close report: the prepared fixture's "Current Petrinaut correspondence" section was fixture-authored rather than produced by any skill directive, so this fixture is a viability proof and is not promoted into the provenance pair; and the fenced `runbook-ir` block plus message-id-and-hash selection is a Mission 6 contract that Mission 7 replaces for model-produced revisions, keeping the tagged prepared signal for test-authored material only. This mission's constraint that construction tools stay out of ordinary conversations is amended by the Mission 7 cut, not here. Remote replacement durability and release infrastructure remain in the historical Mission 8 handoff, and a Mission 8 successor must be scheduled before any remote claim. Multi-tab concurrent editing, a durable cross-store commit protocol, explicit localStorage failure injection and refusal of a concurrent write from a tab holding an older revision (distinct from the readiness-gate refusal to reopen onto stale or mismatched artifacts), and promotion of this prepared fixture into a reusable product seed re-enter only if the automatic mirror loses or overwrites state, a later consumer requires atomic bundle identity, or this mission otherwise exposes concrete strain; their current planning home and re-entry conditions remain in [`MISSION.next.md`](../../MISSION.next.md) and the linked Mission 7/9 drafts. diff --git a/libs/@hashintel/brunch-agent/packages/core/package.json b/libs/@hashintel/brunch-agent/packages/core/package.json index 1b2634eb375..70593a8eb94 100644 --- a/libs/@hashintel/brunch-agent/packages/core/package.json +++ b/libs/@hashintel/brunch-agent/packages/core/package.json @@ -18,6 +18,10 @@ "types": "./src/flue.ts", "import": "./dist/flue.js" }, + "./question-marker": { + "types": "./src/question-marker.ts", + "import": "./dist/question-marker.js" + }, "./storage": { "types": "./src/storage.ts", "import": "./dist/storage.js" diff --git a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts index cb020aaf9e0..9d57e9ea9e1 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts @@ -1,6 +1,20 @@ -import { useModel, useSkill } from "@flue/runtime"; +import { + defineTool, + useDataWriter, + useModel, + useSkill, + useTool, +} from "@flue/runtime"; +import * as v from "valibot"; import systemPrompt from "./prompts/SYSTEM.md?raw"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + type BrunchQuestionData, +} from "./question-marker"; import { ELICITATION_SKILL_NAME, elicitationSkill, @@ -10,14 +24,32 @@ import { skillFromMarkdown } from "./skills/skill-markdown"; /** * Mount the contributions owned by Brunch core and return its system prompt. * - * Core contributes the always-on universal prompt and one `elicitation` - * capability skill. It owns no model-facing tool; add one here only when it - * applies independently of the selected modelling formalism and host. + * Core contributes the always-on universal prompt, one `elicitation` + * capability skill, and the formalism-independent question marker. */ export function useBrunchAgent(model: string): string { useModel(model); useSkill(elicitationSkill); + const writeQuestion = useDataWriter(BRUNCH_QUESTION_DATA_NAME, { + schema: BrunchQuestionDataSchema, + }); + useTool(createBrunchQuestionMarkerTool(writeQuestion)); return systemPrompt.replace(/^\s+|\s+$/gu, ""); } +export const createBrunchQuestionMarkerTool = ( + writeQuestion: (question: BrunchQuestionData) => void, +) => + defineTool({ + name: BRUNCH_QUESTION_TOOL_NAME, + description: + "Mark the exact text of a direct question for accessible replay. Call this immediately before including that exact question in ordinary assistant prose. This marker does not ask or answer the question itself.", + input: BrunchQuestionInputSchema, + output: v.object({ marked: v.literal(true) }), + run({ data, toolCallId }) { + writeQuestion({ question: data.question, toolCallId }); + return { output: { marked: true as const } }; + }, + }); + export { ELICITATION_SKILL_NAME, elicitationSkill, skillFromMarkdown }; diff --git a/libs/@hashintel/brunch-agent/packages/core/src/index.ts b/libs/@hashintel/brunch-agent/packages/core/src/index.ts index f4021f55ef0..25fd3fdc75c 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/index.ts @@ -41,6 +41,14 @@ export { toolPrefix, type Operation, } from "./conversation/naming"; +export { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + parseBrunchQuestionData, + type BrunchQuestionData, +} from "./question-marker"; export { type HarnessReplyEvent, type ReplyPartKind, diff --git a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md index 6c8049566df..99a6c65a011 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md +++ b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md @@ -10,6 +10,8 @@ Establish what the result must help the person decide, answer, compare, explain, Use the person's vocabulary and follow concrete cases rather than traversing a schema, template, or target representation. Do not open with a battery of independent questions; deepen one answerable thread at a time and group questions only when they share one frame. +Before asking the person a direct question, call `brunch_mark_question` with the exact question text. Then include the exact same question text in ordinary assistant prose. The marker only makes that text available for accessible replay; it does not wait for or accept the answer, so continue the same response normally after calling it. Do not mark headings, rhetorical questions, or prose that you will not present verbatim. + ## Authorship and uncertainty Keep what the person said distinct from your normalization, inference, assumption, proposal, transformation, or default. Do not invent content, silently increase precision, or treat assent to wording you supplied as independent evidence. When accounts differ, establish whether the relationship is correction, conflict, or contextual coexistence before reconciling them. diff --git a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts new file mode 100644 index 00000000000..ba6194c63f5 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts @@ -0,0 +1,28 @@ +import * as v from "valibot"; + +export const BRUNCH_QUESTION_TOOL_NAME = "brunch_mark_question"; +export const BRUNCH_QUESTION_DATA_NAME = "brunch-question"; + +const NonBlankStringSchema = v.pipe( + v.string(), + v.check((value) => /\S/u.test(value), "Expected a non-blank string."), +); + +export const BrunchQuestionInputSchema = v.object({ + question: NonBlankStringSchema, +}); + +export const BrunchQuestionDataSchema = v.object({ + question: NonBlankStringSchema, + toolCallId: NonBlankStringSchema, +}); + +export type BrunchQuestionData = v.InferOutput; + +export const parseBrunchQuestionData = ( + value: unknown, +): BrunchQuestionData | undefined => { + const result = v.safeParse(BrunchQuestionDataSchema, value); + + return result.success ? result.output : undefined; +}; diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts new file mode 100644 index 00000000000..b1da49f3049 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -0,0 +1,93 @@ +import { readFile } from "node:fs/promises"; + +import * as v from "valibot"; +import { describe, expect, test, vi } from "vitest"; + +import { createBrunchQuestionMarkerTool } from "../src/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + parseBrunchQuestionData, + type BrunchQuestionData, +} from "../src/question-marker"; + +import type { FlueLogger } from "@flue/runtime"; + +describe("the Brunch question marker", () => { + test("defines one non-interactive tool and data-part identity", () => { + expect(BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); + expect(BRUNCH_QUESTION_DATA_NAME).toBe("brunch-question"); + }); + + test("preserves exact non-blank question text and tool-call identity", () => { + const question = " Which line should run this order? "; + + expect( + v.parse(BrunchQuestionInputSchema, { + question, + }), + ).toEqual({ question }); + expect( + v.parse(BrunchQuestionDataSchema, { + question, + toolCallId: "tool-question-1", + }), + ).toEqual({ question, toolCallId: "tool-question-1" }); + }); + + test("writes the exact marker without terminating or waiting for an answer", async () => { + const writeQuestion = vi.fn<(question: BrunchQuestionData) => void>(); + const tool = createBrunchQuestionMarkerTool(writeQuestion); + + const result = await tool.run({ + data: { question: "Which line should run this order?" }, + log: { + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, + toolCallId: "tool-question-1", + }); + + expect(writeQuestion).toHaveBeenCalledOnce(); + expect(writeQuestion).toHaveBeenCalledWith({ + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }); + expect(result).toEqual({ output: { marked: true } }); + }); + + test.each([ + { question: "" }, + { question: " " }, + { question: "What matters?", toolCallId: "" }, + { question: "What matters?", toolCallId: " " }, + ])("rejects an incomplete marker: %j", (marker) => { + expect(v.safeParse(BrunchQuestionDataSchema, marker).success).toBe(false); + expect(parseBrunchQuestionData(marker)).toBeUndefined(); + }); + + test("parses exact question data at the client projection boundary", () => { + const marker = { + question: " Which line should run this order? ", + toolCallId: "tool-question-1", + }; + + expect(parseBrunchQuestionData(marker)).toEqual(marker); + expect(parseBrunchQuestionData(null)).toBeUndefined(); + }); + + test("instructs the model to mark and then reproduce the exact question in ordinary prose", async () => { + const systemPrompt = await readFile( + new URL("../src/prompts/SYSTEM.md", import.meta.url), + "utf8", + ); + + expect(systemPrompt).toContain("brunch_mark_question"); + expect(systemPrompt).toContain("exact same question text"); + expect(systemPrompt).toContain("ordinary assistant prose"); + expect(systemPrompt).toContain("does not wait for or accept the answer"); + }); +}); diff --git a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts index 986fb63fdc4..d3583e3c839 100644 --- a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts @@ -13,6 +13,9 @@ export default defineConfig({ ), flue: fileURLToPath(new URL("src/flue.ts", import.meta.url)), index: fileURLToPath(new URL("src/index.ts", import.meta.url)), + "question-marker": fileURLToPath( + new URL("src/question-marker.ts", import.meta.url), + ), storage: fileURLToPath(new URL("src/storage.ts", import.meta.url)), workpiece: fileURLToPath(new URL("src/workpiece.ts", import.meta.url)), }, diff --git a/libs/@hashintel/brunch-agent/packages/plugin-sdcpn/src/flue.ts b/libs/@hashintel/brunch-agent/packages/plugin-sdcpn/src/flue.ts index 455f64ef893..b0ff2b8e948 100644 --- a/libs/@hashintel/brunch-agent/packages/plugin-sdcpn/src/flue.ts +++ b/libs/@hashintel/brunch-agent/packages/plugin-sdcpn/src/flue.ts @@ -64,7 +64,7 @@ This is a construct-only headless conversation. Use only the supplied runbook IR delivery.type === preparedWorkpieceSignalType; useInstruction( ` -This is a visibly labelled prepared-fixture conversation. Treat its tagged prepared runbook-ir dispatch as test-authored revision zero, maintain the full Markdown workpiece in later responses, preserve explicit unknowns, and do not relabel prepared material as model-produced. The prepared dispatch only initializes the fixture: acknowledge it without emitting a workpiece or beginning construction, then wait for a later user message to supply confirmed evidence. After receiving that evidence, emit the full current workpiece in a fenced runbook-ir block before the first construction tool call and again before final delivery. Every later assistant-authored workpiece is model-produced: label that revision accordingly and do not copy revision zero's claim that the current revision is test-authored. Use only the mounted canonical Petrinaut read and least arc mutation when confirmed evidence calls for that change. Read the live document before mutating it, report rejected or no-op outcomes honestly, and do not construct unrelated net content. +This is a visibly labelled prepared-fixture conversation. Treat its tagged prepared runbook-ir dispatch as test-authored revision zero, maintain the full Markdown workpiece in later responses, preserve explicit unknowns, and do not relabel prepared material as model-produced. The prepared dispatch only initializes the fixture: acknowledge it without emitting a workpiece or beginning construction, then wait for a later true-user message to supply confirmed evidence. A fragment, topic label, request to inspect or explain, or unrelated message is not confirmation and must not authorize a mutation; ask for the missing confirmation instead. After receiving explicit evidence that confirms or corrects the operational fact requiring a net change, emit the full current workpiece in a fenced runbook-ir block before the first construction tool call and again before final delivery. Every later assistant-authored workpiece is model-produced: label that revision accordingly and do not copy revision zero's claim that the current revision is test-authored. Use only the mounted canonical Petrinaut read and least arc mutation when confirmed evidence calls for that change. Read the live document before mutating it, report rejected or no-op outcomes honestly, and do not construct unrelated net content. `.replace(/^\s+|\s+$/gu, ""), ); if (!isPreparedFixtureInitialization) { diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts new file mode 100644 index 00000000000..16b440c79cc --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts @@ -0,0 +1,81 @@ +const maxErrorTextLength = 10_000; + +const nonEmptyText = (value: string): string | null => + value.trim().length > 0 ? value : null; + +const isPlainObject = (value: unknown): value is Record => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value) as unknown; + return prototype === Object.prototype || prototype === null; +}; + +const serializePlainObject = ( + value: Record, + seen: WeakSet, +): string | null => { + try { + const serialized: unknown = JSON.stringify( + value, + (_key, nestedValue: unknown) => { + if (typeof nestedValue === "bigint") { + return nestedValue.toString(); + } + if (typeof nestedValue !== "object" || nestedValue === null) { + return nestedValue; + } + if (seen.has(nestedValue)) { + return "[Circular]"; + } + seen.add(nestedValue); + return nestedValue; + }, + ); + return typeof serialized === "string" ? serialized : null; + } catch { + return null; + } +}; + +const serializeErrorValue = ( + value: unknown, + seen: WeakSet, +): string | null => { + if (typeof value === "string") { + return nonEmptyText(value); + } + if (value instanceof Error) { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + + const message = nonEmptyText(value.message); + const cause = + value.cause === undefined ? null : serializeErrorValue(value.cause, seen); + if (message !== null && cause !== null) { + return `${message}\nCaused by: ${cause}`; + } + return message ?? cause; + } + if (isPlainObject(value)) { + return serializePlainObject(value, seen); + } + return null; +}; + +export const serializeErrorText = ( + error: unknown, + fallback = "The chat turn failed.", +): string => { + const serialized = serializeErrorValue(error, new WeakSet()); + if (serialized === null) { + return fallback; + } + if (serialized.length <= maxErrorTextLength) { + return serialized; + } + return `${serialized.slice(0, maxErrorTextLength - 1)}…`; +}; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index 573dd5d2647..350bd172fa8 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -2,9 +2,15 @@ import { FlueApiError, FlueExecutionError } from "@flue/sdk"; import { getToolName, isToolUIPart } from "ai"; import { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; +import { serializeErrorText } from "./error-text"; import { createFlueUiStream } from "./ui-stream"; -import type { AgentSendResult, DeliveredMessage, FlueClient } from "@flue/sdk"; +import type { + AgentSendResult, + ConversationStreamChunk, + DeliveredMessage, + FlueClient, +} from "@flue/sdk"; import type { ChatTransport, UIMessage, UIMessageChunk } from "ai"; export { @@ -33,6 +39,26 @@ export interface ClientToolResult { readonly toolCallId: string; readonly toolName: string; readonly output: unknown; + readonly source?: "voice"; +} + +export interface FlueChatResponseMessageEvent { + readonly messageId: string; + readonly submissionId: AgentSendResult["submissionId"]; +} + +export interface FlueChatResponseMessageStartedEvent extends FlueChatResponseMessageEvent { + readonly position: Extract< + ConversationStreamChunk, + { type: "message-started" } + >["position"]; +} + +export interface FlueChatResponseMessageCompletedEvent extends FlueChatResponseMessageEvent { + readonly position: Extract< + ConversationStreamChunk, + { type: "message-completed" } + >["position"]; } export interface FlueChatTransportOptions { @@ -42,17 +68,61 @@ export interface FlueChatTransportOptions { readonly input: unknown; readonly toolName: string; }) => unknown; + readonly hiddenToolNames?: ReadonlySet; readonly onAdmission?: (event: { readonly admission: AgentSendResult; readonly kind: "client-tool-result" | "user"; readonly messageId: string; }) => void; - readonly onResponseMessage?: (event: { - readonly messageId: string; - readonly submissionId: AgentSendResult["submissionId"]; - }) => void; + readonly onResponseMessage?: ( + event: FlueChatResponseMessageStartedEvent, + ) => void; + readonly onResponseMessageCompleted?: ( + event: FlueChatResponseMessageCompletedEvent, + ) => void; +} + +export type FlueChatAdmissionFailure = + | { readonly kind: "aborted" } + | { readonly kind: "ambiguous" } + | { readonly kind: "rejected"; readonly status: number } + | { + readonly kind: "submission-conflict"; + readonly status: 409; + readonly submissionId: AgentSendResult["submissionId"]; + }; + +const admissionFailureMessage = (failure: FlueChatAdmissionFailure): string => { + switch (failure.kind) { + case "aborted": + return "The local chat submission was cancelled."; + case "ambiguous": + return "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again."; + case "rejected": + return `Brunch rejected the message before admission (HTTP ${failure.status}).`; + case "submission-conflict": + return `The delivery key already belongs to admitted submission ${failure.submissionId}; the changed payload was not admitted.`; + } +}; + +export class FlueChatAdmissionError extends Error { + public readonly failure: FlueChatAdmissionFailure; + + public constructor( + failure: FlueChatAdmissionFailure, + options?: { readonly cause?: unknown }, + ) { + super(admissionFailureMessage(failure), options); + this.name = "FlueChatAdmissionError"; + this.failure = failure; + } } +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; + const completedClientToolResults = ( messages: readonly UIMessage[], assistantMessageId: string, @@ -65,6 +135,17 @@ const completedClientToolResults = ( if (assistantMessage === undefined) { return []; } + const metadata = asRecord(assistantMessage.metadata); + const voiceToolCallIds = new Set( + Array.isArray(metadata?.voiceToolCallIds) + ? metadata.voiceToolCallIds.filter( + (toolCallId): toolCallId is string => typeof toolCallId === "string", + ) + : [], + ); + if (typeof metadata?.toolCallId === "string") { + voiceToolCallIds.add(metadata.toolCallId); + } const steps = assistantMessage.parts.reduce< (typeof assistantMessage.parts)[] >( @@ -102,6 +183,9 @@ const completedClientToolResults = ( toolCallId: part.toolCallId, toolName, output: part.output, + ...(voiceToolCallIds.has(part.toolCallId) + ? { source: "voice" as const } + : {}), }, ]; }); @@ -129,20 +213,49 @@ const finalUserMessage = ( const isAbortError = (error: unknown): boolean => error instanceof Error && error.name === "AbortError"; -const admissionError = (error: unknown): Error => { - if (isAbortError(error)) { - return error as Error; +const conflictingSubmissionId = (error: FlueApiError): string | null => { + if (error.status !== 409) return null; + const body = asRecord(error.body); + const errorBody = asRecord(body?.error); + const metadata = asRecord(errorBody?.meta); + return errorBody?.type === "submission_conflict" && + typeof metadata?.submissionId === "string" && + metadata.submissionId.length > 0 + ? metadata.submissionId + : null; +}; + +const documentedPreAdmissionStatuses = new Set([ + 400, 401, 403, 404, 405, 409, 415, +]); + +const admissionError = ( + error: unknown, + signal: AbortSignal | undefined, +): FlueChatAdmissionError => { + if (signal?.aborted || isAbortError(error)) { + return new FlueChatAdmissionError({ kind: "aborted" }, { cause: error }); } if (error instanceof FlueApiError) { - return new Error( - `Brunch rejected the message before admission (HTTP ${error.status}).`, - { cause: error }, - ); + const existingSubmissionId = conflictingSubmissionId(error); + if (existingSubmissionId !== null) { + return new FlueChatAdmissionError( + { + kind: "submission-conflict", + status: 409, + submissionId: existingSubmissionId, + }, + { cause: error }, + ); + } + if (documentedPreAdmissionStatuses.has(error.status)) { + return new FlueChatAdmissionError( + { kind: "rejected", status: error.status }, + { cause: error }, + ); + } } - return new Error( - "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", - { cause: error }, - ); + return new FlueChatAdmissionError({ kind: "ambiguous" }, { cause: error }); }; const streamFailureChunk = ( @@ -168,7 +281,7 @@ const streamFailureChunk = ( error instanceof FlueExecutionError && error.failure === "terminal_event_missing" ? "The chat stream ended before the turn settled." - : "The chat turn failed.", + : serializeErrorText(error), }; }; @@ -191,6 +304,12 @@ const streamSubmission = ( return new ReadableStream({ start(controller) { let terminalEmitted = false; + let responseMessage: + | { + readonly effectiveId: string; + readonly flueId: string; + } + | undefined; const close = (): void => { if (closed) return; closed = true; @@ -215,6 +334,7 @@ const streamSubmission = ( submissionId: admission.submissionId, clientToolNames: options.clientToolNames, mapClientToolInput: options.mapClientToolInput, + hiddenToolNames: options.hiddenToolNames, write, }); @@ -228,12 +348,27 @@ const streamSubmission = ( ) { // Report the id the consumer sees: a client-tool continuation is // projected onto the assistant message it resumes. + responseMessage = { + effectiveId: continuationMessageId ?? event.messageId, + flueId: event.messageId, + }; options.onResponseMessage?.({ - messageId: continuationMessageId ?? event.messageId, + messageId: responseMessage.effectiveId, + position: event.position, submissionId: admission.submissionId, }); } projector.accept(event); + if ( + event.type === "message-completed" && + event.messageId === responseMessage?.flueId + ) { + options.onResponseMessageCompleted?.({ + messageId: responseMessage.effectiveId, + position: event.position, + submissionId: admission.submissionId, + }); + } }, }) .then(close) @@ -269,6 +404,12 @@ export const createFlueChatTransport = < messages, messageId, options.clientToolNames, + ).toSorted((left, right) => + left.toolCallId < right.toolCallId + ? -1 + : left.toolCallId > right.toolCallId + ? 1 + : 0, ); const userMessage = messageId === undefined ? finalUserMessage(messages) : undefined; @@ -295,6 +436,14 @@ export const createFlueChatTransport = < toolCallIds: toolResults .map((result) => result.toolCallId) .join(","), + ...(toolResults.some(({ source }) => source === "voice") + ? { + voiceToolCallIds: toolResults + .filter(({ source }) => source === "voice") + .map(({ toolCallId }) => toolCallId) + .join(","), + } + : {}), }, }; })(); @@ -305,6 +454,9 @@ export const createFlueChatTransport = < .map(({ toolCallId }) => toolCallId) .sort() .join(",")}`; + if (Array.from(idempotencyKey).length > 256) { + throw new Error("The submitted message identity is too long."); + } let admission: AgentSendResult; try { @@ -314,7 +466,7 @@ export const createFlueChatTransport = < signal: abortSignal, }); } catch (error) { - throw admissionError(error); + throw admissionError(error, abortSignal); } options.onAdmission?.({ admission, diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts index 001d50d5de7..03a4dce1ce7 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts @@ -9,10 +9,17 @@ import type { UIMessage } from "ai"; type UiMessagePart = UIMessage["parts"][number]; +export interface UiHistoryMessageMetadata { + readonly source?: "voice"; + readonly voiceToolCallIds?: readonly string[]; + readonly stopped?: true; +} + export type UiHistoryMessage = Omit< - UIMessage, + UIMessage, "metadata" | "parts" | "role" > & { + metadata?: UiHistoryMessageMetadata; role: Extract; parts: UiMessagePart[]; }; @@ -23,6 +30,7 @@ export interface SnapshotToUiMessagesOptions { readonly input: unknown; readonly toolName: string; }) => unknown; + readonly hiddenToolNames?: ReadonlySet; } const unhandledConversationPart = (part: never): never => { @@ -37,11 +45,16 @@ const isFlueDataPart = ( const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; +interface ClientToolResult { + readonly output: unknown; + readonly source?: "voice"; +} + const clientToolResultsFrom = ( snapshot: Pick, signalName: string, -): ReadonlyMap => { - const outputsByCallId = new Map(); +): ReadonlyMap => { + const resultsByCallId = new Map(); for (const message of snapshot.messages) { if (message.purpose !== "dispatch") continue; if (message.signal?.tagName !== signalName) continue; @@ -67,19 +80,22 @@ const clientToolResultsFrom = ( ) { continue; } - outputsByCallId.set(result.toolCallId, result.output); + resultsByCallId.set(result.toolCallId, { + output: result.output, + ...(result.source === "voice" ? { source: "voice" } : {}), + }); } } - return outputsByCallId; + return resultsByCallId; }; const toolPartFrom = ( part: Extract, options: SnapshotToUiMessagesOptions, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart => { const isClientTool = options.clientToolNames.has(part.toolName); - const hasClientOutput = clientOutputs.has(part.toolCallId); + const hasClientOutput = clientResults.has(part.toolCallId); const input = isClientTool && options.mapClientToolInput !== undefined ? options.mapClientToolInput({ @@ -106,7 +122,7 @@ const toolPartFrom = ( }; } const output = isClientTool - ? clientOutputs.get(part.toolCallId) + ? clientResults.get(part.toolCallId)?.output : part.state === "output-available" ? part.output : undefined; @@ -132,7 +148,7 @@ const toolPartFrom = ( const partsFrom = ( message: FlueConversationMessage, options: SnapshotToUiMessagesOptions, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart[] => { const parts: UiMessagePart[] = []; for (const part of message.parts) { @@ -145,7 +161,8 @@ const partsFrom = ( continue; } if (part.type === "dynamic-tool") { - parts.push(toolPartFrom(part, options, clientOutputs)); + if (options.hiddenToolNames?.has(part.toolName) === true) continue; + parts.push(toolPartFrom(part, options, clientResults)); continue; } if (part.type === "file") { @@ -167,14 +184,24 @@ const partsFrom = ( }; export const snapshotToUiMessages = ( - snapshot: Pick, + snapshot: Pick & + Partial>, options: SnapshotToUiMessagesOptions, ): UiHistoryMessage[] => { - const clientOutputs = clientToolResultsFrom( + const clientResults = clientToolResultsFrom( snapshot, CLIENT_TOOL_RESULT_SIGNAL, ); const messages: UiHistoryMessage[] = []; + const abortedSubmissions = new Set( + snapshot.settlements + ?.filter(({ outcome }) => outcome === "aborted") + .flatMap(({ submissionId, answeredBySubmissionId }) => + answeredBySubmissionId === undefined + ? [submissionId] + : [submissionId, answeredBySubmissionId], + ), + ); // The live stream projects a client-tool continuation onto the assistant // message it resumes; the snapshot records that continuation as a separate // Flue message behind the `client-tool-result` dispatch, so fold it back. @@ -193,21 +220,64 @@ export const snapshotToUiMessages = ( if (message.display !== "visible") continue; if (message.purpose !== "user" && message.purpose !== "assistant") continue; if (message.role !== "user" && message.role !== "assistant") continue; - const parts = partsFrom(message, options, clientOutputs); + const parts = partsFrom(message, options, clientResults); if (message.role === "user") { resumableAssistant = undefined; awaitingClientResult = false; continuationPending = false; } if (parts.length === 0) continue; - if ( + const foldsIntoPrevious = message.role === "assistant" && (awaitingClientResult || continuationPending) && - resumableAssistant !== undefined - ) { + resumableAssistant !== undefined; + const voiceToolCallIds = + message.role === "assistant" + ? message.parts.flatMap((part) => + part.type === "dynamic-tool" && + clientResults.get(part.toolCallId)?.source === "voice" + ? [part.toolCallId] + : [], + ) + : []; + const stopped = + message.role === "assistant" && + message.submissionId !== undefined && + abortedSubmissions.has(message.submissionId); + const metadata: UiHistoryMessageMetadata = { + ...(voiceToolCallIds.length > 0 + ? { source: "voice" as const, voiceToolCallIds } + : {}), + ...(stopped ? { stopped: true as const } : {}), + }; + awaitingClientResult = + message.role === "assistant" && + !stopped && + message.parts.some( + (part) => + part.type === "dynamic-tool" && + options.clientToolNames.has(part.toolName) && + !clientResults.has(part.toolCallId), + ); + if (foldsIntoPrevious && resumableAssistant !== undefined) { // Live continuations start a new step. Keep that boundary after reopen // so completedClientToolResults still selects only the latest step. resumableAssistant.parts.push({ type: "step-start" }, ...parts); + if (voiceToolCallIds.length > 0 || stopped) { + const combinedOrigins = [ + ...new Set([ + ...(resumableAssistant.metadata?.voiceToolCallIds ?? []), + ...voiceToolCallIds, + ]), + ]; + resumableAssistant.metadata = { + ...resumableAssistant.metadata, + ...metadata, + ...(combinedOrigins.length > 0 + ? { voiceToolCallIds: combinedOrigins } + : {}), + }; + } continuationPending = false; continue; } @@ -215,17 +285,10 @@ export const snapshotToUiMessages = ( id: message.id, role: message.role, parts, + ...(voiceToolCallIds.length > 0 || stopped ? { metadata } : {}), }; messages.push(projected); - if (message.role === "assistant") { - resumableAssistant = projected; - awaitingClientResult = message.parts.some( - (part) => - part.type === "dynamic-tool" && - options.clientToolNames.has(part.toolName) && - !clientOutputs.has(part.toolCallId), - ); - } + if (message.role === "assistant") resumableAssistant = projected; } return messages; }; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts index 250942d8886..90f8c3ee1f4 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts @@ -1,3 +1,5 @@ +import { serializeErrorText } from "./error-text"; + import type { AgentSendResult, ConversationStreamChunk } from "@flue/sdk"; import type { UIMessageChunk } from "ai"; @@ -8,6 +10,7 @@ export interface FlueUiStreamOptions { readonly input: unknown; readonly toolName: string; }) => unknown; + readonly hiddenToolNames?: ReadonlySet; readonly write: (chunk: UIMessageChunk) => void; } @@ -30,6 +33,7 @@ export const createFlueUiStream = ( let turnId: string | undefined; let partOrdinal = 0; let streamingPart: StreamingPart | undefined; + const hiddenToolCallIds = new Set(); const pendingClientToolCallIds = new Set(); const finishPart = (): void => { @@ -97,7 +101,7 @@ export const createFlueUiStream = ( case "failed": options.write({ type: "error", - errorText: "The chat turn failed.", + errorText: serializeErrorText(chunk.error), }); break; case "aborted": @@ -134,6 +138,10 @@ export const createFlueUiStream = ( if (!accepting || messageId === undefined) return; if (chunk.messageId !== messageId) return; finishPart(); + if (options.hiddenToolNames?.has(chunk.toolName) === true) { + hiddenToolCallIds.add(chunk.toolCallId); + return; + } const isClientTool = options.clientToolNames.has(chunk.toolName); if (isClientTool) pendingClientToolCallIds.add(chunk.toolCallId); options.write({ @@ -153,6 +161,7 @@ export const createFlueUiStream = ( } case "tool-output": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-available", @@ -164,6 +173,7 @@ export const createFlueUiStream = ( } case "tool-output-error": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-error", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts index 60d2958efc4..e314abfb6c6 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -278,6 +278,33 @@ test("after snapshot fold, submits only the latest client-tool step", async () = ); }); +test("keeps reordered cumulative tool results byte-identical for idempotent retry", async () => { + const { client, send } = clientWith(completedEvents); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(["readPetrinautDoc"]), + }); + const parts: UIMessage["parts"] = ["tool-b", "tool-a"].map((toolCallId) => ({ + type: "dynamic-tool", + toolName: "readPetrinautDoc", + toolCallId, + state: "output-available", + input: {}, + output: toolCallId, + })); + for (const ordered of [parts, [...parts].reverse()]) { + await readChunks( + await transport.sendMessages( + sendOptions( + [{ id: "assistant-original", role: "assistant", parts: ordered }], + "assistant-original", + ), + ), + ); + } + expect(send.mock.calls[0]?.[0]).toEqual(send.mock.calls[1]?.[0]); +}); + test("admits one user message and projects a finite per-turn stream", async () => { const { client, send } = clientWith(completedEvents); const transport = createFlueChatTransport({ @@ -403,12 +430,14 @@ test("starts with history-only reconnection", async () => { test.each([ [ "failed", - new FlueExecutionError({ - target: "agent_submission", - targetId: admission.submissionId, - failure: "failed", + new Error("Elicitor tool failed.", { + cause: { field: "answer", reason: "Required" }, }), - { type: "error", errorText: "The chat turn failed." }, + { + type: "error", + errorText: + 'Elicitor tool failed.\nCaused by: {"field":"answer","reason":"Required"}', + }, ], [ "aborted", @@ -492,7 +521,7 @@ test("keeps caller cancellation distinct from durable abort", async () => { ]); }); -test("surfaces rejected and ambiguous admission without retrying", async () => { +test("classifies documented rejection and ambiguous admission without retrying", async () => { const rejectedSend = vi.fn(async () => { throw new FlueApiError(403, ""); }); @@ -514,25 +543,97 @@ test("surfaces rejected and ambiguous admission without retrying", async () => { await expect( createTransport(rejectedSend).sendMessages(options), - ).rejects.toThrow("rejected the message before admission (HTTP 403)"); + ).rejects.toMatchObject({ + failure: { kind: "rejected", status: 403 }, + message: "Brunch rejected the message before admission (HTTP 403).", + name: "FlueChatAdmissionError", + }); await expect( createTransport(ambiguousSend).sendMessages(options), - ).rejects.toThrow("may have accepted the message"); + ).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + name: "FlueChatAdmissionError", + }); expect(rejectedSend).toHaveBeenCalledOnce(); expect(ambiguousSend).toHaveBeenCalledOnce(); }); -test("reports one admission and its correlated response message", async () => { +test.each([ + ["server failure", new FlueApiError(500, "")], + ["unknown response", new FlueApiError(418, "")], +] as const)( + "treats a %s after request write as ambiguous", + async (_label, error) => { + const send = vi.fn(async () => { + throw error; + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-ambiguous", + role: "user", + parts: [{ type: "text", text: "Do not retry this." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); + }, +); + +test("classifies an explicit local admission abort without retrying", async () => { + const send = vi.fn(async () => { + throw new DOMException("cancelled", "AbortError"); + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-aborted", + role: "user", + parts: [{ type: "text", text: "Cancel locally." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { kind: "aborted" }, + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); +}); + +test("reports one admission and its correlated response message completion", async () => { const { client } = clientWith(completedEvents); const onAdmission = vi.fn>(); const onResponseMessage = vi.fn>(); + const onResponseMessageCompleted = + vi.fn< + NonNullable + >(); const transport = createFlueChatTransport({ client, clientToolNames: new Set(), onAdmission, onResponseMessage, + onResponseMessageCompleted, }); const stream = await transport.sendMessages( @@ -555,6 +656,13 @@ test("reports one admission and its correlated response message", async () => { expect(onResponseMessage).toHaveBeenCalledOnce(); expect(onResponseMessage).toHaveBeenCalledWith({ messageId: "assistant-1", + position: position(0), + submissionId: admission.submissionId, + }); + expect(onResponseMessageCompleted).toHaveBeenCalledOnce(); + expect(onResponseMessageCompleted).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: position(2), submissionId: admission.submissionId, }); }); @@ -597,14 +705,19 @@ test("stays silent after the consumer cancels the per-turn stream", async () => await expect(reader.closed).resolves.toBeUndefined(); }); -test("reports a client-tool continuation against the resumed assistant id", async () => { +test("reports a client-tool continuation and completion against the resumed assistant id", async () => { const { client } = clientWith(completedEvents); const onResponseMessage = vi.fn>(); + const onResponseMessageCompleted = + vi.fn< + NonNullable + >(); const transport = createFlueChatTransport({ client, clientToolNames: new Set(["readPetrinautDoc"]), onResponseMessage, + onResponseMessageCompleted, }); const stream = await transport.sendMessages( @@ -633,6 +746,114 @@ test("reports a client-tool continuation against the resumed assistant id", asyn expect(onResponseMessage).toHaveBeenCalledOnce(); expect(onResponseMessage).toHaveBeenCalledWith({ messageId: "assistant-original", + position: position(0), submissionId: admission.submissionId, }); + expect(onResponseMessageCompleted).toHaveBeenCalledOnce(); + expect(onResponseMessageCompleted).toHaveBeenCalledWith({ + messageId: "assistant-original", + position: position(2), + submissionId: admission.submissionId, + }); +}); + +test("replays a stable typed or Voice message with the same idempotency key", async () => { + const seenKeys = new Set(); + let admittedTurns = 0; + const send = vi.fn(async (options) => { + const key = options.idempotencyKey; + if (key === undefined || !seenKeys.has(key)) { + admittedTurns += 1; + if (key !== undefined) seenKeys.add(key); + return admission; + } + return { ...admission, deduplicated: true }; + }); + const wait = vi.fn(async () => undefined); + const onAdmission = + vi.fn>(); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + onAdmission, + }); + const typedTurn = sendOptions([ + { + id: "typed-message-1", + role: "user", + parts: [{ type: "text", text: "Admit this once." }], + }, + ]); + + const firstStream = await transport.sendMessages(typedTurn); + const replayedStream = await transport.sendMessages(typedTurn); + await Promise.all([readChunks(firstStream), readChunks(replayedStream)]); + + expect(send).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ idempotencyKey: "ai-sdk:user:typed-message-1" }), + ); + expect(send).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ idempotencyKey: "ai-sdk:user:typed-message-1" }), + ); + expect(admittedTurns).toBe(1); + expect(onAdmission).toHaveBeenNthCalledWith(2, { + admission: { ...admission, deduplicated: true }, + kind: "user", + messageId: "typed-message-1", + }); + + const voiceTurn = sendOptions([ + { + id: "voice-realtime:7:item%2F1:0", + role: "user", + parts: [{ type: "text", text: "Voice transcript." }], + }, + ]); + await readChunks(await transport.sendMessages(voiceTurn)); + expect(send).toHaveBeenLastCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:user:voice-realtime:7:item%2F1:0", + }), + ); +}); + +test("reports an idempotency conflict as a definite existing admission", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-conflict", + role: "user", + parts: [{ type: "text", text: "Changed payload." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + }, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); }); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts index 5cec5e7a1c5..161827b7ae7 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -31,8 +31,104 @@ const snapshotWithPendingClientTool: FlueConversationSnapshot = { const projectionOptions = { clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames: new Set(["brunch_mark_question"]), }; +test("retains Voice origins from folded continuation messages", () => { + const messages: FlueConversationSnapshot["messages"] = []; + for (const ordinal of [1, 2]) { + messages.push( + { + id: `assistant-${ordinal}`, + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: `tool-${ordinal}`, + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: `signal-${ordinal}`, + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + state: "done", + text: JSON.stringify([ + { + toolCallId: `tool-${ordinal}`, + output: "A spoken answer", + source: "voice", + }, + ]), + }, + ], + }, + ); + } + expect(snapshotToUiMessages({ messages }, projectionOptions)).toMatchObject([ + { + id: "assistant-1", + metadata: { source: "voice", voiceToolCallIds: ["tool-1", "tool-2"] }, + }, + ]); +}); + +test("marks only the durably aborted assistant response stopped after reopen", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + id: "partial", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "stopped-turn", + parts: [{ type: "text", state: "done", text: "Partial reply" }], + }, + { + id: "next-user", + role: "user", + purpose: "user", + display: "visible", + parts: [{ type: "text", state: "done", text: "Continue" }], + }, + { + id: "complete", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "next-turn", + parts: [{ type: "text", state: "done", text: "Complete reply" }], + }, + ], + settlements: [ + { submissionId: "stopped-turn", outcome: "aborted" }, + { submissionId: "next-turn", outcome: "completed" }, + ], + }; + const projected = snapshotToUiMessages(snapshot, projectionOptions); + expect(projected.find(({ id }) => id === "partial")?.metadata).toEqual({ + stopped: true, + }); + expect( + projected.find(({ id }) => id === "complete")?.metadata, + ).toBeUndefined(); + expect( + projected.find(({ id }) => id === "next-user")?.metadata, + ).toBeUndefined(); +}); + test("leaves an unfinished client tool available to run", () => { expect( snapshotToUiMessages(snapshotWithPendingClientTool, projectionOptions), @@ -128,6 +224,65 @@ test("uses a recorded browser result even when it is null", () => { ]); }); +test("reconstructs durable voice provenance for each browser result", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + ...snapshotWithPendingClientTool.messages[0]!, + parts: [ + ...snapshotWithPendingClientTool.messages[0]!.parts, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + expect.objectContaining({ + id: "assistant-1", + metadata: { + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }, + }), + ]); +}); + test("keeps Flue data parts on the AI SDK message", () => { const snapshot: FlueConversationSnapshot = { v: 1, @@ -279,3 +434,50 @@ test("folds a client-tool continuation into the assistant message it resumed", ( }, ]); }); + +test("hides a question-marker tool while retaining its durable data", () => { + const question = "Which line should run this order?"; + const snapshot: FlueConversationSnapshot = { + v: 1, + conversationId: "conversation-1", + offset: "0", + messages: [ + { + id: "assistant-question", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + state: "output-available", + input: { question }, + output: { marked: true }, + }, + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ], + settlements: [], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + { + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ]); +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index 6a0b30936a4..b145c4d896a 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -9,11 +9,13 @@ const position = (index: number) => ({ batch: 1, index }); const project = ( chunks: readonly ConversationStreamChunk[], + hiddenToolNames: ReadonlySet = new Set(), ): UIMessageChunk[] => { const written: UIMessageChunk[] = []; const projector = createFlueUiStream({ submissionId: "submission-1", clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames, write: (chunk) => written.push(chunk), }); for (const chunk of chunks) projector.accept(chunk); @@ -64,6 +66,72 @@ test("projects data and metadata onto the AI SDK stream", () => { }); }); +test("hides an implementation tool while preserving its data marker", () => { + const written = project( + [ + { + type: "message-started", + conversationId: "conversation-1", + messageId: "message-1", + submissionId: "submission-1", + turnId: "turn-1", + position: position(0), + }, + { + type: "tool-input", + conversationId: "conversation-1", + messageId: "message-1", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + input: { question: "Which line should run this order?" }, + position: position(1), + }, + { + type: "data-part", + conversationId: "conversation-1", + messageId: "message-1", + name: "brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + position: position(2), + }, + { + type: "tool-output", + conversationId: "conversation-1", + toolCallId: "tool-question-1", + output: { marked: true }, + position: position(3), + }, + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "completed", + position: position(4), + }, + ], + new Set(["brunch_mark_question"]), + ); + + expect(written).toContainEqual({ + type: "data-brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + }); + expect( + written.some( + (chunk) => + chunk.type === "tool-input-available" || + chunk.type === "tool-output-available" || + chunk.type === "tool-output-error", + ), + ).toBe(false); +}); + test("ignores observation catch-up chunks in a submission stream", () => { const written = project([ { @@ -222,3 +290,71 @@ test("keeps a pending client tool in the final projected step", () => { { type: "finish", finishReason: "tool-calls" }, ]); }); + +test.each([ + { + error: new Error("Elicitor failed.", { + cause: "The requested field is required.", + }), + expected: "Elicitor failed.\nCaused by: The requested field is required.", + shape: "Error with cause", + }, + { + error: "The elicitor rejected the answer.", + expected: "The elicitor rejected the answer.", + shape: "string", + }, + { + error: { field: "answer", reason: "Required" }, + expected: '{"field":"answer","reason":"Required"}', + shape: "plain object", + }, + { + error: 503, + expected: "The chat turn failed.", + shape: "unsupported value", + }, + { + error: "", + expected: "The chat turn failed.", + shape: "empty string", + }, +])("preserves a failed submission's $shape error", ({ error, expected }) => { + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error, + position: position(0), + }, + ]); + + expect(written).toEqual([{ type: "error", errorText: expected }]); + expect(written).not.toContainEqual({ + type: "error", + errorText: "[object Object]", + }); +}); + +test("bounds cyclic failed-submission objects", () => { + const cyclicError: Record = { reason: "Recursive failure" }; + cyclicError.self = cyclicError; + cyclicError.payload = "x".repeat(20_000); + + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error: cyclicError, + position: position(0), + }, + ]); + + const failure = written.find((chunk) => chunk.type === "error"); + expect(failure?.errorText).toContain('"self":"[Circular]"'); + expect(failure?.errorText.length).toBeLessThanOrEqual(10_000); +}); diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 47790bc1cc4..a7209260ae4 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -16,15 +16,15 @@ The assistant panel only renders in **Edit** mode. Switching to **Simulate** mod While a response is streaming you can: - Watch the model's text and reasoning appear live. The **Reasoning** block is collapsible; while it is streaming, it auto-opens, shows a shimmer effect, and (once attached timing information arrives) an elapsed timer. -- Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. A host with durable conversation execution can record that stop before Petrinaut cancels its local stream; without that host capability, Stop is local cancellation only. A Stop pressed while the assistant is reading the net or the docs also withholds the follow-up reply that would otherwise start automatically. +- Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. A host with durable conversation execution can record that stop before Petrinaut cancels its local stream; without that host capability, Stop is local cancellation only. A Stop pressed while the assistant is reading or editing the net also withholds browser tools that have not started and the follow-up reply that would otherwise start automatically. Already-applied changes are not rolled back. - Type your next message in the composer -- it is queued for after the current response ends. The application embedding Petrinaut may place an additional control beside the message box. For example, a host can offer another way to enter finalized text. Text submitted by that control behaves like text sent with the keyboard: it joins the same conversation and, when an inline question is waiting for an answer, completes that question rather than starting an unrelated message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. -If the host offers voice input, a finalized spoken turn is held while an existing response finishes and is submitted when the conversation is ready. +If the host offers voice input, only a finalized transcript captured while Voice owns the input turn can be submitted. Voice waits while an existing response finishes or yields through the host's handoff control. -If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to the conversation. Retry from the composer when the assistant is ready. +If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Voice markers attached to client-tool results survive that history. A direct spoken user message remains in the transcript after reopening, but its **Voice** chip may not be restored by the current Brunch host. Durably aborted assistant entries retain their **Response stopped** label even after later completed replies. If Brunch had already completed a tool-call step when Stop withheld its browser follow-up, that local decision has no durable cancellation record: reopening can recover the tool as pending work. Do not treat that local withholding as a reload-safe cancellation. ### Prepared local demo fixture @@ -42,12 +42,15 @@ When the Brunch voice preview is enabled and available, an empty composer shows titled **Start voice mode**. Typing non-whitespace text replaces it with **Send**. The same dynamic action appears in the first-run prompt and the assistant panel; if voice is unavailable, the empty composer retains a disabled **Send** action. Starting Voice mode keeps the transcript in place and -opens the existing one-time disclosure above the composer. Review that OpenAI processes live -audio and speaks the interviewer's words while Petrinaut keeps finalized answers in the conversation -rather than the audio. You can check your microphone before confirming that you understand and -selecting **Start voice mode**. Petrinaut remembers that acknowledgement in this browser for the -current disclosure version, so later uses of **Start voice mode** start directly. If browser storage -is unavailable or the disclosure changes, Petrinaut asks again. +opens the existing one-time disclosure. Voice selected from the first-run prompt starts compact: the +disclosure and microphone check appear in a card immediately above a **Voice setup** dock, while the +AI header, transcript, and composer stay hidden. Select **Expand voice setup** to restore the full +panel. Voice started from the composer keeps that full panel visible. Review that OpenAI processes +live audio and speaks the interviewer's words while Petrinaut keeps finalized answers in the +conversation rather than the audio. You can check your microphone before confirming that you +understand and selecting **Start voice mode**. Petrinaut remembers that acknowledgement in this +browser for the current disclosure version, so later uses of **Start voice mode** start directly. If +browser storage is unavailable or the disclosure changes, Petrinaut asks again. While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel: a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**, @@ -58,32 +61,42 @@ flicker above the line. While the assistant speaks the ribbon takes on a restrai motion instead, colour crossfading as the turn changes hands, so which side holds it is readable at a glance. It flattens to near a line whenever nobody holds the turn. -The conversation itself stays still. Spoken turns are written to it as they happen, because that is -what runs the tools that edit the net, but they stay hidden until the session ends rather than -scrolling the transcript mid-sentence. **Show transcription in chat** lets them through as they land -instead; turning it off holds them back again, and it starts off with each session. Two things are -never held back either way: anything you typed, and any inline question waiting for your answer. When -the session ends, the held turns appear together under a **Voice session · N turns** divider. Only -finalized answers and canonical Brunch text become chat history; provisional transcription and -Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of -the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice -provenance remains visible without duplicating an answer. - -The microphone stays on while the interviewer speaks, so speaking naturally interrupts the audio -and starts listening to you; you do not need to select an interrupt action. Semantic voice detection -finishes each answer automatically after a natural pause and is tuned to allow longer thinking -pauses. There is no required done-speaking action. - -Every session control lives in the dock: **Show transcription in chat** on the left, and on the right -**Mute microphone** (**Unmute microphone** once muted) beside **End voice mode**. Muting stops -sending audio without ending the turn, so the assistant plays out whatever it is saying and unmuting -drops you straight back into the conversation. **Resume voice mode** replaces the microphone action -while a session is paused, and **Reconnect voice mode** replaces it after a failure. Nothing is added -to the canvas toolbar. Sending non-empty typed text from the +Spoken turns appear in the conversation as soon as their finalized text arrives, so the transcript +stays current while the session runs and tools that edit the net remain visible. Select **Collapse +voice session** to reduce the panel to the Voice dock alone; this hides the AI header, transcript, and +host Voice region without ending the session. Select **Expand voice session** to restore them. Ending +Voice while collapsed also closes the AI panel; ending Voice while expanded returns to the text +composer. Only finalized answers and canonical Brunch text become chat history; provisional +transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small +**Voice** chip in front of the words themselves, and the exact inline answer completed by speech +carries the same chip, so Voice provenance remains visible without duplicating an answer while the +session is mounted. + +Voice is half-duplex. The microphone is closed while the interviewer speaks or the assistant is +working, which prevents playback from becoming a false answer. Select **Your turn** to interrupt: +the dock shows the handoff as thinking while it clears pending audio and waits for the provider to +finish cancellation, then opens a fresh input turn. Audio captured before that completed handoff is +discarded. Semantic voice detection finishes your answer automatically after a natural pause, so +there is no required done-speaking action. Duplicate, empty, failed, or unavailable transcripts are +not submitted; the dock asks you to try again. An overlong answer instead asks for a shorter response. +Provisional words remain display-only until the provider completes their transcript. + +Every session control lives in the dock: **Collapse voice session** / **Expand voice session** and +**Voice playback options** on the left, and the available handoff, microphone, recovery, and end +actions on the right. +**Read full response** becomes available after the matching response and speech have both finished +and replays every exact retained canonical segment in order. **Repeat question** uses the same +availability gates and replays only exact question text explicitly marked by Brunch. It stays +disabled when that marker is missing or does not match finalized assistant text rather than +guessing that the final segment is a question. +Playback stays unavailable during active capture, submission, cancellation, pause, and errors. **Mute microphone** becomes +**Unmute microphone** once muted, and your latest choice applies when a handoff settles. **Resume voice mode** +replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it +after a failure. Nothing is added to the canvas toolbar. Sending non-empty typed text from the composer or first-run prompt ends Voice mode before it sends the message once through the same conversation; repeated send actions are ignored while that short handoff completes. -The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only delivers its words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. +The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are @@ -91,8 +104,8 @@ ready. **Clear AI chat** is unavailable while a Voice session is active. If voice cannot continue, the status reads **Voice interrupted** and the actionable error arrives as -a toast that names the microphone, connection, or Voice failure in one sentence, followed by any -diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until +a persistent toast that names the microphone, connection, or Voice failure in one sentence, followed +by any diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until the session recovers. For microphone permission or device errors, allow access or connect/select a microphone before reconnecting. For an interrupted request, network error, or timeout, check the connection and reconnect. If the preview is unavailable, continue with the text composer. An invalid @@ -105,7 +118,7 @@ When no interview is active and the host permits clearing, **Clear AI chat** via ## What the assistant can do -The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation: +The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation. A failed tool card leads with its complete error instead of hiding it behind a hover tooltip: - **Read tools** (neutral, expandable) –– for checking the current net state and active Petrinaut extensions at any point, for compilation errors, and for reading the user guide. - **Mutation tools** (green for additions/updates, red for deletions) -- "Added place X", "Updated transition Y", "Removed metric Z", and so on. Multiple successive mutations group under a collapsible "N changes" header. diff --git a/libs/@hashintel/petrinaut/src/panda-preset.ts b/libs/@hashintel/petrinaut/src/panda-preset.ts index 2cdc120abf8..7d4ab7bd23c 100644 --- a/libs/@hashintel/petrinaut/src/panda-preset.ts +++ b/libs/@hashintel/petrinaut/src/panda-preset.ts @@ -122,10 +122,6 @@ export const petrinautPandaPreset = { from: { opacity: "1", transform: "translateX(0)" }, to: { opacity: "0", transform: "translateX(100px)" }, }, - petrinautVoiceReveal: { - from: { opacity: "0", transform: "translateY(10px)" }, - to: { opacity: "1", transform: "translateY(0)" }, - }, petrinautVoiceSwap: { from: { opacity: "0" }, to: { opacity: "1" }, diff --git a/libs/@hashintel/petrinaut/src/react/notifications/context.ts b/libs/@hashintel/petrinaut/src/react/notifications/context.ts index 9911f709022..5f508982ae4 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/context.ts +++ b/libs/@hashintel/petrinaut/src/react/notifications/context.ts @@ -3,6 +3,7 @@ import { createContext } from "react"; export type NotificationTone = "error" | "neutral" | "success"; export type AddNotificationInput = { + detail?: string; message: string; tone?: NotificationTone; durationMs?: number; diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx new file mode 100644 index 00000000000..1dda6521daa --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx @@ -0,0 +1,76 @@ +/** + * @vitest-environment jsdom + */ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { use } from "react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { NotificationsContext } from "./context"; +import { NotificationsProvider } from "./provider"; +import { notificationsToaster } from "./toaster"; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +test("keeps error notifications open while preserving the default for other tones", async () => { + const createToast = vi.spyOn(notificationsToaster, "create"); + const Trigger = () => { + const { addNotification } = use(NotificationsContext); + + return ( + <> + + + + ); + }; + + render( + + + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Error" })); + fireEvent.click(screen.getByRole("button", { name: "Success" })); + + await waitFor(() => expect(createToast).toHaveBeenCalledTimes(2)); + expect(createToast).toHaveBeenNthCalledWith(1, { + description: "The complete elicitor failure.", + duration: Infinity, + id: "notification-0", + title: "AI assistant error", + type: "error", + }); + expect(createToast).toHaveBeenNthCalledWith(2, { + description: undefined, + duration: 3000, + id: "notification-1", + title: "Saved", + type: "success", + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx index f6d39467b8b..945584fe104 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx @@ -16,22 +16,28 @@ export const NotificationsProvider = ({ }: { children: ReactNode; }) => { - function dismissNotification(id: string) { + const dismissNotification = (id: string) => { queueMicrotask(() => { notificationsToaster.dismiss(id); }); - } + }; - function addNotification({ + const addNotification = ({ + detail, durationMs, message, tone = "success", - }: AddNotificationInput) { - const id = `notification-${nextNotificationId++}`; - const effectiveDurationMs = durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS; + }: AddNotificationInput) => { + const id = `notification-${nextNotificationId}`; + nextNotificationId += 1; + const effectiveDurationMs = + tone === "error" + ? Infinity + : (durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS); queueMicrotask(() => { notificationsToaster.create({ + description: detail, duration: effectiveDurationMs, id, title: message, @@ -40,7 +46,7 @@ export const NotificationsProvider = ({ }); return id; - } + }; useEffect(() => { return () => { diff --git a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx index bc26dc8403b..fd2261b2033 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx @@ -5,7 +5,7 @@ import { createToaster, } from "@ark-ui/react/toast"; -import { usePortalContainerRef } from "@hashintel/ds-components"; +import { Button, usePortalContainerRef } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; export const notificationsToaster = createToaster({ @@ -24,10 +24,11 @@ const toastRootStyle = css({ transition: "[translate 300ms, scale 300ms, opacity 300ms, box-shadow 300ms]", transitionTimingFunction: "[cubic-bezier(0.21, 1.02, 0.73, 1)]", display: "flex", - alignItems: "center", + alignItems: "flex-start", + gap: "2", minHeight: "[26px]", width: "[max-content]", - maxWidth: "[320px]", + maxWidth: "[min(480px, calc(100vw - 32px))]", borderRadius: "lg", boxShadow: "[0 8px 24px rgba(0, 0, 0, 0.24)]", paddingX: "4", @@ -44,23 +45,93 @@ const toastRootStyle = css({ }, }); +const toastContentStyle = css({ + display: "flex", + flex: "[1]", + minWidth: "[0]", + flexDirection: "column", + gap: "1", +}); + const toastTitleStyle = css({ overflow: "hidden", - textOverflow: "ellipsis", - whiteSpace: "nowrap", + overflowWrap: "anywhere", + lineClamp: "4", fontSize: "xs", fontWeight: "medium", lineHeight: "[14px]", }); +const toastDescriptionStyle = css({ + maxHeight: "[240px]", + overflow: "auto", + overflowWrap: "anywhere", + whiteSpace: "pre-wrap", + fontSize: "xs", + lineHeight: "[18px]", + userSelect: "text", +}); + +const toastActionsStyle = css({ + display: "flex", + flexShrink: "[0]", + gap: "1", +}); + +const toastActionStyle = css({ + color: "neutral.s00", + _hover: { + color: "neutral.s00", + }, +}); + export const NotificationsToaster = () => ( - {(toast) => ( - - {toast.title} - - )} + {(toast) => { + const detail = + typeof toast.description === "string" ? toast.description : undefined; + + return ( + +
+ + {toast.title} + + {detail && ( + + {detail} + + )} +
+
+ {detail && ( +
+ + ); + }} ); diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bcdc4e0ee48..73e502c20bb 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -7,9 +7,12 @@ import type { export type VoiceSessionActions = { end: () => void; pause: () => void; + readFullResponse?: () => void; reconnect: () => void; + repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; + takeTurn?: () => Promise | void; }; export type VoiceSessionSnapshot = { diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index bd1a26f425e..b72bee75f8c 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -16,10 +16,18 @@ export type PetrinautAiVoiceSessionPhase = * effect: it changes at microphone-sampling rate. */ export type PetrinautAiVoiceSessionState = { + /** Whether the current canonical assistant response is safe to replay. */ + canReadFullResponse?: boolean; + /** Whether the final segment of the canonical response is safe to repeat. */ + canRepeatQuestion?: boolean; + /** Whether the user can cancel Voice output and start their turn. */ + canTakeTurn?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; /** Normalized 0–1 input level driving the listening indicator. */ microphoneLevel: number; + /** Recoverable feedback about an utterance which was not submitted. */ + notice?: string | null; phase: PetrinautAiVoiceSessionPhase; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index a95671ce61d..3f88e964621 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -62,3 +62,43 @@ export const useVoiceSessionActions = (): VoiceSessionActions | null => { () => null, ); }; + +export const useVoiceSessionCanReadFullResponse = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canReadFullResponse ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanRepeatQuestion = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRepeatQuestion ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanTakeTurn = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canTakeTurn ?? false, + () => false, + ); +}; + +export const useVoiceSessionNotice = (): string | null => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.notice ?? null, + () => null, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 0df936f5fc3..0f7f8da56f0 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -41,6 +41,8 @@ export type PetrinautAiComposerControlContext = { conversationId: string; messages: PetrinautAiMessage[]; status: PetrinautAiComposerStatus; + /** Logical response stopped, including a withheld follow-up; not a Flue settlement claim. */ + stopped?: boolean; /** Call from an event handler or effect, never while rendering. */ stop: () => Promise; /** Call from an event handler or effect, never while rendering. */ @@ -65,12 +67,18 @@ export type PetrinautAiVoiceModeControls = { reconnect: () => void; /** Resumes microphone capture after `pause`. */ resume: () => void; + /** Replays the exact retained canonical assistant response when available. */ + readFullResponse?: () => void; + /** Replays only the exact question selected by the host's canonical marker. */ + repeatQuestion?: () => void; /** * Stops or restarts microphone capture while the session keeps running, so * the assistant carries on speaking. Unlike `pause`, which suspends the * whole session when Petrinaut closes the panel. */ setMicrophoneMuted: (muted: boolean) => void; + /** Cancels Voice output and hands the live microphone turn to the user. */ + takeTurn?: () => Promise | void; }; /** Stable controls and conversation state supplied to a host-owned Voice mode. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index eaf5dabb078..0e2f843924b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -26,10 +26,23 @@ export const voiceSessionStatusLabel = ( }; export const voiceSessionActionLabels = { + collapse: "Collapse voice session", end: "End voice mode", + expand: "Expand voice session", mute: "Mute microphone", pause: "Pause voice mode", + playbackOptions: "Voice playback options", + readFullResponse: "Read full response", reconnect: "Reconnect voice mode", + repeatQuestion: "Repeat question", resume: "Resume voice mode", + takeTurn: "Your turn", unmute: "Unmute microphone", } as const; + +export const voiceSetupLabels = { + collapse: "Collapse voice setup", + expand: "Expand voice setup", + region: "Voice setup", + status: "Voice setup", +} as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 14c3ffc6656..a680b181f08 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -7,10 +7,11 @@ import { fireEvent, render, screen, + within, waitFor, } from "@testing-library/react"; -import { useEffect } from "react"; -import { afterEach, describe, expect, test, vi } from "vitest"; +import { StrictMode, useEffect } from "react"; +import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS, @@ -32,7 +33,11 @@ import { type SDCPNContextValue, } from "../../../../react/state/sdcpn-context"; import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive-tool"; -import { addMappedToolOutput, AiAssistantPanel } from "./ai-assistant-panel"; +import { + addMappedToolOutput, + AiAssistantPanel, + safelyAddToolOutput, +} from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { @@ -49,6 +54,18 @@ import type { UIMessageChunk } from "ai"; let voiceModeMounts = 0; let voiceModeUnmounts = 0; +beforeAll(() => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); +}); + const emptySDCPN: SDCPN = { places: [], transitions: [], @@ -165,6 +182,7 @@ const renderTestPanel = ({ initialMessage, onInitialInteractionModeConsumed, petriNetDefinition = emptySDCPN, + strictMode = false, }: { aiAssistant: PetrinautAiAssistant; editorContext?: EditorContextValue; @@ -172,6 +190,7 @@ const renderTestPanel = ({ initialMessage?: string; onInitialInteractionModeConsumed?: () => void; petriNetDefinition?: SDCPN; + strictMode?: boolean; }) => { const handle = createJsonDocHandle({ id: "ai-assistant-panel-test", @@ -215,10 +234,14 @@ const renderTestPanel = ({ ); - const rendered = render(renderPanel(aiAssistant, editorContext)); + const rendered = render( + renderPanel(aiAssistant, editorContext), + strictMode ? { wrapper: StrictMode } : undefined, + ); return { ...rendered, + instance, rerenderPanel: ( nextAiAssistant: PetrinautAiAssistant, nextEditorContext = editorContext, @@ -326,6 +349,145 @@ describe("AiAssistantPanel composer submissions", () => { expect(sendMessages).not.toHaveBeenCalled(); }); + test("replays StrictMode effects without stranding an initially recovered tool", async () => { + const sendMessages = vi.fn(async () => + streamChunks([ + ...textChunks("reply", "Done."), + { type: "finish", finishReason: "stop" }, + ]), + ); + renderTestPanel({ + strictMode: true, + aiAssistant: { + conversationId: "strict-history", + messages: [ + { + id: "strict-call", + role: "assistant", + parts: [ + { + type: "tool-readPetrinautDoc", + toolCallId: "strict-read", + state: "input-available", + input: { doc: "ai-assistant" }, + }, + ], + }, + ], + transport: { reconnectToStream: async () => null, sendMessages }, + }, + }); + await waitFor(() => expect(sendMessages).toHaveBeenCalledOnce()); + }); + + test("does not carry a stopped browser generation into a different conversation", async () => { + let latest: PetrinautAiComposerControlContext | undefined; + const sendMessages = vi.fn(async () => + streamChunks([ + ...textChunks("reply", "Done."), + { type: "finish", finishReason: "stop" }, + ]), + ); + const config = (conversationId: string): PetrinautAiAssistant => ({ + conversationId, + messages: [ + { + id: `${conversationId}-call`, + role: "assistant", + parts: [ + { + type: "tool-readPetrinautDoc", + toolCallId: `${conversationId}-read`, + state: "input-available", + input: { doc: "ai-assistant" }, + }, + ], + }, + ], + transport: { reconnectToStream: async () => null, sendMessages }, + requestStop: async () => "already-settled", + renderComposerControl: (context) => { + latest = context; + return null; + }, + }); + const { rerenderPanel } = renderTestPanel({ aiAssistant: config("first") }); + await act(async () => { + await latest?.stop(); + }); + await waitFor(() => expect(latest?.stopped).toBe(true)); + expect(sendMessages).not.toHaveBeenCalled(); + rerenderPanel(config("second")); + await waitFor(() => expect(sendMessages).toHaveBeenCalledOnce()); + expect(sendMessages.mock.calls[0]?.[0].chatId).toBe("second"); + }); + + test("does not deliver a previous conversation's asynchronous browser result into its replacement", async () => { + let releaseLayout: (() => void) | undefined; + const sendMessages = vi.fn(async () => + streamChunks([ + ...textChunks("reply", "Done."), + { type: "finish", finishReason: "stop" }, + ]), + ); + const transport: PetrinautAiTransport = { + reconnectToStream: async () => null, + sendMessages, + }; + const { instance, rerenderPanel } = renderTestPanel({ + aiAssistant: { + conversationId: "old-layout", + messages: [ + { + id: "layout-call", + role: "assistant", + parts: [ + { + type: "tool-applyAutoLayout", + toolCallId: "old-layout", + state: "input-available", + input: { askUserFirst: false }, + }, + ], + }, + ], + transport, + }, + }); + const applyAutoLayout = instance.commands.applyAutoLayout.bind( + instance.commands, + ); + const layout = vi + .spyOn(instance.commands, "applyAutoLayout") + .mockImplementation(async () => { + await new Promise((resolve) => { + releaseLayout = resolve; + }); + return applyAutoLayout(); + }); + await waitFor(() => expect(releaseLayout).toBeDefined()); + rerenderPanel({ + conversationId: "replacement", + messages: [ + { + id: "replacement-history", + role: "assistant", + parts: [{ type: "text", text: "Settled replacement" }], + }, + ], + transport, + }); + await act(async () => { + releaseLayout?.(); + await layout.mock.results[0]?.value; + // Drain the explicit continuation timer following the awaited command. + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + }); + expect(sendMessages).not.toHaveBeenCalled(); + }); + test("executes one automatic tool call recovered from host history", async () => { const requestMessages: PetrinautAiMessage[][] = []; const sendMessages = vi.fn( @@ -1408,6 +1570,181 @@ describe("AiAssistantPanel composer submissions", () => { expect(voiceModeUnmounts).toBe(0); }); + test("forwards optional host Voice actions to the production dock", async () => { + const takeTurn = vi.fn(); + const repeatQuestion = vi.fn(); + const readFullResponse = vi.fn(); + const VoiceMode = ({ + context, + replayAllowed, + }: { + context: PetrinautAiVoiceModeContext; + replayAllowed: boolean; + }) => { + const { registerVoiceModeControls, reportVoiceSessionState } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + readFullResponse, + reconnect: vi.fn(), + repeatQuestion, + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + takeTurn, + }), + [registerVoiceModeControls], + ); + useEffect(() => { + reportVoiceSessionState({ + canReadFullResponse: replayAllowed, + canRepeatQuestion: replayAllowed, + canTakeTurn: true, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "speaking", + }); + return () => reportVoiceSessionState(null); + }, [replayAllowed, reportVoiceSessionState]); + + return null; + }; + const aiAssistant = (replayAllowed: boolean): PetrinautAiAssistant => ({ + renderVoiceMode: (context) => ( + + ), + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }); + + const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); + + fireEvent.click(await screen.findByRole("button", { name: "Your turn" })); + expect(takeTurn).toHaveBeenCalledOnce(); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestionItem = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + expect(repeatQuestionItem.getAttribute("aria-disabled")).not.toBe("true"); + const repeatQuestionMenu = screen.getByRole("menu"); + fireEvent.keyDown(repeatQuestionMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(repeatQuestionMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestionItem.id, + ), + ); + fireEvent.keyDown(repeatQuestionMenu, { key: "Enter" }); + await waitFor(() => expect(repeatQuestion).toHaveBeenCalledOnce()); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponseItem = await screen.findByRole("menuitem", { + name: "Read full response", + }); + expect(readFullResponseItem.getAttribute("aria-disabled")).not.toBe("true"); + const readFullResponseMenu = screen.getByRole("menu"); + fireEvent.keyDown(readFullResponseMenu, { key: "End" }); + await waitFor(() => + expect(readFullResponseMenu.getAttribute("aria-activedescendant")).toBe( + readFullResponseItem.id, + ), + ); + fireEvent.keyDown(readFullResponseMenu, { key: "Enter" }); + await waitFor(() => expect(readFullResponse).toHaveBeenCalledOnce()); + + rendered.rerenderPanel(aiAssistant(false), editorContextValue); + fireEvent.click( + await screen.findByRole("button", { name: "Voice playback options" }), + ); + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).getAttribute("aria-disabled"), + ).toBe("true"); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + }); + + test("retires missing and unmounted optional host Voice actions", async () => { + const VoiceMode = ({ + context, + }: { + context: PetrinautAiVoiceModeContext; + }) => { + const { registerVoiceModeControls, reportVoiceSessionState } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }), + [registerVoiceModeControls], + ); + useEffect(() => { + reportVoiceSessionState({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "speaking", + }); + return () => reportVoiceSessionState(null); + }, [reportVoiceSessionState]); + + return null; + }; + const aiAssistant = (mounted: boolean): PetrinautAiAssistant => ({ + renderVoiceMode: (context) => + mounted ? : null, + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }); + const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); + + expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); + fireEvent.click( + await screen.findByRole("button", { name: "Voice playback options" }), + ); + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).getAttribute("aria-disabled"), + ).toBe("true"); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + + rendered.rerenderPanel(aiAssistant(false), editorContextValue); + + await waitFor(() => + expect( + screen.queryByRole("region", { name: "Voice session" }), + ).toBeNull(), + ); + }); + test("ends active Voice mode when the unified composer returns to text", () => { voiceModeMounts = 0; voiceModeUnmounts = 0; @@ -1455,6 +1792,10 @@ describe("AiAssistantPanel composer submissions", () => { fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); expect(screen.getByText("Voice mode voice")).not.toBeNull(); + expect(screen.queryByRole("region", { name: "Voice setup" })).toBeNull(); + expect( + screen.getByRole("textbox", { name: "Message AI assistant" }), + ).not.toBeNull(); fireEvent.click(screen.getByRole("button", { name: "Voice mode voice" })); expect( @@ -1466,7 +1807,7 @@ describe("AiAssistantPanel composer submissions", () => { expect(sendMessages).not.toHaveBeenCalled(); }); - test("defers and consumes an initial Voice mode once, then falls back to text", () => { + test("opens initial Voice setup compact once, then falls back to text", () => { let latestInputMode = "text"; const onInitialInteractionModeConsumed = vi.fn(); const aiAssistant: PetrinautAiAssistant = { @@ -1497,6 +1838,27 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestInputMode).toBe("voice"); expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + expect(screen.getByText("Voice mode")).not.toBeNull(); + const composer = screen.getByRole("textbox", { + hidden: true, + name: "Message AI assistant", + }); + const composerWrap = composer.closest("form")?.parentElement; + expect(composerWrap?.className).toContain("d_none"); + + const setupDock = screen.getByRole("region", { name: "Voice setup" }); + fireEvent.click( + within(setupDock).getByRole("button", { name: "Expand voice setup" }), + ); + + expect(screen.queryByRole("region", { name: "Voice setup" })).toBeNull(); + expect(composerWrap?.className).not.toContain("d_none"); + expect(screen.getByRole("textbox", { name: "Message AI assistant" })).toBe( + composer, + ); + expect( + screen.getByRole("button", { name: "Close AI assistant" }), + ).not.toBeNull(); const unavailableAssistant: PetrinautAiAssistant = { transport: aiAssistant.transport, @@ -1510,6 +1872,78 @@ describe("AiAssistantPanel composer submissions", () => { expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); }); + test("ends collapsed Voice and closes the panel without pausing", async () => { + const events: string[] = []; + const endVoice = vi.fn(async () => { + events.push("end"); + }); + const pauseVoice = vi.fn(() => events.push("pause")); + const setAiAssistantOpen = vi.fn(() => events.push("close")); + const VoiceMode = ({ + context, + }: { + context: PetrinautAiVoiceModeContext; + }) => { + const { + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: endVoice, + pause: pauseVoice, + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }), + [registerVoiceModeControls], + ); + useEffect(() => { + if (inputMode !== "voice") { + return; + } + setVoiceActive(true); + reportVoiceSessionState({ + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "listening", + }); + }, [inputMode, reportVoiceSessionState, setVoiceActive]); + + return null; + }; + + renderTestPanel({ + aiAssistant: { + renderVoiceMode: (context) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }, + editorContext: { + ...editorContextValue, + setAiAssistantOpen, + }, + initialInteractionMode: "voice", + }); + + const dock = await screen.findByRole("region", { name: "Voice session" }); + fireEvent.click( + within(dock).getByRole("button", { name: "End voice mode" }), + ); + + expect(events).toEqual(["end", "close"]); + expect(endVoice).toHaveBeenCalledOnce(); + expect(pauseVoice).not.toHaveBeenCalled(); + expect(setAiAssistantOpen).toHaveBeenCalledWith(false); + }); + test("accepts one voice input while generic chat is streaming and submits it after settlement", async () => { let firstStreamController: | ReadableStreamDefaultController @@ -1612,7 +2046,10 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestVoiceContext?.status).toBe("streaming"); expect(latestVoiceContext?.canAcceptVoiceInput).toBe(true); expect(requests[1]?.at(-1)).toMatchObject({ - metadata: { source: "voice", toolCallId: "queued-question" }, + metadata: { + source: "voice", + voiceToolCallIds: ["queued-question"], + }, role: "assistant", }); expect( @@ -2039,9 +2476,9 @@ describe("AiAssistantPanel composer submissions", () => { streamController?.enqueue({ type: "text-end", id: "preamble" }); streamController?.enqueue({ type: "tool-input-available", - toolCallId: "net-read-1", - toolName: "getLatestNetDefinition", - input: {}, + toolCallId: "stopped-mutation", + toolName: "updatePlace", + input: { placeId: "place-1", update: { name: "MustNotApply" } }, }); streamController?.enqueue({ type: "finish-step" }); streamController?.enqueue({ type: "finish", finishReason: "tool-calls" }); @@ -2066,8 +2503,12 @@ describe("AiAssistantPanel composer submissions", () => { await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); expect(await screen.findByText("Response stopped")).not.toBeNull(); - // Let any automatic follow-up the SDK might schedule drain first. + // Let both deferred execution and any follow-up drain: withholding only + // the send is insufficient if the mutation already ran after Stop. await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + expect(testInstances.at(-1)?.definition.get().places[0]?.name).toBe( + "PlaceOne", + ); expect(sendMessages).toHaveBeenCalledOnce(); expect(screen.getByRole("button", { name: "Send message" })).toHaveProperty( "disabled", @@ -2075,6 +2516,77 @@ describe("AiAssistantPanel composer submissions", () => { ); }); + test("reports a textless automatic browser failure to hosts and its matching tool", async () => { + let latest: PetrinautAiComposerControlContext | undefined; + const sendMessages = vi.fn(async () => + streamChunks([ + { type: "start-step" }, + { + type: "tool-input-available", + toolCallId: "invalid-doc", + toolName: "readPetrinautDoc", + input: { doc: "not-a-guide-page" }, + }, + { type: "finish-step" }, + { type: "finish", finishReason: "tool-calls" }, + ]), + ); + renderTestPanel({ + aiAssistant: { + transport: { reconnectToStream: async () => null, sendMessages }, + renderComposerControl: (context) => { + latest = context; + return null; + }, + }, + initialMessage: "Read the guide", + }); + await waitFor(() => expect(latest?.status).toBe("error")); + await waitFor(() => + expect(latest?.messages.at(-1)?.parts).toContainEqual( + expect.objectContaining({ + toolCallId: "invalid-doc", + state: "output-error", + errorText: expect.any(String) as unknown, + }), + ), + ); + expect(sendMessages).toHaveBeenCalledOnce(); + }); + + test("does not execute tools from a durably stopped reopened response", async () => { + const sendMessages = vi.fn(async () => + streamChunks([]), + ); + renderTestPanel({ + aiAssistant: { + messages: [ + { + id: "stopped", + role: "assistant", + metadata: { stopped: true }, + parts: [ + { + type: "tool-updatePlace", + toolCallId: "stopped-mutation", + state: "input-available", + input: { placeId: "place-1", update: { name: "MustNotApply" } }, + }, + ], + }, + ], + transport: { reconnectToStream: async () => null, sendMessages }, + }, + petriNetDefinition: nonEmptySDCPN, + }); + await act(() => new Promise((resolve) => setTimeout(resolve, 30))); + expect(testInstances.at(-1)?.definition.get().places[0]?.name).toBe( + "PlaceOne", + ); + expect(sendMessages).not.toHaveBeenCalled(); + expect(screen.getByText("Response stopped")).not.toBeNull(); + }); + test("keeps hosts seeing a busy conversation between a tool-calls step and its follow-up", async () => { const observedStatuses: PetrinautAiComposerControlContext["status"][] = []; let requestCount = 0; @@ -2785,6 +3297,51 @@ describe("AiAssistantPanel composer submissions", () => { ); }); + test("preserves the already-normalized Voice payload at the panel boundary", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requestMessages.push(structuredClone(messages)); + return Promise.resolve( + streamChunks(textChunks("voice-response", "Voice message accepted")), + ); + }), + }; + + renderTestPanel({ + aiAssistant: { + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit normalized Voice payload" }), + ); + await screen.findByText("Voice message accepted"); + + expect(requestMessages[0]?.at(-1)).toMatchObject({ + id: "voice-realtime:3:item-1:0", + metadata: { source: "voice" }, + parts: [{ text: " Already normalized upstream ", type: "text" }], + role: "user", + }); + }); + test("marks the exact pending tool as voice-origin without a user message", async () => { const requestMessages: PetrinautAiMessage[][] = []; const onMessages = vi.fn(); @@ -2863,7 +3420,10 @@ describe("AiAssistantPanel composer submissions", () => { ), ); expect(containingMessage).toMatchObject({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }); expect( containingMessage?.parts.find( @@ -2887,12 +3447,139 @@ describe("AiAssistantPanel composer submissions", () => { expect(onMessages.mock.lastCall?.[0]).toEqual( expect.arrayContaining([ expect.objectContaining({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }), ]), ); }); + test("retains every voice tool origin on one assistant message", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + const addToolOutput = vi.fn().mockResolvedValue(undefined); + + for (const toolCallId of ["voice-question-1", "voice-question-2"]) { + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: toolCallId }, + tool: "answerQuestion", + toolCallId, + }, + source: "voice", + updateMessages, + }); + } + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-1", "voice-question-2"], + }); + }); + + test("preserves sibling voice provenance when another tool output rejects", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + let rejectFirstSubmission: ((reason?: unknown) => void) | undefined; + const addToolOutput = vi + .fn() + .mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectFirstSubmission = reject; + }), + ) + .mockResolvedValueOnce(undefined); + + const firstSubmission = addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The shift lead" }, + tool: "answerQuestion", + toolCallId: "voice-question-1", + }, + source: "voice", + updateMessages, + }); + const firstSubmissionRejection = expect(firstSubmission).rejects.toThrow( + "First voice tool output rejected.", + ); + + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The release manager" }, + tool: "answerQuestion", + toolCallId: "voice-question-2", + }, + source: "voice", + updateMessages, + }); + rejectFirstSubmission?.(new Error("First voice tool output rejected.")); + await firstSubmissionRejection; + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-2"], + }); + }); + test("rolls back failed tool provenance before a typed retry", async () => { let latestMessages = [ { @@ -2967,6 +3654,34 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestMessages[0]?.metadata).toBeUndefined(); }); + test("reports browser tool-output rejections through the AI SDK error state", async () => { + const addToolOutput = vi + .fn() + .mockRejectedValueOnce(new Error("The browser tool rejected its output.")) + .mockResolvedValueOnce(undefined); + + safelyAddToolOutput( + addToolOutput as Parameters[0], + { + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + output: { + definition: emptySDCPN, + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + title: "Failure fixture", + }, + }, + ); + + await waitFor(() => expect(addToolOutput).toHaveBeenCalledTimes(2)); + expect(addToolOutput).toHaveBeenLastCalledWith({ + errorText: "The browser tool rejected its output.", + state: "output-error", + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + }); + }); + test("sends review chips as messages while an interactive tool is pending", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index ffc2d9f1495..0979fd72005 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -148,6 +148,7 @@ const hasRunnableStaticToolCalls = ( const message = messages.at(-1); return ( message?.role === "assistant" && + !message.metadata?.stopped && message.parts.some((part) => isRunnableStaticToolPart(part)) ); }; @@ -166,10 +167,30 @@ const markVoiceToolOrigin = ( ): PetrinautAiMessage[] => messages.map((message) => message.id === messageId - ? { - ...message, - metadata: { ...message.metadata, source: "voice", toolCallId }, - } + ? (() => { + const previousToolCallIds = + message.metadata?.source === "voice" + ? [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ] + : []; + const { toolCallId: _legacyToolCallId, ...previousMetadata } = + message.metadata ?? {}; + + return { + ...message, + metadata: { + ...previousMetadata, + source: "voice", + voiceToolCallIds: [ + ...new Set([...previousToolCallIds, toolCallId]), + ], + }, + }; + })() : message, ); @@ -182,7 +203,25 @@ const isPetrinautAiCommandToolName = ( toolName: string, ): toolName is AiCommandActionName => toolName in aiCommandActionInputSchemas; -const safelyAddToolOutput = ( +const browserToolErrorText = (error: unknown): string => { + if (error instanceof Error && error.message.trim().length > 0) { + return error.message; + } + if (typeof error === "string" && error.trim().length > 0) { + return error; + } + try { + const serialized: unknown = JSON.stringify(error); + if (typeof serialized === "string" && serialized.length > 0) { + return serialized; + } + } catch { + // Fall through to the stable fallback for cyclic values. + } + return "The browser tool failed."; +}; + +export const safelyAddToolOutput = ( addToolOutput: ReturnType< typeof useChat >["addToolOutput"], @@ -190,10 +229,16 @@ const safelyAddToolOutput = ( ReturnType>["addToolOutput"] >[0], ) => { - // Failures here surface in the UI as an errored tool call (with the - // error message on hover), so we just swallow the rejection to avoid an - // unhandled-promise warning. - void Promise.resolve(addToolOutput(params)).catch(() => {}); + void Promise.resolve(addToolOutput(params)).catch((error: unknown) => { + void Promise.resolve( + addToolOutput({ + errorText: browserToolErrorText(error), + state: "output-error", + tool: params.tool, + toolCallId: params.toolCallId, + }), + ).catch(() => {}); + }); }; const addDynamicToolOutput = ( @@ -258,8 +303,42 @@ export const addMappedToolOutput = async ({ latestMessages.map((message) => message.id === containingMessage.id && message.metadata?.source === "voice" && - message.metadata.toolCallId === params.toolCallId - ? { ...message, metadata: previousMetadata } + (message.metadata.voiceToolCallIds?.includes(params.toolCallId) === + true || + message.metadata.toolCallId === params.toolCallId) + ? (() => { + const attributionAlreadyPresent = + previousMetadata?.source === "voice" && + (previousMetadata.voiceToolCallIds?.includes( + params.toolCallId, + ) === true || + previousMetadata.toolCallId === params.toolCallId); + const voiceToolCallIds = [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ]; + const remainingVoiceToolCallIds = attributionAlreadyPresent + ? voiceToolCallIds + : voiceToolCallIds.filter( + (candidateToolCallId) => + candidateToolCallId !== params.toolCallId, + ); + if (remainingVoiceToolCallIds.length === 0) { + return { ...message, metadata: previousMetadata }; + } + const { toolCallId: _legacyToolCallId, ...metadata } = + message.metadata; + + return { + ...message, + metadata: { + ...metadata, + voiceToolCallIds: [...new Set(remainingVoiceToolCallIds)], + }, + }; + })() : message, ), ); @@ -372,12 +451,19 @@ const ConversationAiAssistantPanel = ({ const [composerFocusRequest, setComposerFocusRequest] = useState(0); const [interactionMode, setInteractionMode] = useState("text"); + const [voiceDockCollapsed, setVoiceDockCollapsed] = useState(false); const interactionModeRef = useRef("text"); const selectInteractionMode = useCallback( - (nextMode: PetrinautAiInputMode) => { + ( + nextMode: PetrinautAiInputMode, + options: { collapseVoiceDock?: boolean } = {}, + ) => { const previousMode = interactionModeRef.current; interactionModeRef.current = nextMode; setInteractionMode(nextMode); + setVoiceDockCollapsed( + nextMode === "voice" && options.collapseVoiceDock === true, + ); if (previousMode === "voice" && nextMode === "text") { setComposerFocusRequest((request) => request + 1); } @@ -530,9 +616,16 @@ const ConversationAiAssistantPanel = ({ // invalidates the host's active generation. end: () => requestInputMode("text"), pause: () => controls.pause(), + ...(controls.readFullResponse + ? { readFullResponse: () => controls.readFullResponse?.() } + : {}), reconnect: () => controls.reconnect(), + ...(controls.repeatQuestion + ? { repeatQuestion: () => controls.repeatQuestion?.() } + : {}), resume: () => controls.resume(), setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), + ...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}), }); return () => { @@ -547,13 +640,21 @@ const ConversationAiAssistantPanel = ({ ); const stopRequestedRef = useRef(false); - // Advances on every composer submission so an asynchronous Stop can tell - // whether the turn it was pressed for is still the current one. + // Advances on composer submissions and conversation changes so late work + // cannot settle, stop, or continue a replacement turn. const submissionGenerationRef = useRef(0); + const toolHostIdentityRef = useRef(null); const pendingSubmissionRecoveryRef = useRef<(() => void) | null>(null); const hydratedConversationIdRef = useRef(null); const automaticToolCallExecutionsRef = useRef(new Set()); const pendingAutomaticToolCallExecutionsRef = useRef(new Set()); + const automaticToolTerminationRef = useRef<{ + generation: number; + kind: "stopped" | "failed"; + } | null>(null); + const automaticToolExecutionTimersRef = useRef( + new Map, string>(), + ); // AI SDK's implicit addToolOutput continuation races its stream-to-ready // cleanup. Static browser tools instead await output and explicitly continue. const automaticToolContinuationTimerRef = useRef { + const termination = automaticToolTerminationRef.current; + if (termination?.generation !== generation) return false; + if (termination.kind === "stopped") withholdContinuationForStop(); + return true; + }; + const automaticToolTurnIsTerminatedRef = useLatest( + automaticToolTurnIsTerminated, + ); + const addAutomaticToolOutputForGeneration = async ( params: Parameters< ReturnType>["addToolOutput"] >[0], + generation: number, + executionConversationId: string, ): Promise => { + const executionKey = `${executionConversationId}:${params.toolCallId}`; + const canContinue = () => + generation === submissionGenerationRef.current && + executionConversationId === toolHostIdentityRef.current && + !automaticToolTurnIsTerminated(generation); + if (!canContinue()) { + pendingAutomaticToolCallExecutionsRef.current.delete(executionKey); + return; + } const currentAddToolOutput = addToolOutputRef.current; if (currentAddToolOutput === null) { throw new Error("The AI assistant tool host is not ready."); } - const executionKey = `${aiAssistant.conversationId ?? "local"}:${params.toolCallId}`; pendingAutomaticToolCallExecutionsRef.current.delete(executionKey); // Prevent addToolOutput's fire-and-forget continuation from racing the // explicit continuation chained to its promise below. @@ -600,11 +720,26 @@ const ConversationAiAssistantPanel = ({ throw caught; }, ); + + if (!canContinue()) return; }; const executeToolCall: ChatOnToolCallCallback = async ({ toolCall, }) => { + const generation = submissionGenerationRef.current; + const executionConversationId = toolHostIdentityRef.current; + if (executionConversationId === null) { + throw new Error("The AI assistant tool host is not ready."); + } + const addAutomaticToolOutput = ( + params: Parameters[0], + ) => + addAutomaticToolOutputForGeneration( + params, + generation, + executionConversationId, + ); if (!instance) { throw new Error( "The AI assistant cannot run without an editor instance.", @@ -885,6 +1020,7 @@ const ConversationAiAssistantPanel = ({ toolCall.dynamic ? executeToolCall({ toolCall }) : undefined, }); useLayoutEffect(() => { + toolHostIdentityRef.current = conversationId; addToolOutputRef.current = addToolOutput; sendAutomaticToolContinuationRef.current = () => sendMessage(); return () => { @@ -892,8 +1028,9 @@ const ConversationAiAssistantPanel = ({ addToolOutputRef.current = null; } sendAutomaticToolContinuationRef.current = null; + toolHostIdentityRef.current = null; }; - }, [addToolOutput, sendMessage]); + }, [addToolOutput, conversationId, sendMessage]); useEffect(() => { if ( chatStatus !== "ready" || @@ -905,10 +1042,14 @@ const ConversationAiAssistantPanel = ({ return; } + const generation = submissionGenerationRef.current; automaticToolContinuationTimerRef.current = setTimeout(() => { automaticToolContinuationTimerRef.current = null; - if (stopRequestedRef.current) { - withholdContinuationForStop(); + if ( + generation !== submissionGenerationRef.current || + toolHostIdentityRef.current !== conversationId || + automaticToolTurnIsTerminatedRef.current(generation) + ) { return; } const sendContinuation = sendAutomaticToolContinuationRef.current; @@ -918,25 +1059,56 @@ const ConversationAiAssistantPanel = ({ return; } void sendContinuation().catch((caught: unknown) => { + if (generation !== submissionGenerationRef.current) return; setContinuationPending(false); setStreamError( caught instanceof Error ? caught : new Error(String(caught)), ); }); }, 0); - }, [chatStatus, continuationPending, messages]); + }, [ + automaticToolTurnIsTerminatedRef, + chatStatus, + continuationPending, + conversationId, + messages, + ]); useEffect( () => () => { + for (const [ + timer, + executionKey, + ] of automaticToolExecutionTimersRef.current) { + clearTimeout(timer); + // Cancelled-before-start work is claimable on StrictMode's next setup. + automaticToolCallExecutionsRef.current.delete(executionKey); + pendingAutomaticToolCallExecutionsRef.current.delete(executionKey); + } + automaticToolExecutionTimersRef.current.clear(); if (automaticToolContinuationTimerRef.current !== null) { clearTimeout(automaticToolContinuationTimerRef.current); + automaticToolContinuationTimerRef.current = null; } }, - [], + [conversationId], ); const executeToolCallRef = useLatest(executeToolCall); - + const submissionConversationIdRef = useRef(conversationId); + useLayoutEffect(() => { + if (submissionConversationIdRef.current === conversationId) return; + submissionConversationIdRef.current = conversationId; + submissionGenerationRef.current += 1; + stopRequestedRef.current = false; + setContinuationPending(false); + setStreamError(null); + setStopped(false); + }, [conversationId]); const status: PetrinautAiComposerStatus = - continuationPending && chatStatus === "ready" ? "submitted" : chatStatus; + chatStatus === "ready" && streamError !== null + ? "error" + : continuationPending && chatStatus === "ready" + ? "submitted" + : chatStatus; useEffect(() => { if ( @@ -975,6 +1147,7 @@ const ConversationAiAssistantPanel = ({ } for (const message of messages) { + if (message.metadata?.stopped) continue; for (const part of message.parts) { if (!isRunnableStaticToolPart(part)) { continue; @@ -985,32 +1158,79 @@ const ConversationAiAssistantPanel = ({ input: part.input, toolCallId: part.toolCallId, toolName: getStaticToolName(part), - } as PetrinautAiToolCall; - const executionKey = `${aiAssistant.conversationId ?? "local"}:${toolCall.toolCallId}`; + } as Extract; + const executionKey = `${conversationId}:${toolCall.toolCallId}`; if (automaticToolCallExecutionsRef.current.has(executionKey)) continue; automaticToolCallExecutionsRef.current.add(executionKey); pendingAutomaticToolCallExecutionsRef.current.add(executionKey); - // React can publish the ready render before AI SDK has finished its - // internal request cleanup. Cross that boundary before updating the - // message; addAutomaticToolOutput then submits the explicit continuation. - setTimeout(() => { - // A resumed hydrated call has no onFinish to mark the turn busy. - setContinuationPending(true); - void Promise.resolve(executeToolCallRef.current({ toolCall })).catch( - (caught: unknown) => { + const generation = submissionGenerationRef.current; + // Hydrated calls have no onFinish; claim both their execution and busy + // state before scheduling, so another render cannot queue them again. + setContinuationPending(true); + const timer = setTimeout(() => { + automaticToolExecutionTimersRef.current.delete(timer); + if ( + generation !== submissionGenerationRef.current || + toolHostIdentityRef.current !== conversationId + ) { + pendingAutomaticToolCallExecutionsRef.current.delete(executionKey); + return; + } + if (automaticToolTurnIsTerminatedRef.current(generation)) { + pendingAutomaticToolCallExecutionsRef.current.delete(executionKey); + return; + } + void Promise.resolve() + .then(() => executeToolCallRef.current({ toolCall })) + .catch(async (caught: unknown) => { pendingAutomaticToolCallExecutionsRef.current.delete( executionKey, ); + if ( + generation !== submissionGenerationRef.current || + toolHostIdentityRef.current !== conversationId + ) + return; + if (automaticToolTurnIsTerminatedRef.current(generation)) return; + automaticToolTerminationRef.current = { + generation, + kind: "failed", + }; setContinuationPending(false); setStreamError( - caught instanceof Error ? caught : new Error(String(caught)), + caught instanceof Error + ? caught + : new Error(browserToolErrorText(caught)), ); - }, - ); + // A static failure belongs to this call, not just the toast. Do + // not let recording its error trigger an implicit continuation. + suppressedAutomaticSendsRef.current += 1; + await Promise.resolve() + .then(() => + addToolOutputRef.current?.({ + tool: toolCall.toolName, + toolCallId: toolCall.toolCallId, + state: "output-error", + errorText: browserToolErrorText(caught), + }), + ) + .catch(() => {}) + .then(() => { + suppressedAutomaticSendsRef.current -= 1; + }); + }); }, 0); + automaticToolExecutionTimersRef.current.set(timer, executionKey); } } - }, [aiAssistant.conversationId, chatStatus, executeToolCallRef, messages]); + }, [ + automaticToolTurnIsTerminatedRef, + chatStatus, + conversationId, + executeToolCallRef, + messages, + toolHostIdentityRef, + ]); const composerSubmissionStateRef = useLatest({ addToolOutput, @@ -1054,8 +1274,8 @@ const ConversationAiAssistantPanel = ({ target?: "auto" | "message"; text: string; }): Promise => { - const trimmed = text.trim(); - if (!trimmed) { + const submissionText = source === "voice" ? text : text.trim(); + if (!submissionText.trim()) { const submissionError = new Error( "AI assistant text must not be empty.", ); @@ -1134,7 +1354,7 @@ const ConversationAiAssistantPanel = ({ try { output = mappedToolCall.mapText({ input: mappedToolCall.input, - text: trimmed, + text: submissionText, }); } catch (caught) { const submissionError = @@ -1188,7 +1408,7 @@ const ConversationAiAssistantPanel = ({ await submitMessage({ id: messageId, ...(source === "voice" ? { metadata: { source } } : {}), - parts: [{ text: trimmed, type: "text" }], + parts: [{ text: submissionText, type: "text" }], role: "user", }); return { kind: "message", messageId }; @@ -1298,6 +1518,7 @@ const ConversationAiAssistantPanel = ({ } const generation = submissionGenerationRef.current; + automaticToolTerminationRef.current = { generation, kind: "stopped" }; stopRequestedRef.current = true; if (requestStop !== undefined) { try { @@ -1421,12 +1642,14 @@ const ConversationAiAssistantPanel = ({ return; } - selectInteractionMode( + const nextMode = initialInteractionMode === "voice" && - aiAssistant.renderVoiceMode === undefined + aiAssistant.renderVoiceMode === undefined ? "text" - : initialInteractionMode, - ); + : initialInteractionMode; + selectInteractionMode(nextMode, { + collapseVoiceDock: nextMode === "voice", + }); consumedInitialInteractionModeRef.current = initialInteractionMode; onInitialInteractionModeConsumed?.(); }, [ @@ -1499,6 +1722,7 @@ const ConversationAiAssistantPanel = ({ conversationId, messages, status, + stopped, stop: stopComposer, submitText, }; @@ -1535,6 +1759,7 @@ const ConversationAiAssistantPanel = ({ isOpen={isAiAssistantOpen} messages={messages} onClearMessages={() => { + submissionGenerationRef.current += 1; // Clearing aborts any in-flight response too, which fires `onFinish` // with `isAbort`. Drop the stop flag first so that handler treats this // as an incidental abort and doesn't repopulate or persist the @@ -1555,6 +1780,7 @@ const ConversationAiAssistantPanel = ({ voiceModeControlsRef.current?.pause(); setAiAssistantOpen(false); }} + onCollapsedVoiceEnd={() => setAiAssistantOpen(false)} onInputChange={setInput} onInputModeChange={selectInteractionMode} onInteractiveToolSubmit={({ toolCallId, toolName, output }) => { @@ -1633,11 +1859,13 @@ const ConversationAiAssistantPanel = ({ void stopComposer(); }} onSubmit={submitComposerInput} + onVoiceDockCollapsedChange={setVoiceDockCollapsed} promptChips={promptChips} rightOffset={hasSelection ? propertiesPanelWidth + PANEL_MARGIN : 0} status={status} stopped={stopped} voiceHandoffPending={voiceHandoffPending} + voiceDockCollapsed={voiceDockCollapsed} voiceMode={voiceMode} voiceModeAvailable={aiAssistant.renderVoiceMode !== undefined} /> diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c21fe5af927..c6e9c98b7b3 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -1,4 +1,5 @@ import { type ReactNode, useState } from "react"; +import { userEvent, within } from "storybook/test"; import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; @@ -244,6 +245,7 @@ const HostVoiceSlotPreview = () => ( const Frame = ({ error, + initialVoiceDockCollapsed = false, inputMode = "text", messages, status = "ready", @@ -253,6 +255,7 @@ const Frame = ({ voiceSession, }: { error?: Error; + initialVoiceDockCollapsed?: boolean; inputMode?: "text" | "voice"; messages: PetrinautAiMessage[]; status?: "submitted" | "streaming" | "ready" | "error"; @@ -262,6 +265,9 @@ const Frame = ({ voiceSession?: PetrinautAiVoiceSessionState; }) => { const [input, setInput] = useState(""); + const [voiceDockCollapsed, setVoiceDockCollapsed] = useState( + initialVoiceDockCollapsed, + ); // Stands in for the host, which reports session state rather than rendering // the live surfaces itself. const [voiceSessionStore] = useState(() => { @@ -291,8 +297,10 @@ const Frame = ({ onInputModeChange={() => {}} onStop={() => {}} onSubmit={() => setInput("")} + onVoiceDockCollapsedChange={setVoiceDockCollapsed} status={status} stopped={stopped} + voiceDockCollapsed={voiceDockCollapsed} voiceMode={voiceMode} voiceModeAvailable={voiceModeAvailable} /> @@ -330,6 +338,18 @@ export const VoiceModeAwaitingConsent: Story = { ), }; +export const VoiceModeAwaitingConsentCompact: Story = { + render: () => ( + } + voiceModeAvailable + /> + ), +}; + export const VoiceSessionListening: Story = { render: () => ( ( + } + voiceModeAvailable + voiceSession={liveSession({ microphoneLevel: 0.6 })} + /> + ), + play: async ({ canvasElement }) => { + await userEvent.click( + within(canvasElement).getByRole("button", { + name: "Collapse voice session", + }), + ); + }, +}; + export const VoiceSessionSpeaking: Story = { render: () => ( + ...singleToolCallMessage, + parts: singleToolCallMessage.parts.map((part) => part.type.startsWith("tool-") ? { ...part, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ee2d2d8c6f7..7bfb76a8df4 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -44,26 +44,84 @@ vi.mock("react-markdown", async (importOriginal) => { }); const noop = () => {}; +const initialClipboardDescriptor = Object.getOwnPropertyDescriptor( + navigator, + "clipboard", +); // The voice ribbon asks for a 2D context on mount. jsdom has no canvas, and // answering with `null` takes the same branch a browser without one would, // instead of letting jsdom log a not-implemented error per render. beforeAll(() => { vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); }); afterEach(() => { cleanup(); vi.clearAllMocks(); vi.useRealTimers(); + if (initialClipboardDescriptor === undefined) { + Reflect.deleteProperty(navigator, "clipboard"); + } else { + Object.defineProperty(navigator, "clipboard", initialClipboardDescriptor); + } }); describe("AiAssistantContents", () => { + test("labels stopped history after a later completed reply without global Stop state", () => { + render( + + + , + ); + expect(screen.getByText("Partial reply")).not.toBeNull(); + expect(screen.getByText("Later completed reply")).not.toBeNull(); + expect(screen.getAllByText("Response stopped")).toHaveLength(1); + }); + test("shows assistant errors as toasts instead of transcript messages", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); render( { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe("Failed to fetch"); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("AI assistant error"); + expect( + toast.querySelector('[data-scope="toast"][data-part="description"]') + ?.textContent, + ).toBe( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); + fireEvent.click( + within(toast).getByRole("button", { name: "Copy details" }), + ); + expect(writeText).toHaveBeenCalledWith( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); expect( within(screen.getByTestId("ai-transcript")).queryByText( - "Failed to fetch", + "AI assistant error", ), ).toBeNull(); + fireEvent.click( + within(toast).getByRole("button", { name: "Close notification" }), + ); + await waitFor(() => + expect(toast.getAttribute("data-state")).toBe("closed"), + ); }); test("keeps one Voice mode slot mounted above the composer when the panel closes", () => { @@ -137,8 +216,9 @@ describe("AiAssistantContents", () => { expect(voiceModeUnmounts).toBe(0); }); - test("swaps the composer for the dock while a session runs, and defers its spoken turns", () => { + test("shows spoken turns live and collapses an active session without unmounting the panel", () => { const store = createVoiceSessionStore(); + const onCollapsedVoiceEnd = vi.fn(); const actions = { end: vi.fn(), pause: vi.fn(), @@ -160,21 +240,55 @@ describe("AiAssistantContents", () => { parts: [{ type: "text", text: "Earlier answer" }], }, ] as PetrinautAiMessage[]; - const renderWith = (messages: PetrinautAiMessage[]) => ( - - - - ); + const liveMessages = [ + ...earlierMessages, + { + id: "spoken-user", + metadata: { source: "voice" }, + role: "user", + parts: [{ type: "text", text: "Spoken request" }], + }, + { + id: "spoken-assistant", + role: "assistant", + parts: [{ type: "text", text: "Spoken reply" }], + }, + { + id: "typed-user", + role: "user", + parts: [{ type: "text", text: "Typed aside" }], + }, + ] as PetrinautAiMessage[]; + const VoiceContents = ({ + inputMode = "voice", + messages, + }: { + inputMode?: "text" | "voice"; + messages: PetrinautAiMessage[]; + }) => { + const [collapsed, setCollapsed] = useState(false); + + return ( + + Host Voice controls} + /> + + ); + }; - const { rerender } = render(renderWith(earlierMessages)); + const { rerender } = render(); const dock = screen.getByRole("region", { name: "Voice session" }); expect(within(dock).getByText("Listening")).not.toBeNull(); @@ -182,69 +296,139 @@ describe("AiAssistantContents", () => { screen.queryByRole("textbox", { name: "Message AI assistant" }), ).toBeNull(); expect(screen.getByText("Earlier answer")).not.toBeNull(); + expect( + within(dock) + .getByRole("button", { name: "Collapse voice session" }) + .getAttribute("aria-expanded"), + ).toBeNull(); - rerender( - renderWith([ - ...earlierMessages, - { - id: "spoken-user", - metadata: { source: "voice" }, - role: "user", - parts: [{ type: "text", text: "Spoken request" }], - }, - { - id: "spoken-assistant", - role: "assistant", - parts: [{ type: "text", text: "Spoken reply" }], - }, - { - id: "typed-user", - role: "user", - parts: [{ type: "text", text: "Typed aside" }], - }, - ] as PetrinautAiMessage[]), - ); + rerender(); - expect(screen.queryByText("Spoken request")).toBeNull(); - expect(screen.queryByText("Spoken reply")).toBeNull(); + expect(screen.getByText("Spoken request")).not.toBeNull(); + expect(screen.getByText("Spoken reply")).not.toBeNull(); expect(screen.getByText("Typed aside")).not.toBeNull(); - // The dock's transcription action writes the held turns into the chat - // mid-session, and holds them back again when it is turned off. + const transcript = screen.getByTestId("ai-transcript"); + const voiceMode = screen.getByTestId("ai-voice-mode"); + const header = screen + .getByRole("button", { name: "Close AI assistant" }) + .closest("div")!; + + fireEvent.click( + within(dock).getByRole("button", { name: "Collapse voice session" }), + ); + + expect(screen.getByTestId("ai-transcript")).toBe(transcript); + expect(screen.getByTestId("ai-voice-mode")).toBe(voiceMode); + expect( + screen + .getByRole("button", { name: "Close AI assistant", hidden: true }) + .closest("div"), + ).toBe(header); + expect(transcript.className).toContain("d_none"); + expect(voiceMode.className).toContain("d_none"); + expect(header.className).toContain("d_none"); + fireEvent.click( - within(dock).getByRole("button", { name: "Show transcription in chat" }), + within(dock).getByRole("button", { name: "End voice mode" }), ); + + expect(actions.end).toHaveBeenCalledOnce(); + expect(onCollapsedVoiceEnd).toHaveBeenCalledOnce(); + + fireEvent.click( + within(dock).getByRole("button", { name: "Expand voice session" }), + ); + + expect(transcript.className).not.toContain("d_none"); + expect(voiceMode.className).not.toContain("d_none"); + expect(header.className).not.toContain("d_none"); expect(screen.getByText("Spoken request")).not.toBeNull(); - expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.queryByText("Voice session · 1 turn")).toBeNull(); fireEvent.click( - within(dock).getByRole("button", { name: "Hide transcription in chat" }), + within(dock).getByRole("button", { name: "End voice mode" }), ); - expect(screen.queryByText("Spoken request")).toBeNull(); + + expect(actions.end).toHaveBeenCalledTimes(2); + expect(onCollapsedVoiceEnd).toHaveBeenCalledOnce(); act(() => store.setState(null)); + rerender(); expect(screen.getByText("Spoken request")).not.toBeNull(); expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.getByText("Voice session · 1 turn")).not.toBeNull(); expect(screen.queryByRole("region", { name: "Voice session" })).toBeNull(); expect( screen.getByRole("textbox", { name: "Message AI assistant" }), ).not.toBeNull(); }); - test("keeps the session's controls in the dock", () => { + test("stacks Voice setup above its compact dock while keeping the full panel mounted", () => { + const onVoiceDockCollapsedChange = vi.fn(); + render( + Permission + } + />, + ); + + const permission = screen.getByRole("region", { + name: "Voice mode consent", + }); + const setupDock = screen.getByRole("region", { name: "Voice setup" }); + expect(permission.parentElement?.nextElementSibling).toBe( + setupDock.parentElement, + ); + expect(screen.getByTestId("ai-transcript").className).toContain("d_none"); + expect( + screen + .getByRole("button", { name: "Close AI assistant", hidden: true }) + .closest("div")?.className, + ).toContain("d_none"); + expect( + screen.getByRole("textbox", { + hidden: true, + name: "Message AI assistant", + }), + ).not.toBeNull(); + + const expandButton = within(setupDock).getByRole("button", { + name: "Expand voice setup", + }); + expect(expandButton.getAttribute("aria-expanded")).toBeNull(); + fireEvent.click(expandButton); + + expect(onVoiceDockCollapsedChange).toHaveBeenCalledWith(false); + }); + + test("keeps handoff and canonical playback controls in the Voice dock", async () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), + takeTurn: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, errorMessage: null, microphoneLevel: 0.4, microphoneMuted: false, @@ -273,9 +457,45 @@ describe("AiAssistantContents", () => { fireEvent.click( within(dock).getByRole("button", { name: "End voice mode" }), ); + fireEvent.click(within(dock).getByRole("button", { name: "Your turn" })); expect(actions.setMicrophoneMuted).toHaveBeenCalledWith(true); expect(actions.end).toHaveBeenCalledOnce(); + expect(actions.takeTurn).toHaveBeenCalledOnce(); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestion = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const repeatMenu = screen.getByRole("menu"); + fireEvent.keyDown(repeatMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(repeatMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestion.id, + ), + ); + fireEvent.keyDown(repeatMenu, { key: "Enter" }); + await waitFor(() => expect(actions.repeatQuestion).toHaveBeenCalledOnce()); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponse = await screen.findByRole("menuitem", { + name: "Read full response", + }); + const fullResponseMenu = screen.getByRole("menu"); + fireEvent.keyDown(fullResponseMenu, { key: "End" }); + await waitFor(() => + expect(fullResponseMenu.getAttribute("aria-activedescendant")).toBe( + readFullResponse.id, + ), + ); + fireEvent.keyDown(fullResponseMenu, { key: "Enter" }); + await waitFor(() => + expect(actions.readFullResponse).toHaveBeenCalledOnce(), + ); act(() => { store.setState({ @@ -292,6 +512,20 @@ describe("AiAssistantContents", () => { ); expect(actions.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + + act(() => { + store.setState({ + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + notice: "We didn't catch that. Please try again.", + phase: "listening", + }); + }); + expect( + within(dock).getAllByText("We didn't catch that. Please try again."), + ).not.toHaveLength(0); + expect(dock.getAttribute("data-voice-notice")).toBe("visible"); }); test("shows a voice recovery failure as a toast", async () => { @@ -319,15 +553,16 @@ describe("AiAssistantContents", () => { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe( - "Microphone unavailable. Check your browser permissions.", - ); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("Microphone unavailable. Check your browser permissions."); }); test("does not repeat a voice error toast until the session recovers", () => { @@ -554,7 +789,7 @@ describe("AiAssistantContents", () => { ).toBeNull(); }); - test("marks only the exact submitted interactive-tool answer named by voice metadata", () => { + test("marks every submitted interactive-tool answer named by voice metadata", () => { const hostTool = definePetrinautAiInteractiveTool({ toolName: "answerQuestion", inputSchema: { @@ -570,7 +805,10 @@ describe("AiAssistantContents", () => { const messages = [ { id: "assistant-questions", - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice-1", "question-voice-2"], + }, role: "assistant", parts: [ { @@ -585,10 +823,18 @@ describe("AiAssistantContents", () => { type: "dynamic-tool", toolName: "answerQuestion", state: "output-available", - toolCallId: "question-voice", + toolCallId: "question-voice-1", input: { question: "Who approves it?" }, output: { answer: "The shift lead" }, }, + { + type: "dynamic-tool", + toolName: "answerQuestion", + state: "output-available", + toolCallId: "question-voice-2", + input: { question: "Who acts next?" }, + output: { answer: "The dispatcher" }, + }, ], }, ] as unknown as PetrinautAiMessage[]; @@ -606,13 +852,18 @@ describe("AiAssistantContents", () => { />, ); - expect( - within( - screen - .getByText("question-voice: The shift lead") - .closest("[data-tool-call-id]")!, - ).getByTestId("voice-input-provenance"), - ).not.toBeNull(); + for (const [toolCallId, answer] of [ + ["question-voice-1", "The shift lead"], + ["question-voice-2", "The dispatcher"], + ]) { + expect( + within( + screen + .getByText(`${toolCallId}: ${answer}`) + .closest("[data-tool-call-id]")!, + ).getByTestId("voice-input-provenance"), + ).not.toBeNull(); + } expect( within( screen @@ -620,7 +871,7 @@ describe("AiAssistantContents", () => { .closest("[data-tool-call-id]")!, ).queryByTestId("voice-input-provenance"), ).toBeNull(); - expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(1); + expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(2); expect(screen.queryByText("The shift lead", { exact: true })).toBeNull(); expect(container.querySelectorAll('[data-role="user"]')).toHaveLength(0); }); @@ -1594,7 +1845,7 @@ describe("AiAssistantContents", () => { expect(screen.getByRole("button", { name: /2 changes/u })).not.toBeNull(); }); - test("labels failed tool calls as errored", () => { + test("shows failed tool-call errors inline", () => { const messages: PetrinautAiMessage[] = [ { id: "assistant-1", @@ -1625,9 +1876,11 @@ describe("AiAssistantContents", () => { />, ); - expect( - screen.getByRole("button", { name: /deleteItemsByIds errored/u }), - ).not.toBeNull(); + const tool = screen.getByRole("button", { + name: /Validation failed.*deleteItemsByIds/u, + }); + expect(tool).not.toBeNull(); + expect(tool.getAttribute("title")).toBeNull(); }); test("expands deleted item summaries", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 22c62fa18a5..9ab974abe20 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -1,5 +1,4 @@ import { - Fragment, memo, type ReactNode, type RefObject, @@ -14,9 +13,11 @@ import ReactMarkdown from "react-markdown"; import { Button, Icon } from "@hashintel/ds-components"; import { css, cva } from "@hashintel/ds-helpers/css"; -import { NotificationsContext } from "../../../../../react/notifications/context"; +import { + NotificationsContext, + type AddNotificationInput, +} from "../../../../../react/notifications/context"; import { EditorContext } from "../../../../../react/state/editor-context"; -import { VoiceSessionContext } from "../../../../../react/voice-session/context"; import { useVoiceSessionErrorMessage, useVoiceSessionPhase, @@ -24,7 +25,7 @@ import { import { AiAssistantIcon } from "../../../../components/ai-assistant-icon"; import { ResizeHandle } from "../../../../resize/resize-handle"; import { AiVoiceModeIcon } from "../../components/ai-voice-mode-button"; -import { partitionVoiceSessionMessages } from "./ai-assistant-contents/defer-voice-messages"; +import { voiceSetupLabels } from "../../components/voice-session-labels"; import { aiFooterMinHeight } from "./ai-assistant-contents/footer-height"; import { getMessageRenderItems } from "./ai-assistant-contents/get-message-render-items"; import { @@ -37,7 +38,7 @@ import { AiAssistantToolList, type OnInteractiveToolSubmit, } from "./ai-assistant-contents/tool-list"; -import { LiveVoiceDock } from "./ai-assistant-contents/voice-dock"; +import { LiveVoiceDock, VoiceDock } from "./ai-assistant-contents/voice-dock"; import { VoiceInputProvenance } from "./ai-assistant-contents/voice-input-provenance"; import type { PetrinautAiInputMode } from "../../../../types/ai-assistant-composer-control"; @@ -49,6 +50,11 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; +const errorNotification = ( + message: string, + detail?: string, +): AddNotificationInput => ({ detail, message, tone: "error" }); + export type AiAssistantContentsProps = { clearMessagesDisabled?: boolean; composerControl?: ReactNode; @@ -61,6 +67,7 @@ export type AiAssistantContentsProps = { messages: PetrinautAiMessage[]; onClearMessages?: () => void; onClose: () => void; + onCollapsedVoiceEnd?: () => void; onInputModeChange?: (mode: PetrinautAiInputMode) => void; onInputChange: (value: string) => void; onInteractiveToolSubmit?: OnInteractiveToolSubmit; @@ -68,11 +75,13 @@ export type AiAssistantContentsProps = { onSendPrompt?: (prompt: string) => void; onStop: () => void; onSubmit: () => void; + onVoiceDockCollapsedChange?: (collapsed: boolean) => void; promptChips?: PromptChip[]; rightOffset?: number; status: AiAssistantStatus; stopped?: boolean; voiceHandoffPending?: boolean; + voiceDockCollapsed?: boolean; voiceMode?: ReactNode; voiceModeAvailable?: boolean; }; @@ -89,6 +98,9 @@ const shellStyle = cva({ }, }, variants: { + collapsed: { + true: {}, + }, open: { true: { top: "0", @@ -115,6 +127,16 @@ const shellStyle = cva({ }, }, }, + compoundVariants: [ + { + collapsed: true, + open: true, + css: { + top: "[auto]", + height: "auto", + }, + }, + ], }); // Tracks the card's inset within the padded shell, so the resize handle @@ -259,40 +281,6 @@ const messageStyle = cva({ textAlign: "right", }, }, - // Spoken turns land in the transcript together once the session ends, so - // they arrive with a single entrance rather than appearing out of nowhere. - revealed: { - true: { - animationName: "[petrinautVoiceReveal]", - animationDuration: "[420ms]", - animationTimingFunction: "[cubic-bezier(0.22, 0.9, 0.3, 1)]", - "@media (prefers-reduced-motion: reduce)": { - animationName: "[none]", - }, - }, - }, - }, -}); - -const voiceSessionMetaStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", - paddingX: "1", - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - _before: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', - }, - _after: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', }, }); @@ -439,12 +427,10 @@ const AiAssistantMessage = memo( handlersRef, interactiveTools, message, - revealed = false, }: { handlersRef: MessageHandlersRef; interactiveTools: readonly PetrinautAiInteractiveTool[]; message: PetrinautAiMessage; - revealed?: boolean; }) => { const role = message.role === "user" ? "user" : "assistant"; const renderItems = getMessageRenderItems(message, interactiveTools); @@ -457,7 +443,7 @@ const AiAssistantMessage = memo( return (
@@ -506,6 +492,9 @@ const AiAssistantMessage = memo( } })} {hasVoiceOrigin && firstTextKey === null && } + {role === "assistant" && message.metadata?.stopped && ( +
Response stopped
+ )}
); }, @@ -524,6 +513,7 @@ export const AiAssistantContents = ({ messages, onClearMessages, onClose, + onCollapsedVoiceEnd, onInputModeChange, onInputChange, onInteractiveToolSubmit, @@ -531,16 +521,17 @@ export const AiAssistantContents = ({ onSendPrompt, onStop, onSubmit, + onVoiceDockCollapsedChange, promptChips, rightOffset = 0, status, stopped = false, voiceHandoffPending = false, + voiceDockCollapsed = false, voiceMode, voiceModeAvailable = false, }: AiAssistantContentsProps) => { const { addNotification } = use(NotificationsContext); - const voiceSessionStore = use(VoiceSessionContext); const voiceSessionPhase = useVoiceSessionPhase(); const voiceSessionErrorMessage = useVoiceSessionErrorMessage(); const isVoiceSessionLive = voiceSessionPhase !== null; @@ -599,76 +590,8 @@ export const AiAssistantContents = ({ variant: "solid", }; - // Index of the first message belonging to the current or most recent voice - // session. Everything from here on is held back while that session runs, and - // revealed together once it ends. - const [sessionBaselineIndex, setSessionBaselineIndex] = useState< - number | null - >(() => - voiceSessionStore.getSnapshot().state === null ? null : messages.length, - ); - - // Off by default: the dock's transcription action writes spoken turns into - // the conversation as they land instead of holding them to the end. - const [transcriptionShown, setTranscriptionShown] = useState(false); - - const messageCountRef = useRef(messages.length); - useEffect(() => { - messageCountRef.current = messages.length; - }, [messages]); - - // Read from the store rather than from a render effect, so the baseline is - // captured on the event that starts the session instead of a render that - // happens to observe it. - useEffect(() => { - let wasLive = voiceSessionStore.getSnapshot().state !== null; - - return voiceSessionStore.subscribe(() => { - const isLive = voiceSessionStore.getSnapshot().state !== null; - if (isLive === wasLive) { - return; - } - wasLive = isLive; - - if (isLive) { - setSessionBaselineIndex(messageCountRef.current); - setTranscriptionShown(false); - } - }); - }, [voiceSessionStore]); - - const isHoldingVoiceTurns = isVoiceSessionLive && !transcriptionShown; - - const sessionPartition = - sessionBaselineIndex === null - ? null - : partitionVoiceSessionMessages({ - deferredFromIndex: sessionBaselineIndex, - interactiveTools, - messages, - }); - - const visibleMessages = - isHoldingVoiceTurns && sessionPartition !== null - ? sessionPartition.visible - : messages; - - // Held turns become "revealed" once they are let through — by the - // transcription action mid-session, or by the session ending — so they carry - // the entrance animation either way. - const revealedIds = new Set( - isHoldingVoiceTurns || sessionPartition === null - ? [] - : sessionPartition.deferred.map((message) => message.id), - ); - // The divider counts a finished session, so it waits for the session to end - // rather than growing a turn at a time under a live transcription. - const firstRevealedMessageId = isVoiceSessionLive - ? undefined - : visibleMessages.find((message) => revealedIds.has(message.id))?.id; - const revealedVoiceTurnCount = visibleMessages.filter( - (message) => revealedIds.has(message.id) && message.role === "user", - ).length; + const isVoiceDockCollapsed = + voiceDockCollapsed && (isVoiceSessionLive || inputMode === "voice"); // Held in editor state, not here: the bottom toolbar and the viewport // controls have to keep clear of this panel, and cannot read a local value. @@ -689,10 +612,7 @@ export const AiAssistantContents = ({ return; } notifiedErrorRef.current = error; - addNotification({ - message: error.message, - tone: "error", - }); + addNotification(errorNotification("AI assistant error", error.message)); }, [addNotification, error]); // Voice failures (microphone denied, connection dropped) are reported by the @@ -712,10 +632,7 @@ export const AiAssistantContents = ({ } notifiedVoiceErrorRef.current = voiceSessionErrorMessage; - addNotification({ - message: voiceSessionErrorMessage, - tone: "error", - }); + addNotification(errorNotification(voiceSessionErrorMessage)); }, [addNotification, voiceSessionErrorMessage, voiceSessionPhase]); const inputRef = useRef(null); @@ -813,7 +730,10 @@ export const AiAssistantContents = ({