FE-1580: Reconcile Voice turn behavior on the shared Brunch conversation

.changeset/flue-voice-safety.md

@@ -0,0 +1,5 @@
+---
+"@hashintel/petrinaut": patch
+---
+
+Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history.

apps/brunch-agent/petrinaut-local.vite.config.ts

@@ -9,13 +9,34 @@
 
 import { join, resolve } from "node:path";
 
-import { defineConfig, loadConfigFromFile, mergeConfig } from "vite";
+import {
+  defineConfig,
+  loadConfigFromFile,
+  mergeConfig,
+  type UserConfig,
+} from "vite";
 
 import {
   defaultChatOrigin,
   petrinautLocalServer,
 } from "./src/http/local-origins.ts";
 
+interface PetrinautPanelConfigOptions {
+  readonly chatOrigin: string;
+  readonly loadedConfig: UserConfig;
+  readonly root: string;
+}
+
+export const mergePetrinautPanelConfig = ({
+  chatOrigin,
+  loadedConfig,
+  root,
+}: PetrinautPanelConfigOptions): UserConfig =>
+  mergeConfig(loadedConfig, {
+    root,
+    server: petrinautLocalServer(chatOrigin),
+  });
+
 export default defineConfig(async (environment) => {
   const websiteRoot = process.env.PETRINAUT_WEBSITE_ROOT;
   if (!websiteRoot) {
@@ -36,8 +57,9 @@ export default defineConfig(async (environment) => {
     throw new Error(`Could not load Petrinaut's Vite config from ${root}.`);
 
   const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin;
-  return mergeConfig(loaded.config, {
+  return mergePetrinautPanelConfig({
+    chatOrigin,
+    loadedConfig: loaded.config,
     root,
-    server: petrinautLocalServer(chatOrigin),
   });
 });

apps/brunch-agent/test/architecture/boundaries.integration.ts

@@ -385,6 +385,7 @@ describe("core auxiliary subpaths stay in their assigned lanes", () => {
       ".",
       "./client-tools",
       "./flue",
+      "./question-marker",
       "./storage",
       "./workpiece",
     ]);
@@ -424,6 +425,8 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12
    * path enters here by review only.
    */
   const SUBSTRATE_INTEGRATION_ENTRY_POINTS: Readonly<Record<string, string>> = {
+    "libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts":
+      "Types the Flue logger and calls the core marker tool with a mocked data-part writer and logger; no runtime boot, provider, key or socket.",
     "apps/brunch-agent/test/brunch-turn.test.ts":
       "Types Flue's client, admission, and conversation snapshot and constructs FlueExecutionError so the persona bridge can be unit-tested against a stubbed client — no provider key, no socket, no model call, no runtime boot.",
     "apps/brunch-agent/test/flue-transcript.test.ts":

apps/brunch-agent/test/local-dev-origins.test.ts

@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
 
 import { expect, test } from "vitest";
 
+import { mergePetrinautPanelConfig } from "../petrinaut-local.vite.config.ts";
 import {
   defaultChatOrigin,
   localChatListen,
@@ -21,7 +22,7 @@ test("one documented root command starts the Brunch server and Petrinaut panel",
   };
 
   expect(rootPackage.scripts["dev:brunch"]).toBe(
-    "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel",
+    "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel",
   );
   expect(rootPackage.scripts["dev:brunch:server"]).toBe(
     "yarn workspace @apps/brunch-agent dev",
@@ -64,3 +65,19 @@ test("petrinaut:dev proxies the mounted Flue conversation route", () => {
     'VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"',
   );
 });
+
+test("petrinaut:dev retains the website API handlers needed by Voice", () => {
+  const config = mergePetrinautPanelConfig({
+    chatOrigin: defaultChatOrigin,
+    loadedConfig: {
+      plugins: [{ name: "petrinaut-api-dev" }],
+    },
+    root: "/test/petrinaut-website",
+  });
+
+  expect(config.plugins).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining({ name: "petrinaut-api-dev" }),
+    ]),
+  );
+});

apps/brunch-agent/test/petrinaut-chat-result.ts

@@ -21,6 +21,10 @@ export interface PetrinautChatResult {
   readonly resumedStatus: number;
   readonly resumedText: string;
   readonly resumedFinish: UIMessageChunk | undefined;
+  readonly questionMarkerLive: unknown;
+  readonly questionMarkerHistory: unknown;
+  readonly questionToolVisibleLive: boolean;
+  readonly questionToolVisibleHistory: boolean;
   readonly historyUserEntryCount: number;
   readonly historyClientToolResultCount: number;
   readonly historyGetStatus: number;
@@ -51,5 +55,7 @@ export interface PetrinautChatResult {
 export interface PetrinautResumeResult {
   readonly historyGetStatus: number;
   readonly historyUserText: string;
+  readonly questionMarkerHistory: unknown;
+  readonly questionToolVisibleHistory: boolean;
   readonly transcript: string;
 }

apps/brunch-agent/test/petrinaut-chat.integration.ts

@@ -18,6 +18,10 @@ import {
   snapshotToUiMessages,
 } from "@hashintel/brunch-agent-transport-aisdk";
 import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue";
+import {
+  BRUNCH_QUESTION_DATA_NAME,
+  BRUNCH_QUESTION_TOOL_NAME,
+} from "@hashintel/brunch-agent/question-marker";
 
 import { PING_TOOL_NAME } from "../src/agents/chat-agent/tools/ping.ts";
 import { applyCaptureSweep } from "../src/capture/apply-sweep.ts";
@@ -43,6 +47,7 @@ const ACTIVATE_SKILL_TOOL_NAME = "activate_skill";
 const CHAT_MODEL_ID = "claude-haiku-4-5";
 const RUNBOOK_SKILL_NAME = "sdcpn-modelling";
 const READ_SKILL_RESOURCE_TOOL_NAME = "read_skill_resource";
+const question = "Which documentation page should we inspect next?";
 
 const principalKey = "principal-mission-1";
 const conversationId = "conversation-mission-1";
@@ -79,6 +84,35 @@ const userTextFromHistory = (
     .map((part) => part.text)
     .join("");
 
+const questionMarkerFromHistory = (
+  messages: ReturnType<typeof snapshotToUiMessages>,
+): unknown => {
+  const marker = messages
+    .flatMap((message) => message.parts)
+    .find(
+      (part) =>
+        part.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in part,
+    );
+  return marker !== undefined && "data" in marker ? marker.data : undefined;
+};
+
+const questionMarkerFromChunks = (
+  chunks: readonly UIMessageChunk[],
+): unknown => {
+  const marker = chunks.find(
+    (chunk) =>
+      chunk.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in chunk,
+  );
+  return marker !== undefined && "data" in marker ? marker.data : undefined;
+};
+
+const questionToolVisibleInHistory = (
+  messages: ReturnType<typeof snapshotToUiMessages>,
+): boolean =>
+  messages
+    .flatMap((message) => message.parts)
+    .some((part) => part.type === `tool-${BRUNCH_QUESTION_TOOL_NAME}`);
+
 const faux = fauxProvider({
   provider: "anthropic",
   models: [{ id: CHAT_MODEL_ID, reasoning: true }],
@@ -98,19 +132,24 @@ try {
   const panelTransport = createFlueChatTransport({
     client: historyClient,
     clientToolNames,
+    hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]),
   });
   const projectHistory = (
     snapshot: Awaited<ReturnType<typeof historyClient.history>>,
   ) =>
     snapshotToUiMessages(snapshot, {
       clientToolNames,
+      hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]),
     });
 
   if (process.env.BRUNCH_RESUME_PHASE === "1") {
     const snapshot = await historyClient.history();
+    const historyMessages = projectHistory(snapshot);
     const result: PetrinautResumeResult = {
       historyGetStatus: 200,
-      historyUserText: userTextFromHistory(projectHistory(snapshot)),
+      historyUserText: userTextFromHistory(historyMessages),
+      questionMarkerHistory: questionMarkerFromHistory(historyMessages),
+      questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages),
       transcript: formatFlueTranscript(snapshot),
     };
     process.stdout.write(`PETRINAUT_RESUME_RESULT ${JSON.stringify(result)}\n`);
@@ -196,9 +235,19 @@ try {
         ],
         { stopReason: "toolUse" },
       ),
+      fauxAssistantMessage(
+        [
+          fauxToolCall(
+            BRUNCH_QUESTION_TOOL_NAME,
+            { question },
+            { id: "tool-question-1" },
+          ),
+        ],
+        { stopReason: "toolUse" },
+      ),
       fauxAssistantMessage([
         fauxText(
-          "The guide says the assistant can read its own documentation pages.",
+          `The guide says the assistant can read its own documentation pages. ${question}`,
         ),
       ]),
       fauxAssistantMessage([
@@ -400,6 +449,14 @@ try {
         .map((chunk) => chunk.delta)
         .join(""),
       resumedFinish: resumedChunks.at(-1),
+      questionMarkerLive: questionMarkerFromChunks(resumedChunks),
+      questionMarkerHistory: questionMarkerFromHistory(historyMessages),
+      questionToolVisibleLive: resumedChunks.some(
+        (chunk) =>
+          chunk.type === "tool-input-available" &&
+          chunk.toolName === BRUNCH_QUESTION_TOOL_NAME,
+      ),
+      questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages),
       historyUserEntryCount: userEntryIds.length,
       historyClientToolResultCount: clientToolResultCount,
       historyGetStatus: 200,

apps/brunch-agent/test/petrinaut-chat.test.ts

@@ -72,6 +72,16 @@ test("the browser transport streams the mounted Flue agent through server and cl
       type: "finish",
       finishReason: "stop",
     });
+    expect(result.questionMarkerLive).toEqual({
+      question: "Which documentation page should we inspect next?",
+      toolCallId: "tool-question-1",
+    });
+    expect(result.questionToolVisibleLive).toBe(false);
+    expect(result.questionMarkerHistory).toEqual({
+      question: "Which documentation page should we inspect next?",
+      toolCallId: "tool-question-1",
+    });
+    expect(result.questionToolVisibleHistory).toBe(false);
     expect(result.historyUserEntryCount).toBe(1);
     expect(result.historyClientToolResultCount).toBe(1);
 
@@ -107,6 +117,7 @@ test("the browser transport streams the mounted Flue agent through server and cl
     expect(result.interviewerToolNames).toContain("read_skill_resource");
     expect(result.interviewerToolNames).toContain("ping");
     expect(result.interviewerToolNames).toContain("readPetrinautDoc");
+    expect(result.interviewerToolNames).toContain("brunch_mark_question");
     expect(result.interviewerToolNames).not.toContain("brunch_ask");
     expect(result.interviewerToolNames).not.toContain("sweep");
     expect(result.interviewerToolNames).not.toContain("brunch_sweep");
@@ -151,10 +162,16 @@ test("the browser transport streams the mounted Flue agent through server and cl
     expect(resumeResult.historyUserText).toContain(
       "Run the FE-1435 transport probe.",
     );
+    expect(resumeResult.questionMarkerHistory).toEqual({
+      question: "Which documentation page should we inspect next?",
+      toolCallId: "tool-question-1",
+    });
+    expect(resumeResult.questionToolVisibleHistory).toBe(false);
     expect(resumeResult.transcript).toContain("tool ping");
     expect(resumeResult.transcript).toContain("tool readPetrinautDoc");
     expect(resumeResult.transcript).toContain("tool activate_skill");
     expect(resumeResult.transcript).toContain("tool read_skill_resource");
+    expect(resumeResult.transcript).toContain("tool brunch_mark_question");
   } finally {
     await rm(dbDirectory, { recursive: true, force: true });
   }

apps/petrinaut-website/README.md

@@ -97,7 +97,22 @@ disclosure before requesting microphone access. The disclosure also provides a
 microphone check and is remembered in browser storage only after Voice mode
 starts.
 
-When Brunch is selected, typed and finalized spoken turns both enter the same mounted Flue conversation route. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Closing or speaking over Voice playback only stops local media; it does not alter canonical conversation history. Reopening the same net restores its observed Flue conversation without resubmitting a turn or replaying settled audio.
+When Brunch is selected, typed turns and completed Voice transcripts both enter
+the same mounted Flue conversation route. Each logical turn carries a stable
+delivery key so a replayed request converges on the existing admission instead
+of creating another turn. If admission cannot be confirmed, the UI reports the
+ambiguity and does not retry automatically. **Stop** requests a durable Brunch
+abort before the panel cancels its local response stream. Local playback
+cancellation remains separate and does not alter canonical history. Canonical
+Flue history is the source used when the same net is reopened. Automated
+coverage guards a locally submitted turn from an older hydration snapshot and
+does not resubmit turns or replay settled audio. The real hard-reload witness is
+still pending, so reload parity is not yet claimed for this preview.
+Voice-origin client-tool results retain their markers in Flue history. Direct
+spoken user turns remain canonical text, but Flue 2.0.3 does not yet expose the
+caller delivery metadata needed to restore their Voice chip after reopening.
+
+Browser execution and its continuation keep the shared composer busy; a local tool failure reaches Voice as an error rather than an apparently completed response. Durably aborted history entries retain their stopped label. If the Flue step has already completed, Stop can withhold not-yet-started browser work locally but cannot durably record that withholding: a reopen can recover those calls as pending. This cancellation/reopen limitation remains unresolved; the local guard is not a durable cancellation claim.
 
 An active session stays at the end of the transcript. Its compact divider shows
 a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a
@@ -111,24 +126,43 @@ The text composer remains available. Sending typed text ends Voice mode first,
 then submits the draft exactly once through the same conversation; a failed
 handoff restores the draft. Closing the assistant pauses capture and speech
 before hiding it. Reopening preserves the mounted session in **Paused** state.
-**Pause** and **End voice mode** live under **Voice mode actions**, while
-**Resume** or **Reconnect** appears as the primary action when applicable.
+The dock exposes **Your turn** while canonical audio owns the turn. That action
+clears pending input and output, waits for the provider's matching
+acknowledgements and response terminal event, and only then opens the
+microphone for fresh capture. Its playback menu offers **Repeat question** and
+**Read full response**. Full-response replay becomes available once the matching
+response and audio output have both finished, enqueues all exact retained
+canonical segments in order, and is disabled during capture, submission,
+cancellation, pause, and errors. **Repeat question** has the same safety gates
+and replays only exact question text carrying Brunch's non-interactive marker;
+if the marker is missing, malformed, or does not match finalized prose, the
+action stays disabled rather than guessing from the final segment.
 
 The browser sends its SDP offer to this app; the server initializes a trusted
 `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified
-Realtime call endpoint. The provider key, model, instructions, tools, language,
-and vocabulary policy stay server-side. The session uses semantic VAD with low
-eagerness so natural thinking pauses are less likely to end an answer early.
-
-Realtime is the disposable media plane: it carries continuous microphone and remote audio, detects complete turns, and handles barge-in. Brunch remains the control plane and sole authority for questions, captures, state, completion, and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-question correlation.
+Realtime call endpoint. The provider key, model, instructions, language, and
+vocabulary policy stay server-side. Realtime exposes no tools, uses
+`tool_choice: "none"`, and configures semantic VAD to detect an input boundary
+without creating a model response.
+
+Realtime is the disposable media plane: it carries microphone and remote audio,
+detects complete turns, and transcribes input. Brunch remains the control plane
+and sole authority for questions, captures, state, completion, and durable
+history. The bridge accepts only
+`conversation.item.input_audio_transcription.completed` as an answer, ignores
+model function arguments, and submits the normalized transcript through
+Petrinaut's shared composer path. Connection epoch, item id, and content index
+form its stable identity. Duplicate, empty, failed, unavailable, and over-limit
+transcripts never submit; recoverable failures leave a not-heard or too-long
+notice in the dock. Provisional transcription remains display-only.
 
 The bridge waits for the correlated Brunch turn before returning canonical
-speech segments to Realtime. It then requests audio with tools disabled and
-instructs Realtime to speak only those segments. Generated audio is not a
-verbatim record: canonical Brunch text remains visible and authoritative. The
-microphone stays active while the interviewer speaks and while Brunch is
-working. Speaking over assistant audio interrupts playback automatically;
-WebRTC truncates provider-side unheard audio without changing Brunch history.
+speech segments to Realtime. It instructs Realtime to speak only those
+segments. Generated audio is not a verbatim recording: canonical Brunch text
+remains visible and authoritative. Voice is half-duplex: the physical
+microphone is closed while the interviewer speaks, while Brunch is working, and
+through cancellation. Audio captured before a **Your turn** handoff is
+discarded and cannot become a later answer.
 
 The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**.