diff --git a/.changeset/durable-brunch-stop.md b/.changeset/durable-brunch-stop.md new file mode 100644 index 00000000000..fc8ff7e0628 --- /dev/null +++ b/.changeset/durable-brunch-stop.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Let hosts make the assistant's Stop action durable before Petrinaut cancels its local response stream, while preserving local-only cancellation for hosts that do not provide the new stop request. diff --git a/.changeset/voice-input-withdrawal-and-stop-correlation.md b/.changeset/voice-input-withdrawal-and-stop-correlation.md new file mode 100644 index 00000000000..7d371bd6dbe --- /dev/null +++ b/.changeset/voice-input-withdrawal-and-stop-correlation.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Let hosts withdraw a retained voice input through an `AbortSignal`, keep a late durable Stop result from cancelling a newer turn, replace all conversation-owned assistant state when the conversation identity changes, hydrate host-owned history only once it carries every locally streamed reply, keep the composer status busy across the automatic follow-up to a client-tool step, and let a Stop pressed during that step withhold the follow-up. diff --git a/apps/brunch-agent/README.md b/apps/brunch-agent/README.md index 9f630adf764..8b50562e904 100644 --- a/apps/brunch-agent/README.md +++ b/apps/brunch-agent/README.md @@ -8,9 +8,7 @@ From the repository root, make `ANTHROPIC_API_KEY` available in the environment yarn dev:brunch ``` -The first step builds the Petrinaut libraries the panel imports (`dist/` and design-system -codegen). Then it starts the Brunch server at `http://127.0.0.1:4321` and the real Petrinaut -website at `http://127.0.0.1:4915`. The website proxies `/api/chat` to Brunch. The panel talks to one Flue chat agent composed from the context-independent core prompt in `@hashintel/brunch-agent/flue`, the SDCPN/Petrinaut instructions, modelling runbook skill, and `readPetrinautDoc` client tool in `@hashintel/brunch-agent-plugin-sdcpn`, and app-owned deployment/transport material. The skill is activated via `activate_skill`, with supporting resources disclosed via `read_skill_resource`; the app's only model-facing diagnostic tool is `ping`. There is no generalized elicitation loop, sweep tool, or `brunch_ask` on this path. Capture is a harness-side pipe: an explicit settled range of Flue history is applied into a JSON store beside the conversation database, not by the interviewer. +The first step builds the Petrinaut libraries the panel imports (`dist/` and design-system codegen). Then it starts the Brunch server at `http://127.0.0.1:4321` and the real Petrinaut website at `http://127.0.0.1:4915`. The website proxies `/agents/chat/*` to Brunch without changing the request origin or Flue protocol. The typed panel and Voice mode talk to one Flue chat agent composed from the context-independent core prompt in `@hashintel/brunch-agent/flue`, the SDCPN/Petrinaut instructions, modelling runbook skill, and `readPetrinautDoc` client tool in `@hashintel/brunch-agent-plugin-sdcpn`, and app-owned deployment material. The skill is activated via `activate_skill`, with supporting resources disclosed via `read_skill_resource`; the app's only model-facing diagnostic tool is `ping`. There is no generalized elicitation loop, sweep tool, or `brunch_ask` on this path. Capture is a harness-side pipe: an explicit settled range of Flue history is applied into a JSON store beside the conversation database, not by the interviewer. A headless Mission 3 drive (simulated expert, same `ChatAgent` door): @@ -18,22 +16,13 @@ A headless Mission 3 drive (simulated expert, same `ChatAgent` door): yarn workspace @apps/brunch-agent runbook:headless ``` -`ANTHROPIC_API_KEY` is required. `BRUNCH_CHAT_MODEL` selects the interviewer -(default `claude-sonnet-4-5` for this script only). Artifacts write under -`libs/@hashintel/brunch-agent/docs/evidence/evaluations/vestera-runbook-headless/` -unless `BRUNCH_RUNBOOK_OUTPUT_DIR` is set. +`ANTHROPIC_API_KEY` is required. `BRUNCH_CHAT_MODEL` selects the interviewer (default `claude-sonnet-4-5` for this script only). Artifacts write under `libs/@hashintel/brunch-agent/docs/evidence/evaluations/vestera-runbook-headless/` unless `BRUNCH_RUNBOOK_OUTPUT_DIR` is set. -Conversations persist in `apps/brunch-agent/.data-wipe-me/conversations.db`. `BRUNCH_DEV_DB_PATH` -overrides that local path. Capture envelopes for one Flue conversation sit beside that sqlite -file, named by the hashed instance id (`.json`). The hermetic `/api/chat` test uses -`BRUNCH_CHAT_DB_PATH` and writes the capture file in that same directory. Flue history is the -conversation log; the capture store is not a second transcript. The browser may cache messages -but reload hydrates from `GET /api/chat?id=`. +Conversations persist in `apps/brunch-agent/.data-wipe-me/conversations.db`. `BRUNCH_DEV_DB_PATH` overrides that local path. Capture envelopes for one Flue conversation sit beside that sqlite file, named by the hashed instance id (`.json`). The hermetic browser-transport test uses `BRUNCH_CHAT_DB_PATH` and writes the capture file in that same directory. Flue history is the conversation log; the capture store is not a second transcript. The panel rehydrates from the SDK's canonical conversation observation and does not resubmit or replay settled turns. -The mounted Flue URL `/agents/chat/:id` requires the same principal and conversation identity (`x-brunch-principal` and `x-brunch-conversation`) as `/api/chat`; the path id is the hash of those, not a bearer token. +The mounted Flue URL `/agents/chat/:instanceId` requires the principal and logical conversation identity in `x-brunch-principal` and `x-brunch-conversation`. The path id is the hash of those values, not a bearer token or trusted authentication. -Print a human-readable transcript of one conversation from that same Flue history (server already -running): +Print a human-readable transcript of one conversation from that same Flue history (server already running): ```sh yarn workspace @apps/brunch-agent transcript -- --principal --id @@ -110,17 +99,16 @@ yarn workspace @apps/brunch-agent smoke:deployment BRUNCH_SMOKE_MODE=history yarn workspace @apps/brunch-agent smoke:deployment ``` -## Voice dock +## Panel and Voice conversation route -A second input modality joins the same chat door. It is not a voice route and does not own -provider audio or session state. +Voice is a second input modality over the panel's conversation. It is not a Voice route and does not own provider audio or durable conversation state. -| | | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| URL | `POST /api/chat` (and `GET /api/chat?id=` to hydrate) | -| Identity | `x-brunch-principal` plus body `id` (the conversation id). The server hashes those into the Flue instance id. | -| Initial turn | JSON `{ id, trigger: "submit-message", messages }` whose last user text part is the utterance. | -| Client-tool follow-up | Same POST, with `messageId` of the assistant message and completed client-tool parts (`providerExecuted` not true). Correlated by `toolCallId`. | -| Response | AI SDK UI-message stream (SSE). | +| | | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| URL | `/agents/chat/:instanceId`, called through the public Flue browser client and the same-origin local proxy | +| Identity | `x-brunch-principal` plus `x-brunch-conversation`; the server verifies that their hash matches the mounted instance id | +| Initial turn | One `FlueClient.send()` carrying `{ kind: "user", body }` | +| Client-tool follow-up | One `FlueClient.send()` carrying the `client-tool-result` signal for completed client-tool parts, correlated by `toolCallId` | +| Response | `FlueClient.wait()` chunks projected into one finite AI SDK UI-message stream; observation/history provides canonical rehydration | -`OPTIONS /api/chat` is the CORS preflight for that same contract. +Typed and finalized spoken turns use this same route. The panel's explicit **Stop** requests a conversation-wide Flue abort before cancelling its local stream. Local Voice interruption stops playback only and leaves canonical history unchanged. diff --git a/apps/brunch-agent/package.json b/apps/brunch-agent/package.json index d27fa1bad08..5e4e48a0924 100644 --- a/apps/brunch-agent/package.json +++ b/apps/brunch-agent/package.json @@ -2,7 +2,7 @@ "name": "@apps/brunch-agent", "version": "0.0.0-private", "private": true, - "description": "Remote Brunch server, local development loop, and Petrinaut /api/chat door.", + "description": "Remote Brunch server and local Petrinaut development loop.", "license": "AGPL-3.0", "type": "module", "scripts": { diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index b265537076c..3b3a53c3142 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -1,49 +1,28 @@ /** * Local FE-1436 panel launcher. * - * Loads the real hash Petrinaut website config, removes only its incumbent - * `/api/chat` dev handler, and sends that same-origin route to brunch's - * committed application server. The real panel, wrappers, and editor stay untouched; - * hash's tracked checkout stays clean. + * Loads the real hash Petrinaut website config and proxies Brunch's mounted + * Flue route to the committed application server. The real panel, API + * handlers, wrappers, and editor stay untouched; hash's tracked checkout + * stays clean. */ import { join, resolve } from "node:path"; -import { - defineConfig, - loadConfigFromFile, - mergeConfig, - type PluginOption, -} from "vite"; +import { defineConfig, loadConfigFromFile, mergeConfig } from "vite"; import { defaultChatOrigin, petrinautLocalServer, } from "./src/http/local-origins.ts"; -const withoutIncumbentChatHandler = ( - plugins: readonly PluginOption[], -): PluginOption[] => - plugins.filter((plugin) => { - if ( - plugin === false || - plugin === null || - plugin === undefined || - Array.isArray(plugin) || - typeof plugin !== "object" || - !("name" in plugin) - ) { - return true; - } - return plugin.name !== "petrinaut-api-dev"; - }); - export default defineConfig(async (environment) => { const websiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; if (!websiteRoot) { throw new Error("PETRINAUT_WEBSITE_ROOT is required."); } const root = resolve(websiteRoot); + process.env.VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"; // Babel resolves the React compiler plugin from the launched project's cwd, // not from the imported config file. Match a native hash launch before the // plugin begins transforming the real panel source. @@ -57,14 +36,8 @@ export default defineConfig(async (environment) => { throw new Error(`Could not load Petrinaut's Vite config from ${root}.`); const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin; - return mergeConfig( - { - ...loaded.config, - plugins: withoutIncumbentChatHandler(loaded.config.plugins ?? []), - }, - { - root, - server: petrinautLocalServer(chatOrigin), - }, - ); + return mergeConfig(loaded.config, { + root, + server: petrinautLocalServer(chatOrigin), + }); }); diff --git a/apps/brunch-agent/src/app.ts b/apps/brunch-agent/src/app.ts index 6a35669ac64..5a6f4acbd19 100644 --- a/apps/brunch-agent/src/app.ts +++ b/apps/brunch-agent/src/app.ts @@ -1,11 +1,4 @@ -/** - * The app's route map — one plain Flue chat agent plus Petrinaut's /api/chat door. - * - * Both doors require principal + conversation id. `/api/chat` takes the principal - * header and body `id`; `/agents/chat/:id` takes the same principal plus - * `x-brunch-conversation` and admits the request only when those re-derive the - * path id. The Flue instance id is derived, not a bearer token. - */ +/** The app's route map — one ownership-guarded Flue conversation door. */ import "./telemetry-bootstrap.ts"; import { readFile } from "node:fs/promises"; @@ -17,26 +10,14 @@ import { ChatAgent } from "./agents/chat-agent/agent.ts"; import { healthHandler } from "./health.ts"; import { assetHandler } from "./http/assets.ts"; import { agentOwnershipGuard } from "./http/ownership.ts"; -import { createPetrinautChatHandler } from "./http/petrinaut-chat.ts"; -import { - CHAT_AGENT_ROUTE, - HEALTH_ROUTE, - PETRINAUT_CHAT_ROUTE, -} from "./http/routes.ts"; +import { CHAT_AGENT_ROUTE, HEALTH_ROUTE } from "./http/routes.ts"; const app = new Hono(); -const appTransport: typeof fetch = async (input, init) => - app.fetch(input instanceof Request ? input : new Request(input, init)); -const petrinautChatHandler = createPetrinautChatHandler(appTransport); const chatAgentMount = `/agents/${CHAT_AGENT_ROUTE}`; app.use(`${chatAgentMount}/*`, agentOwnershipGuard(`${chatAgentMount}/`)); app.route(chatAgentMount, createAgentRouter(ChatAgent)); -app.on(["GET", "POST", "OPTIONS"], PETRINAUT_CHAT_ROUTE, (c) => - petrinautChatHandler(c.req.raw), -); - app.get(HEALTH_ROUTE, healthHandler); const uiRoot = new URL( diff --git a/apps/brunch-agent/src/conversation/client-tools.ts b/apps/brunch-agent/src/conversation/client-tools.ts index 2826f13f7f5..c238c9e3215 100644 --- a/apps/brunch-agent/src/conversation/client-tools.ts +++ b/apps/brunch-agent/src/conversation/client-tools.ts @@ -1,11 +1,11 @@ /** Flue-side client-tool signal contract: awaiting sentinel, result signal, tool names. */ import { READ_PETRINAUT_DOC_TOOL_NAME } from "@hashintel/brunch-agent-plugin-sdcpn/flue"; +import { CLIENT_TOOL_RESULT_SIGNAL } from "@hashintel/brunch-agent-transport-aisdk"; import { AWAITING_CLIENT } from "@hashintel/brunch-agent/client-tools"; export { AWAITING_CLIENT }; - -export const CLIENT_TOOL_RESULT_SIGNAL = "client-tool-result"; +export { CLIENT_TOOL_RESULT_SIGNAL }; export const clientToolNames: ReadonlySet = new Set([ READ_PETRINAUT_DOC_TOOL_NAME, diff --git a/apps/brunch-agent/src/conversation/identity-web.ts b/apps/brunch-agent/src/conversation/identity-web.ts deleted file mode 100644 index 68d7da8536d..00000000000 --- a/apps/brunch-agent/src/conversation/identity-web.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** Browser-safe instance-id hash; must match `flueConversationId` byte-for-byte. */ - -import { hexFromDigest, identityPayload } from "./payload.ts"; - -export const flueConversationIdWeb = async ( - principalKey: string, - conversationId: string, -): Promise => { - const payload = identityPayload(principalKey, conversationId); - const bytes = new ArrayBuffer(payload.byteLength); - new Uint8Array(bytes).set(payload); - const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes); - return hexFromDigest(digest); -}; diff --git a/apps/brunch-agent/src/conversation/identity.ts b/apps/brunch-agent/src/conversation/identity.ts index ffaa4738819..da87b3a377c 100644 --- a/apps/brunch-agent/src/conversation/identity.ts +++ b/apps/brunch-agent/src/conversation/identity.ts @@ -3,47 +3,32 @@ import { createHash, timingSafeEqual } from "node:crypto"; import { - BRUNCH_CONVERSATION_HEADER, - BRUNCH_PRINCIPAL_HEADER, identityPayload, -} from "./payload.ts"; - -import type { ConversationIdentity } from "@hashintel/brunch-agent-transport-aisdk"; + type ConversationIdentity, +} from "@hashintel/brunch-agent-transport-aisdk"; export { + agentOwnershipHeaders, BRUNCH_CONVERSATION_HEADER, BRUNCH_PRINCIPAL_HEADER, - LOCAL_UI_PRINCIPAL, -} from "./payload.ts"; +} from "@hashintel/brunch-agent-transport-aisdk"; +export { LOCAL_UI_PRINCIPAL } from "./payload.ts"; export type { ConversationIdentity }; -export const flueConversationId = ( - principalKey: string, - conversationId: string, -): string => - createHash("sha256") - .update(identityPayload(principalKey, conversationId)) - .digest("hex"); +export const flueConversationId = (identity: ConversationIdentity): string => + createHash("sha256").update(identityPayload(identity)).digest("hex"); export const flueConversationIdFrom = ( identity: ConversationIdentity, -): string => flueConversationId(identity.principalKey, identity.conversationId); +): string => flueConversationId(identity); export const ownsFlueInstance = ( - principalKey: string, - conversationId: string, + identity: ConversationIdentity, instanceId: string, ): boolean => { - const expected = flueConversationId(principalKey, conversationId); + const expected = flueConversationId(identity); const expectedBytes = Buffer.from(expected); const presentedBytes = Buffer.from(instanceId); if (expectedBytes.length !== presentedBytes.length) return false; return timingSafeEqual(expectedBytes, presentedBytes); }; - -export const agentOwnershipHeaders = ( - identity: ConversationIdentity, -): Record => ({ - [BRUNCH_PRINCIPAL_HEADER]: identity.principalKey, - [BRUNCH_CONVERSATION_HEADER]: identity.conversationId, -}); diff --git a/apps/brunch-agent/src/conversation/payload.ts b/apps/brunch-agent/src/conversation/payload.ts index 2507070f3ed..20de535d2fe 100644 --- a/apps/brunch-agent/src/conversation/payload.ts +++ b/apps/brunch-agent/src/conversation/payload.ts @@ -1,29 +1,2 @@ -/** Identity headers and payload encoding shared by Node and the local Flue UI. */ - -export { BRUNCH_PRINCIPAL_HEADER } from "@hashintel/brunch-agent-transport-aisdk/headers"; - -export const BRUNCH_CONVERSATION_HEADER = "x-brunch-conversation"; - /** Principal for the stock Flue UI at `/`. Not a second ownership rule. */ export const LOCAL_UI_PRINCIPAL = "local"; - -export const identityPayload = ( - principalKey: string, - conversationId: string, -): Uint8Array => { - const encoder = new TextEncoder(); - const principalBytes = encoder.encode(principalKey); - const conversationBytes = encoder.encode(conversationId); - const payload = new Uint8Array( - principalBytes.length + 1 + conversationBytes.length, - ); - payload.set(principalBytes, 0); - payload[principalBytes.length] = 0; - payload.set(conversationBytes, principalBytes.length + 1); - return payload; -}; - -export const hexFromDigest = (digest: ArrayBuffer): string => - [...new Uint8Array(digest)] - .map((byte) => byte.toString(16).padStart(2, "0")) - .join(""); diff --git a/apps/brunch-agent/src/conversation/transcript.ts b/apps/brunch-agent/src/conversation/transcript.ts index cbaf34b0c33..23bfb5f8b18 100644 --- a/apps/brunch-agent/src/conversation/transcript.ts +++ b/apps/brunch-agent/src/conversation/transcript.ts @@ -1,4 +1,4 @@ -/** Human-readable and UI-message projections of Flue's public conversation snapshot. */ +/** Human-readable projection of Flue's public conversation snapshot. */ import { type FlueConversationMessage, @@ -6,38 +6,7 @@ import { type FlueConversationSnapshot, } from "@flue/sdk"; -import { - CLIENT_TOOL_RESULT_SIGNAL, - isAwaitingClient, - providerExecutedFor, -} from "./client-tools.ts"; - -type UiMessagePart = - | { readonly type: "text"; readonly text: string; readonly state: "done" } - | { - readonly type: "reasoning"; - readonly text: string; - readonly state: "done"; - } - | { - readonly type: `data-${string}`; - readonly data: unknown; - } - | { - readonly type: "file"; - readonly mediaType: string; - readonly url: string; - readonly filename?: string; - } - | { - readonly type: `tool-${string}`; - readonly toolCallId: string; - readonly state: "output-available" | "output-error" | "input-available"; - readonly input: unknown; - readonly output?: unknown; - readonly errorText?: string; - readonly providerExecuted?: boolean; - }; +import { CLIENT_TOOL_RESULT_SIGNAL, isAwaitingClient } from "./client-tools.ts"; const unhandledConversationPart = (part: never): never => { throw new Error(`Unhandled Flue conversation part: ${JSON.stringify(part)}`); @@ -48,12 +17,6 @@ const isFlueDataPart = ( ): part is Extract => part.type.startsWith("data-"); -export interface UiHistoryMessage { - readonly id: string; - readonly role: "user" | "assistant"; - readonly parts: readonly UiMessagePart[]; -} - const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -108,95 +71,6 @@ const resolveToolOutput = ( return clientOutput; }; -const toolPartFrom = ( - part: Extract, - clientOutputs: ReadonlyMap, -): UiMessagePart => { - const output = resolveToolOutput(part, clientOutputs); - const providerExecuted = - part.state === "output-available" - ? providerExecutedFor(isAwaitingClient(part.output)) - : undefined; - if (part.state === "output-error") { - return { - type: `tool-${part.toolName}`, - toolCallId: part.toolCallId, - state: "output-error", - input: part.input, - errorText: part.errorText, - ...(providerExecuted === undefined ? {} : { providerExecuted }), - }; - } - if (output !== undefined) { - return { - type: `tool-${part.toolName}`, - toolCallId: part.toolCallId, - state: "output-available", - input: part.input, - output, - ...(providerExecuted === undefined ? {} : { providerExecuted }), - }; - } - return { - type: `tool-${part.toolName}`, - toolCallId: part.toolCallId, - state: "input-available", - input: part.input, - }; -}; - -const partsFrom = ( - message: FlueConversationMessage, - clientOutputs: ReadonlyMap, -): UiMessagePart[] => { - const parts: UiMessagePart[] = []; - for (const part of message.parts) { - if (part.type === "text") { - parts.push({ type: "text", text: part.text, state: "done" }); - continue; - } - if (part.type === "reasoning") { - parts.push({ type: "reasoning", text: part.text, state: "done" }); - continue; - } - if (part.type === "dynamic-tool") { - parts.push(toolPartFrom(part, clientOutputs)); - continue; - } - if (part.type === "file") { - parts.push({ - type: "file", - mediaType: part.mediaType, - url: part.url ?? "", - ...(part.filename === undefined ? {} : { filename: part.filename }), - }); - continue; - } - if (isFlueDataPart(part)) { - parts.push({ type: part.type, data: part.data }); - continue; - } - unhandledConversationPart(part); - } - return parts; -}; - -export const snapshotToUiMessages = ( - snapshot: FlueConversationSnapshot, -): UiHistoryMessage[] => { - const clientOutputs = clientToolResultsFrom(snapshot); - const messages: UiHistoryMessage[] = []; - for (const message of snapshot.messages) { - if (message.display !== "visible") continue; - if (message.purpose !== "user" && message.purpose !== "assistant") continue; - if (message.role !== "user" && message.role !== "assistant") continue; - const parts = partsFrom(message, clientOutputs); - if (parts.length === 0) continue; - messages.push({ id: message.id, role: message.role, parts }); - } - return messages; -}; - const textOf = (message: FlueConversationMessage): string => message.parts .filter( diff --git a/apps/brunch-agent/src/diagnostics/transcript-cli.ts b/apps/brunch-agent/src/diagnostics/transcript-cli.ts index 9e2fabb3274..e38007c0999 100644 --- a/apps/brunch-agent/src/diagnostics/transcript-cli.ts +++ b/apps/brunch-agent/src/diagnostics/transcript-cli.ts @@ -5,9 +5,9 @@ * * yarn workspace @apps/brunch-agent transcript -- --principal --id * - * Identity matches POST /api/chat: principal + conversation id hash to the - * Flue instance. The mounted URL requires those same values as headers. This - * is a read of canonical Flue history, not a second log. + * Identity matches the mounted Flue route: principal + conversation id hash + * to the Flue instance, with those values also sent as ownership headers. + * This is a read of canonical Flue history, not a second log. */ import { createFlueClient } from "@flue/sdk"; diff --git a/apps/brunch-agent/src/http/local-origins.ts b/apps/brunch-agent/src/http/local-origins.ts index 797fc52b346..ea299ed66c7 100644 --- a/apps/brunch-agent/src/http/local-origins.ts +++ b/apps/brunch-agent/src/http/local-origins.ts @@ -14,17 +14,12 @@ export const localPanelListen = { export const defaultChatOrigin = `http://${localChatListen.host}:${localChatListen.port}`; -export const defaultPanelOrigins = [ - `http://${localPanelListen.host}:${localPanelListen.port}`, - `http://localhost:${localPanelListen.port}`, -] as const; - export const petrinautLocalServer = (chatOrigin: string) => ({ ...localPanelListen, proxy: { - "/api/chat": { + "/agents/chat": { target: chatOrigin, - changeOrigin: true, + changeOrigin: false, }, }, }); diff --git a/apps/brunch-agent/src/http/ownership.ts b/apps/brunch-agent/src/http/ownership.ts index 277d83f2227..dbcb1463fdf 100644 --- a/apps/brunch-agent/src/http/ownership.ts +++ b/apps/brunch-agent/src/http/ownership.ts @@ -1,9 +1,11 @@ -/** Hono middleware: the mounted Flue route enforces the same ownership rule as `/api/chat`. */ +/** Hono middleware for the mounted Flue conversation route. */ -import { BRUNCH_PRINCIPAL_HEADER } from "@hashintel/brunch-agent-transport-aisdk/headers"; +import { + BRUNCH_CONVERSATION_HEADER, + BRUNCH_PRINCIPAL_HEADER, +} from "@hashintel/brunch-agent-transport-aisdk/headers"; import { ownsFlueInstance } from "../conversation/identity.ts"; -import { BRUNCH_CONVERSATION_HEADER } from "../conversation/payload.ts"; import type { MiddlewareHandler } from "hono"; @@ -27,7 +29,7 @@ export const agentOwnershipGuard = (mountPrefix: string): MiddlewareHandler => { .find((segment) => segment.length > 0); if ( instanceId === undefined || - !ownsFlueInstance(principalKey, conversationId, instanceId) + !ownsFlueInstance({ principalKey, conversationId }, instanceId) ) { return context.json({ error: "forbidden" }, 403); } diff --git a/apps/brunch-agent/src/http/petrinaut-chat.ts b/apps/brunch-agent/src/http/petrinaut-chat.ts deleted file mode 100644 index e76ba0fbbdd..00000000000 --- a/apps/brunch-agent/src/http/petrinaut-chat.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** Application composition for Petrinaut's stock AI SDK chat transport. */ - -import { init } from "@flue/runtime"; -import { createFlueClient, type FlueConversationSnapshot } from "@flue/sdk"; - -import { - createAiSdkChatHandler, - type ChatResumeInput, - type ChatTurnInput, - type ConversationIdentity, - type TransportInspectionEvent, -} from "@hashintel/brunch-agent-transport-aisdk"; - -import { ChatAgent } from "../agents/chat-agent/agent.ts"; -import { - clientToolNames, - CLIENT_TOOL_RESULT_SIGNAL, -} from "../conversation/client-tools.ts"; -import { - agentOwnershipHeaders, - flueConversationIdFrom, -} from "../conversation/identity.ts"; -import { snapshotToUiMessages } from "../conversation/transcript.ts"; -import { createFlueUiStream } from "../conversation/ui-stream.ts"; -import { defaultPanelOrigins } from "./local-origins.ts"; -import { CHAT_AGENT_ROUTE } from "./routes.ts"; - -import type { UIMessageChunk } from "ai"; - -const inspect = - process.env.BRUNCH_TRANSPORT_AISDK_INSPECT === "1" - ? (event: TransportInspectionEvent): void => { - process.stdout.write(`TRANSPORT_AISDK ${JSON.stringify(event)}\n`); - } - : undefined; - -const conversationUrl = (instanceId: string): string => - `http://brunch.local/agents/${CHAT_AGENT_ROUTE}/${instanceId}`; - -const historyClient = ( - identity: ConversationIdentity, - appTransport: typeof fetch, -) => - createFlueClient({ - url: conversationUrl(flueConversationIdFrom(identity)), - fetch: appTransport, - headers: agentOwnershipHeaders(identity), - }); - -const streamTurn = async ( - instanceId: string, - dispatch: Parameters["dispatch"]>[0], - write: (chunk: UIMessageChunk) => void, -): Promise => { - const agent = init(ChatAgent, { id: instanceId }); - const receipt = await agent.dispatch(dispatch); - const projector = createFlueUiStream({ - submissionId: receipt.submissionId, - clientToolNames, - write, - }); - await agent.read(receipt, { onEvent: (chunk) => projector.accept(chunk) }); -}; - -const runUserTurn = ( - input: ChatTurnInput, - write: (chunk: UIMessageChunk) => void, -): Promise => - streamTurn( - flueConversationIdFrom(input), - { message: input.userMessage.text, idempotencyKey: input.idempotencyKey }, - write, - ); - -const runClientToolResume = ( - input: ChatResumeInput, - write: (chunk: UIMessageChunk) => void, -): Promise => - streamTurn( - flueConversationIdFrom(input), - { - message: { - kind: "signal", - type: CLIENT_TOOL_RESULT_SIGNAL, - tagName: CLIENT_TOOL_RESULT_SIGNAL, - body: JSON.stringify(input.toolResults), - attributes: { - toolCallIds: input.toolResults - .map((result) => result.toolCallId) - .join(","), - }, - }, - idempotencyKey: input.idempotencyKey, - }, - write, - ); - -export const createPetrinautChatHandler = (appTransport: typeof fetch) => { - const loadHistory = async ( - identity: ConversationIdentity, - ): Promise<{ readonly messages: readonly unknown[] }> => { - let snapshot: FlueConversationSnapshot; - try { - snapshot = await historyClient(identity, appTransport).history(); - } catch { - return { messages: [] }; - } - return { messages: snapshotToUiMessages(snapshot) }; - }; - - return createAiSdkChatHandler({ - allowedOrigins: ( - process.env.BRUNCH_PETRINAUT_ORIGINS ?? defaultPanelOrigins.join(",") - ) - .split(",") - .map((origin) => origin.trim()) - .filter((origin) => origin.length > 0), - inspect, - runTurn: runUserTurn, - resumeTurn: runClientToolResume, - loadHistory, - }); -}; diff --git a/apps/brunch-agent/src/http/routes.ts b/apps/brunch-agent/src/http/routes.ts index 008c73437ee..63637318d8f 100644 --- a/apps/brunch-agent/src/http/routes.ts +++ b/apps/brunch-agent/src/http/routes.ts @@ -4,6 +4,3 @@ export const CHAT_AGENT_ROUTE = "chat"; /** Cheap process-liveness probe; dependency readiness is established before listen. */ export const HEALTH_ROUTE = "/health"; - -/** Stock `DefaultChatTransport` endpoint used by Petrinaut's local panel. */ -export const PETRINAUT_CHAT_ROUTE = "/api/chat"; diff --git a/apps/brunch-agent/src/ui/chat.tsx b/apps/brunch-agent/src/ui/chat.tsx index e3581f59903..faf71094c85 100644 --- a/apps/brunch-agent/src/ui/chat.tsx +++ b/apps/brunch-agent/src/ui/chat.tsx @@ -6,12 +6,12 @@ import { } from "@flue/sdk"; import { useEffect, useMemo, useState, type FormEvent } from "react"; -import { flueConversationIdWeb } from "../conversation/identity-web.ts"; import { - BRUNCH_CONVERSATION_HEADER, - BRUNCH_PRINCIPAL_HEADER, - LOCAL_UI_PRINCIPAL, -} from "../conversation/payload.ts"; + agentOwnershipHeaders, + flueConversationIdWeb, +} from "@hashintel/brunch-agent-transport-aisdk"; + +import { LOCAL_UI_PRINCIPAL } from "../conversation/payload.ts"; import { CHAT_AGENT_ROUTE } from "../http/routes.ts"; type ChatConfiguration = @@ -165,18 +165,12 @@ export function Chat() { if (configuration.mode === "observer-error") return; let cancelled = false; - void flueConversationIdWeb( - configuration.principalKey, - configuration.conversationId, - ).then((instanceId) => { + void flueConversationIdWeb(configuration).then((instanceId) => { if (cancelled) return; setClient( createFlueClient({ url: `/agents/${CHAT_AGENT_ROUTE}/${instanceId}`, - headers: { - [BRUNCH_PRINCIPAL_HEADER]: configuration.principalKey, - [BRUNCH_CONVERSATION_HEADER]: configuration.conversationId, - }, + headers: agentOwnershipHeaders(configuration), }), ); }); diff --git a/apps/brunch-agent/test/agent-ownership.test.ts b/apps/brunch-agent/test/agent-ownership.test.ts index 9a2f19921ba..8c9143ce65e 100644 --- a/apps/brunch-agent/test/agent-ownership.test.ts +++ b/apps/brunch-agent/test/agent-ownership.test.ts @@ -9,9 +9,9 @@ import { expect, test } from "vitest"; import { agentOwnershipHeaders, + BRUNCH_CONVERSATION_HEADER, flueConversationIdFrom, } from "../src/conversation/identity.ts"; -import { BRUNCH_CONVERSATION_HEADER } from "../src/conversation/payload.ts"; import { agentOwnershipGuard } from "../src/http/ownership.ts"; import { CHAT_AGENT_ROUTE } from "../src/http/routes.ts"; diff --git a/apps/brunch-agent/test/architecture/boundaries.integration.ts b/apps/brunch-agent/test/architecture/boundaries.integration.ts index 08a40a09bf7..11d2f9ae83c 100644 --- a/apps/brunch-agent/test/architecture/boundaries.integration.ts +++ b/apps/brunch-agent/test/architecture/boundaries.integration.ts @@ -160,17 +160,20 @@ describe("dependency direction", () => { } }); - test("transports consume their wire encoder only — never core, a binding, or Flue", () => { + test("transports consume their wire encoder and the public Flue client only — never core, a binding, or the runtime", () => { const transports = byRole("transport"); expect(transports.length).toBeGreaterThan(0); for (const transport of transports) { - expect(runtimeDependencies(transport).sort()).toEqual(["ai", "valibot"]); + expect(runtimeDependencies(transport).sort()).toEqual([ + "@flue/sdk", + "ai", + ]); for (const file of sourceFiles(transport).filter((file) => file.path.startsWith(join(transport.path, "src")), )) { for (const specifier of importedPackages(file)) { if (specifier.startsWith("node:")) continue; - expect(["ai", "valibot"]).toContain(packageOf(specifier)); + expect(["@flue/sdk", "ai"]).toContain(packageOf(specifier)); } } } @@ -424,10 +427,8 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12 "Types Flue's client, admission, and conversation snapshot and constructs FlueExecutionError so the persona bridge can be unit-tested against a stubbed client — no provider key, no socket, no model call, no runtime boot.", "apps/brunch-agent/test/flue-transcript.test.ts": "Types Flue's public conversation snapshot so the transcript projector can be unit-tested; the import is type-only — no provider key, no socket, no model call, no runtime boot.", - "apps/brunch-agent/test/flue-ui-stream.test.ts": - "Types Flue conversation-stream chunks so the AI SDK projector can be unit-tested; the import is type-only — no provider key, no socket, no model call, no runtime boot.", "apps/brunch-agent/test/petrinaut-chat.integration.ts": - "Boots the plain Flue chat agent on Flue's node runtime with pi-ai's faux provider, drives the committed /api/chat door over app.fetch, and proves streamed reasoning/text, one server tool, one stub skill activation, one read-only client-tool resume, GET history ownership, SQLite restart, and harness-side idempotent apply-sweep into a capture store keyed by Flue conversation identity — no provider key, no socket, no extraction model call. Run as a child process by petrinaut-chat.test.ts.", + "Boots the plain Flue chat agent on Flue's node runtime with pi-ai's faux provider, drives the browser ChatTransport against the mounted Flue route over app.fetch, and proves streamed reasoning/text, server tools, client-tool resume, SDK history ownership, SQLite restart, and harness-side idempotent apply-sweep — no provider key, no socket, no extraction model call. Run as a child process by petrinaut-chat.test.ts.", "apps/brunch-agent/test/proof-artifacts.test.ts": "Types Flue's public conversation snapshot so canonical trace derivation, workpiece binding, and atomic evidence retention can be unit-tested against an in-memory fixture — no provider key, no socket, no model call, no runtime boot.", "apps/brunch-agent/test/runbook-artifacts.test.ts": @@ -438,6 +439,12 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12 "Boots the built Flue ChatAgent with pi-ai's faux provider and a headless Petrinaut client to prove validated construct-only tool flow without a provider key, socket, or network model call.", "apps/brunch-agent/test/telemetry.test.ts": "Constructs Flue's content-free OpenTelemetry instrumentation with an injected exporter setup to prove disposal order; it registers no global instrumentation, opens no socket, and makes no provider call.", + "libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts": + "Types a stubbed public Flue client and stream chunks to prove finite AI SDK projection and client-tool signal admission — no runtime boot, provider key, socket, or model call.", + "libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts": + "Types Flue's public conversation snapshot so history can be projected into AI SDK messages without a runtime boot, provider key, socket, or model call.", + "libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts": + "Types Flue conversation-stream chunks so the finite AI SDK projector can be unit-tested without a runtime boot, provider key, socket, or model call.", }; test("no test file carries a live model credential", () => { diff --git a/apps/brunch-agent/test/build-artifact.test.ts b/apps/brunch-agent/test/build-artifact.test.ts index 013ada40b22..6c1cd8840bc 100644 --- a/apps/brunch-agent/test/build-artifact.test.ts +++ b/apps/brunch-agent/test/build-artifact.test.ts @@ -19,6 +19,8 @@ import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, test } from "vitest"; +import { loadBuiltBrunchApplication } from "../src/evaluations/runbook/load-built-application"; + const DEV_APP = fileURLToPath(new URL("..", import.meta.url)).replace( /[/\\]$/u, "", @@ -117,6 +119,19 @@ describe("the emitted server bundle", () => { expect(bundle).toContain(".data-wipe-me"); }); + test("serves only the guarded Flue conversation door", async () => { + const application = await loadBuiltBrunchApplication(); + const [legacyResponse, flueResponse] = await Promise.all([ + application.fetch(new Request("http://brunch.test/api/chat")), + application.fetch( + new Request("http://brunch.test/agents/chat/missing-identity"), + ), + ]); + + expect(legacyResponse.status).toBe(404); + expect(flueResponse.status).toBe(401); + }); + test("packages the authored skill without the retired filesystem loader", () => { expect(bundle).toContain("defineSkill"); expect(bundle).toContain("sdcpn-modelling"); diff --git a/apps/brunch-agent/test/conversation-identity.test.ts b/apps/brunch-agent/test/conversation-identity.test.ts index acf6d6c361b..aa78794f5a2 100644 --- a/apps/brunch-agent/test/conversation-identity.test.ts +++ b/apps/brunch-agent/test/conversation-identity.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "vitest"; -import { flueConversationIdWeb } from "../src/conversation/identity-web.ts"; +import { flueConversationIdWeb } from "@hashintel/brunch-agent-transport-aisdk"; + import { flueConversationId, flueConversationIdFrom, @@ -8,14 +9,22 @@ import { } from "../src/conversation/identity.ts"; test("the same principal and conversation id always hash to the same Flue instance", () => { - expect(flueConversationId("principal-a", "conversation-1")).toBe( - flueConversationId("principal-a", "conversation-1"), - ); + const identity = { + principalKey: "principal-a", + conversationId: "conversation-1", + }; + expect(flueConversationId(identity)).toBe(flueConversationId(identity)); }); test("a different principal cannot address the same Flue instance", () => { - const owned = flueConversationId("principal-a", "conversation-1"); - const foreign = flueConversationId("principal-b", "conversation-1"); + const owned = flueConversationId({ + principalKey: "principal-a", + conversationId: "conversation-1", + }); + const foreign = flueConversationId({ + principalKey: "principal-b", + conversationId: "conversation-1", + }); expect(owned).not.toBe(foreign); expect(owned).toMatch(/^[0-9a-f]{64}$/); expect(foreign).toMatch(/^[0-9a-f]{64}$/); @@ -27,30 +36,25 @@ test("ownsFlueInstance admits only the principal and conversation that hashed th conversationId: "conversation-1", }; const instanceId = flueConversationIdFrom(identity); + expect(ownsFlueInstance(identity, instanceId)).toBe(true); expect( - ownsFlueInstance( - identity.principalKey, - identity.conversationId, - instanceId, - ), - ).toBe(true); - expect( - ownsFlueInstance("principal-b", identity.conversationId, instanceId), - ).toBe(false); - expect( - ownsFlueInstance(identity.principalKey, "conversation-2", instanceId), + ownsFlueInstance({ ...identity, principalKey: "principal-b" }, instanceId), ).toBe(false); expect( ownsFlueInstance( - identity.principalKey, - identity.conversationId, - "not-a-hash", + { ...identity, conversationId: "conversation-2" }, + instanceId, ), ).toBe(false); + expect(ownsFlueInstance(identity, "not-a-hash")).toBe(false); }); test("the browser hash matches the Node hash", async () => { - expect(await flueConversationIdWeb("principal-a", "conversation-1")).toBe( - flueConversationId("principal-a", "conversation-1"), + const identity = { + principalKey: "principal-a", + conversationId: "conversation-1", + }; + expect(await flueConversationIdWeb(identity)).toBe( + flueConversationId(identity), ); }); diff --git a/apps/brunch-agent/test/flue-transcript.test.ts b/apps/brunch-agent/test/flue-transcript.test.ts index 0b6827cfbe1..051e9792b3c 100644 --- a/apps/brunch-agent/test/flue-transcript.test.ts +++ b/apps/brunch-agent/test/flue-transcript.test.ts @@ -1,58 +1,9 @@ import { expect, test } from "vitest"; -import { - formatFlueTranscript, - snapshotToUiMessages, -} from "../src/conversation/transcript.ts"; +import { formatFlueTranscript } from "../src/conversation/transcript"; import type { FlueConversationSnapshot } from "@flue/sdk"; -const snapshotWithPendingClientTool: FlueConversationSnapshot = { - v: 1, - conversationId: "conversation-1", - offset: "0", - messages: [ - { - id: "assistant-1", - role: "assistant", - purpose: "assistant", - display: "visible", - parts: [ - { - type: "dynamic-tool", - toolCallId: "tool-doc-1", - toolName: "readPetrinautDoc", - state: "output-available", - input: { doc: "ai-assistant" }, - output: { awaiting: "client" }, - }, - ], - }, - ], - settlements: [], -}; - -const snapshotWithCompletedClientTool: FlueConversationSnapshot = { - ...snapshotWithPendingClientTool, - messages: [ - ...snapshotWithPendingClientTool.messages, - { - id: "signal-1", - role: "system", - purpose: "dispatch", - display: "hidden", - signal: { tagName: "client-tool-result" }, - parts: [ - { - type: "text", - text: '[{"toolCallId":"tool-doc-1","toolName":"readPetrinautDoc","output":null}]', - state: "done", - }, - ], - }, - ], -}; - const snapshotWithDataPart: FlueConversationSnapshot = { v: 1, conversationId: "conversation-1", @@ -72,49 +23,6 @@ const snapshotWithDataPart: FlueConversationSnapshot = { settlements: [], }; -test("history reconstruction leaves an unfinished client tool available to run", () => { - expect(snapshotToUiMessages(snapshotWithPendingClientTool)).toEqual([ - { - id: "assistant-1", - role: "assistant", - parts: [ - { - type: "tool-readPetrinautDoc", - toolCallId: "tool-doc-1", - state: "input-available", - input: { doc: "ai-assistant" }, - }, - ], - }, - ]); -}); - -test("history reconstruction uses the browser result even when it is null", () => { - const [message] = snapshotToUiMessages(snapshotWithCompletedClientTool); - expect(message?.parts).toEqual([ - { - type: "tool-readPetrinautDoc", - toolCallId: "tool-doc-1", - state: "output-available", - input: { doc: "ai-assistant" }, - output: null, - }, - ]); -}); - -test("history reconstruction keeps Flue data-* parts on the AI SDK message", () => { - expect(snapshotToUiMessages(snapshotWithDataPart)).toEqual([ - { - id: "assistant-1", - role: "assistant", - parts: [ - { type: "text", text: "Here is the order.", state: "done" }, - { type: "data-orderCard", data: { orderId: "42", status: "loaded" } }, - ], - }, - ]); -}); - test("the human transcript names Flue data parts instead of omitting them", () => { expect(formatFlueTranscript(snapshotWithDataPart)).toContain( '- data orderCard: {"orderId":"42","status":"loaded"}', diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index f85c76e97bb..ddf0d734ec3 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -4,7 +4,6 @@ import { expect, test } from "vitest"; import { defaultChatOrigin, - defaultPanelOrigins, localChatListen, localPanelListen, petrinautLocalServer, @@ -43,11 +42,7 @@ test("dev listens on the chat origin the panel proxy already assumes", () => { expect(readAppFile("vite.config.ts")).toContain("localChatListen"); }); -test("petrinaut:dev listens on the panel origin chat CORS already assumes", () => { - expect(defaultPanelOrigins).toEqual([ - "http://127.0.0.1:4915", - "http://localhost:4915", - ]); +test("petrinaut:dev proxies the mounted Flue conversation route", () => { expect(localPanelListen).toEqual({ host: "127.0.0.1", port: 4915, @@ -56,16 +51,16 @@ test("petrinaut:dev listens on the panel origin chat CORS already assumes", () = expect(petrinautLocalServer(defaultChatOrigin)).toEqual({ ...localPanelListen, proxy: { - "/api/chat": { + "/agents/chat": { target: defaultChatOrigin, - changeOrigin: true, + changeOrigin: false, }, }, }); expect(readAppFile("petrinaut-local.vite.config.ts")).toContain( "petrinautLocalServer", ); - expect(readAppFile("src/http/petrinaut-chat.ts")).toContain( - "defaultPanelOrigins", + expect(readAppFile("petrinaut-local.vite.config.ts")).toContain( + 'VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"', ); }); diff --git a/apps/brunch-agent/test/petrinaut-chat-result.ts b/apps/brunch-agent/test/petrinaut-chat-result.ts index 761d4fade07..c7cc142689c 100644 --- a/apps/brunch-agent/test/petrinaut-chat-result.ts +++ b/apps/brunch-agent/test/petrinaut-chat-result.ts @@ -21,13 +21,11 @@ export interface PetrinautChatResult { readonly resumedStatus: number; readonly resumedText: string; readonly resumedFinish: UIMessageChunk | undefined; - readonly retriedStatus: number; - readonly retriedResumeStatus: number; readonly historyUserEntryCount: number; readonly historyClientToolResultCount: number; readonly historyGetStatus: number; readonly historyUserText: string; - readonly foreignHistoryMessages: number; + readonly legacyRouteStatus: number; readonly unauthenticatedHistoryStatus: number; readonly foreignAgentHistoryStatus: number; readonly transcript: string; diff --git a/apps/brunch-agent/test/petrinaut-chat.integration.ts b/apps/brunch-agent/test/petrinaut-chat.integration.ts index 111f119aefa..20750ca68a4 100644 --- a/apps/brunch-agent/test/petrinaut-chat.integration.ts +++ b/apps/brunch-agent/test/petrinaut-chat.integration.ts @@ -1,7 +1,6 @@ import { mkdtemp } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { fileURLToPath } from "node:url"; import { fauxAssistantMessage, @@ -14,11 +13,18 @@ import { setProvider } from "@flue/runtime"; import { createFlueClient, FlueApiError } from "@flue/sdk"; import { READ_PETRINAUT_DOC_TOOL_NAME } from "@hashintel/brunch-agent-plugin-sdcpn/flue"; +import { + createFlueChatTransport, + snapshotToUiMessages, +} from "@hashintel/brunch-agent-transport-aisdk"; import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue"; import { PING_TOOL_NAME } from "../src/agents/chat-agent/tools/ping.ts"; import { applyCaptureSweep } from "../src/capture/apply-sweep.ts"; -import { CLIENT_TOOL_RESULT_SIGNAL } from "../src/conversation/client-tools.ts"; +import { + clientToolNames, + CLIENT_TOOL_RESULT_SIGNAL, +} from "../src/conversation/client-tools.ts"; import { agentOwnershipHeaders, flueConversationIdFrom, @@ -31,7 +37,7 @@ import type { PetrinautChatResult, PetrinautResumeResult, } from "./petrinaut-chat-result"; -import type { UIMessageChunk } from "ai"; +import type { UIMessage, UIMessageChunk } from "ai"; const ACTIVATE_SKILL_TOOL_NAME = "activate_skill"; const CHAT_MODEL_ID = "claude-haiku-4-5"; @@ -51,26 +57,26 @@ const dbFile = dbPath.endsWith(".db") process.env.BRUNCH_CHAT_MODEL = CHAT_MODEL_ID; process.env.BRUNCH_DEV_DB_PATH = dbFile; -process.env.BRUNCH_TRANSPORT_AISDK_INSPECT ??= "1"; - -const chunksFrom = (body: string): UIMessageChunk[] => - body - .trim() - .split("\n\n") - .slice(0, -1) - .map((frame) => JSON.parse(frame.slice("data: ".length)) as UIMessageChunk); +const chunksFrom = async ( + stream: ReadableStream, +): Promise => { + const chunks: UIMessageChunk[] = []; + const reader = stream.getReader(); + for (;;) { + const result = await reader.read(); + if (result.done) return chunks; + chunks.push(result.value); + } +}; const userTextFromHistory = ( - messages: readonly { - role?: string; - parts?: { type?: string; text?: string }[]; - }[], + messages: ReturnType, ): string => messages .filter((message) => message.role === "user") - .flatMap((message) => message.parts ?? []) + .flatMap((message) => message.parts) .filter((part) => part.type === "text") - .map((part) => part.text ?? "") + .map((part) => part.text) .join(""); const faux = fauxProvider({ @@ -89,27 +95,22 @@ try { fetch: appTransport, headers: agentOwnershipHeaders(identity), }); + const panelTransport = createFlueChatTransport({ + client: historyClient, + clientToolNames, + }); + const projectHistory = ( + snapshot: Awaited>, + ) => + snapshotToUiMessages(snapshot, { + clientToolNames, + }); if (process.env.BRUNCH_RESUME_PHASE === "1") { const snapshot = await historyClient.history(); - const historyGet = await app.fetch( - new Request( - `http://brunch.test/api/chat?id=${encodeURIComponent(conversationId)}`, - { - method: "GET", - headers: { "x-brunch-principal": principalKey }, - }, - ), - ); - const historyBody = (await historyGet.json()) as { - messages?: { - role?: string; - parts?: { type?: string; text?: string }[]; - }[]; - }; const result: PetrinautResumeResult = { - historyGetStatus: historyGet.status, - historyUserText: userTextFromHistory(historyBody.messages ?? []), + historyGetStatus: 200, + historyUserText: userTextFromHistory(projectHistory(snapshot)), transcript: formatFlueTranscript(snapshot), }; process.stdout.write(`PETRINAUT_RESUME_RESULT ${JSON.stringify(result)}\n`); @@ -208,36 +209,21 @@ try { ]), ]); - const fixturePath = fileURLToPath( - new URL( - "../../../libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-initial.post.json", - import.meta.url, - ), - ); - const { readFile } = await import("node:fs/promises"); - const initialBody = JSON.parse(await readFile(fixturePath, "utf8")) as { - id: string; - messages: { id: string; role: string; parts: unknown[] }[]; - trigger: string; - }; - initialBody.id = conversationId; - const userMessage = initialBody.messages[0]; - if (userMessage === undefined) { - throw new Error("panel-initial.post.json is missing the user message"); - } + const userMessage = { + id: "user-mission-1", + role: "user", + parts: [{ type: "text", text: "Run the FE-1435 transport probe." }], + } satisfies UIMessage; - const initialResponse = await app.fetch( - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { - "content-type": "application/json", - "x-brunch-principal": principalKey, - "x-request-id": "request-mission-1", - }, - body: JSON.stringify(initialBody), + const initialChunks = await chunksFrom( + await panelTransport.sendMessages({ + trigger: "submit-message", + chatId: conversationId, + messageId: undefined, + messages: [userMessage], + abortSignal: undefined, }), ); - const initialChunks = chunksFrom(await initialResponse.text()); const startChunk = initialChunks.find((chunk) => chunk.type === "start"); const pingCall = initialChunks.find( @@ -277,82 +263,44 @@ try { chunk.toolName === READ_PETRINAUT_DOC_TOOL_NAME, ) ?? null; - const pendingHistoryResponse = await app.fetch( - new Request( - `http://brunch.test/api/chat?id=${encodeURIComponent(conversationId)}`, - { - method: "GET", - headers: { "x-brunch-principal": principalKey }, - }, - ), - ); - const pendingHistoryBody = (await pendingHistoryResponse.json()) as { - messages?: { - parts?: { toolCallId?: string; state?: string }[]; - }[]; - }; - const pendingHistoryClientToolState = pendingHistoryBody.messages - ?.flatMap((message) => message.parts ?? []) - .find((part) => part.toolCallId === clientToolCall?.toolCallId)?.state; + const pendingHistory = projectHistory(await historyClient.history()); + const pendingHistoryClientToolState = pendingHistory + .flatMap((message) => message.parts) + .find( + (part) => + "toolCallId" in part && + part.toolCallId === clientToolCall?.toolCallId, + ); - const resumeBody = { - id: conversationId, - trigger: "submit-message", - messageId: startChunk?.messageId, - messages: [ - userMessage, - { - id: startChunk?.messageId, - role: "assistant", - parts: [ - { - type: `tool-${READ_PETRINAUT_DOC_TOOL_NAME}`, - toolCallId: clientToolCall?.toolCallId, - state: "output-available", - input: { doc: "ai-assistant" }, - output: - "# AI Assistant\nThe assistant can read its own documentation pages.", - }, - ], - }, - ], - }; - const resumeResponse = await app.fetch( - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { - "content-type": "application/json", - "x-brunch-principal": principalKey, - "x-request-id": "request-mission-1-resume", - }, - body: JSON.stringify(resumeBody), - }), - ); - const resumedChunks = chunksFrom(await resumeResponse.text()); - const retriedResumeResponse = await app.fetch( - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { - "content-type": "application/json", - "x-brunch-principal": principalKey, - "x-request-id": "request-mission-1-resume-retry", - }, - body: JSON.stringify(resumeBody), - }), - ); - await retriedResumeResponse.text(); - const retriedResponse = await app.fetch( - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { - "content-type": "application/json", - "x-brunch-principal": principalKey, - "x-request-id": "request-mission-1-retry", - }, - body: JSON.stringify(initialBody), + if (startChunk?.type !== "start" || clientToolCall === null) { + throw new Error("initial stream did not reach the client-tool pause"); + } + const resumeMessages = [ + userMessage, + { + id: startChunk.messageId, + role: "assistant" as const, + parts: [ + { + type: `tool-${READ_PETRINAUT_DOC_TOOL_NAME}`, + toolCallId: clientToolCall.toolCallId, + state: "output-available", + input: { doc: "ai-assistant" }, + output: + "# AI Assistant\nThe assistant can read its own documentation pages.", + }, + ], + }, + ] as UIMessage[]; + const resumedChunks = await chunksFrom( + await panelTransport.sendMessages({ + trigger: "submit-message", + chatId: conversationId, + messageId: startChunk.messageId, + messages: resumeMessages, + abortSignal: undefined, }), ); - await retriedResponse.text(); const snapshot = await historyClient.history(); const userEntryIds = snapshot.messages .filter( @@ -407,37 +355,14 @@ try { foreignAgentHistoryStatus = error instanceof FlueApiError ? error.status : -1; } - const historyGet = await app.fetch( - new Request( - `http://brunch.test/api/chat?id=${encodeURIComponent(conversationId)}`, - { - method: "GET", - headers: { "x-brunch-principal": principalKey }, - }, - ), + const historyMessages = projectHistory(snapshot); + const legacyRoute = await app.fetch( + new Request("http://brunch.test/api/chat"), ); - const historyBody = (await historyGet.json()) as { - messages?: { - role?: string; - parts?: { type?: string; text?: string }[]; - }[]; - }; - const foreignHistory = await app.fetch( - new Request( - `http://brunch.test/api/chat?id=${encodeURIComponent(conversationId)}`, - { - method: "GET", - headers: { "x-brunch-principal": "principal-other" }, - }, - ), - ); - const foreignBody = (await foreignHistory.json()) as { - messages?: unknown[]; - }; const result: PetrinautChatResult = { - status: initialResponse.status, - messageId: startChunk?.messageId, + status: 200, + messageId: startChunk.messageId, partIds: initialChunks .filter( (chunk) => @@ -461,23 +386,25 @@ try { clientToolOutputsOnInitial: initialChunks.filter( (chunk) => chunk.type === "tool-output-available" && - chunk.toolCallId === clientToolCall?.toolCallId, + chunk.toolCallId === clientToolCall.toolCallId, ), initialFinish: initialChunks.at(-1), - pendingHistoryClientToolState, - resumedStatus: resumeResponse.status, + pendingHistoryClientToolState: + pendingHistoryClientToolState === undefined || + !("state" in pendingHistoryClientToolState) + ? undefined + : pendingHistoryClientToolState.state, + resumedStatus: 200, resumedText: resumedChunks .filter((chunk) => chunk.type === "text-delta") .map((chunk) => chunk.delta) .join(""), resumedFinish: resumedChunks.at(-1), - retriedStatus: retriedResponse.status, - retriedResumeStatus: retriedResumeResponse.status, historyUserEntryCount: userEntryIds.length, historyClientToolResultCount: clientToolResultCount, - historyGetStatus: historyGet.status, - historyUserText: userTextFromHistory(historyBody.messages ?? []), - foreignHistoryMessages: foreignBody.messages?.length ?? -1, + historyGetStatus: 200, + historyUserText: userTextFromHistory(historyMessages), + legacyRouteStatus: legacyRoute.status, unauthenticatedHistoryStatus, foreignAgentHistoryStatus, transcript: formatFlueTranscript(snapshot), @@ -486,12 +413,7 @@ try { activateSkillCall, readSkillResourceCall, interviewerToolNames, - captureUserText: userTextFromHistory( - snapshot.messages.map((message) => ({ - role: message.role, - parts: message.parts, - })), - ), + captureUserText: userTextFromHistory(historyMessages), captureIds: firstSweep.captures.map((capture) => capture.id), recaptureIds: secondSweep.captures.map((capture) => capture.id), skippedDedupKeys: secondSweep.skippedDedupKeys, diff --git a/apps/brunch-agent/test/petrinaut-chat.test.ts b/apps/brunch-agent/test/petrinaut-chat.test.ts index 712f5cac82d..daf97db27ea 100644 --- a/apps/brunch-agent/test/petrinaut-chat.test.ts +++ b/apps/brunch-agent/test/petrinaut-chat.test.ts @@ -10,11 +10,10 @@ import type { PetrinautChatResult, PetrinautResumeResult, } from "./petrinaut-chat-result"; -import type { TransportInspectionEvent } from "@hashintel/brunch-agent-transport-aisdk"; const testDirectory = import.meta.dirname; -test("the committed /api/chat door streams a plain Flue agent through server and client tools", async () => { +test("the browser transport streams the mounted Flue agent through server and client tools", async () => { const dbDirectory = await mkdtemp(join(tmpdir(), "brunch-chat-")); const dbPath = join(dbDirectory, "conversations.db"); @@ -26,15 +25,6 @@ test("the committed /api/chat door streams a plain Flue agent through server and ); expect(exitCode, stderr || stdout).toBe(0); - const inspectionLines = stdout - .split("\n") - .filter((line) => line.startsWith("TRANSPORT_AISDK ")) - .map( - (line) => - JSON.parse( - line.slice("TRANSPORT_AISDK ".length), - ) as TransportInspectionEvent, - ); const resultLine = stdout .split("\n") .find((line) => line.startsWith("PETRINAUT_CHAT_RESULT ")); @@ -82,8 +72,6 @@ test("the committed /api/chat door streams a plain Flue agent through server and type: "finish", finishReason: "stop", }); - expect(result.retriedStatus).toBe(200); - expect(result.retriedResumeStatus).toBe(200); expect(result.historyUserEntryCount).toBe(1); expect(result.historyClientToolResultCount).toBe(1); @@ -91,7 +79,7 @@ test("the committed /api/chat door streams a plain Flue agent through server and expect(result.historyUserText).toContain( "Run the FE-1435 transport probe.", ); - expect(result.foreignHistoryMessages).toBe(0); + expect(result.legacyRouteStatus).toBe(404); expect(result.unauthenticatedHistoryStatus).toBe(401); expect(result.foreignAgentHistoryStatus).toBe(403); expect(result.transcript).toContain("Run the FE-1435 transport probe."); @@ -143,41 +131,6 @@ test("the committed /api/chat door streams a plain Flue agent through server and "Run the FE-1435 transport probe.", ); - expect(inspectionLines[0]).toMatchObject({ - type: "request-start", - requestId: "request-mission-1", - }); - expect(inspectionLines.some((event) => event.type === "resume-start")).toBe( - true, - ); - expect( - inspectionLines.filter((event) => event.type === "request-finish"), - ).toEqual([ - { - type: "request-finish", - requestId: "request-mission-1", - terminal: "completed", - }, - { - type: "request-finish", - requestId: "request-mission-1-resume", - terminal: "completed", - }, - { - type: "request-finish", - requestId: "request-mission-1-resume-retry", - terminal: "completed", - }, - { - type: "request-finish", - requestId: "request-mission-1-retry", - terminal: "completed", - }, - ]); - expect( - inspectionLines.filter((event) => event.type === "history-read"), - ).toHaveLength(3); - const resumed = await runNodeScript( join(testDirectory, "petrinaut-chat.integration.ts"), join(testDirectory, "../../.."), diff --git a/apps/brunch-agent/turbo.json b/apps/brunch-agent/turbo.json index c99ce816844..15876422c0b 100644 --- a/apps/brunch-agent/turbo.json +++ b/apps/brunch-agent/turbo.json @@ -13,7 +13,6 @@ "ANTHROPIC_API_KEY", "BRUNCH_CHAT_MODEL", "BRUNCH_DEV_DB_PATH", - "BRUNCH_PETRINAUT_ORIGINS", "BRUNCH_TRANSPORT_AISDK_INSPECT", "HASH_OTLP_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 30cc5d6a7e3..10a694237c6 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -76,7 +76,7 @@ provides a fake optimizer for isolated UI development. | `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice outside production. | | `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | | `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | -| `VITE_BRUNCH_CHAT_ENDPOINT` | for voice input | website | Full Brunch Petrinaut chat endpoint used by the panel. | +| `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | | `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | | `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | @@ -97,6 +97,8 @@ disclosure before requesting microphone access. The disclosure also provides a microphone check and is remembered in browser storage only after Voice mode starts. +When Brunch is selected, typed and finalized spoken turns both enter the same mounted Flue conversation route. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Closing or speaking over Voice playback only stops local media; it does not alter canonical conversation history. Reopening the same net restores its observed Flue conversation without resubmitting a turn or replaying settled audio. + An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a recovery state. Listening levels follow microphone input; provisional words @@ -118,13 +120,7 @@ Realtime call endpoint. The provider key, model, instructions, tools, language, and vocabulary policy stay server-side. The session uses semantic VAD with low eagerness so natural thinking pauses are less likely to end an answer early. -Realtime is the disposable media plane: it carries continuous microphone and -remote audio, detects complete turns, and handles barge-in. Brunch remains the -control plane and sole authority for questions, captures, state, completion, -and durable history. The browser bridge accepts only the configured -`continue_interview` function, validates and serializes its arguments, rejects -duplicate or stale calls, and submits the answer through Petrinaut's shared -composer path with pending-`brunch_ask` correlation. +Realtime is the disposable media plane: it carries continuous microphone and remote audio, detects complete turns, and handles barge-in. Brunch remains the control plane and sole authority for questions, captures, state, completion, and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-question correlation. The bridge waits for the correlated Brunch turn before returning canonical speech segments to Realtime. It then requests audio with tools disabled and @@ -134,13 +130,7 @@ microphone stays active while the interviewer speaks and while Brunch is working. Speaking over assistant audio interrupts playback automatically; WebRTC truncates provider-side unheard audio without changing Brunch history. -The Brunch deployment must allow the website origin through its -`BRUNCH_PETRINAUT_ORIGINS` setting. Denying microphone permission leaves the -text composer available and submits nothing to Brunch. When Voice mode cannot -continue, the inline recovery state distinguishes microphone, connection, and -other Voice failures, explains the next action, and offers **Reconnect** where -appropriate. Sanitized error codes and diagnostic references remain collapsed -under **Technical details**. +The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**. Realtime connection, transcription, and canonical speech timings use random request IDs, and the existing Brunch transport provides its own request diff --git a/apps/petrinaut-website/package.json b/apps/petrinaut-website/package.json index ca2eb5ecde5..2f9314e4ce9 100644 --- a/apps/petrinaut-website/package.json +++ b/apps/petrinaut-website/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@ai-sdk/openai": "3.0.63", + "@flue/sdk": "2.0.3", "@hashintel/brunch-agent": "workspace:*", "@hashintel/brunch-agent-transport-aisdk": "workspace:*", "@hashintel/ds-components": "workspace:*", diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts new file mode 100644 index 00000000000..e3226db07ec --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts @@ -0,0 +1,14 @@ +import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; + +/** + * The one catalog of tools the browser answers on Brunch's behalf. The panel + * transport admits their results, the history projection leaves them runnable, + * and every interactive widget the demo registers must name one of them so a + * composer answer reaches Flue as a `client-tool-result` rather than an error. + * Kept free of React imports so the transport can load outside the DOM. + */ +export const brunchClientToolNames: ReadonlySet = new Set([ + readPetrinautDocToolName, + ASK_TOOL_NAME, +]); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts index fcf0482e253..6df757d8614 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts @@ -1,34 +1,177 @@ -import { expect, test } from "vitest"; +import { expect, test, vi } from "vitest"; -import { createBrunchPanelTransport } from "./brunch-panel-transport"; +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "./brunch-panel-transport"; -import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; +import type { AgentSendResult, FlueClient } from "@flue/sdk"; -test("pins send and reconnect to the stable conversation id", async () => { - const seenChatIds: string[] = []; - const sourceTransport: PetrinautAiChatTransport = { - reconnectToStream: async (options) => { - seenChatIds.push(options.chatId); - return null; +test("delegates one typed message to the supplied Flue conversation", async () => { + const admission: AgentSendResult = { + streamUrl: "http://brunch.test/stream", + offset: "offset-1", + submissionId: "submission-1", + uid: "uid-1", + }; + const send = vi.fn(async () => admission); + const wait = vi.fn(async (_admission, options) => { + await options?.onEvent?.({ + type: "message-started", + conversationId: "conversation-stable", + messageId: "assistant-1", + submissionId: admission.submissionId, + turnId: "turn-1", + position: { batch: 1, index: 0 }, + }); + await options?.onEvent?.({ + type: "submission-settled", + conversationId: "conversation-stable", + submissionId: admission.submissionId, + outcome: "completed", + position: { batch: 1, index: 1 }, + }); + }); + const client = { + send, + wait, + } as Pick as FlueClient; + const tracker = new BrunchPanelConversationTracker(); + const admissionListener = vi.fn(); + tracker.subscribeToAdmission( + { kind: "user", messageId: "user-1" }, + admissionListener, + ); + const onAdmission = vi.fn(); + const transport = createBrunchPanelTransport( + Promise.resolve(client), + tracker, + { onAdmission }, + ); + const stream = await transport.sendMessages({ + trigger: "submit-message", + chatId: "conversation-stable", + messageId: undefined, + messages: [ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Typed tracer." }], + }, + ], + abortSignal: undefined, + }); + expect(admissionListener).toHaveBeenCalledOnce(); + expect(admissionListener).toHaveBeenCalledWith({ + admission, + kind: "user", + messageId: "user-1", + }); + await stream.pipeTo(new WritableStream()); + + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith({ + message: { kind: "user", body: "Typed tracer." }, + signal: undefined, + }); + expect(tracker.submissionForInput("user-1")).toBe("submission-1"); + expect(tracker.submissionsForResponse("assistant-1")).toEqual([ + "submission-1", + ]); + expect(onAdmission).toHaveBeenCalledOnce(); + expect(onAdmission).toHaveBeenCalledWith(admission); +}); + +test("matches client-tool admissions once and supports unsubscribe", () => { + const admission: AgentSendResult = { + streamUrl: "http://brunch.test/stream", + offset: "offset-1", + submissionId: "submission-1", + uid: "uid-1", + }; + const tracker = new BrunchPanelConversationTracker(); + const matchingListener = vi.fn(); + const unsubscribedListener = vi.fn(); + tracker.subscribeToAdmission( + { + kind: "client-tool-result", + messageId: "assistant-question", }, - sendMessages: async (options) => { - seenChatIds.push(options.chatId); - return new ReadableStream({ - start(controller) { - controller.close(); - }, - }); + matchingListener, + ); + const unsubscribe = tracker.subscribeToAdmission( + { + kind: "client-tool-result", + messageId: "assistant-question", }, + unsubscribedListener, + ); + unsubscribe(); + + tracker.recordAdmission({ + admission, + kind: "user", + messageId: "assistant-question", + }); + tracker.recordAdmission({ + admission, + kind: "client-tool-result", + messageId: "assistant-other", + }); + expect(matchingListener).not.toHaveBeenCalled(); + + const matchedAdmission = { + ...admission, + submissionId: "submission-tool-result", + }; + const event = { + admission: matchedAdmission, + kind: "client-tool-result" as const, + messageId: "assistant-question", }; + tracker.recordAdmission(event); + tracker.recordAdmission(event); - const transport = createBrunchPanelTransport( - sourceTransport, - "conversation-stable", + expect(matchingListener).toHaveBeenCalledOnce(); + expect(matchingListener).toHaveBeenCalledWith(event); + expect(unsubscribedListener).not.toHaveBeenCalled(); +}); + +test("records every submission that wrote a resumed assistant message", () => { + const tracker = new BrunchPanelConversationTracker(); + tracker.recordResponse("assistant-1", "submission-1"); + tracker.recordResponse("assistant-1", "submission-continuation"); + tracker.recordResponse("assistant-1", "submission-continuation"); + + expect(tracker.submissionsForResponse("assistant-1")).toEqual([ + "submission-1", + "submission-continuation", + ]); + expect(tracker.submissionsForResponse("assistant-2")).toBeUndefined(); +}); + +test("settles in-flight submissions before a durable abort can target them", async () => { + const tracker = new BrunchPanelConversationTracker(); + let admit: (() => void) | undefined; + void tracker.trackSubmission( + new Promise((resolve) => { + admit = resolve; + }), ); - await transport.sendMessages({ chatId: "generated-by-use-chat" } as never); - await transport.reconnectToStream({ - chatId: "generated-by-use-chat", - } as never); + let settled = false; + void tracker.settleInFlightSubmissions().then(() => { + settled = true; + }); + + await Promise.resolve(); + expect(settled).toBe(false); - expect(seenChatIds).toEqual(["conversation-stable", "conversation-stable"]); + admit?.(); + await vi.waitFor(() => expect(settled).toBe(true)); + + const rejected = tracker.trackSubmission( + Promise.reject(new Error("rejected admission")), + ); + await expect(rejected).rejects.toThrow("rejected admission"); + await expect(tracker.settleInFlightSubmissions()).resolves.toBeUndefined(); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index 3cdc3e19459..973075debb3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -1,15 +1,118 @@ +import { createFlueChatTransport } from "@hashintel/brunch-agent-transport-aisdk"; import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; import { sweepOutputSchema } from "../brunch-sweep-output"; +import { brunchClientToolNames } from "./brunch-client-tools"; import type { SweepCapture, SweepCompletionFailure, SweepCompletionReport, } from "../brunch-sweep-output"; +import type { AgentSendResult, FlueClient } from "@flue/sdk"; +import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; import type { UIMessageChunk } from "ai"; +export type BrunchPanelAdmission = Parameters< + NonNullable +>[0]; +export type BrunchPanelAdmissionTarget = Pick< + BrunchPanelAdmission, + "kind" | "messageId" +>; + +export class BrunchPanelConversationTracker { + readonly #admissionSubscriptions = new Set<{ + readonly listener: (admission: BrunchPanelAdmission) => void; + readonly target: BrunchPanelAdmissionTarget; + }>(); + readonly #inFlightSubmissions = new Set>(); + readonly #inputSubmissions = new Map< + string, + AgentSendResult["submissionId"] + >(); + readonly #responseSubmissions = new Map< + string, + AgentSendResult["submissionId"][] + >(); + + public recordAdmission(admission: BrunchPanelAdmission): void { + if (admission.kind === "user") { + this.#inputSubmissions.set( + admission.messageId, + admission.admission.submissionId, + ); + } + for (const subscription of this.#admissionSubscriptions) { + if ( + subscription.target.kind === admission.kind && + subscription.target.messageId === admission.messageId + ) { + this.#admissionSubscriptions.delete(subscription); + subscription.listener(admission); + } + } + } + + /** + * A client-tool continuation is projected onto the assistant message it + * resumes, so one message can be written by several submissions. Keep them + * all: Voice correlates a reply by membership, whichever side admitted the + * continuation. + */ + public recordResponse( + messageId: string, + submissionId: AgentSendResult["submissionId"], + ): void { + const recorded = this.#responseSubmissions.get(messageId); + if (recorded === undefined) { + this.#responseSubmissions.set(messageId, [submissionId]); + } else if (!recorded.includes(submissionId)) { + recorded.push(submissionId); + } + } + + /** + * Resolves once every submission currently between `send()` and its + * admission has been admitted or rejected, so a conversation-wide abort + * issued afterwards has a settled target rather than racing the admission. + */ + public settleInFlightSubmissions(): Promise { + return Promise.allSettled(this.#inFlightSubmissions).then(() => undefined); + } + + public trackSubmission(submission: Promise): Promise { + this.#inFlightSubmissions.add(submission); + const release = (): void => { + this.#inFlightSubmissions.delete(submission); + }; + submission.then(release, release); + return submission; + } + + public submissionForInput( + messageId: string, + ): AgentSendResult["submissionId"] | undefined { + return this.#inputSubmissions.get(messageId); + } + + public submissionsForResponse( + messageId: string, + ): readonly AgentSendResult["submissionId"][] | undefined { + return this.#responseSubmissions.get(messageId); + } + + public subscribeToAdmission( + target: BrunchPanelAdmissionTarget, + listener: (admission: BrunchPanelAdmission) => void, + ): () => void { + const subscription = { listener, target }; + this.#admissionSubscriptions.add(subscription); + return () => this.#admissionSubscriptions.delete(subscription); + } +} + const formatFailure = (failure: SweepCompletionFailure): string => { const location = failure.nodeId === undefined @@ -126,26 +229,30 @@ const decorateBrunchStream = ( ); }; -/** - * Pin Petrinaut's stock transport to one stable conversation id so reload, - * client-tool follow-up, and the voice dock share Flue's conversation. - */ +/** Adapt one mounted Flue conversation to Petrinaut's AI SDK rendering contract. */ export const createBrunchPanelTransport = ( - transport: PetrinautAiChatTransport, - conversationId: string, -): PetrinautAiChatTransport => ({ - reconnectToStream: async (options) => { - const stream = await transport.reconnectToStream({ - ...options, - chatId: conversationId, - }); - return stream === null ? null : decorateBrunchStream(stream); + clientPromise: Promise, + tracker: BrunchPanelConversationTracker, + hooks?: { + readonly onAdmission?: (admission: AgentSendResult) => void; }, - sendMessages: async (options) => - decorateBrunchStream( - await transport.sendMessages({ - ...options, - chatId: conversationId, - }), +): PetrinautAiChatTransport => ({ + reconnectToStream: async () => null, + sendMessages: (sendOptions) => + tracker.trackSubmission( + (async () => { + const client = await clientPromise; + const transport = createFlueChatTransport({ + client, + clientToolNames: brunchClientToolNames, + onAdmission: (event) => { + tracker.recordAdmission(event); + hooks?.onAdmission?.(event.admission); + }, + onResponseMessage: ({ messageId, submissionId }) => + tracker.recordResponse(messageId, submissionId), + }); + return decorateBrunchStream(await transport.sendMessages(sendOptions)); + })(), ), }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 02e1bfe1b7f..4744abfe074 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -8,11 +8,16 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/react"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; +import { brunchClientToolNames } from "./brunch-client-tools"; +import { BrunchPanelConversationTracker } from "./brunch-panel-transport"; import { + brunchInteractiveTools, getBrunchVoiceMode, LocalStorageDemoApp, + requestFlueStop, } from "./local-storage-demo-app"; +import type { FlueClient } from "@flue/sdk"; import type { PetrinautNavigationController } from "@hashintel/petrinaut/react"; const defaultTransportOptions = vi.hoisted(() => ({ @@ -87,6 +92,13 @@ describe("local storage demo Brunch voice integration", () => { }); }); + test("registers only interactive widgets that answer declared client tools", () => { + expect(brunchInteractiveTools.length).toBeGreaterThan(0); + for (const tool of brunchInteractiveTools) { + expect(brunchClientToolNames.has(tool.toolName)).toBe(true); + } + }); + test("correlates the existing Brunch transport request", () => { const options = defaultTransportOptions.current as { readonly headers: () => Record; @@ -96,6 +108,46 @@ describe("local storage demo Brunch voice integration", () => { /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u, ); }); + + test.each([ + [true, "stop-requested"], + [false, "already-settled"], + ] as const)( + "maps Flue abort result %s onto the host Stop contract", + async (aborted, expected) => { + const abort = vi.fn(async () => ({ aborted })); + const client = { abort } as Pick as FlueClient; + + await expect( + requestFlueStop( + Promise.resolve(client), + new BrunchPanelConversationTracker(), + ), + ).resolves.toBe(expected); + expect(abort).toHaveBeenCalledOnce(); + }, + ); + + test("lets an in-flight admission land before requesting the durable abort", async () => { + const abort = vi.fn(async () => ({ aborted: true })); + const client = { abort } as Pick as FlueClient; + const tracker = new BrunchPanelConversationTracker(); + let admit: (() => void) | undefined; + void tracker.trackSubmission( + new Promise((resolve) => { + admit = resolve; + }), + ); + + const stop = requestFlueStop(Promise.resolve(client), tracker); + await Promise.resolve(); + await Promise.resolve(); + expect(abort).not.toHaveBeenCalled(); + + admit?.(); + await expect(stop).resolves.toBe("stop-requested"); + expect(abort).toHaveBeenCalledOnce(); + }); }); /** diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 28f385e4ef1..2fb825cb30b 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -3,10 +3,14 @@ * @role Editable demo shell: nets in local storage, one live document handle */ +import { createFlueClient, type FlueConversationSettlement } from "@flue/sdk"; import { castDraft, produce } from "immer"; import { useEffect, useMemo, useState } from "react"; -import { BRUNCH_PRINCIPAL_HEADER } from "@hashintel/brunch-agent-transport-aisdk/headers"; +import { + agentOwnershipHeaders, + flueConversationIdWeb, +} from "@hashintel/brunch-agent-transport-aisdk"; import { createJsonDocHandle, type MinimalNetMetadata, @@ -21,7 +25,9 @@ import { import { DefaultChatTransport, Petrinaut, + type PetrinautAiInteractiveTool, type PetrinautAiMessage, + type PetrinautAiStopResult, type PetrinautAiVoiceMode, type PetrinautAiVoiceModeContext, WalkthroughProvider, @@ -41,7 +47,10 @@ import { } from "../voice-interview/voice-interview-control"; import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { getOrCreateBrunchConversationId } from "./brunch-conversation-id"; -import { createBrunchPanelTransport } from "./brunch-panel-transport"; +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "./brunch-panel-transport"; import { resolveBrunchPreviewConfig } from "./brunch-preview-config"; import { getOrCreateBrunchPrincipal } from "./brunch-principal"; import { useFlueChatHistory } from "./use-flue-chat-history"; @@ -72,18 +81,32 @@ const brunchPreviewConfig = resolveBrunchPreviewConfig( import.meta.env.VITE_BRUNCH_CHAT_ENDPOINT, ); -// Only Brunch keeps a conversation to hydrate from; the generic fallback route -// has no history door. -const brunchHistoryEndpoint = brunchPreviewConfig.isBrunchConfigured - ? brunchPreviewConfig.chatEndpoint - : null; - export const getBrunchVoiceMode = ( config: OpenAIVoiceConfig | null | undefined, + tracker?: BrunchPanelConversationTracker, + settlements?: readonly FlueConversationSettlement[], ): PetrinautAiVoiceMode | undefined => config ? (context: PetrinautAiVoiceModeContext) => ( - + + tracker?.submissionForInput(messageId) + } + resolveResponseSubmission={(messageId) => + tracker?.submissionsForResponse(messageId) + } + subscribeToAdmission={ + tracker === undefined + ? undefined + : (target, listener) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ) + } + /> ) : undefined; @@ -96,14 +119,52 @@ const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => const brunchPrincipal = getOrCreateBrunchPrincipal(); +/** Every widget here must answer a tool named in `brunchClientToolNames`. */ +export const brunchInteractiveTools: readonly PetrinautAiInteractiveTool[] = [ + brunchAskInteractiveTool, +]; + const stockChatTransport = new DefaultChatTransport({ api: brunchPreviewConfig.chatEndpoint, headers: () => ({ - [BRUNCH_PRINCIPAL_HEADER]: brunchPrincipal, [VOICE_REQUEST_ID_HEADER]: crypto.randomUUID(), }), }); +const createBrunchFlueClient = async (conversationId: string) => { + const identity = { conversationId, principalKey: brunchPrincipal }; + const instanceId = await flueConversationIdWeb(identity); + const mountUrl = new URL( + brunchPreviewConfig.chatEndpoint, + window.location.origin, + ); + mountUrl.pathname = `${mountUrl.pathname.replace(/\/+$/u, "")}/${instanceId}`; + return createFlueClient({ + url: mountUrl.href, + headers: agentOwnershipHeaders(identity), + }); +}; + +/** + * Flue's `abort()` is conversation-wide and only reaches unsettled work, so a + * Stop pressed while `send()` is still in flight must first let that admission + * land; otherwise `aborted: false` would read as "already settled" while the + * admitted turn keeps running. + */ +export const requestFlueStop = async ( + clientPromise: Promise>, + tracker: BrunchPanelConversationTracker, +): Promise => { + const client = await clientPromise; + await tracker.settleInFlightSubmissions(); + const result = await client.abort(); + return result.aborted ? "stop-requested" : "already-settled"; +}; + +const createConversationTrackerFor = ( + _conversationId: string | null, +): BrunchPanelConversationTracker => new BrunchPanelConversationTracker(); + const getStoredSDCPNsForDisplay = ( storedSDCPNs: Record, ): Record => { @@ -127,6 +188,64 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ fallbackNet: net, }); +type FlueChatHistory = ReturnType; + +const errorStatus = (error: Error | undefined): number | undefined => { + if ( + error !== undefined && + "status" in error && + typeof error.status === "number" + ) { + return error.status; + } + return undefined; +}; + +const BrunchConversationStatus = ({ + error, + latestSettlement, + phase, + refresh, +}: Pick< + FlueChatHistory, + "error" | "latestSettlement" | "phase" | "refresh" +>) => { + if (phase === undefined) return null; + + const label = + phase === "loading" + ? "Loading Brunch conversation…" + : phase === "connecting" + ? "Reconnecting to Brunch…" + : phase === "absent" + ? "New Brunch conversation" + : phase === "error" + ? errorStatus(error) === 401 || errorStatus(error) === 403 + ? "Brunch access was denied." + : "Brunch conversation unavailable." + : phase === "closed" + ? "Brunch conversation closed." + : latestSettlement?.outcome === "aborted" + ? "Last Brunch response stopped." + : latestSettlement?.outcome === "failed" + ? "Last Brunch response failed." + : "Brunch conversation ready."; + + return ( + + {label} + {phase === "error" && ( + <> + {" "} + + + )} + + ); +}; + /** * The demo's own palette command, registered beside Petrinaut's: picking it * in the palette starts a fresh net. @@ -166,8 +285,9 @@ export const LocalStorageDemoApp = ({ search: SharedExampleSearch; }) => { const sentryFeedbackAction = useSentryFeedbackAction(); - const [openAIVoiceConfig, setOpenAIVoiceConfig] = - useState(); + const [openAIVoiceConfig, setOpenAIVoiceConfig] = useState< + OpenAIVoiceConfig | null | undefined + >(() => (brunchPreviewConfig.isBrunchConfigured ? undefined : null)); /** * History is left to the library's default on purpose. That default already * replaces rather than pushes while an intent continues, so a drag-select @@ -201,8 +321,6 @@ export const LocalStorageDemoApp = ({ useEffect(() => { if (!brunchPreviewConfig.isBrunchConfigured) { - // eslint-disable-next-line react-hooks-js/set-state-in-effect -- Resolve the loading sentinel when voice is not configured. - setOpenAIVoiceConfig(null); return; } @@ -219,11 +337,6 @@ export const LocalStorageDemoApp = ({ return () => abortController.abort(); }, []); - const brunchVoiceMode = useMemo( - () => getBrunchVoiceMode(openAIVoiceConfig), - [openAIVoiceConfig], - ); - // Pick the most recently modified net const mostRecentlyModifiedNet = Object.values(storedSDCPNsForDisplay).sort( @@ -371,31 +484,73 @@ export const LocalStorageDemoApp = ({ const conversationId = currentNetId ? getOrCreateBrunchConversationId(currentNetId) : null; + const flueClientPromise = useMemo( + () => + brunchPreviewConfig.isBrunchConfigured && conversationId !== null + ? createBrunchFlueClient(conversationId) + : null, + [conversationId], + ); + const conversationTracker = useMemo( + // Correlation state belongs to one conversation and must not cross a net switch. + () => createConversationTrackerFor(conversationId), + [conversationId], + ); const flueHistory = useFlueChatHistory( - brunchHistoryEndpoint, + flueClientPromise, conversationId ?? "", - brunchPrincipal, + ); + const brunchVoiceMode = useMemo( + () => + getBrunchVoiceMode( + openAIVoiceConfig, + conversationTracker, + flueHistory.settlements, + ), + [conversationTracker, flueHistory.settlements, openAIVoiceConfig], ); const petrinautAiChatTransport = useMemo( () => - conversationId === null - ? createBrunchPanelTransport(stockChatTransport, "") - : createBrunchPanelTransport(stockChatTransport, conversationId), - [conversationId], + flueClientPromise === null + ? stockChatTransport + : createBrunchPanelTransport(flueClientPromise, conversationTracker, { + onAdmission: flueHistory.refresh, + }), + [conversationTracker, flueClientPromise, flueHistory.refresh], ); const aiAssistant = useMemo( () => ({ ...(conversationId === null ? {} : { conversationId }), - interactiveTools: [brunchAskInteractiveTool], + canClearMessages: flueClientPromise === null, + interactiveTools: brunchInteractiveTools, transport: petrinautAiChatTransport, - messages: flueHistory.ready - ? flueHistory.messages - : currentNetId - ? aiMessagesByNetId[currentNetId] - : undefined, + ...(flueClientPromise === null + ? {} + : { + requestStop: () => + requestFlueStop(flueClientPromise, conversationTracker), + }), + ...(flueClientPromise === null + ? {} + : { + renderComposerControl: () => ( + + ), + }), + messages: + flueClientPromise === null + ? currentNetId + ? aiMessagesByNetId[currentNetId] + : undefined + : flueHistory.messages, onMessages: (messages: PetrinautAiMessage[]) => { - if (!currentNetId) { + if (!currentNetId || flueClientPromise !== null) { return; } @@ -405,7 +560,7 @@ export const LocalStorageDemoApp = ({ })); }, onClearMessages: () => { - if (!currentNetId) { + if (!currentNetId || flueClientPromise !== null) { return; } @@ -424,10 +579,15 @@ export const LocalStorageDemoApp = ({ [ aiMessagesByNetId, brunchVoiceMode, + conversationTracker, conversationId, currentNetId, + flueClientPromise, + flueHistory.error, + flueHistory.latestSettlement, flueHistory.messages, - flueHistory.ready, + flueHistory.phase, + flueHistory.refresh, petrinautAiChatTransport, setAiMessagesByNetId, ], diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts index d06490a7235..f21c7a30033 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts @@ -1,69 +1,209 @@ +import { FlueApiError } from "@flue/sdk"; /** * @vitest-environment jsdom */ import { cleanup, renderHook, waitFor } from "@testing-library/react"; import { afterEach, expect, test, vi } from "vitest"; -import { BRUNCH_PRINCIPAL_HEADER } from "@hashintel/brunch-agent-transport-aisdk/headers"; - import { useFlueChatHistory } from "./use-flue-chat-history"; -const brunchEndpoint = "https://brunch.test/api/chat"; +import type { + AgentConversationObservation, + AgentConversationObservationSnapshot, + FlueClient, +} from "@flue/sdk"; afterEach(() => { cleanup(); - vi.unstubAllGlobals(); }); -test("hydrates from the configured Brunch endpoint", async () => { - const fetchMock = vi.fn(async () => - Response.json({ - messages: [{ id: "assistant-1", role: "assistant", parts: [] }], - }), - ); - vi.stubGlobal("fetch", fetchMock); +const createObservationHarness = ( + initialSnapshot: AgentConversationObservationSnapshot, +) => { + let snapshot = initialSnapshot; + const listeners = new Set<() => void>(); + const refresh = vi.fn(); + const close = vi.fn(); + const observation: AgentConversationObservation = { + close, + getSnapshot: () => snapshot, + refresh, + subscribe: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; + const observe = vi.fn(() => observation); + return { + clientPromise: Promise.resolve({ + observe, + } as Pick as FlueClient), + close, + observe, + publish(next: AgentConversationObservationSnapshot) { + snapshot = next; + for (const listener of listeners) listener(); + }, + refresh, + }; +}; +test("hydrates through the public Flue observation projection", async () => { + const harness = createObservationHarness({ + conversation: { + conversationId: "conversation-1", + settlements: [], + messages: [ + { + id: "assistant-1", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [{ type: "text", text: "Canonical reply.", state: "done" }], + }, + ], + }, + offset: "offset-1", + phase: "live", + error: undefined, + }); const { result } = renderHook(() => - useFlueChatHistory(brunchEndpoint, "conversation-1", "principal-1"), + useFlueChatHistory(harness.clientPromise, "conversation-1"), ); await waitFor(() => expect(result.current.ready).toBe(true)); expect(result.current.messages).toStrictEqual([ - { id: "assistant-1", role: "assistant", parts: [] }, + { + id: "assistant-1", + role: "assistant", + parts: [{ type: "text", text: "Canonical reply.", state: "done" }], + }, ]); + expect(result.current.phase).toBe("live"); + expect(harness.observe).toHaveBeenCalledWith({ live: "sse" }); +}); + +test("exposes the canonical settlement index for Voice correlation", async () => { + const harness = createObservationHarness({ + conversation: { + conversationId: "conversation-1", + settlements: [{ submissionId: "submission-1", outcome: "aborted" }], + messages: [], + }, + offset: "offset-1", + phase: "live", + error: undefined, + }); + const { result } = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); - const [url, init] = fetchMock.mock.calls[0] as unknown as [ - URL, - { headers: Record }, - ]; - expect(url.toString()).toBe("https://brunch.test/api/chat?id=conversation-1"); - expect(init.headers[BRUNCH_PRINCIPAL_HEADER]).toBe("principal-1"); + await waitFor(() => expect(result.current.ready).toBe(true)); + expect(result.current.settlements).toEqual([ + { submissionId: "submission-1", outcome: "aborted" }, + ]); }); test("asks nothing of the generic chat route, which keeps no history", () => { - const fetchMock = vi.fn(); - vi.stubGlobal("fetch", fetchMock); - const { result } = renderHook(() => - useFlueChatHistory(null, "conversation-1", "principal-1"), + useFlueChatHistory(null, "conversation-1"), ); - expect(fetchMock).not.toHaveBeenCalled(); expect(result.current.ready).toBe(false); expect(result.current.messages).toBeUndefined(); }); -test("leaves the panel on its local cache when hydration fails", async () => { - const fetchMock = vi.fn( - async () => new Response("Method not allowed", { status: 405 }), +test("represents an absent conversation as an empty canonical history", async () => { + const harness = createObservationHarness({ + conversation: undefined, + offset: undefined, + phase: "absent", + error: undefined, + }); + const { result } = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), ); - vi.stubGlobal("fetch", fetchMock); + await waitFor(() => expect(result.current.phase).toBe("absent")); + expect(result.current.ready).toBe(true); + expect(result.current.messages).toEqual([]); +}); + +test("retains canonical messages while the SDK reconnects", async () => { + const conversation = { + conversationId: "conversation-1", + settlements: [ + { submissionId: "submission-1", outcome: "completed" as const }, + ], + messages: [ + { + id: "assistant-1", + role: "assistant" as const, + purpose: "assistant" as const, + display: "visible" as const, + parts: [ + { type: "text" as const, text: "Settled.", state: "done" as const }, + ], + }, + ], + }; + const harness = createObservationHarness({ + conversation, + offset: "offset-1", + phase: "live", + error: undefined, + }); const { result } = renderHook(() => - useFlueChatHistory(brunchEndpoint, "conversation-1", "principal-1"), + useFlueChatHistory(harness.clientPromise, "conversation-1"), ); + await waitFor(() => expect(result.current.phase).toBe("live")); - await waitFor(() => expect(fetchMock).toHaveBeenCalled()); - expect(result.current.ready).toBe(false); - expect(result.current.messages).toBeUndefined(); + harness.publish({ + conversation, + offset: "offset-1", + phase: "connecting", + error: new TypeError("network unavailable"), + }); + + await waitFor(() => expect(result.current.phase).toBe("connecting")); + expect(result.current.messages?.[0]?.id).toBe("assistant-1"); + expect(result.current.latestSettlement?.outcome).toBe("completed"); +}); + +test.each([401, 403])( + "surfaces fatal ownership status %s and exposes SDK refresh", + async (status) => { + const failure = new FlueApiError(status, ""); + const harness = createObservationHarness({ + conversation: undefined, + offset: undefined, + phase: "error", + error: failure, + }); + const { result } = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + + await waitFor(() => expect(result.current.phase).toBe("error")); + expect(result.current.error).toBe(failure); + result.current.refresh(); + expect(harness.refresh).toHaveBeenCalledOnce(); + }, +); + +test("closes the SDK observation on unmount", async () => { + const harness = createObservationHarness({ + conversation: undefined, + offset: undefined, + phase: "loading", + error: undefined, + }); + const { unmount } = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + await waitFor(() => expect(harness.observe).toHaveBeenCalledOnce()); + + unmount(); + + expect(harness.close).toHaveBeenCalledOnce(); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts index d33e4f1deac..7e3bf752055 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts @@ -1,69 +1,115 @@ -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; -import { BRUNCH_PRINCIPAL_HEADER } from "@hashintel/brunch-agent-transport-aisdk/headers"; +import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; +import { brunchClientToolNames } from "./brunch-client-tools"; + +import type { + AgentConversationObservation, + AgentConversationObservationPhase, + AgentConversationObservationSnapshot, + FlueClient, + FlueConversationSettlement, + FlueConversationState, +} from "@flue/sdk"; import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; -/** - * Hydrates the panel from the Brunch agent's `GET ?id=` door. - * - * `endpoint` is null whenever the preview runs against the generic OpenAI - * route instead: that route keeps no conversation history and answers anything - * but POST with 405, so asking it is pure noise. - */ +const noSettlements: readonly FlueConversationSettlement[] = []; + +const projectPetrinautMessages = ( + conversation: FlueConversationState, +): PetrinautAiMessage[] => + // The host owns this narrowing: its configured client-tool catalog is the + // same catalog Petrinaut's message type exposes. + snapshotToUiMessages(conversation, { + clientToolNames: brunchClientToolNames, + }) as PetrinautAiMessage[]; + export const useFlueChatHistory = ( - endpoint: string | null, + clientPromise: Promise | null, conversationId: string, - principal: string, ): { + readonly error: Error | undefined; + readonly latestSettlement: FlueConversationSettlement | undefined; readonly messages: PetrinautAiMessage[] | undefined; + readonly phase: AgentConversationObservationPhase | undefined; readonly ready: boolean; + readonly refresh: () => void; + readonly settlements: readonly FlueConversationSettlement[]; } => { - const [loaded, setLoaded] = useState<{ + const observationRef = useRef(null); + const [observed, setObserved] = useState<{ readonly conversationId: string; - readonly messages: PetrinautAiMessage[]; + readonly snapshot: AgentConversationObservationSnapshot; }>(); + const refresh = useCallback(() => observationRef.current?.refresh(), []); + useEffect(() => { - if (endpoint === null || conversationId.length === 0) { + if (clientPromise === null || conversationId.length === 0) { + observationRef.current = null; return; } let cancelled = false; - const load = async (): Promise => { + let unsubscribe: (() => void) | undefined; + let observation: AgentConversationObservation | undefined; + const observe = async (): Promise => { try { - // The endpoint is a full Brunch URL in every configured deployment, - // but resolving it against the page keeps a relative one working. - const url = new URL(endpoint, window.location.origin); - url.searchParams.set("id", conversationId); - const response = await fetch(url, { - headers: { [BRUNCH_PRINCIPAL_HEADER]: principal }, - }); - if (!response.ok) { - return; - } - const body = (await response.json()) as { - messages?: PetrinautAiMessage[]; + const client = await clientPromise; + if (cancelled) return; + observation = client.observe({ live: "sse" }); + observationRef.current = observation; + const publish = (): void => { + if (!cancelled && observation !== undefined) { + setObserved({ + conversationId, + snapshot: observation.getSnapshot(), + }); + } }; - if (!cancelled) { - setLoaded({ - conversationId, - messages: body.messages ?? [], - }); - } - } catch { - // Leave `loaded` stale so the panel keeps using its localStorage cache. + publish(); + unsubscribe = observation.subscribe(publish); + } catch (caught) { + if (cancelled) return; + setObserved({ + conversationId, + snapshot: { + conversation: undefined, + offset: undefined, + phase: "error", + error: caught instanceof Error ? caught : new Error(String(caught)), + }, + }); } }; - void load(); + void observe(); return () => { cancelled = true; + unsubscribe?.(); + observation?.close(); + if (observationRef.current === observation) { + observationRef.current = null; + } }; - }, [conversationId, endpoint, principal]); + }, [clientPromise, conversationId]); - const ready = - conversationId.length > 0 && loaded?.conversationId === conversationId; + const snapshot = + observed?.conversationId === conversationId ? observed.snapshot : undefined; + const conversation = snapshot?.conversation; + const absent = snapshot?.phase === "absent"; + const ready = absent || conversation !== undefined; return { - messages: ready ? loaded.messages : undefined, + error: snapshot?.error, + latestSettlement: conversation?.settlements.at(-1), + messages: + conversation === undefined + ? absent + ? [] + : undefined + : projectPetrinautMessages(conversation), + phase: snapshot?.phase, ready, + refresh, + settlements: conversation?.settlements ?? noSettlements, }; }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index bb111385b7e..b99a0afdf30 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -5,6 +5,7 @@ import { import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; +import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; export { hashCanonicalSpeechText }; @@ -15,6 +16,11 @@ export interface CanonicalSpeechSegment { readonly messageId: string; readonly partId: string; readonly source: "assistant-text" | "brunch-ask"; + /** + * Every Flue submission that wrote to this segment's message: the one that + * started it plus any client-tool continuation projected back onto it. + */ + readonly submissionIds?: readonly AgentSendResult["submissionId"][]; readonly text: string; } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 2e486daae5d..70f7bc4088f 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -249,6 +249,7 @@ describe("OpenAIRealtimeSession", () => { { connectionEpoch: 1, responseId: "response-canonical", + speechRequestId: "canonical-1-1", type: "output-started", }, { @@ -354,6 +355,55 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("closes a stopped function call without requesting speech", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const sentBefore = sentEvents(channel).length; + + harness.session.completeFunctionCallWithoutResponse( + "call-stopped", + "aborted", + ); + + expect(sentEvents(channel).slice(sentBefore)).toEqual([ + { + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-stopped", + output: JSON.stringify({ response_text: [], outcome: "aborted" }), + }, + }, + ]); + }); + + test("preserves exact canonical whitespace while rejecting blank speech", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + + harness.session.completeFunctionCall("call-exact", [ + canonicalSegment("ask-exact", " Exact Brunch text.\n"), + ]); + + expect(sentEvents(channel)[0]).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-exact", + output: JSON.stringify({ + response_text: [" Exact Brunch text.\n"], + }), + }, + }); + const sentCount = sentEvents(channel).length; + expect(() => + harness.session.speakCanonical([canonicalSegment("ask-blank", " \n\t")]), + ).toThrow(); + expect(sentEvents(channel)).toHaveLength(sentCount); + }); + test("queues canonical speech behind an active Realtime response", async () => { const harness = createHarness(); await harness.session.connect(); @@ -368,6 +418,9 @@ describe("OpenAIRealtimeSession", () => { ]); expect(sentEvents(channel)).toEqual([]); + expect(harness.events).not.toContainEqual( + expect.objectContaining({ type: "canonical-speech-requested" }), + ); channel.receive({ response: { id: "response-active", @@ -384,6 +437,11 @@ describe("OpenAIRealtimeSession", () => { metadata: { petrinaut_kind: "canonical-speech" }, }, }); + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + speechRequestId: "canonical-1-1", + type: "canonical-speech-requested", + }); expect(harness.events).toContainEqual({ connectionEpoch: 1, responseId: "response-active", @@ -707,12 +765,10 @@ describe("OpenAIRealtimeSession", () => { type: "response.done", }); - expect(harness.events).toEqual([ - expect.objectContaining({ - code: "invalid-response", - type: "error", - }), - ]); + expect(harness.events.at(-1)).toMatchObject({ + code: "invalid-response", + type: "error", + }); expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index f7c13d9c28c..99b66aac151 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -50,8 +50,14 @@ export type OpenAIRealtimeSessionEvent = | { readonly connectionEpoch: number; readonly responseId: string; + readonly speechRequestId: string; readonly type: "output-started"; } + | { + readonly connectionEpoch: number; + readonly speechRequestId: string; + readonly type: "canonical-speech-requested"; + } | { readonly connectionEpoch: number; readonly responseId: string; @@ -198,6 +204,7 @@ export class OpenAIRealtimeSession { readonly #pendingClientEvents = new Map(); readonly #pendingSpeechRequests = new Map(); readonly #remoteStreams = new Set(); + readonly #speechRequestIds = new Map(); readonly #speechTimings = new Map(); readonly #transcriptionTimings = new Map(); #abortController: AbortController | null = null; @@ -421,6 +428,28 @@ export class OpenAIRealtimeSession { this.#requestCanonicalSpeech(segments, false); } + /** + * Close a Realtime function call whose Brunch turn settled without a reply. + * The call output records the settlement so the model does not wait on it, + * and no speech is requested: a stopped turn has no canonical text to speak. + */ + public completeFunctionCallWithoutResponse( + callId: string, + outcome: "aborted" | "failed", + ): void { + if (!callId) { + throw new VoiceError("speech", "invalid-response", ""); + } + this.#send({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: callId, + output: JSON.stringify({ response_text: [], outcome }), + }, + }); + } + public cancelOutput(): void { if (!this.#connected || this.#dataChannel?.readyState !== "open") { return; @@ -464,9 +493,12 @@ export class OpenAIRealtimeSession { #canonicalResponseText(segments: CanonicalSpeechSegment[]): string[] { const responseText = segments .slice(0, MAX_CANONICAL_SEGMENTS) - .map(({ text }) => text.trim()) - .filter(Boolean); - if (responseText.length === 0 || responseText.length !== segments.length) { + .map(({ text }) => text); + if ( + responseText.length === 0 || + responseText.length !== segments.length || + responseText.some((text) => text.trim().length === 0) + ) { throw new VoiceError("speech", "invalid-response", ""); } return responseText; @@ -573,6 +605,13 @@ export class OpenAIRealtimeSession { response: request.response, type: "response.create", }); + if (this.#activeEpoch !== null) { + this.#emit({ + connectionEpoch: this.#activeEpoch, + speechRequestId: request.speechRequestId, + type: "canonical-speech-requested", + }); + } } catch (error) { this.#responseCreateEventId = null; this.#pendingClientEvents.delete(eventId); @@ -682,6 +721,7 @@ export class OpenAIRealtimeSession { } this.#pendingSpeechRequests.delete(speechRequestId); this.#authorizedResponseIds.add(responseId); + this.#speechRequestIds.set(responseId, speechRequestId); this.#speechTimings.set(responseId, timing); } @@ -910,7 +950,17 @@ export class OpenAIRealtimeSession { return; } this.#speakingResponseId = responseId; - this.#emit({ connectionEpoch, responseId, type: "output-started" }); + const speechRequestId = this.#speechRequestIds.get(responseId); + if (!speechRequestId) { + this.#handleConnectionFailure("invalid-response", "connection"); + return; + } + this.#emit({ + connectionEpoch, + responseId, + speechRequestId, + type: "output-started", + }); return; } const wasSpeaking = this.#speakingResponseId === responseId; @@ -1004,6 +1054,7 @@ export class OpenAIRealtimeSession { errorCode, ); } + this.#speechRequestIds.delete(responseId); this.#authorizedResponseIds.delete(responseId); if (this.#speakingResponseId === responseId) { this.#speakingResponseId = null; @@ -1285,6 +1336,7 @@ export class OpenAIRealtimeSession { this.#pendingClientEvents.clear(); this.#pendingSpeechRequests.clear(); this.#speechTimings.clear(); + this.#speechRequestIds.clear(); this.#authorizedResponseIds.clear(); this.#canonicalResponseIds.clear(); this.#responseCreateEventId = null; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index a723f2dfe5a..b65fc510f9c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -26,6 +26,7 @@ const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { completeFunctionCall: vi.fn(), + completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; @@ -34,15 +35,14 @@ const createHarness = () => { }; }), }; - const submitInterviewAnswer = vi.fn( - async (): Promise< - | { kind: "interactive-tool"; toolCallId: string } - | { kind: "message"; messageId: string } - > => ({ - kind: "interactive-tool", - toolCallId: "ask-current", - }), - ); + const submitInterviewAnswer = vi.fn< + ConstructorParameters< + typeof RealtimeBrunchBridge + >[0]["submitInterviewAnswer"] + >(async () => ({ + kind: "interactive-tool", + toolCallId: "ask-current", + })); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -126,6 +126,30 @@ describe("RealtimeBrunchBridge", () => { ]); }); + test("rehydrates the settled Voice turn without resubmission or playback", () => { + const harness = createHarness(); + const settledResponse = { + ...segment( + "settled-response", + "This canonical response was already delivered.", + "assistant-text", + ), + submissionIds: ["submission-settled"], + }; + + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [settledResponse], + status: "ready", + }); + harness.bridge.start(9); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.events).toEqual([]); + }); + test("streams and validates one tool call, preserves ask correlation, and waits for canonical Brunch output", async () => { const harness = createHarness(); const question = segment("ask-current", "What happens after approval?"); @@ -144,10 +168,16 @@ describe("RealtimeBrunchBridge", () => { await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith({ - id: createRealtimeSubmissionId(7, "call-1"), - text: "The supervisor approves it.", - }); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + admissionTarget: { + kind: "client-tool-result", + messageId: "message-ask-current", + }, + id: createRealtimeSubmissionId(7, "call-1"), + text: "The supervisor approves it.", + }), + ); expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); harness.bridge.updateChat({ @@ -180,15 +210,88 @@ describe("RealtimeBrunchBridge", () => { expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", "submission-accepted", + "canonical-text-ready", + "submission-settled", "canonical-response-ready", ]); }); - test("uses the first spoken turn to start Brunch when no question exists", async () => { + test("emits the real admission before composer submission completes", async () => { + const harness = createHarness(); + let finishSubmission: (() => void) | undefined; + harness.submitInterviewAnswer.mockImplementationOnce(async () => { + await new Promise((resolve) => { + finishSubmission = resolve; + }); + return { + kind: "interactive-tool", + toolCallId: "ask-current", + }; + }); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [segment("ask-current", "Question")], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + const submission = harness.submitInterviewAnswer.mock.calls[0]?.[0]; + expect(submission).toBeDefined(); + submission?.onAdmission("submission-early"); + submission?.onAdmission("submission-early"); + + expect(harness.events).toEqual([ + expect.objectContaining({ type: "submission-started" }), + { + callId: "call-1", + submissionId: "submission-early", + type: "submission-admitted", + }, + ]); + + finishSubmission?.(); + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + const question = segment("ask-current", "Question"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question], + status: "streaming", + }); + const unrelated = segment("unrelated", "Do not select this."); + const correlated = { + ...segment("correlated", "Select this response."), + submissionIds: ["submission-early"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question, unrelated, correlated], + status: "ready", + }); + expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( + "call-1", + [correlated], + ); + harness.bridge.stop(); + submission?.onAdmission("submission-stale"); + expect( + harness.events.filter(({ type }) => type === "submission-admitted"), + ).toHaveLength(1); + }); + + test("admits one finalized Realtime answer through Flue once", async () => { const harness = createHarness(); harness.submitInterviewAnswer.mockResolvedValueOnce({ kind: "message", messageId: "message-kickoff", + submissionId: "submission-voice-1", }); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, @@ -197,26 +300,43 @@ describe("RealtimeBrunchBridge", () => { }); harness.bridge.start(7); + harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith({ - id: createRealtimeSubmissionId(7, "call-1"), - text: "Battery charger workflow", - }), + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + admissionTarget: { + kind: "user", + messageId: createRealtimeSubmissionId(7, "call-1"), + }, + id: createRealtimeSubmissionId(7, "call-1"), + text: "Battery charger workflow", + }), + ), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, canonicalSegments: [], status: "submitted", }); - const firstQuestion = segment( - "ask-first", - "What starts the battery charger workflow?", - ); + const unrelated = { + ...segment("unrelated", "Do not speak this response."), + submissionIds: ["submission-other"], + }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [firstQuestion], + canonicalSegments: [unrelated], + status: "ready", + }); + expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + const firstQuestion = { + ...segment("ask-first", "What starts the battery charger workflow?"), + submissionIds: ["submission-voice-1"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [unrelated, firstQuestion], status: "ready", }); @@ -227,10 +347,40 @@ describe("RealtimeBrunchBridge", () => { expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", "submission-accepted", + "canonical-text-ready", + "submission-settled", "canonical-response-ready", ]); }); + test("rejects a submission result that disagrees with the transport admission", async () => { + const harness = createHarness(); + harness.submitInterviewAnswer.mockImplementationOnce(async (input) => { + input.onAdmission("submission-early"); + return { + kind: "message", + messageId: input.id, + submissionId: "submission-other", + }; + }); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ + code: "interview-correlation", + type: "error", + }), + ), + ); + }); + test("requires a correlated Brunch busy cycle before accepting new canonical segments", async () => { const harness = createHarness(); const question = segment("ask-current", "What happens after approval?"); @@ -350,10 +500,16 @@ describe("RealtimeBrunchBridge", () => { await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith({ - id: createRealtimeSubmissionId(3, "call-2"), - text: "Accepted", - }); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + admissionTarget: { + kind: "client-tool-result", + messageId: "message-ask-current", + }, + id: createRealtimeSubmissionId(3, "call-2"), + text: "Accepted", + }), + ); expect(harness.events).not.toContainEqual( expect.objectContaining({ type: "error" }), ); @@ -452,6 +608,225 @@ describe("RealtimeBrunchBridge", () => { expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); }); + test("records first canonical text before the turn settles", async () => { + const harness = createHarness(); + const question = segment("ask-current", "Question"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( + "submission-text", + ); + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question], + status: "streaming", + }); + const firstText = { + ...segment("first", "First completed block.", "assistant-text"), + submissionIds: ["submission-text"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question, firstText], + status: "streaming", + }); + + const typesWhileStreaming = harness.events.map(({ type }) => type); + expect(typesWhileStreaming).toContain("canonical-text-ready"); + expect(typesWhileStreaming).not.toContain("submission-settled"); + expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question, firstText], + status: "ready", + }); + expect( + harness.events.filter(({ type }) => type === "canonical-text-ready"), + ).toHaveLength(1); + expect(harness.events.map(({ type }) => type)).toContain( + "submission-settled", + ); + expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( + "call-1", + [firstText], + ); + }); + + test("closes a durably stopped Voice turn without speaking", async () => { + const harness = createHarness(); + const question = segment("ask-current", "Question"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( + "submission-stopped", + ); + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question], + status: "streaming", + }); + + // A completed step with no text is not a stop: the panel may still be + // sending the client-tool follow-up that carries the reply. + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + settlements: [ + { outcome: "completed", submissionId: "submission-stopped" }, + ], + status: "ready", + }); + expect( + harness.session.completeFunctionCallWithoutResponse, + ).not.toHaveBeenCalled(); + expect(harness.events.map(({ type }) => type)).not.toContain( + "submission-settled", + ); + + const speechRequestsBeforeStop = + harness.session.speakCanonical.mock.calls.length; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + settlements: [{ outcome: "aborted", submissionId: "submission-stopped" }], + status: "ready", + }); + expect( + harness.session.completeFunctionCallWithoutResponse, + ).toHaveBeenCalledWith("call-1", "aborted"); + expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.session.speakCanonical).toHaveBeenCalledTimes( + speechRequestsBeforeStop, + ); + expect(harness.events.map(({ type }) => type)).toEqual( + expect.arrayContaining(["submission-settled", "submission-stopped"]), + ); + expect(harness.events.some(({ type }) => type === "error")).toBe(false); + }); + + test("speaks the folded continuation that answers a Voice brunch_ask follow-up", async () => { + const harness = createHarness(); + const question = { + ...segment("ask-current", "Question"), + submissionIds: ["submission-question"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( + "submission-answer", + ); + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question], + status: "streaming", + }); + + // The continuation is projected onto the message that asked, so that + // message is now written by both submissions. + const nextQuestion = { + ...segment("ask-next", "Next question"), + messageId: question.messageId, + submissionIds: ["submission-question", "submission-answer"], + }; + const askedAgain = { + ...question, + submissionIds: nextQuestion.submissionIds, + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [askedAgain, nextQuestion], + status: "ready", + }); + + expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( + "call-1", + [nextQuestion], + ); + }); + + test("speaks a reply that arrives through a client-tool follow-up", async () => { + const harness = createHarness(); + harness.submitInterviewAnswer.mockResolvedValueOnce({ + kind: "message", + messageId: "message-kickoff", + submissionId: "submission-voice-1", + }); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7, '{"answer":"Read the guide first."}')); + await vi.waitFor(() => + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "submitted", + }); + + // Brunch read a doc mid-turn; the panel's follow-up submission finished + // the same assistant message. + const reply = { + ...segment("reply", "The guide says hello.", "assistant-text"), + submissionIds: ["submission-voice-1", "submission-doc-follow-up"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [reply], + status: "ready", + }); + + expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( + "call-1", + [reply], + ); + }); + test("speaks new canonical text turns without creating a Realtime tool result", () => { const harness = createHarness(); const question = segment("ask-current", "Question"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 1a84fa74c62..a7b0014cc59 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,12 +1,23 @@ import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; - -type ChatStatus = "ready" | "submitted" | "streaming" | "error"; +import type { AgentSendResult, FlueConversationSettlement } from "@flue/sdk"; +import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; +import type { + PetrinautAiComposerSubmitTextResult, + PetrinautAiVoiceModeContext, +} from "@hashintel/petrinaut/ui"; + +export type VoiceSubmissionSettlement = Pick< + FlueConversationSettlement, + "outcome" | "submissionId" +>; interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; - readonly status: ChatStatus; + /** Flue's settlement index: the only witness that a turn ended short of a reply. */ + readonly settlements?: readonly VoiceSubmissionSettlement[]; + readonly status: PetrinautAiVoiceModeContext["status"]; } interface RealtimeBridgeSession { @@ -14,18 +25,37 @@ interface RealtimeBridgeSession { callId: string, segments: CanonicalSpeechSegment[], ): void; + completeFunctionCallWithoutResponse( + callId: string, + outcome: Exclude, + ): void; speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } -interface SubmitInterviewAnswerInput { +type SubmitVoiceInput = Parameters< + PetrinautAiVoiceModeContext["submitVoiceInput"] +>[0]; +type FlueChatAdmission = Parameters< + NonNullable +>[0]; +export type RealtimeBrunchAdmissionTarget = Pick< + FlueChatAdmission, + "kind" | "messageId" +>; + +type SubmitInterviewAnswerInput = Pick & { + readonly admissionTarget: RealtimeBrunchAdmissionTarget; readonly id: string; - readonly text: string; -} + readonly onAdmission: (submissionId: AgentSendResult["submissionId"]) => void; + readonly signal: AbortSignal; +}; type SubmitInterviewAnswerResult = - | { readonly kind: "interactive-tool"; readonly toolCallId: string } - | { readonly kind: "message"; readonly messageId: string }; + | Extract + | (Extract & { + readonly submissionId?: AgentSendResult["submissionId"]; + }); interface RealtimeBrunchBridgeDependencies { readonly session: RealtimeBridgeSession; @@ -35,12 +65,16 @@ interface RealtimeBrunchBridgeDependencies { } interface ActiveSubmission { + readonly abortController: AbortController; readonly baselineSegmentIds: ReadonlySet; readonly callId: string; readonly epoch: number; readonly pendingQuestionId: string | null; + readonly pendingQuestionMessageId: string | null; correlated: boolean; + firstTextEmitted: boolean; sawBusyChatStatus: boolean; + submissionId: AgentSendResult["submissionId"] | null; } interface ArgumentStream { @@ -65,11 +99,32 @@ export type RealtimeBrunchBridgeEvent = readonly callId: string; readonly type: "submission-accepted"; } + | { + readonly callId: string; + readonly submissionId: AgentSendResult["submissionId"]; + readonly type: "submission-admitted"; + } | { readonly callId: string; readonly segments: CanonicalSpeechSegment[]; readonly type: "canonical-response-ready"; } + | { + readonly callId: string; + readonly type: "canonical-text-ready"; + } + | { + readonly callId: string; + readonly type: "submission-settled"; + } + | { + readonly callId: string; + readonly outcome: Exclude< + VoiceSubmissionSettlement["outcome"], + "completed" + >; + readonly type: "submission-stopped"; + } | { readonly code: RealtimeBridgeErrorCode; readonly message: string; @@ -146,6 +201,7 @@ export class RealtimeBrunchBridge { public start(connectionEpoch: number): void { ++this.#generation; + this.#activeSubmission?.abortController.abort(); this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; this.#argumentDeltas.clear(); @@ -168,6 +224,7 @@ export class RealtimeBrunchBridge { public stop(): void { ++this.#generation; + this.#activeSubmission?.abortController.abort(); this.#activeEpoch = null; this.#activeSubmission = null; this.#argumentDeltas.clear(); @@ -224,6 +281,7 @@ export class RealtimeBrunchBridge { code: RealtimeBridgeErrorCode = "interview-correlation", ): void { ++this.#generation; + this.#activeSubmission?.abortController.abort(); this.#activeSubmission = null; this.#argumentDeltas.clear(); this.#emit({ code, message, type: "error" }); @@ -315,14 +373,18 @@ export class RealtimeBrunchBridge { const generation = this.#generation; this.#activeSubmission = { + abortController: new AbortController(), baselineSegmentIds: new Set( this.#chat.canonicalSegments.map(({ id }) => id), ), callId: event.callId, correlated: false, epoch: event.connectionEpoch, + firstTextEmitted: false, pendingQuestionId: question?.partId ?? null, + pendingQuestionMessageId: question?.messageId ?? null, sawBusyChatStatus: false, + submissionId: null, }; this.#emit({ answer, callId: event.callId, type: "submission-started" }); void this.#submit(event, answer, generation); @@ -353,8 +415,45 @@ export class RealtimeBrunchBridge { generation: number, ): Promise { try { + const activeAtSubmission = this.#activeSubmission; + if (!activeAtSubmission) return; + const voiceMessageId = createRealtimeSubmissionId( + event.connectionEpoch, + event.callId, + ); const result = await this.#submitInterviewAnswer({ - id: createRealtimeSubmissionId(event.connectionEpoch, event.callId), + admissionTarget: + activeAtSubmission.pendingQuestionMessageId === null + ? { kind: "user", messageId: voiceMessageId } + : { + kind: "client-tool-result", + messageId: activeAtSubmission.pendingQuestionMessageId, + }, + id: voiceMessageId, + onAdmission: (submissionId) => { + const active = this.#activeSubmission; + if ( + generation !== this.#generation || + !active || + active.callId !== event.callId || + active.epoch !== event.connectionEpoch + ) { + return; + } + if (active.submissionId !== null) { + if (active.submissionId !== submissionId) { + this.#fail(INVALID_BRIDGE_EVENT); + } + return; + } + active.submissionId = submissionId; + this.#emit({ + callId: event.callId, + submissionId, + type: "submission-admitted", + }); + }, + signal: activeAtSubmission.abortController.signal, text: answer, }); const active = this.#activeSubmission; @@ -375,6 +474,17 @@ export class RealtimeBrunchBridge { this.#fail(INVALID_BRIDGE_EVENT); return; } + const resultSubmissionId = + result.kind === "message" ? (result.submissionId ?? null) : null; + if ( + active.submissionId !== null && + resultSubmissionId !== null && + active.submissionId !== resultSubmissionId + ) { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + active.submissionId ??= resultSubmissionId; active.correlated = true; this.#emit({ answer, @@ -394,20 +504,34 @@ export class RealtimeBrunchBridge { #completeCorrelatedSubmission(): void { const active = this.#activeSubmission; - if ( - !active?.correlated || - !active.sawBusyChatStatus || - this.#chat.status !== "ready" - ) { + if (!active?.correlated || !active.sawBusyChatStatus) { return; } + // A reply may be written by the admitted submission itself or by a + // client-tool continuation projected onto the same message, and an ask + // follow-up writes into the message that asked; so match membership and + // exclude only what was already there when this answer was submitted. const responseSegments = this.#chat.canonicalSegments.filter( - ({ id }) => !active.baselineSegmentIds.has(id), + (segment) => + !active.baselineSegmentIds.has(segment.id) && + (active.submissionId === null || + (segment.submissionIds?.includes(active.submissionId) ?? false)), ); + if (responseSegments.length > 0 && !active.firstTextEmitted) { + // Completed canonical text can land while the turn is still streaming; + // record that instant separately from settlement. + active.firstTextEmitted = true; + this.#emit({ callId: active.callId, type: "canonical-text-ready" }); + } + if (this.#chat.status !== "ready") { + return; + } if (responseSegments.length === 0) { + this.#completeStoppedSubmission(active); return; } + this.#emit({ callId: active.callId, type: "submission-settled" }); try { this.#session.completeFunctionCall(active.callId, responseSegments); } catch { @@ -424,4 +548,36 @@ export class RealtimeBrunchBridge { type: "canonical-response-ready", }); } + + /** + * A turn that settled short of a reply leaves no canonical text behind. Only + * Flue's settlement index distinguishes it from a turn still in progress or + * a completed step whose client-tool follow-up the panel is about to send, + * so wait for that record and never treat silence alone as a stop. + */ + #completeStoppedSubmission(active: ActiveSubmission): void { + if (active.submissionId === null) return; + const settlement = this.#chat.settlements?.find( + ({ submissionId }) => submissionId === active.submissionId, + ); + if (settlement === undefined || settlement.outcome === "completed") { + return; + } + this.#emit({ callId: active.callId, type: "submission-settled" }); + try { + this.#session.completeFunctionCallWithoutResponse( + active.callId, + settlement.outcome, + ); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + this.#activeSubmission = null; + this.#emit({ + callId: active.callId, + outcome: settlement.outcome, + type: "submission-stopped", + }); + } } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index b4e37880a96..6e94c159fe0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -17,11 +17,13 @@ import { acknowledgeVoiceInterviewDisclosure, isVoiceInterviewDisclosureAcknowledged, loadOpenAIVoiceConfig, + submitVoiceInputWithAdmission, VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, VoiceInterviewControl, } from "./voice-interview-control"; import { VoiceTurnController } from "./voice-turn-controller"; +import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext, PetrinautAiVoiceModeControls, @@ -180,6 +182,93 @@ afterEach(() => { }); describe("voice interview control", () => { + test("keeps an interactive-tool submission pending until Flue admits its continuation", async () => { + const events: string[] = []; + let notifyAdmission: + | ((submissionId: AgentSendResult["submissionId"]) => void) + | undefined; + const unsubscribe = vi.fn(); + const subscribeToAdmission = vi.fn( + ( + _target: { + readonly kind: "client-tool-result" | "user"; + readonly messageId: string; + }, + listener: (submissionId: AgentSendResult["submissionId"]) => void, + ) => { + notifyAdmission = listener; + return unsubscribe; + }, + ); + const submitVoiceInput = vi.fn< + PetrinautAiVoiceModeContext["submitVoiceInput"] + >(async () => { + events.push("composer-result"); + return { + kind: "interactive-tool", + toolCallId: "ask-current", + }; + }); + const resultPromise = submitVoiceInputWithAdmission({ + input: { + admissionTarget: { + kind: "client-tool-result", + messageId: "assistant-question", + }, + id: "voice-realtime:1:call-1", + onAdmission: () => events.push("admitted"), + signal: new AbortController().signal, + text: "Approved", + }, + submitVoiceInput, + subscribeToAdmission, + }); + let completed = false; + void resultPromise.then(() => { + completed = true; + }); + + await vi.waitFor(() => expect(submitVoiceInput).toHaveBeenCalledOnce()); + expect(completed).toBe(false); + expect(unsubscribe).not.toHaveBeenCalled(); + + notifyAdmission?.("submission-1"); + + await expect(resultPromise).resolves.toEqual({ + kind: "interactive-tool", + toolCallId: "ask-current", + }); + expect(events).toEqual(["composer-result", "admitted"]); + expect(unsubscribe).toHaveBeenCalledOnce(); + }); + + test("releases a pending admission subscription when the bridge cancels", async () => { + const abortController = new AbortController(); + const unsubscribe = vi.fn(); + const resultPromise = submitVoiceInputWithAdmission({ + input: { + admissionTarget: { + kind: "client-tool-result", + messageId: "assistant-question", + }, + id: "voice-realtime:1:call-1", + onAdmission: vi.fn(), + signal: abortController.signal, + text: "Approved", + }, + submitVoiceInput: async () => ({ + kind: "interactive-tool", + toolCallId: "ask-current", + }), + subscribeToAdmission: () => unsubscribe, + }); + + abortController.abort(); + + await expect(resultPromise).rejects.toMatchObject({ name: "AbortError" }); + expect(unsubscribe).toHaveBeenCalledOnce(); + }); + test("stores and reads the versioned disclosure acknowledgement", () => { const values = new Map(); const storage = { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index ee911bc6d4d..5206ac7d8c1 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -12,7 +12,11 @@ import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeechSegments } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; -import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; +import { + RealtimeBrunchBridge, + type RealtimeBrunchAdmissionTarget, + type VoiceSubmissionSettlement, +} from "./realtime-brunch-bridge"; import { toVoiceSessionState } from "./voice-session-state"; import { VoiceTurnController, @@ -20,8 +24,81 @@ import { type VoiceTurnSnapshot, } from "./voice-turn-controller"; +import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; +type ResolveSubmission = ( + messageId: string, +) => AgentSendResult["submissionId"] | undefined; +type ResolveSubmissions = ( + messageId: string, +) => readonly AgentSendResult["submissionId"][] | undefined; +type SubscribeToAdmission = ( + target: RealtimeBrunchAdmissionTarget, + listener: (submissionId: AgentSendResult["submissionId"]) => void, +) => () => void; +type SubmitInterviewAnswer = ConstructorParameters< + typeof RealtimeBrunchBridge +>[0]["submitInterviewAnswer"]; +type SubmitInterviewAnswerInput = Parameters[0]; +type SubmitInterviewAnswerResult = Awaited>; + +export const submitVoiceInputWithAdmission = async ({ + input, + resolveInputSubmission, + submitVoiceInput, + subscribeToAdmission, +}: { + readonly input: SubmitInterviewAnswerInput; + readonly resolveInputSubmission?: ResolveSubmission; + readonly submitVoiceInput: PetrinautAiVoiceModeContext["submitVoiceInput"]; + readonly subscribeToAdmission?: SubscribeToAdmission; +}): Promise => { + let unsubscribe = () => {}; + let removeAbortListener = () => {}; + const cancelled = new Promise((_resolve, reject) => { + const rejectForAbort = () => + reject(new DOMException("Voice admission cancelled", "AbortError")); + if (input.signal.aborted) { + rejectForAbort(); + return; + } + input.signal.addEventListener("abort", rejectForAbort, { once: true }); + removeAbortListener = () => + input.signal.removeEventListener("abort", rejectForAbort); + }); + const admissionObserved = + subscribeToAdmission === undefined + ? Promise.resolve() + : new Promise((resolve) => { + unsubscribe = subscribeToAdmission( + input.admissionTarget, + (submissionId) => { + input.onAdmission(submissionId); + resolve(); + }, + ); + }); + try { + const [result] = await Promise.race([ + Promise.all([submitVoiceInput(input), admissionObserved]), + cancelled, + ]); + if (result.kind !== "message") return result; + const submissionId = resolveInputSubmission?.(result.messageId); + if (resolveInputSubmission !== undefined && submissionId === undefined) { + throw new Error("The Flue admission could not be correlated."); + } + return { + ...result, + ...(submissionId === undefined ? {} : { submissionId }), + }; + } finally { + removeAbortListener(); + unsubscribe(); + } +}; + export interface OpenAIVoiceConfig { readonly available: true; readonly connectionTimeoutMs: number; @@ -213,7 +290,7 @@ const VoiceInterviewDisclosure = ({ const recordLatency = (event: VoiceLatencyEvent): void => { try { performance.measure(`voice-interview:${event.name}`, { - detail: { questionId: event.questionId }, + detail: { correlationId: event.correlationId }, duration: event.elapsedMs, start: 0, }); @@ -225,9 +302,17 @@ const recordLatency = (event: VoiceLatencyEvent): void => { const AvailableVoiceInterviewControl = ({ config, context, + resolveInputSubmission, + resolveResponseSubmission, + settlements, + subscribeToAdmission, }: { config: OpenAIVoiceConfig; context: PetrinautAiVoiceModeContext; + resolveInputSubmission?: ResolveSubmission; + resolveResponseSubmission?: ResolveSubmissions; + settlements?: readonly VoiceSubmissionSettlement[]; + subscribeToAdmission?: SubscribeToAdmission; }) => { "use no memo"; @@ -236,6 +321,8 @@ const AvailableVoiceInterviewControl = ({ // bridge, so these callbacks read what the layout effect below installs // rather than what was captured here. let latestSubmitVoiceInput = context.submitVoiceInput; + let latestResolveInputSubmission = resolveInputSubmission; + let latestSubscribeToAdmission = subscribeToAdmission; const session = new OpenAIRealtimeSession({ cancelAnimationFrame: (handle) => globalThis.cancelAnimationFrame(handle), connectionTimeoutMs: config.connectionTimeoutMs, @@ -251,7 +338,13 @@ const AvailableVoiceInterviewControl = ({ }); const bridge = new RealtimeBrunchBridge({ session, - submitInterviewAnswer: (input) => latestSubmitVoiceInput(input), + submitInterviewAnswer: (input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: latestResolveInputSubmission, + submitVoiceInput: latestSubmitVoiceInput, + subscribeToAdmission: latestSubscribeToAdmission, + }), }); const controller = new VoiceTurnController({ bridge, @@ -266,8 +359,14 @@ const AvailableVoiceInterviewControl = ({ controller.subscribe(listener), updateSubmissionContext: ( nextSubmitVoiceInput: PetrinautAiVoiceModeContext["submitVoiceInput"], + nextResolveInputSubmission: + | ((messageId: string) => string | undefined) + | undefined, + nextSubscribeToAdmission: SubscribeToAdmission | undefined, ) => { latestSubmitVoiceInput = nextSubmitVoiceInput; + latestResolveInputSubmission = nextResolveInputSubmission; + latestSubscribeToAdmission = nextSubscribeToAdmission; }, }; }); @@ -289,10 +388,22 @@ const AvailableVoiceInterviewControl = ({ } = context; useLayoutEffect(() => { - store.updateSubmissionContext(context.submitVoiceInput); + store.updateSubmissionContext( + context.submitVoiceInput, + resolveInputSubmission, + subscribeToAdmission, + ); store.controller.updateChat({ canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages), + canonicalSegments: selectCanonicalSpeechSegments(context.messages).map( + (segment) => { + const submissionIds = resolveResponseSubmission?.(segment.messageId); + return submissionIds === undefined || submissionIds.length === 0 + ? segment + : { ...segment, submissionIds }; + }, + ), + settlements, status: context.status, }); }, [ @@ -300,6 +411,10 @@ const AvailableVoiceInterviewControl = ({ context.messages, context.status, context.submitVoiceInput, + resolveInputSubmission, + resolveResponseSubmission, + settlements, + subscribeToAdmission, store, ]); @@ -412,13 +527,25 @@ const AvailableVoiceInterviewControl = ({ export const VoiceInterviewControl = ({ config, + resolveInputSubmission, + resolveResponseSubmission, + settlements, + subscribeToAdmission, ...context }: PetrinautAiVoiceModeContext & { readonly config: OpenAIVoiceConfig; + readonly resolveInputSubmission?: ResolveSubmission; + readonly resolveResponseSubmission?: ResolveSubmissions; + readonly settlements?: readonly VoiceSubmissionSettlement[]; + readonly subscribeToAdmission?: SubscribeToAdmission; }) => ( ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 596389aad50..6e4a0a63095 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -5,11 +5,18 @@ import { VOICE_REQUEST_ID_HEADER, type VoiceDiagnosticEvent, } from "../../../voice-diagnostics"; +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "../local-storage-demo/brunch-panel-transport"; import { selectCanonicalSpeechSegments } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { VoiceTurnController } from "./voice-turn-controller"; +import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; +import type { AgentSendResult, FlueClient } from "@flue/sdk"; import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; const origin = "https://petrinaut.test"; @@ -274,10 +281,16 @@ describe("controlled voice preview", () => { }); await vi.waitFor(() => - expect(submitInterviewAnswer).toHaveBeenCalledWith({ - id: "voice-realtime:1:call-1", - text: spokenAnswer, - }), + expect(submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + admissionTarget: { + kind: "client-tool-result", + messageId: "initial-question-message", + }, + id: "voice-realtime:1:call-1", + text: spokenAnswer, + }), + ), ); expect(controller.getSnapshot()).toMatchObject({ input: "submitting", @@ -390,4 +403,135 @@ describe("controlled voice preview", () => { expect(remoteAudio.pause).toHaveBeenCalledOnce(); expect(peer.close).toHaveBeenCalledOnce(); }); + + test("admits a Voice turn only through the Flue route", async () => { + const admission: AgentSendResult = { + streamUrl: "https://petrinaut.test/agents/chat/instance-1", + offset: "offset-1", + submissionId: "submission-voice-1", + uid: "uid-1", + }; + const send = vi.fn(async () => admission); + let settleSubmission: (() => void) | undefined; + const wait = vi.fn( + async (_admission, options) => + new Promise((resolve) => { + settleSubmission = () => { + void Promise.resolve( + options?.onEvent?.({ + type: "submission-settled", + conversationId: "conversation-1", + submissionId: admission.submissionId, + outcome: "completed", + position: { batch: 1, index: 0 }, + }), + ).then(() => resolve()); + }; + }), + ); + const client = { + send, + wait, + } as Pick as FlueClient; + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve(client), + tracker, + ); + let realtimeListener: + | ((event: OpenAIRealtimeSessionEvent) => void) + | undefined; + const bridge = new RealtimeBrunchBridge({ + session: { + completeFunctionCall: vi.fn(), + completeFunctionCallWithoutResponse: vi.fn(), + speakCanonical: vi.fn(), + subscribe: (listener) => { + realtimeListener = listener; + return () => { + realtimeListener = undefined; + }; + }, + }, + submitInterviewAnswer: async ({ + admissionTarget, + id, + onAdmission, + text, + }) => { + const unsubscribe = tracker.subscribeToAdmission( + admissionTarget, + ({ admission: admitted }) => onAdmission(admitted.submissionId), + ); + const stream = await transport.sendMessages({ + trigger: "submit-message", + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + role: "user", + metadata: { source: "voice" }, + parts: [{ type: "text", text }], + }, + ], + abortSignal: undefined, + }); + try { + await stream.pipeTo(new WritableStream()); + const submissionId = tracker.submissionForInput(id); + if (submissionId === undefined) { + throw new Error("missing Flue admission"); + } + return { kind: "message", messageId: id, submissionId }; + } finally { + unsubscribe(); + } + }, + }); + const bridgeEvents: RealtimeBrunchBridgeEvent[] = []; + bridge.subscribe((event) => bridgeEvents.push(event)); + bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + bridge.start(1); + + const finalized = { + arguments: '{"answer":"The supervisor approves it."}', + callId: "call-1", + connectionEpoch: 1, + itemId: "function-item-1", + name: "continue_interview", + responseId: "response-1", + type: "tool-arguments-done" as const, + }; + realtimeListener?.(finalized); + realtimeListener?.(finalized); + + await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); + await vi.waitFor(() => + expect(bridgeEvents).toContainEqual({ + callId: "call-1", + submissionId: admission.submissionId, + type: "submission-admitted", + }), + ); + expect(bridgeEvents).not.toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ); + expect(send).toHaveBeenCalledWith({ + message: { kind: "user", body: "The supervisor approves it." }, + signal: undefined, + }); + expect(admission.streamUrl).toContain("/agents/chat/"); + + settleSubmission?.(); + await vi.waitFor(() => + expect(bridgeEvents).toContainEqual( + expect.objectContaining({ type: "submission-accepted" }), + ), + ); + }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 76c09ccf495..08dbffb2e7a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -6,9 +6,11 @@ import { VoiceTurnController } from "./voice-turn-controller"; import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; +import type { VoiceLatencyEvent } from "./voice-turn-controller"; const createHarness = () => { let epoch = 0; + let now = 0; let sessionListener: | ((event: OpenAIRealtimeSessionEvent) => void) | undefined; @@ -39,14 +41,25 @@ const createHarness = () => { updateChat: vi.fn(), }; const submitText = vi.fn(async () => ({ kind: "message" as const })); - const controller = new VoiceTurnController({ bridge, session, submitText }); + const latencyEvents: VoiceLatencyEvent[] = []; + const controller = new VoiceTurnController({ + bridge, + now: () => now, + onLatencyEvent: (event) => latencyEvents.push(event), + session, + submitText, + }); return { + advanceTime: (elapsedMs: number) => { + now += elapsedMs; + }, bridge, controller, emitBridge: (event: RealtimeBrunchBridgeEvent) => bridgeListener?.(event), emitSession: (event: OpenAIRealtimeSessionEvent) => sessionListener?.(event), + latencyEvents, session, submitText, }; @@ -65,6 +78,102 @@ const question = ( }); describe("VoiceTurnController", () => { + test("records the content-free Voice lifecycle once in causal order", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitBridge({ + answer: "Private finalized answer", + callId: "call-opaque", + type: "submission-started", + }); + harness.advanceTime(10); + harness.emitBridge({ + callId: "call-opaque", + submissionId: "submission-opaque", + type: "submission-admitted", + }); + harness.emitBridge({ + callId: "call-opaque", + submissionId: "submission-opaque", + type: "submission-admitted", + }); + harness.advanceTime(10); + harness.emitBridge({ + answer: "Private finalized answer", + callId: "call-opaque", + type: "submission-accepted", + }); + harness.emitBridge({ + callId: "call-opaque", + type: "canonical-text-ready", + }); + harness.advanceTime(10); + harness.emitBridge({ + callId: "call-opaque", + type: "submission-settled", + }); + harness.advanceTime(10); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-opaque", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-duplicate", + type: "canonical-speech-requested", + }); + harness.advanceTime(10); + const outputStarted: OpenAIRealtimeSessionEvent = { + connectionEpoch: 1, + responseId: "response-opaque", + speechRequestId: "speech-opaque", + type: "output-started", + }; + harness.emitSession(outputStarted); + harness.emitSession(outputStarted); + + expect(harness.latencyEvents).toEqual([ + { + correlationId: "call-opaque", + elapsedMs: 10, + name: "submission-admitted", + }, + { + correlationId: "call-opaque", + elapsedMs: 20, + name: "first-canonical-text", + }, + { + correlationId: "call-opaque", + elapsedMs: 30, + name: "submission-settled", + }, + { + correlationId: "call-opaque", + elapsedMs: 40, + name: "first-tts-request", + }, + { + correlationId: "call-opaque", + elapsedMs: 50, + name: "first-tts-audio", + }, + ]); + expect(JSON.stringify(harness.latencyEvents)).not.toContain( + "Private finalized answer", + ); + + await harness.controller.end(); + harness.emitBridge({ + callId: "call-opaque", + type: "submission-settled", + }); + harness.emitSession(outputStarted); + expect(harness.latencyEvents).toHaveLength(5); + }); + test("opens a continuous microphone before starting canonical question speech", async () => { const harness = createHarness(); const order: string[] = []; @@ -97,6 +206,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-1", + speechRequestId: "speech-1", type: "output-started", }); expect(harness.controller.getSnapshot()).toMatchObject({ @@ -155,6 +265,40 @@ describe("VoiceTurnController", () => { ); }); + test("returns to listening after a durably stopped turn without speaking", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitBridge({ + answer: "Stop this one.", + callId: "call-1", + type: "submission-started", + }); + harness.emitBridge({ + answer: "Stop this one.", + callId: "call-1", + type: "submission-accepted", + }); + harness.advanceTime(40); + harness.emitBridge({ callId: "call-1", type: "submission-settled" }); + harness.emitBridge({ + callId: "call-1", + outcome: "aborted", + type: "submission-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + lastAnswerDelivery: "delivered", + output: "idle", + }); + expect(harness.latencyEvents).toContainEqual({ + correlationId: "call-1", + elapsedMs: 40, + name: "submission-settled", + }); + }); + test("restores submission state when resumed before Brunch releases the turn", async () => { const harness = createHarness(); await harness.controller.start(); @@ -380,6 +524,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-1", + speechRequestId: "speech-1", type: "output-started", }); @@ -407,6 +552,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-1", + speechRequestId: "speech-1", type: "output-started", }); @@ -490,6 +636,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-after-pause", + speechRequestId: "speech-after-pause", type: "output-started", }); @@ -509,6 +656,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "stale-response", + speechRequestId: "stale-speech", type: "output-started", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 4277717f7aa..0f4e42cbac6 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -5,7 +5,12 @@ import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBridgeErrorCode, RealtimeBrunchBridgeEvent, + VoiceSubmissionSettlement, } from "./realtime-brunch-bridge"; +import type { + PetrinautAiComposerControlContext, + PetrinautAiVoiceModeContext, +} from "@hashintel/petrinaut/ui"; export type VoiceConnectionState = | "idle" @@ -37,17 +42,20 @@ export interface VoiceTurnSnapshot { } export interface VoiceLatencyEvent { + readonly correlationId: string; readonly elapsedMs: number; readonly name: + | "submission-admitted" + | "submission-settled" + | "first-canonical-text" + | "first-tts-request" + | "first-tts-audio" | "question-visible" | "question-spoken-started" | "question-spoken" | "answer-ready"; - readonly questionId: string; } -type ChatStatus = "ready" | "submitted" | "streaming" | "error"; - interface RealtimeSession { cancelOutput(): void; connect(): Promise; @@ -63,9 +71,12 @@ interface RealtimeBridge { updateChat(update: ChatUpdate): void; } -interface SubmitTextInput { +type ComposerSubmitTextInput = Parameters< + PetrinautAiComposerControlContext["submitText"] +>[0]; + +interface SubmitTextInput extends Pick { readonly target: "message"; - readonly text: string; } interface VoiceTurnControllerDependencies { @@ -79,7 +90,8 @@ interface VoiceTurnControllerDependencies { interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; - readonly status: ChatStatus; + readonly settlements?: readonly VoiceSubmissionSettlement[]; + readonly status: PetrinautAiVoiceModeContext["status"]; } type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; @@ -119,12 +131,15 @@ export class VoiceTurnController { #currentQuestionId: string | null = null; #generation = 0; #inputStateOnResume: Exclude | null = null; + #latencyCorrelationId: string | null = null; #pauseRequested = false; + readonly #recordedLatencyEvents = new Set(); #snapshot = initialSnapshot; #submittingQuestionId: string | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; #transcriptKey: string | null = null; + #ttsSpeechRequestId: string | null = null; public constructor({ bridge, @@ -224,10 +239,13 @@ export class VoiceTurnController { this.#bridgeStarted = false; this.#currentQuestionId = null; this.#inputStateOnResume = null; + this.#latencyCorrelationId = null; + this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = null; this.#pauseRequested = false; this.#transcriptItemId = null; this.#transcriptKey = null; + this.#ttsSpeechRequestId = null; this.#bridge.stop(); this.#session.setMicrophoneEnabled(false); const teardownPromise = this.#teardownPromise ?? this.#session.disconnect(); @@ -389,9 +407,12 @@ export class VoiceTurnController { this.#inputStateOnResume = "submitting"; } this.#answerFinalizedAt = this.#now(); + this.#latencyCorrelationId = event.callId; + this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = this.#currentQuestionId; this.#transcriptItemId = null; this.#transcriptKey = null; + this.#ttsSpeechRequestId = null; this.#update({ input: paused ? "paused" : "submitting", lastAnswerDelivery: "pending", @@ -407,6 +428,31 @@ export class VoiceTurnController { this.#update({ lastAnswerDelivery: "delivered" }); return; } + if (event.type === "submission-admitted") { + this.#recordLatency("submission-admitted", event.callId); + return; + } + if (event.type === "canonical-text-ready") { + this.#recordLatency("first-canonical-text", event.callId); + return; + } + if (event.type === "submission-settled") { + this.#recordLatency("submission-settled", event.callId); + return; + } + if (event.type === "submission-stopped") { + // Brunch was stopped before it replied: nothing to speak, and the + // interviewer is free to listen again. + const pausedWhileStopped = this.#snapshot.input === "paused"; + if (pausedWhileStopped) { + this.#inputStateOnResume = "listening"; + } + this.#update({ + input: pausedWhileStopped ? "paused" : "listening", + output: "idle", + }); + return; + } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; @@ -439,6 +485,16 @@ export class VoiceTurnController { ) { return; } + if (event.type === "canonical-speech-requested") { + if ( + this.#latencyCorrelationId !== null && + this.#ttsSpeechRequestId === null + ) { + this.#ttsSpeechRequestId = event.speechRequestId; + this.#recordLatency("first-tts-request", this.#latencyCorrelationId); + } + return; + } if (event.type === "output-started") { if (this.#snapshot.input === "paused") { this.#session.cancelOutput(); @@ -446,6 +502,12 @@ export class VoiceTurnController { return; } this.#update({ output: "speaking" }); + if ( + this.#latencyCorrelationId !== null && + event.speechRequestId === this.#ttsSpeechRequestId + ) { + this.#recordLatency("first-tts-audio", this.#latencyCorrelationId); + } if (this.#currentQuestionId) { this.#recordLatency("question-spoken-started", this.#currentQuestionId); } @@ -513,9 +575,12 @@ export class VoiceTurnController { ++this.#generation; this.#activeEpoch = null; this.#inputStateOnResume = null; + this.#latencyCorrelationId = null; + this.#recordedLatencyEvents.clear(); this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; + this.#ttsSpeechRequestId = null; this.#bridge.stop(); this.#session.setMicrophoneEnabled(false); void this.#session.disconnect(); @@ -536,12 +601,15 @@ export class VoiceTurnController { }); } - #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { + #recordLatency(name: VoiceLatencyEvent["name"], correlationId: string): void { if (this.#answerFinalizedAt === null) return; + const eventKey = `${correlationId}:${name}`; + if (this.#recordedLatencyEvents.has(eventKey)) return; + this.#recordedLatencyEvents.add(eventKey); this.#onLatencyEvent?.({ + correlationId, elapsedMs: Math.max(0, this.#now() - this.#answerFinalizedAt), name, - questionId, }); } diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 468aad51347..80ffa308dc6 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,7 +1,164 @@ -# No live Brunch mission on this branch +# Mission 5 — one Flue conversation route for Voice and the typed panel ## Status -**Mission 4 closed by owner adjudication on 2026-09-03.** Its accepted implementation, bounded evidence, observed S4 failure, missing full-run candidate, and deferred concerns are preserved in [`4-core-plugin-elicitation-proof-of-life.md`](docs/mission-archive/4-core-plugin-elicitation-proof-of-life.md) and [`mission-4-closure-and-deferral-2026-09-03.md`](docs/evidence/decisions/mission-4-closure-and-deferral-2026-09-03.md). +**Live as of 2026-09-04** for [FE-1574](https://linear.app/hash/issue/FE-1574/let-voice-speak-through-canonical-brunch-conversations) on `ln/fe-1574-direct-voice-flue`, stacked directly on the closed Mission 4 branch. This is the sole execution authority for the branch. The builder implementation now routes typed panel and finalized Voice turns through one browser `FlueClient` at `/agents/chat/:instanceId`, projects canonical replies into Petrinaut and TTS, uses durable Flue abort for explicit Stop, and rehydrates canonical conversation state through SDK observation. The former Brunch `/api/chat` handler and projector are deleted. A human witness passed typed and Voice admission, spoken playback, barge-in, and explicit durable Stop, but failed faithful reopen: per-message typed/Voice provenance disappeared and the stopped assistant entry returned as ordinary truncated content rather than a stopped turn. The owner deferred remediation to Mission 6 rather than expanding this branch. Mission acceptance remains open; proof leaf 8 is not complete. -This file is a closure pointer, not execution authority. No implementation may begin here until an owner-authorized issue/branch mission is re-evaluated and installed. Future planning lives in [`MISSION.next.md`](MISSION.next.md). Voice reconciliation starts from [`mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md) without reopening Mission 4. Restacked commit navigation and the content-hash evidence rule are recorded in [`mission-4-final-restack-provenance-2026-09-03.md`](docs/evidence/decisions/mission-4-final-restack-provenance-2026-09-03.md). +The accepted departure base remains Mission 4's package-composed `ChatAgent`: `useBrunchAgent()` mounts core's independent `elicitation` capability and `useSdcpnPlugin()` mounts the SDCPN job contribution. `@hashintel/brunch-agent-transport-aisdk` is now the browser adapter over public `@flue/sdk`, not a server handler. The current external Voice evidence remains PR [#9496](https://github.com/hashintel/hash/pull/9496) at `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82`, PR [#9507](https://github.com/hashintel/hash/pull/9507) at `252b9dbb0c77fae8cee45a506f09cac3e20c381c`, and PR [#9512](https://github.com/hashintel/hash/pull/9512) at `d13535d1077b3a78d6a1411031b7d0a0a78e3144`. They are read-only source evidence, not branches to rewrite or wholesale architecture to restore. + +Cold-start reads are [`docs/evidence/implementations/mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md), [`packages/transport-aisdk/src/index.ts`](packages/transport-aisdk/src/index.ts), [`packages/transport-aisdk/src/ui-stream.ts`](packages/transport-aisdk/src/ui-stream.ts), [`packages/transport-aisdk/src/transcript.ts`](packages/transport-aisdk/src/transcript.ts), [`apps/brunch-agent/src/app.ts`](../../../apps/brunch-agent/src/app.ts), [`apps/brunch-agent/src/http/ownership.ts`](../../../apps/brunch-agent/src/http/ownership.ts), [`apps/brunch-agent/test/petrinaut-chat.integration.ts`](../../../apps/brunch-agent/test/petrinaut-chat.integration.ts), [`apps/brunch-agent/test/architecture/boundaries.integration.ts`](../../../apps/brunch-agent/test/architecture/boundaries.integration.ts), [`apps/petrinaut-website/src/main/app/local-storage-demo/`](../../../apps/petrinaut-website/src/main/app/local-storage-demo/), [`apps/petrinaut-website/src/main/app/voice-interview/`](../../../apps/petrinaut-website/src/main/app/voice-interview/), and the installed Flue 2.0.3 documentation at `node_modules/@flue/sdk/docs/sdk/flue-client.md`, `node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, and `node_modules/@flue/sdk/docs/guide/react.md`. + +## Imperative + +Make the mounted Flue conversation route the only product door into a Brunch conversation, and make Voice a faithful audio projection of that one canonical conversation. One finalized spoken answer and one typed panel message must both enter the owning Flue conversation through `@flue/sdk` against `/agents/chat/:instanceId`, and the corresponding canonical Brunch response must reach visible text and TTS without another model rewriting the text. Do this now because Mission 4 established the canonical agent composition while two transports still exist to the same conversation: the Voice preview's AI SDK composer path and the server-side `/api/chat` adapter, which admits through a different code path than the SDK does. Routing Voice onto Flue while keeping `/api/chat` for typed text would harden the split into two routes, two ownership rules, and two protocols; the least mechanism is one route, with the AI SDK reduced to the panel's rendering contract behind a host-supplied browser `ChatTransport`. + +### Product-manager litmus + +Adopted on restack onto the parent spine's 2026-09-03 litmus reframing. A product manager who did not watch the work must be able to notice the advance; the single-route consolidation, the browser `ChatTransport`, the deleted `/api/chat` door, and the repurposed transport package are internal sequencing and must not be presented as the advance. + +**Release note:** in the Petrinaut Brunch panel you can type or speak to Brunch in one conversation; what you hear is exactly what Brunch wrote; **Stop** really stops Brunch rather than just hiding its answer; and reopening the panel shows the same conversation you left, without re-sending or replaying anything. + +**Demo script (no engineer present), on the deployment posture available at cut time — the local `yarn dev:brunch` pair with the Brunch preview selected:** open the panel and type one message; read the reply. Start Voice mode and speak one answer; see exactly one new user message appear, then see Brunch's reply appear as text and hear the same words read aloud. Speak over it once; playback stops and the text stays. Ask a second question and press **Stop** while Brunch is still working; the conversation shows that turn as stopped, not as an answer. Close the panel and reopen the same conversation: the typed turn, the spoken turn, and the stopped turn are all there exactly as you saw them, nothing replays, and nothing is sent again. + +**Previously impossible:** Stop only cancelled the browser request while Brunch kept working, so reopening the panel showed a full answer you had stopped; typed and spoken turns entered Brunch through different doors, so a spoken turn could be held or ordered differently from a typed one. + +**Completion:** the mission is complete at the contract stratum below — when a product manager can run this demo script end to end and proof leaf 8's witness bundle records it — not when the first typed or spoken turn crosses the route. The first green typed-panel tracer and the first green Voice tracer are internal milestones. + +### Recut rationale + +Inspected at the real boundary on 2026-09-03 (`node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, `packages/transport-aisdk/src/index.ts`, `packages/transport-aisdk/src/ui-stream.ts`, `apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts`, `node_modules/ai/dist/index.d.ts` `ChatTransport`): + +- Flue's SSE does not remove the need for translation while the panel speaks `useChat`: Flue admits one `DeliveredMessage` with a 202 and streams `ConversationStreamChunk` batches on a separate, never-ending offset-resumed read; the AI SDK posts the whole `messages` array and expects one finite `UIMessageChunk` stream per turn. Request shape, vocabulary, and lifecycle all differ, and `@flue/*` ships no AI SDK adapter. +- At the recut, the adapter was one translation cut across two homes by the transport topology gate: AI SDK request framing, CORS, and principal parsing in `transport-aisdk` (then Flue-free), and the load-bearing `ConversationStreamChunk → UIMessageChunk` projection plus snapshot → UI messages in `apps/brunch-agent/src/conversation/`. +- A Hono-level relocation would have re-expressed the same admit → follow → project → terminate join as a server handler; it would have removed nothing and kept two routes. It was rejected. +- AI SDK `ChatTransport` is a client-side interface, and Petrinaut already accepts and wraps host-supplied transports. The landed browser transport over the same `createFlueClient()` that Voice uses owns the projector, removes `/api/chat` and its handler, and makes typed and spoken turns share one admission path, one ownership guard, and one protocol. + +## Throughline + +The real boundary is the local Petrinaut surface driven by `yarn dev:brunch` — both its typed panel and its Voice control — through one same-origin-proxied Flue route into the mounted production `ChatAgent`, back to visible Petrinaut text and, for Voice, audible playback: + +```text + Petrinaut typed panel (`useChat`) Petrinaut microphone + → host-supplied browser `ChatTransport` → OpenAI Realtime provisional STT + turn detection + (`sendMessages` → one user text or → one validated finalized `continue_interview` answer + one client-tool-result signal) + └──────────────┬────────────────────────────┘ +→ one browser `createFlueClient()` per selected principal + logical conversation id +→ one supported `send()` admission at the same-origin proxied `/agents/chat/:instanceId` route + (Voice may enter via the panel's transport — preferred, one visible store — or call `send()` directly; see fog-line) +→ `agentOwnershipGuard` (the only ownership check) → `createAgentRouter(ChatAgent)` +→ current `ChatAgent` with `useBrunchAgent()` + `useSdcpnPlugin()` +→ SDK reads: `wait(admission, { onEvent })` for the panel's finite per-turn stream, + `observe({ live: "sse" })` for canonical state and reopen +→ response parts correlated by server-issued `submissionId` +→ panel: existing `ConversationStreamChunk → UIMessageChunk` projector, terminated on `submission-settled` +→ Voice: canonical completed Brunch text displayed and passed unchanged as TTS input +→ local playback/observation cancellation or explicit conversation-wide `abort()` +→ observation rehydration after reopening the same logical conversation +``` + +The `/api/chat` route, `createPetrinautChatHandler`, the in-process `init().dispatch()/read()` admission path, the `GET ?id=` history door, and the `/api/chat` Vite proxy are removed from the Brunch app; the local launcher proxies `/agents/chat/*` instead. `@hashintel/brunch-agent-transport-aisdk` is repurposed as the browser-side adapter: it exports the projector, the snapshot → UI-message projection, the header names, and a `ChatTransport` factory over a caller-supplied `FlueClient`, and depends on `ai` and the public `@flue/sdk` client only. The Petrinaut panel itself stays on `useChat`; it is not rewritten onto `@flue/react`. + +`conversationId` is the stable logical reference selected by the Petrinaut host. The current principal plus that id mechanically derives the Flue instance path and ownership headers. `submissionId` correlates one admitted answer and its settlement. Flue offsets remain opaque. `uid` identifies one current incarnation and may guard a known continuation, but it must not replace the logical conversation id or be presented as durable user identity. + +For maintained Voice state beyond what the panel already holds, use the SDK's materialized `observe()` surface, or `useFlueAgent()` over the same memoized client if the React seam earns it. Use `readSubmissionReply()`, `read()`, or `wait({ onEvent })` only for submission-scoped extraction, not as a parallel transcript reducer; the panel transport's per-submission projection into `useChat`'s own store is that submission-scoped use, not a second transcript. Do not parse SSE, calculate offsets, retry stream chunks, or hand-pick the latest message. The tracer may stream canonical text visibly through the observation, but TTS begins only from completed speakable segments and may remain settlement-gated; token-by-token speech is not part of this claim. + +The first tracer is text-turn-only at the Brunch boundary. It does not require the temporary `brunch_ask` client-tool shim: a finalized spoken answer is a direct Flue user message, and canonical plain assistant text is sufficient to prove the transport. If the real tracer cannot preserve answer correlation without structured questions, stop and present that observed strain before mounting the suspended capability. + +### Contract stratum and readiness gate + +Close the **one-route conversation transport stratum**: one typed turn and one finalized Voice turn each admitted once through `@flue/sdk` at the mounted route, canonical output, client-tool follow-up as a signal, local playback cancellation, durable abort, visible failure, and same-conversation reopen — with no second server-side door remaining. + +Order the tracers so the cheaper one proves the route first: the typed panel over the browser transport (it reuses the existing projector and has an existing integration scenario to re-express), then Voice. After each end-to-end turn works, enumerate the lateral obligations it exposes and close those required to make the visible claim true: duplicate finalization, ambiguous admission, submission/reply correlation, client-tool resume correlation, reconnect and replay, local cancellation versus durable abort races, fatal ownership errors, and canonical text/TTS-input correspondence. Carry broader speech ergonomics, multi-turn barge-in tuning, structured questions, and remote identity and exposure only to the named deferred owners below. + +## Proof + +This proof establishes that one real local typed turn and one real local Voice turn each cross the supported Flue conversation protocol at the single mounted route into the current canonical Brunch agent, that the typed turn returns as one finite AI SDK stream and the Voice turn as one visible and spoken canonical response, both with bounded cancellation and recovery semantics, and that no server-side AI SDK door remains. It does **not** establish trusted production authentication, remote deployment, broad Voice UX, structured-question transport, Petrinaut client-tool mutation, workpiece viability, or that the `useChat` panel itself is removable. + +1. **Typed panel over the browser Flue transport.** A typed panel submission calls `send()` exactly once with one `kind: "user"` message; a completed client-tool follow-up calls `send()` exactly once with one `kind: "signal"` `client-tool-result` message and resumes the same assistant message id; the returned `UIMessageChunk` stream carries the same start/step/part/finish sequence the former `/api/chat` integration asserted, and terminates on that submission's `submission-settled`. Reopen hydration comes from `observe({ live: "sse" })` through `snapshotToUiMessages`. Oracle: the current `apps/brunch-agent/test/petrinaut-chat.integration.ts` scenario re-expressed through the browser transport against the in-process `app.fetch` of the real `app.ts` (Flue route, ownership guard, faux provider), preserving its text, reasoning, server-tool, and client-tool-resume assertions; the relocated projector and transcript unit tests; and the outer witness typing one message in the real panel with the network ledger showing only `/agents/chat/:instanceId` traffic. +2. **Direct finalized admission.** A completed Realtime `continue_interview` call invokes Flue `send()` exactly once with one `kind: "user"` message; provisional transcript events, duplicated provider terminal events, stale epochs, and repeated tool-call delivery never enter history. The admitted server `submissionId` becomes the turn correlation key. A lost or ambiguous admission is surfaced and never blindly resent. Oracle: named cases in `apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts`, including `admits one finalized Realtime answer through Flue once`, plus canonical snapshot inspection showing exactly one matching visible user message. +3. **No server-side AI SDK door.** The Brunch app no longer mounts `/api/chat`; `createPetrinautChatHandler`, `PETRINAUT_CHAT_ROUTE`, the in-process `init()` admission path, and the `/api/chat` proxy are deleted; the repurposed transport package contains no `Request`/`Response` handler. The real Voice turn reaches the mounted route through `@flue/sdk` — either through the panel's Flue transport or by a direct `send()` — and makes no submission over any non-Flue protocol. Oracle: `apps/brunch-agent/test/build-artifact.test.ts` asserting the built server answers `/api/chat` with Hono's 404 and still serves the Flue route; the retained browser network ledger from the outer witness; and a focused integration case in `apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts` named `admits a Voice turn only through the Flue route`. +4. **Canonical visible and TTS output.** The materialized response selected for the admitted `submissionId` is the source for visible text and TTS input. Completed visible assistant text is preserved in part order; reasoning and non-speech parts are not promoted to spoken text. The exact string array sent for canonical speech equals the selected Brunch text, and no response-preparation or simplification generation call occurs. Oracle: `apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts`, `openai-realtime-session.test.ts`, and the outer artifact comparison of canonical snapshot text to the recorded TTS request payload; the human witness confirms that playback begins but does not claim synthesized audio is a verbatim recording. +5. **Cancellation and abort stay distinct.** Barge-in, pause, or panel closure stops local playback/observation according to the existing Voice contract without rewriting or deleting canonical history. An explicit durable Stop action uses Flue's conversation-wide `abort()` when work is unsettled, and history/observation exposes an `aborted` settlement; an abort that loses the race to completion remains completed. Oracle: named local-versus-durable cancellation cases in `voice-turn-controller.test.ts` and `voice-preview.integration.test.ts`, plus the retained Voice event ledger and Flue settlement. +6. **Reopen resumes without replay.** Reopening the same selected conversation rehydrates its canonical messages and settlements from Flue, does not submit another user message, and does not automatically replay settled audio. An absent conversation, fatal 401/403, reconnecting stream, and settled conversation are visibly distinguishable. Oracle: a named `rehydrates the settled Voice turn without resubmission or playback` case over the SDK observation in `realtime-brunch-bridge.test.ts`, followed by the outer witness reopening the panel and comparing the second view with the same snapshot. +7. **Architecture and boundary integrity.** The built app still mounts `useBrunchAgent()` plus `useSdcpnPlugin()`, excludes the obsolete app-local stub agent, derives ownership from principal plus logical conversation id in exactly one guard, and reaches Flue locally through a same-origin protocol-preserving proxy rather than a newly public route. The transport package's runtime dependencies are exactly `ai` and `@flue/sdk`; it imports no `@flue/runtime`, core, plugin, or binding module. Oracle: `apps/brunch-agent/test/build-artifact.test.ts`, `apps/brunch-agent/test/agent-ownership.test.ts`, the transport case in `apps/brunch-agent/test/architecture/boundaries.integration.ts` (`transports consume their wire encoder and the public Flue client only — never core, a binding, or the runtime`), SDCPN packaging tests, and browser inspection of the claimed local route and headers. +8. **Real Voice witness and retained proof bundle.** With `yarn dev:brunch`, a human speaks one answer, sees exactly one matching user message, sees and hears the canonical Brunch response begin, interrupts playback once, exercises Stop on one unsettled turn, and reopens the original settled turn. Retain under `docs/evidence/implementations/mission-5-direct-voice-flue/` the witness record, sanitized Voice event ledger, network route summary, canonical Flue snapshot, settlement outcomes, source/build commit, and hashes. Oracle: human adjudication against that bundle; mocked browser or server-only evidence cannot satisfy this leaf. +9. **Focused repository verification and truthful docs.** Brunch app, website, core/plugin, transport, and Petrinaut checks pass; end-user and operator prose describes the single route that actually shipped and preserves the distinction between canonical text and generated audio; no surviving prose or comment names `/api/chat` as a Brunch door. Oracle: `yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk`, `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` if a Petrinaut architectural boundary changes, `rg -n "api/chat" apps/brunch-agent libs/@hashintel/brunch-agent/packages apps/petrinaut-website/src/main/app/local-storage-demo` returning only the stock Petrinaut route and frozen Mission 4 evidence, inspection of `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md`, and a patch changeset if the published Petrinaut package changes. + +## Constraints + +- Preserve Mission 4's current core/plugin/app composition and authored skill packaging. Voice reconciliation must never restore the deleted app-local `ChatAgent`, concise stub prompt, YAML plugin machinery, or a second model-facing agent. +- Flue history is the sole canonical conversation record. Voice owns media capture, provisional display, turn finalization, TTS, playback, and local interaction state; it owns no durable transcript and may not splice into stock-assistant history. +- Use `@flue/sdk`/`@flue/react` directly for shell-facing conversation transport. No adapter may re-specify Flue offsets, retries, materialization, settlement, or recovery. The browser `ChatTransport` is a projection over the public `FlueClient` (`send()`, `wait()`/`observe()`, `history()`), never a second HTTP client; it reads chunks only through the SDK's `onEvent`/observation surfaces. +- One product route. `/agents/chat/:instanceId` behind `agentOwnershipGuard` is the sole door for typed, Voice, diagnostic, and evaluation traffic; no route, handler, or package may accept a conversation turn over another protocol. The stock Petrinaut `/api/chat` (the website's own OpenAI function) is untouched and must not be borrowed. +- The Petrinaut panel remains on `useChat`; the AI SDK is its rendering contract, supplied a transport by the host. Do not rewrite the panel onto `@flue/react`, and do not add a second transcript store beside `useChat`'s messages for the typed panel. +- Submit only the validated finalized answer. Provisional transcription and audio remain ephemeral. One Voice finalization causes at most one `send()` call; because Flue 2.0.3 does not accept a caller idempotency key on `send()`, ambiguous admission must remain visible and must not trigger an automatic retry. +- Brunch owns canonical response content. TTS may synthesize audio from exact selected text, but no second model may summarize, shorten, paraphrase, or select replacement wording for the tracer. +- Local playback cancellation, local observation cancellation, HTTP request cancellation, and Flue's conversation-wide durable abort are distinct operations and must remain distinguishable in code, UI state, evidence, and tests. +- The current browser-minted local principal is an ownership discriminator, not trusted authentication. The outer proof is local and same-origin; it must not expose `/agents/chat/:id` publicly or claim production identity, authorization, CORS, deployment, or recovery. +- `@hashintel/brunch-agent-transport-aisdk` survives only as the browser-side adapter and the home of the projector, snapshot projection, and header names. Its runtime dependencies are `ai` and `@flue/sdk`; it never imports `@flue/runtime`, core, a plugin, or a binding, and client-tool names reach it as caller-supplied options. The boundary test's transport gate is amended to say exactly that — this is the one accepted topology-gate change of the recut. +- Do not mount the suspended `brunch_ask` capability merely to preserve the divergent preview stack. Re-entry requires observed plain-turn correlation strain and an owner decision consistent with the structured-question planning contract. +- External Voice branches and their issues/PRs remain read-only evidence. Port only behavior that serves this mission, preserve relevant provenance in commits, and do not rewrite, close, or represent those records as accepted wholesale. +- Record admission, first canonical text, first TTS request/audio, and settlement latency without transcript, prompt, tool, SDP, audio, credential, or response-body content in ordinary telemetry. +- No implementation begins until this authority cut is committed separately. Material changes to this contract require owner review and another focused authority commit before dependent implementation. + +### Expected touched paths + +```text +~ libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ delete the HTTP handler; add ChatTransport factory over FlueClient; receive ui-stream + snapshotToUiMessages + headers +~ libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json deps become ai + @flue/sdk; drop valibot if unused +~ libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ replace chat-handler/golden with transport + projector + snapshot tests +~ apps/brunch-agent/test/architecture/boundaries.integration.ts amend the transport gate +- apps/brunch-agent/src/http/petrinaut-chat.ts server-side door removed +- apps/brunch-agent/src/conversation/ui-stream.ts moves into the transport package +~ apps/brunch-agent/src/conversation/transcript.ts snapshotToUiMessages moves out; formatFlueTranscript stays for the CLI +~ apps/brunch-agent/src/http/routes.ts, local-origins.ts drop PETRINAUT_CHAT_ROUTE and the /api/chat proxy; proxy /agents/chat/* +~ apps/brunch-agent/src/app.ts remove the /api/chat mount and app-transport closure +~ apps/brunch-agent/package.json drop the `ai` devDependency if nothing else uses it +- apps/brunch-agent/test/petrinaut-chat.test.ts, petrinaut-chat.integration.ts, petrinaut-chat-result.ts, flue-ui-stream.test.ts re-expressed against the browser transport / relocated +~ apps/brunch-agent/test/build-artifact.test.ts, local-dev-origins.test.ts single-route assertions +~ apps/brunch-agent/petrinaut-local.vite.config.ts same-origin Flue-route proxy for the local real surface +~ apps/petrinaut-website/src/main/app/local-storage-demo/ createFlueClient composition, browser transport, delete use-flue-chat-history +~ apps/petrinaut-website/src/main/app/voice-interview/ direct Flue admission, materialized response, cancellation, reopen +~ apps/petrinaut-website/package.json add @flue/sdk +~ yarn.lock workspace dependency update +? apps/brunch-agent/src/http/ownership.ts, src/conversation/identity*.ts only if the identity-contract home (fog-line) moves +? libs/@hashintel/petrinaut/src/ui/ smallest public panel seam only if host composition cannot remain local +~ apps/petrinaut-website/README.md operator-facing route and preview behavior +~ libs/@hashintel/petrinaut/docs/ai-assistant.md user-visible Voice behavior +~ libs/@hashintel/brunch-agent/MISSION.next.md reconcile the production-door, restricted-ingress, and adapter-removal statements ++ libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/ retained proof bundle +? .changeset/ one patch changeset only if the published Petrinaut package changes +``` + +## Fog-line + +- How a finalized Voice answer enters Flue. Two shapes are admissible: **(B, preferred)** Voice submits through the panel's own Flue `ChatTransport` (`useChat.sendMessage` → `send()`), so the panel's `useChat` messages remain the single visible store and Voice's `observe()` shrinks to selecting completed canonical text for TTS — or reads the panel's completed assistant message and drops `observe()` entirely; **(A)** Voice calls `send()` directly and keeps its own `observe()` state, with the panel rehydrating. Start with B; fall back to A only if the existing hold-while-streaming, epoch, or TTS-correlation semantics demonstrably strain under the panel's transport, and record the observed strain. Either way, one finalization is one `send()`, and if the chosen shape would create two mutable transcript stores or a second custom reducer, stop and reorient at the panel boundary. +- The smallest honest home for the browser-safe principal + logical-conversation-id → Flue-instance-id contract and the two ownership header names. `apps/brunch-agent/src/conversation/identity-web.ts` proves the algorithm; the website must not gain an app-to-app source import, and core must not own HTTP header names. The leading candidate is the repurposed transport package, which already exports the principal header; a website-local copy pinned by an equality test against the app is the fallback. Do not create a new package to hold two strings and a hash. +- Whether `wait(admission, { onEvent })` alone gives the browser transport a clean finite per-turn stream, or whether the panel needs `observe()` for reconnect during a turn. `wait()` rejects on failed/aborted settlement and on `terminal_event_missing`; the transport must map those to `error`/`abort` chunks rather than throwing past `useChat`. The first real disconnect mid-turn decides; do not pre-build reconnect machinery. +- Whether `reconnectToStream` should return `null` (observation-only rehydration, the current behavior) or resume an unsettled submission after reload. Start with `null` plus SDK observation; re-enter only if the witness observes a lost in-flight turn. +- The exact subset of PRs #9496, #9507, and #9512 to port after semantic comparison with the current branch. Their useful Voice state-machine behavior is evidence; their app-local agent topology, temporary ask shim, and generative preparation are not presumed requirements. +- The bounded speech-selection policy if the exercised Brunch response contains multiple completed text blocks or an interactive part. Begin with canonical completed visible text in order; if this produces duplicate, misleading, or unspeakable output, retain the mismatch and seek a Brunch-owned deterministic presentation rule rather than another generator. +- Whether the existing Stop affordance can express both local Voice interruption and explicit conversation-wide durable abort without misleading the user. The first real race decides the smallest UI distinction. +- Whether a same-origin local proxy can carry every SDK history/SSE/send/abort route unchanged. A crisp protocol or middleware blocker permits the thinnest route correction; it does not permit rebuilding the AI SDK adapter under another name. + +## Stop or reorient + +Stop and surface the evidence if the implementation creates a second conversation authority, keeps or re-adds any server-side route that accepts a conversation turn over a non-Flue protocol, submits provisional STT, automatically retries an ambiguous admission, rewrites canonical text through another model, hand-rolls stream recovery, computes offsets, restores the old stub agent, or activates `brunch_ask` without observed need and owner approval. + +Stop if the browser transport cannot preserve the current client-tool resume semantics (completed client-tool parts on the referenced assistant message → one signal send → continuation of the same assistant message id) without a server-side helper; that is evidence the resume contract needs redesign, not permission to reintroduce `/api/chat`. + +Stop at the boundary if direct Flue state cannot reach the existing visible panel without duplicated mutable history; decide the UI ownership seam before adding synchronization machinery. Stop if local cancellation accidentally aborts durable work, explicit Stop only cancels a browser request while the provider keeps spending, a stale response is spoken after conversation/epoch change, or reopen resubmits or replays a settled turn. + +Stop rather than widen if the real route requires public unauthenticated exposure, production identity work, remote deployment, Petrinaut mutation tools, workpiece/projection state, or a whole assistant rewrite. Those are not hidden prerequisites to this transport tracer. + +## Deferred + +- **Faithful transcript projection on resume:** the 2026-09-04 human witness found that canonical messages returned after reload without audio replay, but typed/Voice provenance disappeared and a stopped assistant entry rehydrated as ordinary truncated content even though the global stopped settlement remained visible. Mission 6 owns preserving those two distinctions across its second-tab conversation resume and proving them at that real boundary. The same witness also found the small **Show transcript** control and the separation between **Exit voice mode** and the chat composer's durable **Stop** easy to miss; treat those as observed discoverability strain when shaping the resumed surface, without conflating local Voice exit with Flue abort. +- **`useChat` panel removal:** with the server-side door gone, the AI SDK survives only as the Petrinaut panel's rendering contract behind a host-supplied transport. Whether Petrinaut ever drops `useChat` is a Petrinaut product decision, not a Brunch transport question; Brunch carries no further obligation here. +- **Restricted-ingress rule for the Flue route:** Mission 8's landed contract denied `/agents/chat/:id` publicly and routed restricted traffic through `/api/chat`. This recut makes the Flue route the only product route, so that rule must be re-expressed as the FE-1423 gates applying directly to `/agents/chat/:id`. Record the re-expression in `MISSION.next.md`; the release/deployment gate owns its enforcement. +- **Structured questions:** core-owned question semantics, binding, rendering, correlated reply, and resumed tool execution remain in the shared future-planning record. Re-enter when plain Voice turns demonstrably cannot preserve a required interaction. +- **Broader Voice quality:** multi-turn barge-in tuning, long-response ergonomics, optional deterministic spoken presentation, accessibility breadth, and response optimisation re-enter after measured strain on the direct canonical route. +- **Remote/public operation:** trusted identity and authorization, origin policy, hosted Flue reachability, rate/spend controls, replacement recovery, and remote observability remain with the Mission 8 release/deployment gate or a separately cut successor. +- **Product-data work:** prepared workpiece/Petrinaut viability remains Mission 6; capture-backed review remains Mission 7; automatic traceable projection remains Mission 9. This mission carries no document mutation or provenance claim beyond canonical conversation history. +- **Host breadth:** stock/Brunch picker behavior, session switching beyond the selected local Brunch conversation, and HASH embed parity wait for the first visible consumer that makes them load-bearing. diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md index fc8c2c63533..c336cd4ebaf 100644 --- a/libs/@hashintel/brunch-agent/MISSION.next.md +++ b/libs/@hashintel/brunch-agent/MISSION.next.md @@ -1,8 +1,8 @@ # Brunch future mission spine -> Canonical future-planning spine, shared frame, and backlog index only. This file is not execution authority and authorizes no implementation. [`MISSION.md`](MISSION.md) is currently a closure pointer; a future owner-authorized cut must replace it with the sole live authority. Detailed provisional clusters are context repositories, not missions; re-evaluate and convert one into `MISSION.md` before acting. +> Canonical future-planning spine, shared frame, and backlog index only. This file is not execution authority and authorizes no implementation. [`MISSION.md`](MISSION.md) is the live Mission 5 authority on this branch. Detailed provisional clusters are context repositories, not missions; re-evaluate and convert one into `MISSION.md` on its own branch before acting. -This spine and its six linked drafts form one future-planning record. Keep each consequential meaning in one authoritative planning home: shared contracts and unallocated concerns live here; mission-specific detail lives in its draft. A spine pointer is not a second contract. Material omitted from a future cut returns to this record at full fidelity, and the consumed draft is removed. +This spine and its five linked drafts form one future-planning record. Keep each consequential meaning in one authoritative planning home: shared contracts and unallocated concerns live here; mission-specific detail lives in its draft. A spine pointer is not a second contract. Material omitted from a future cut returns to this record at full fidelity, and the consumed draft is removed. ## Current authority and accepted spine @@ -10,14 +10,14 @@ Mission 4 closed on this branch by owner adjudication on 2026-09-03. The accepte A future Mission 4 close-out addendum requires its own issue, branch, PR, and mission authority. It may stack on this closed branch and own broader reliability/hardening if warranted, browser parity, fixture/seed promotion contracts, topology-neutral case allocation, contract/readiness sweeps, archive subtraction, and Mission 8 preparation. It also owns the observed S4 report-versus-immediate-ask decision unless a later numbered mission first makes it load-bearing: re-enter only when a real review must continue immediately or repeated gap-only reports create visible friction; preserve S3 restraint while testing S4 activation and asking under a fresh instrument. Its exact issue/name and minimum scope remain owner decisions; do not create another Mission 4 draft. -Two successor missions are now independently cuttable from Mission 4 under separate issue, branch, PR, worktree, and mission authority. Mission 5 retires the Voice transport uncertainty: finalized speech enters canonical Flue directly and canonical Brunch output reaches TTS without the AI SDK chat composer or a secondary generative simplifier. Mission 6 retires the workpiece/projection uncertainty: one deliberately prepared, honestly labelled fixture joins canonical conversation, session history, Markdown workpiece, and Petrinaut document; one browser-backed read/write change saves and resumes across tabs. Neither requires a Mission 4 full-run candidate, and neither is the other's prerequisite. +Mission 5 is now live on this branch under FE-1574 and owns the direct Voice/Flue transport cut; its full contract lives only in [`MISSION.md`](MISSION.md). Mission 6 remains independently cuttable from Mission 4 under its own issue, branch, PR, worktree, and mission authority: one deliberately prepared, honestly labelled fixture joins canonical conversation, session history, Markdown workpiece, and Petrinaut document; one browser-backed read/write change saves and resumes across tabs. Neither mission requires a Mission 4 full-run candidate, and neither is the other's prerequisite. The earlier capture-backed provenance, automatic-projection, revision, and optimisation drafts remain later readiness/product advances. They are renumbered around the historical Mission 8 deployment track rather than forcing the two uncertainty-retiring tracers to inherit its unproved remote boundary. ```text M4 closed — core/plugin elicitation pattern accepted; S4 transition and full-run candidate deferred M4+ optional successor — broader hardening or source promotion only under separate authority -M5 direct Voice/Flue — one finalized spoken turn, canonical streamed reply, cancellation, and reopen +M5 live on this branch — direct Voice/Flue turn, canonical streamed reply, cancellation, and reopen M6 resumable fixture tracer — conversation → Markdown workpiece → Petrinaut read/write → cross-tab resume M7 capture-backed review — close selected-pair provenance breadth and visible why/refusal M8 deployment handoff — historical branch stopped after local application proof, before infrastructure deployment @@ -26,21 +26,15 @@ M10 revision — ship bounded authorized reviewer revision and a scoped patch M11 optimisation — ship an accepted optimisation handoff after its consumer contract exists ``` -Every numbered product mission after the proof-of-life exception must pass the **product-manager litmus**: a product manager who did not watch the work must be able to notice that the product materially moved forward. Each mission therefore states, in its draft's visible-product-advance section, a release-note sentence, a demo script a product manager can run without an engineer, and the thing that was impossible before. Snapshots, manifests, event ledgers, and negative controls are oracles that belong in the evidence sections; they are not the visible advance. A mission is complete at its readiness gate, when the demo script works for the named scenario, not at the first green throughline tracer, which is an internal milestone inside the mission. Mission 5 names the Voice/Flue surface; Mission 6 names the stable fixture and browser Petrinaut document; Missions 7, 9, and 10 name the Petrinaut Brunch panel. Because Mission 8 stopped before remote deployment, those panel missions must name the deployment posture available at cut time, and a locally run panel is acceptable for the demo; a product-manager-noticeable claim must never depend on infrastructure that does not exist, while remote durability obligations stay in their readiness gates. Architecture, schema repair, fixtures, evaluation, rehearsal, and spikes may support the advance but cannot be the sole outcome. Parallel work means separate issue, branch, PR, worktree, and mission authority; it never means multiple live missions here. +Every numbered product mission after the proof-of-life exception must pass the **product-manager litmus**: a product manager who did not watch the work must be able to notice that the product materially moved forward. Each mission therefore states, in its draft's visible-product-advance section, a release-note sentence, a demo script a product manager can run without an engineer, and the thing that was impossible before. Snapshots, manifests, event ledgers, and negative controls are oracles that belong in the evidence sections; they are not the visible advance. A mission is complete at its readiness gate, when the demo script works for the named scenario, not at the first green throughline tracer, which is an internal milestone inside the mission. Mission 5 names the Petrinaut Brunch panel's typed and Voice surface over one Flue route, with its litmus stated in the live [`MISSION.md`](MISSION.md#product-manager-litmus); Mission 6 names the stable fixture and browser Petrinaut document; Missions 7, 9, and 10 name the Petrinaut Brunch panel. Because Mission 8 stopped before remote deployment, those panel missions must name the deployment posture available at cut time, and a locally run panel is acceptable for the demo; a product-manager-noticeable claim must never depend on infrastructure that does not exist, while remote durability obligations stay in their readiness gates. Architecture, schema repair, fixtures, evaluation, rehearsal, and spikes may support the advance but cannot be the sole outcome. Parallel work means separate issue, branch, PR, worktree, and mission authority; it never means multiple live missions here. ## Successor mission précis -### M5 — Speak directly to canonical Brunch - -Tracker projection: [FE-1574](https://linear.app/hash/issue/FE-1574/let-voice-speak-through-canonical-brunch-conversations). - -A finalized spoken answer enters Flue exactly once and canonical Brunch text streams directly to visible Voice output and TTS, without the AI SDK chat composer or a secondary generative simplifier. This can be cut immediately and independently on the Voice reconciliation worktree. **Visible/usable proof:** [one spoken turn, cancellation, canonical snapshot, and reopen](docs/mission-drafts/5-direct-voice-flue-transport.md#throughline-proof-floor). - ### M6 — Prove the Markdown/workpiece/Petrinaut loop Tracker projection: [FE-1575](https://linear.app/hash/issue/FE-1575/resume-one-brunch-workpiece-and-petrinaut-document-across-tabs). -One honestly prepared fixture links a canonical conversation, session history, Markdown workpiece, and Petrinaut document; Brunch updates the workpiece, performs one meaningful browser-backed document change, saves, and resumes from a second tab. This can be cut immediately and independently of Voice. **Product-manager litmus:** Brunch edits the net you are looking at from the conversation, and the work survives closing the tab. Demo: open the demo fixture, say one new thing about the process, watch the net change, save, reopen in a second tab and continue. Previously impossible: Brunch only produced off-canvas net JSON for manual load. Complete at the [readiness gate](docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md#readiness-gate-after-the-new-throughline), not at the first green mutation; see the [visible product advance](docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md#visible-product-advance). +One honestly prepared fixture links a canonical conversation, session history, Markdown workpiece, and Petrinaut document; Brunch updates the workpiece, performs one meaningful browser-backed document change, saves, and resumes from a second tab. The 2026-09-04 Mission 5 witness adds one bounded resume obligation: preserve each message's typed/Voice provenance and render an aborted assistant entry as stopped rather than ordinary truncated content. This can otherwise be cut immediately and independently of Voice input. **Product-manager litmus:** Brunch edits the net you are looking at from the conversation, and the work survives closing the tab. Demo: open the demo fixture, say one new thing about the process, watch the net change, save, reopen in a second tab and continue. Previously impossible: Brunch only produced off-canvas net JSON for manual load. Complete at the [readiness gate](docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md#readiness-gate-after-the-new-throughline), not at the first green mutation; see the [visible product advance](docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md#visible-product-advance). ### M7 — Make the demo net genuinely explainable @@ -162,7 +156,7 @@ The extraction ladder remains: model-free stub envelopes (proved) → separate c ### Product and host boundary -The production door is Petrinaut panel → AI SDK `useChat`/`onToolCall` transport → long-running Flue `ChatAgent` → Anthropic, with client tools executed through the existing host route. Brunch is a selectable second assistant; stock remains functional when Brunch is absent or unselected. Never splice histories, steal the stock `/api/chat` contract, rewrite the panel onto `@flue/react`, or add a direct canvas/server bypass. +The production door is Petrinaut panel (`useChat`/`onToolCall`) → host-supplied browser `ChatTransport` over `@flue/sdk` → mounted `/agents/chat/:instanceId` behind the ownership guard → long-running Flue `ChatAgent` → Anthropic, with client tools executed in the host and returned as one `client-tool-result` signal send. The recut live Mission 5 landed that route and removed the earlier server-side Brunch `/api/chat` adapter; the stock Petrinaut `/api/chat` remains a separate fallback owned by the website. Brunch is a selectable second assistant; stock remains functional when Brunch is absent or unselected. Never splice histories, steal the stock `/api/chat` contract, rewrite the panel onto `@flue/react`, add a second server-side turn protocol, or add a direct canvas/server bypass. Core owns universal, context/domain/editor/formalism-independent elicitation semantics. Plugins pair one reusable domain typology with one target formalism and own that pairing's recognition/operations/coverage/verification guidance, never concrete scenario nouns. The app is the directive-marked registration and host-composition shell. Flue owns `useInstruction`, `useSkill`, `useTool`, static resource packaging, and runtime lifecycle; binding packages adapt generalized capture mechanics to a substrate. @@ -199,7 +193,7 @@ Landed application contract, retained for successor consumers: - active Flue conversation/submission/recovery/settlement state uses `@flue/postgres` with dedicated fields, verified TLS, RDS-IAM async fresh-token support and runtime-password fallback; URI-only and silent SQLite production fallback are rejected; - OTLP/gRPC is initialized before content-free Flue instrumentation and flushed on shutdown; local disposable collector receipt is proved; - local Docker/Postgres/collector smoke proved non-root execution, packaged resources, no `/repo` writes, TLS Postgres startup/refusal, and bounded graceful shutdown; -- public ingress denies `/`, `/assets/*`, and `/agents/chat/:id`; restricted product traffic uses `/api/chat`; one-live-owner policy remains desired-count one, stop-before-start until overlap safety is proved; +- public ingress denies `/`, `/assets/*`, and `/agents/chat/:id`; restricted product traffic used `/api/chat` at that commit. **Superseded by the recut live Mission 5 (2026-09-03):** `/agents/chat/:instanceId` becomes the only product route, so the restricted-ingress rule must be re-expressed as the FE-1423 gates (authentication, per-conversation authorization, telemetry, state versioning/backup, restart durability) applying directly to the mounted Flue route, with `/api/chat` no longer mounted by the Brunch app. The release/deployment gate owns that re-expression and its enforcement; one-live-owner policy remains desired-count one, stop-before-start until overlap safety is proved; - separate Brunch capture JSON is inactive and non-durable. Do not migrate it speculatively, but any mission that consumes capture must first give it durable owner refusal, atomicity, format validation, and session/capture consistency. Flue's Node target is a long-running service with an in-process coordinator and long-lived streams. Do not deploy it as Lambda, a short-lived function, or scale-to-zero. Shared Postgres does not establish active-active safety; keep one replica until ownership and routing through replacement overlap are proved. @@ -217,7 +211,7 @@ Old Mission 8 reconciliation: | Old subsection | Disposition | Surviving consequence/evidence | | --- | --- | --- | | Observed starting point; application-owned surface; runtime candidates; CI wiring | Superseded proposal where implemented; landed application contract where locally observed | The bullets above and deployment handoff replace the pre-implementation audit. Image slimming, Compose parity, and obsolete workflow cleanup have no surviving requirement without strain. | -| Service/communication contract | Landed locally at the application seam | Long-running `/api/chat` → Flue → Anthropic shape, Postgres state, liveness, restricted routes, and content-free OTel survive; remote crossing remains unproved. | +| Service/communication contract | Landed locally at the application seam; door superseded by recut Mission 5 | Long-running Flue → Anthropic shape, Postgres state, liveness, and content-free OTel survive. The `/api/chat` door that carried it is removed by the live mission in favor of the mounted Flue route; the restricted-route rule is re-expressed above. Remote crossing remains unproved. | | Infrastructure-owned surface | Still-open infrastructure gate | Provisioning and identifiers belong to infra; a deploy-catalog entry cannot create them. | | Restricted smoke/public release; identity; front door; rate limits; streaming/availability | Restricted-threshold proposal partly superseded by the stopped handoff; public decisions still open | No public release. Caller UUID, CORS, obscurity, or rate limiting are not authentication. Keep one replica; measure timeout/reconnect and ownership before widening. | | Persistence, migration, recovery | Flue Postgres application contract landed; replacement/backup and capture durability still open | Process-local proof is not ECS replacement proof. Capture remains inactive and must become durable when consumed. | @@ -233,8 +227,7 @@ These tracks may start only under their own issue, branch, PR, and mission autho | Track | Join gate under the accepted spine | Does not block | | --- | --- | --- | | Frozen prospective baseline | The closed Mission 4 branch preserves it unchanged and makes no quality-improvement claim; because Mission 4 produced no full-run candidate, a successor may inspect this observed one-invalid/two-valid Mission 3 range only if it explicitly selects and promotes an eligible source | Mission 7's source selection and provenance-suitability decision, or the optional Mission 4 addendum | -| Direct Voice/Flue | One finalized spoken turn has one canonical Flue submission and canonical streamed TTS output without AI SDK chat transport or generative simplification | Mission 6 and later product-data work | -| Resumable fixture viability | One prepared fixture crosses conversation → Markdown workpiece → browser Petrinaut read/write → save/reopen | Mission 5 Voice transport work | +| Resumable fixture viability | One prepared fixture crosses conversation → Markdown workpiece → browser Petrinaut read/write → save/reopen | The live Mission 5 Voice transport work | | Inferential observer fold | Decide before Mission 10 whether observed foreground strain earns promotion; otherwise retain phase-boundary synthesis | Missions 4–9 | | Provider-visible nested schema | Mission 6 tests only the least mutation needed by its meaningful fixture; Mission 9 closes the broader canonical projection classes after a crisp blocker or success | Mission 5 and non-construction Voice work | | Provenance interaction fixture | Mission 6 establishes minimal fixture identity; freeze the broader derivation fixture before Mission 7 why and Mission 9 automatic projection diverge | Voice work and prepared-fixture viability | @@ -246,16 +239,15 @@ The deliberately provisional shared-interface names remain `EvidenceBackedWorkpi ## Detailed provisional clusters -Detailed mission-specific boundaries, tracer floors, readiness ratchets, risks, oracles, and stop conditions live only in these six context repositories: +Detailed mission-specific boundaries, tracer floors, readiness ratchets, risks, oracles, and stop conditions live only in these five context repositories: -- [Draft Mission 5 — direct Voice over canonical Flue transport](docs/mission-drafts/5-direct-voice-flue-transport.md) - [Draft Mission 6 — resumable workpiece-to-Petrinaut fixture tracer](docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md) - [Draft Mission 7 — capture-backed review](docs/mission-drafts/7-capture-backed-review.md) - [Draft Mission 9 — automatic traceable projection](docs/mission-drafts/9-traceable-projection.md) - [Draft Mission 10 — bounded reviewer revision](docs/mission-drafts/10-bounded-reviewer-revision.md) - [Draft Mission 11 — optimisation handoff](docs/mission-drafts/11-optimisation-handoff.md) -Do not create Mission 4 or Mission 8 drafts. Missions 5 and 6 are parallel, independent next cuts; each must become the sole live root `MISSION.md` in its own worktree. Mission 11 stays deliberately shallow until Chris and Yannis accept input artifacts, one optimisation question, scenario/parameter representation, execution boundary, expected result, and minimum credibility checks. +Do not create Mission 4 or Mission 8 drafts. Mission 5 is live here; Mission 6 remains an independent next cut that must become the sole live root `MISSION.md` in its own worktree. Mission 11 stays deliberately shallow until Chris and Yannis accept input artifacts, one optimisation question, scenario/parameter representation, execution boundary, expected result, and minimum credibility checks. ## Unallocated backlog @@ -368,17 +360,13 @@ Working session hypothesis: net id discriminates one Flue conversation per princ Before claiming long-running provenance, prove panel/transcript/workpiece recovery across real Flue compaction (`compaction-vs-durable-history` / FE-1386). Current recovery scrapes the last `runbook-ir` fence; summary loss would break it. A short rehearsal may avoid compaction only if the handoff declares uncompacted-history dependence. Compaction is history reconstruction, not projection sequencing; control to compact/show summaries waits for evidence. -### Voice - -Direct Voice/Flue transport is now Draft Mission 5 rather than an unallocated integration constraint. The observed remote stack remains `kostandin/fe-1570-voice-optimized-brunch-responses` → `kah-6763-temporary-brunch-ask` → `kah-6800-improve-petrinaut-voice-turn-taking-and-answer-provenance` (PRs #9496, #9507, and #9512), diverging before Mission 4's app/package restructuring. Reconciliation must preserve current `useBrunchAgent()` + `useSdcpnPlugin()` composition and port only the still-needed Voice behavior onto current seams; never restore the older app-local stub agent. - -Mission 5 treats the AI SDK UI-message transport and existing Petrinaut assistant as optional consumers, not Voice authority. Voice should use supported Flue `send`/`read`/`observe`/`history`/`abort` semantics directly or through the thinnest authenticated protocol-preserving proxy. Brunch owns canonical response text; Voice owns STT, TTS, interruption/cancellation, finalized-answer provenance, and playback. Provisional transcription/audio remains ephemeral, finalized answers enter canonical Flue history once, and the first tracer speaks canonical output without a secondary generative simplifier. Adapter/UI deletion requires separate consumer proof. +### Voice after the live transport cut -The complete inherited seam map is in [`mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md); the revised cut contract and proof target are in [`5-direct-voice-flue-transport.md`](docs/mission-drafts/5-direct-voice-flue-transport.md). +The live Mission 5 contract, recut on 2026-09-03, now owns the single-route consolidation: the typed panel's browser `ChatTransport` over `@flue/sdk`, removal of the server-side `/api/chat` door, repurposing `transport-aisdk` as the browser-side adapter, direct Voice/Flue reconciliation, its selected external-PR evidence, and the bounded local tracer. Its 2026-09-04 human witness passed typed and Voice admission, spoken playback, barge-in, and durable Stop, then failed faithful reopen: per-message typed/Voice provenance disappeared and the stopped entry returned as ordinary truncated content. Mission 6 owns those two defects because its visible boundary is exact conversation resume in a second tab. The small transcript reveal control and the distinction between local **Exit voice mode** and durable composer **Stop** are observed discoverability strain for that surface, not permission to equate Voice exit with conversation abort. This future record otherwise retains only work beyond the direct cut: whether Petrinaut ever drops `useChat` itself is a Petrinaut product decision with no Brunch obligation; the structured-question route re-enters only after plain-turn strain and owner acceptance; broader barge-in, long-response, speech-selection, and accessibility quality require observations from the direct route; and trusted remote identity, origin policy, deployment, and spend controls remain release work. The inherited seam map remains in [`mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md). ### Observability and simulation viewing -Brunch has local content-free OTLP/gRPC export and graceful flush; hosted reachability and real normal/failure attributes remain open. Prove `gen_ai.conversation.id` equals Flue instance id and decide `traceparent` propagation. Keep prompt/response/tool content off until privacy/retention/access policy. Mission 5 needs only tracer latency/tool evidence; broad OTel remains a release gate. +Brunch has local content-free OTLP/gRPC export and graceful flush; hosted reachability and real normal/failure attributes remain open. Prove `gen_ai.conversation.id` equals Flue instance id and decide `traceparent` propagation. Keep prompt/response/tool content off until privacy/retention/access policy. The live Mission 5 contract owns only its bounded tracer latency evidence; broad OTel remains a release gate. Simulated-conversation viewing remains evaluation infrastructure, not a protocol rewrite. The 2026-09-02 [live-observable persona spike](docs/evidence/evaluations/live-observable-persona-spike/README.md) @@ -401,13 +389,13 @@ No evidence yet says a non-empty parser-valid net behaves like the workpiece. Ca ### Other substrate and product hypotheses -AI SDK 7 `HarnessAgent` is undecided: it is the converse of the current door, resuming a harness session by chat id. Flue already owns that session and `transport-aisdk` adapts UI. A Pi/Claude Code harness is another binding substrate or Flue replacement, not the simulation viewer. Re-enter only with a concrete consumer. +AI SDK 7 `HarnessAgent` is undecided: it is the converse of the current door, resuming a harness session by chat id. Flue already owns that session and `transport-aisdk` adapts the panel's `useChat` contract onto the public Flue client in the browser. A Pi/Claude Code harness is another binding substrate or Flue replacement, not the simulation viewer. Re-enter only with a concrete consumer. Exploded-view net prototypes belong on Petrinaut website host routes, not `:4321`. If `ChatAgent` leaves the app, put it under `packages//`; the app stays shell. HASH embed remains stock unless explicitly opted in. Historical Conditions 1/2/4/5 remain batch evidence; no TUI, retired SDCPN elicitor, generalized `useElicitation()` runtime, loader, workflow engine, or second model-facing agent. ## Historical 2026-09-02 migration disposition -This section preserves the planning split as it was accepted on 2026-09-02. It is historical evidence, not the current draft inventory or numbering authority; the current six-draft topology is listed above. At that time, the candidate split had one live authority, one compact spine, four provisional drafts, and no Mission 4 or Mission 8 draft. Its source-to-destination ledger was: +This section preserves the planning split as it was accepted on 2026-09-02. It is historical evidence, not the current draft inventory or numbering authority; the then-current six-draft topology preceded the live Mission 5 cut. At that time, the candidate split had one live authority, one compact spine, four provisional drafts, and no Mission 4 or Mission 8 draft. Its source-to-destination ledger was: | Previous `MISSION.next.md` section | Surviving planning home | Disposition and consequence | | --- | --- | --- | @@ -498,4 +486,4 @@ The owner subsequently changed the integration premise: Voice should use canonic ## 2026-09-03 product-manager litmus reframing -Later on 2026-09-03 the owner replaced the "visible/usable proof" completion criterion with the product-manager litmus defined in the accepted spine above. The observed problem was that each précis pinned completion to an evidence bundle at the first green throughline tracer, which convinces a builder but is invisible to a product manager, and that Draft Mission 9 carried engineering internals in its visible-advance section. The change re-pins completion to each mission's readiness gate for the named demo scenario, moves oracles out of the visible-advance sections, expands Mission 7 from one element to every consequential element of the demo net, and names the deployment posture problem for Missions 7, 9, and 10. Mission 5 was live on its own branch and was not touched. Mission-specific detail lives in the affected drafts' `Visible product advance` and `Throughline proof floor` sections and in the [draft README](docs/mission-drafts/README.md). +Later on 2026-09-03 the owner replaced the "visible/usable proof" completion criterion with the product-manager litmus defined in the accepted spine above. The observed problem was that each précis pinned completion to an evidence bundle at the first green throughline tracer, which convinces a builder but is invisible to a product manager, and that Draft Mission 9 carried engineering internals in its visible-advance section. The change re-pins completion to each mission's readiness gate for the named demo scenario, moves oracles out of the visible-advance sections, expands Mission 7 from one element to every consequential element of the demo net, and names the deployment posture problem for Missions 7, 9, and 10. Mission 5 was live on its own branch and was not touched by that commit; on restack, the live branch adopted the litmus in [`MISSION.md`](MISSION.md#product-manager-litmus), naming Stop-that-really-stops and one shared typed/spoken conversation as its product-manager-noticeable advance and its single-route consolidation as internal sequencing. Mission-specific detail lives in the affected drafts' `Visible product advance` and `Throughline proof floor` sections and in the [draft README](docs/mission-drafts/README.md). diff --git a/libs/@hashintel/brunch-agent/docs/adr/README.md b/libs/@hashintel/brunch-agent/docs/adr/README.md index a4100b80873..1aec2e1af60 100644 --- a/libs/@hashintel/brunch-agent/docs/adr/README.md +++ b/libs/@hashintel/brunch-agent/docs/adr/README.md @@ -8,4 +8,4 @@ re-earn before building further on them. Internal references to retired paths (`docs/control/`, `docs/agents/`, `docs/INDEX.md`) are historical and not maintained. -For the current accepted Brunch architecture, start at the root [`MISSION.md`](../../MISSION.md) closure pointer, [`MISSION.next.md`](../../MISSION.next.md), and the final [Mission 4 archive](../mission-archive/4-core-plugin-elicitation-proof-of-life.md). Mission 4 replaced the generalized YAML/repertoire/plugin machinery described in ADR-0002, ADR-0006, and ADR-0007 with a Flue-native independent core `elicitation` capability, target-pairing plugin job skills, and app-owned composition. Those ADRs remain useful design history, not an integration baseline. +For the current accepted Brunch architecture, start at the live root [`MISSION.md`](../../MISSION.md), [`MISSION.next.md`](../../MISSION.next.md), and the final [Mission 4 archive](../mission-archive/4-core-plugin-elicitation-proof-of-life.md). Mission 4 replaced the generalized YAML/repertoire/plugin machinery described in ADR-0002, ADR-0006, and ADR-0007 with a Flue-native independent core `elicitation` capability, target-pairing plugin job skills, and app-owned composition. Those ADRs remain useful design history, not an integration baseline. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/README.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/README.md new file mode 100644 index 00000000000..6f5dd2bc43c --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/README.md @@ -0,0 +1,87 @@ +# Mission 5 direct Voice over Flue evidence + +## Readiness status + +Automated contract evidence passed on 2026-09-03. A real human Voice witness was attempted on 2026-09-04. It passed typed and Voice admission, exact-once visible Voice input, spoken canonical output, barge-in, and explicit durable Stop, but failed faithful reopen. This record therefore does not claim Mission 5 product acceptance. + +The implementation under test is: + +- `bb4457c558` — browser Flue `ChatTransport`, stream projector, history projection, and ownership headers; +- `daf525e142` — typed Petrinaut panel wiring and same-origin Flue proxy; +- `969808e772` — removal of the Brunch `/api/chat` route; +- `0d5343b069` — Voice submission correlation, canonical response selection, observation-based reopen, and durable Stop; +- `2936a4b3f7` — review fixes for canonical history hydration, exact TTS text, content-free lifecycle latency telemetry, Clear behavior, API simplification, and documentation; +- `05b363649e` — React-compiler-safe tracker lifecycle and final live-authority corrections; +- `f1189523a8` — real Flue admission timing and production-path correlation evidence; +- `a4ec9f28ce` — client-tool-result ordering that keeps the Voice submission pending until its real Flue admission and cancels stale admission waits; +- `2a1bb35775` — durable Stop correlation and aligned live/history conversation projections; and +- `eecbe99e20` — reply correlation across every submission that wrote a resumed assistant message; and +- `2d4e81f3a4` — preservation of Petrinaut's Voice API handlers in the Brunch local preview. + +## Automated verification + +The focused repository command completed with 36 successful tasks out of 36: + +```sh +yarn exec turbo run lint:tsc lint:eslint test:unit build \ + --filter @apps/brunch-agent \ + --filter @apps/petrinaut-website \ + --filter @hashintel/petrinaut \ + --filter @hashintel/brunch-agent \ + --filter @hashintel/brunch-agent-plugin-sdcpn \ + --filter @hashintel/brunch-agent-transport-aisdk +``` + +The unit results included: + +| Workspace | Test files | Tests | +| ---------------------------------------------- | ---------: | ----: | +| `@apps/brunch-agent` | 16 | 79 | +| `@apps/petrinaut-website` | 31 | 212 | +| `@hashintel/petrinaut` | 53 | 478 | +| `@hashintel/brunch-agent` | 9 | 77 | +| `@hashintel/brunch-agent-plugin-sdcpn` | 2 | 8 | +| `@hashintel/brunch-agent-transport-aisdk` | 3 | 14 | + +`yarn install --immutable` passed with the repository's existing peer-dependency warnings. `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` also passed with 62 layers, 297 edges, 613 files, 63 generated pages, and 31 authored pages. The focused ESLint run retained one non-blocking `set-state-in-effect` warning in `voice-interview-control.tsx`. + +The proof-leaf route scan over `apps/brunch-agent`, `packages/`, and the Petrinaut local-storage demo found no production path that sends a Brunch turn through `/api/chat`: its live hits are the stock Petrinaut fallback and negative tests asserting the removed Brunch route returns 404. Current integration and topology references now name `/agents/chat/:instanceId`; archived missions, prior implementation evidence, and historical decision records retain `/api/chat` as provenance for the superseded door. + +The Voice integration holds the finite Flue response stream open and asserts that `submission-admitted` arrives from the real `createFlueChatTransport().onAdmission` callback before composer submission completion. The Voice control tests also prove that a locally completed interactive-tool result remains pending until the subsequent client-tool-result admission and that cancellation releases the one-shot subscription. Bridge tests separately cover direct-message and client-tool-result matching, duplicate delivery, stale cancellation, mismatched ids, and submission-id-based canonical response selection. + +## Human witness still required + +Run `yarn dev:brunch` with `ANTHROPIC_API_KEY`, `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and a dedicated `OPENAI_VOICE_API_KEY`, then perform this witness against source commit `2d4e81f3a4` or a descendant that changes evidence only: + +1. Open one saved net, submit one typed panel turn, and confirm the network ledger contains conversation traffic only under `/agents/chat/:instanceId`. +2. Start Voice mode, accept the disclosure if required, speak one finalized answer, and confirm exactly one corresponding visible user message. +3. Confirm the content-free lifecycle ledger records one ordered admission, first canonical text, settlement, first TTS request, and first TTS audio sequence for the same opaque correlation id. +4. Compare the ordered canonical Brunch text with the exact `response_text` string array queued for speech. Record only matching hashes, lengths, and the boolean result; do not retain the text. +5. Interrupt assistant playback by speaking and confirm canonical history is unchanged. +6. Start another unsettled turn, select **Stop**, and confirm Flue records either an aborted settlement or the documented already-settled race rather than only cancelling the browser stream. +7. Reload or reopen the same net and confirm canonical messages reappear without a duplicate submission, a replayed tool effect, or Voice audio replay. + +Retain these sanitized artifacts here: + +1. `witness.md` — date, adjudicator, source/build commit, and observed outcome; +2. `voice-events.jsonl` — content-free admission, first canonical text, first TTS/audio, interruption, Stop, and settlement timing; +3. `network-routes.json` — method and route summary proving conversation traffic used only `/agents/chat/:instanceId`; +4. `flue-snapshot.json` — sanitized canonical snapshot after reopen; +5. `settlements.json` — settled, aborted, and abort-lost-to-completion outcomes; +6. `manifest.sha256` — hashes for the retained witness artifacts. + +The witness must type one turn, speak one finalized answer, confirm exactly one visible user message, compare canonical text with the exact TTS request input, interrupt playback, durably stop one unsettled turn, and reopen without resubmission or audio replay. Do not retain transcript text, audio, credentials, SDP, prompts, tool payloads, or provider response bodies in ordinary telemetry. + +## Human witness attempt — 2026-09-04 + +The local `yarn dev:brunch` pair ran against source commit `2d4e81f3a4` with the required Voice configuration available. The human observed: + +- one typed turn completed through the Brunch panel; +- one finalized spoken answer produced exactly one visible user message; +- Brunch's visible response also played aloud, and speaking over it stopped playback without removing the visible response; +- the transcript was initially hidden behind a small **Show transcript** control; +- **Exit voice mode** stopped the audio session but did not durably stop an admitted Brunch submission; the partial spoken input was submitted and the response completed as text; +- using the chat composer's actual Stop control durably stopped the unsettled response and displayed **Response stopped** before reload; and +- after closing the panel, reloading, and reopening the same conversation, all message content returned, but typed/Voice provenance was absent and the stopped assistant entry appeared as ordinary truncated content. The global stopped status remained visible. + +The reopen gate failed because the transcript did not return exactly as left. Absence of duplicate submission and audio replay was not fully adjudicated after this failure, and the required content-free event, route, snapshot, settlement, and hash artifacts were not retained. The owner chose not to expand Mission 5 with immediate product remediation. Mission 6 owns preserving per-message typed/Voice provenance and stopped-turn presentation across its second-tab resume proof; its cut should also account for the observed discoverability gap between transcript reveal, local Voice exit, and durable conversation Stop. diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/10-bounded-reviewer-revision.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/10-bounded-reviewer-revision.md index 7c20b04c3a3..87cdb10c326 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/10-bounded-reviewer-revision.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/10-bounded-reviewer-revision.md @@ -8,7 +8,7 @@ A fresh builder must read these sources before cutting or implementing this cluster: -- [`MISSION.md`](../../MISSION.md) — closure pointer for Mission 4. Mission 4 produced no full-run workpiece candidate; consume only the source/workpiece pair explicitly selected and promoted by Mission 7 or a predecessor addendum. +- [`MISSION.md`](../../MISSION.md) — live Mission 5 Voice authority on this branch; it supplies no Mission 10 execution authority or workpiece candidate. Consume only the source/workpiece pair explicitly selected and promoted by Mission 7 or a predecessor addendum. - [`MISSION.next.md`](../../MISSION.next.md) — compact shared frame, standing locks, and current mission joins. - [`README.md`](README.md) — durable draft authority, lifecycle, conversion, and oracle-gap rules. - [`docs/mission-archive/2-mechanical-capture-sweep.md`](../mission-archive/2-mechanical-capture-sweep.md) — exact-evidence capture, idempotency, Flue-history authority, and model-free scheduling. @@ -61,7 +61,7 @@ Stale-base revision and legitimate impact widening are cross-class failure/exten ```text scenario declares reviewer authority + selected region + base revisions → reviewer enters the deployed Petrinaut assistant panel -→ AI SDK /api/chat transport resumes the owning Flue conversation +→ host-supplied browser `ChatTransport` over `@flue/sdk` resumes the owning Flue conversation at the mounted route → foreground Brunch agent conducts 3–5 focused operational-language turns → Flue history retains the canonical conversation → harness-owned mechanical sweep durably captures the settled reviewer range @@ -126,7 +126,7 @@ Breadth beyond the named classes and accepted scenario portfolio remains unearne ## Candidate evidence and oracles -- `apps/brunch-agent/test/petrinaut-chat.test.ts`, test **“the committed /api/chat door streams a plain Flue agent through server and client tools”**, currently proves the production AI SDK/Flue door, client-tool correlation, history recovery, ownership refusal, exact capture excerpts, idempotent recapture, and absence of sweep/construction tools on the interviewer. It does not prove reviewer revision or deployed infrastructure. +- `apps/brunch-agent/test/petrinaut-chat.test.ts`, test **“the committed /api/chat door streams a plain Flue agent through server and client tools”** (re-expressed by recut Mission 5 through the browser Flue transport at the mounted route; cite its successor), currently proves the production AI SDK/Flue door, client-tool correlation, history recovery, ownership refusal, exact capture excerpts, idempotent recapture, and absence of sweep/construction tools on the interviewer. It does not prove reviewer revision or deployed infrastructure. - `apps/brunch-agent/test/headless-petrinaut-client.test.ts`, tests **“constructs a parser-accepted document through the bounded callbacks”** and **“refuses tools outside the side-quest subset”**, currently prove only the six-tool construct subset and parser acceptance. They are evidence for bounded capability/refusal, not a scoped update patch or semantic fidelity. - `evaluations/oracles/ir-quality-ruler-v1.md` supplies stable `CONFLICT-COLLAPSE`, `CONS-MISS`, `CONS-DISTORT`, `INVENT`, `HARDEN`, `SCOPE`, and `GAP-MISCLASS` judgments. Its own scope excludes successive revision and PN construction, so it may seed but cannot settle the revision claim. - `docs/evidence/evaluations/vestera-prospective-baseline-v1/campaign-adjudication.md` records one correction preserved in a workpiece and healthy ordinary-turn timing. It does not prove authorization, successive revision, or patch locality. diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/11-optimisation-handoff.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/11-optimisation-handoff.md index f94547cf6bc..4f163baa4ab 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/11-optimisation-handoff.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/11-optimisation-handoff.md @@ -8,7 +8,7 @@ A fresh builder must read these durable sources before deepening this cluster: -- [`../../MISSION.md`](../../MISSION.md) — current closure pointer. Mission 4 is closed; later accepted mission archives and an owner-authorized live cut become inherited authority before this draft can execute. +- [`../../MISSION.md`](../../MISSION.md) — live Mission 5 Voice authority on this branch. Mission 4 is closed; later accepted mission archives and an owner-authorized Mission 11 cut become inherited authority before this draft can execute. - [`../../MISSION.next.md`](../../MISSION.next.md) and [`README.md`](README.md) — shared frame, standing locks, draft authority, and lifecycle. - [`10-bounded-reviewer-revision.md`](10-bounded-reviewer-revision.md) and the eventual accepted Missions 7, 9, and 10 close evidence — inherited real-path artifacts and proof. Draft promises are not join evidence. - [`../mission-archive/3-structurally-typed-runbook-to-headless-pn.md`](../mission-archive/3-structurally-typed-runbook-to-headless-pn.md) — accepted workpiece leg, falsified real-model construction, and the parser-valid-empty warning. diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/5-direct-voice-flue-transport.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/5-direct-voice-flue-transport.md deleted file mode 100644 index 35e5ba40d2c..00000000000 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/5-direct-voice-flue-transport.md +++ /dev/null @@ -1,129 +0,0 @@ -# Draft Mission 5 — Direct Voice over canonical Flue transport - -> Draft cluster only. Not execution authority. Do not implement until this cluster is re-evaluated and cut into `MISSION.md`. - -## Cold-start reads - -- [FE-1574](https://linear.app/hash/issue/FE-1574/let-voice-speak-through-canonical-brunch-conversations) — tracker projection for this future branch mission; the eventual branch `MISSION.md` remains execution authority. -- [`../../MISSION.md`](../../MISSION.md) — Mission 4 closure pointer; no live Brunch mission exists. -- [`../../MISSION.next.md`](../../MISSION.next.md) — shared contracts, parallel-track rules, and current sequencing. -- [`../evidence/implementations/mission-4-voice-integration-handoff.md`](../evidence/implementations/mission-4-voice-integration-handoff.md) — observed Voice stack, conflict surfaces, and the package-composition invariant. -- [`../../packages/core/src/flue.ts`](../../packages/core/src/flue.ts) and [`../../../../../apps/brunch-agent/src/app.ts`](../../../../../apps/brunch-agent/src/app.ts) — accepted `useBrunchAgent()` + `useSdcpnPlugin()` agent composition and mounted Flue route. -- [`../../../../../apps/brunch-agent/src/http/petrinaut-chat.ts`](../../../../../apps/brunch-agent/src/http/petrinaut-chat.ts), [`../../../../../apps/brunch-agent/src/conversation/ui-stream.ts`](../../../../../apps/brunch-agent/src/conversation/ui-stream.ts), and [`../../packages/transport-aisdk/src/index.ts`](../../packages/transport-aisdk/src/index.ts) — current AI SDK adapter and Flue-to-UI projection; these are terrain, not a required Voice path. -- Flue [`FlueClient`](https://flueframework.com/docs/sdk/flue-client/) and React client documentation — supported `send`, `read`, `observe`, `history`, `abort`, offsets, submission correlation, and conversation incarnation semantics. -- Voice PRs #9496, #9507, and #9512 at their current accepted tips; do not reconstruct their behavior from this draft. - -## Visible product advance - -A person speaks one finalized answer in the Voice surface and hears the canonical Brunch reply begin through TTS while the same answer and reply appear exactly once in the owning Flue conversation. Voice connects through Flue's supported conversation protocol rather than submitting through the Petrinaut AI SDK chat composer, and no secondary model rewrites Brunch's reply before speech. - -The existing Petrinaut AI assistant may remain temporarily present during integration, but it is not the conversation authority or required transport for this proof. Removal of obsolete assistant UI and deletion of `transport-aisdk` are consequences only after dependency inspection proves they have no surviving consumer. - -## Contract stratum - -Close the **one-turn direct Voice/Flue transport stratum** for finalized input, streamed canonical output, cancellation, and conversation resumption. - -The accepted objects are one stable logical conversation id, one admitted submission id, one finalized user message, one canonical assistant response, and one Voice playback lifecycle. `conversationId` is the durable logical reference; `submissionId` correlates one admitted turn and supports reattachment; stream offsets are opaque Flue cursors; `uid` identifies one current Flue incarnation and must not become the durable demo/session id. - -## Boundary crossings and current throughline hypothesis - -```text -microphone → provisional STT (ephemeral) -→ one finalized transcript -→ supported FlueClient send to the owning Brunch conversation -→ accepted useBrunchAgent() + useSdcpnPlugin() composition -→ canonical Flue response chunks and settlement -→ exact canonical text projected to Voice -→ ordinary TTS playback -→ history/observe rehydration of the same conversation after reopen -``` - -Authentication, principal ownership, CORS or a same-origin protocol-preserving proxy remain host obligations. “Direct Flue” means use of `@flue/sdk`/`@flue/react`, not handwritten SSE parsing or an unauthenticated public agent route. - -## Throughline proof floor - -From the real Voice surface, one finalized spoken answer produces exactly one visible user message in canonical Flue history; one canonical Brunch response streams to both visible text and TTS without a generative simplification pass; interruption stops local playback and the selected durable abort action has its documented effect; reopening the conversation reconstructs the same settled turn without duplicate submission or playback. - -The retained proof artifact is the canonical Flue snapshot plus the Voice event ledger for STT finalization, admission/submission id, text/TTS projection, cancellation, settlement, and reopen. This does not prove client-side Petrinaut mutations, workpiece viability, broad Voice UX, remote deployment, or that the AI SDK adapter is removable. - -## Readiness ratchet - -### Inherited stratum closure - -- Preserve Mission 4 package composition; never restore the deleted app-local stub `ChatAgent` to resolve Voice conflicts. -- Preserve canonical Flue history as the sole conversation authority and exact finalized-answer correlation from the Voice work. -- Preserve principal/ownership semantics even if the AI SDK adapter is bypassed. - -### Readiness gate after the new throughline - -Before this one-turn capability is accepted, close provisional-versus-final transcript deduplication, submission correlation, replay/reopen behavior, TTS cancellation, durable abort races, visible failed/aborted settlement, authentication/origin handling at the claimed host boundary, and exact canonical spoken/visible correspondence. Carry only broader speech ergonomics, multi-turn barge-in tuning, and obsolete-adapter/UI deletion, each after observed strain or dependency proof. - -## Candidate evidence and oracles - -| Claim leaf | Candidate oracle | -| --- | --- | -| Finalized speech enters one canonical conversation once | Snapshot inspection shows one user message with the expected text and one admission/submission id; provisional STT never appears in history. | -| Voice bypasses AI SDK UI-message transport | Network/source inspection shows the supported Flue conversation protocol and no Voice request to the AI SDK chat route. | -| Spoken output is canonical | Captured TTS input equals the canonical response text selected by the documented deterministic policy; no secondary generation call occurs. | -| Cancellation and abort remain distinct | Voice event ledger plus Flue settlement/history distinguish local playback cancellation, local observation cancellation, and durable conversation abort. | -| Reopen resumes rather than duplicates | A second surface rehydrates the same conversation and settled submission from `history()`/`observe()` without a new user message or automatic replay. | -| Accepted agent architecture survives reconciliation | Composition/dependency test and code inspection retain `useBrunchAgent()` + `useSdcpnPlugin()` and exclude the older stub agent. | - -## Verification approach - -- **Inner:** deterministic tests for finalized-transcript deduplication, canonical text selection, TTS cancellation, submission correlation, and rehydration. -- **Middle:** run the real Brunch agent behind Flue and drive one Voice turn through `send` plus `observe` or `read`, retaining canonical history and the Voice ledger. -- **Outer:** a human speaks, hears the response begin, interrupts once, reopens the same conversation, and confirms visible/spoken/history agreement. Browser-only mocks or a server-only Flue call do not establish the Voice advance. - -## Inputs and joins - -- This mission may cut from Mission 4 independently of the fixture/workpiece mission; neither mission is a prerequisite for the other's first tracer. -- The Voice branch supplies STT, TTS, playback, and answer-correlation behavior. Mission 4 supplies the current Brunch composition and canonical conversation runtime. -- A later integration mission may reuse this direct client to service Petrinaut client tools, but this mission does not need tool mutation to prove transport. - -## Risks and assumptions - -- If canonical Mission 4 replies remain too long for speech, first try deterministic question-focused presentation or Brunch-owned spoken-mode instruction; re-admit secondary generative preparation only after measured failure and with visible canonical/spoken distinction. -- If browser-to-Flue auth/CORS cannot be made safe directly, use the thinnest same-origin proxy that preserves Flue semantics rather than translating into AI SDK messages. -- If `transport-aisdk` or the existing assistant UI has another live consumer, retain it; this mission establishes that Voice does not require it, not that the repository does not. - -## Accepted constraints and guarded invariants - -- One canonical Flue history; no Voice-side transcript authority. -- One finalized answer submission; provisional speech remains ephemeral. -- No lossy generative simplification in the tracer. -- Voice owns audio interaction and playback; Brunch owns canonical response content. -- Use supported Flue client APIs; do not hand-roll offset, retry, or stream reduction. -- Preserve ownership/authentication and make failures visible. -- Do not restore the old app-local Brunch agent or splice Voice into the stock assistant's history. - -## Cross-cutting obligations - -Record latency to admission, first canonical text, first audio, and settlement; distinguish local cancellation from durable abort; keep content out of ordinary telemetry; and update Voice/Petrinaut user documentation if the visible interaction or assistant surface changes. - -## Expected touched paths - -```text -Voice-stack application paths ~ direct Flue client host, STT/TTS projection, cancellation -apps/brunch-agent/src/app.ts ? protocol/auth mounting only if the existing route is insufficient -apps/brunch-agent/src/agents/chat-agent/ ~ preserve current package composition during reconciliation -apps/brunch-agent/src/conversation/ ? only shared identity/tool-result mechanics actually reused -libs/@hashintel/brunch-agent/packages/transport-aisdk/ ? retain or remove only after consumer inspection -libs/@hashintel/petrinaut/src/ui/ ? remove/replace obsolete assistant surface only if separately admitted -``` - -## Fog-line - -- The exact Voice-stack source after its PRs settle and the selected integration order. -- Whether `FlueClient.read` or a maintained `observe({ live: "sse" })` store best fits the existing Voice state machine. -- The authenticated production URL/proxy and origin policy. -- The deterministic policy for which canonical text is spoken if a response contains multiple text blocks or interactive tool parts. -- Whether any non-Voice consumer still needs `transport-aisdk` or the current assistant UI. - -## Stop or reorient - -Stop if integration creates a second conversation authority, submits provisional STT, rewrites canonical output through another model without an observed need, restores the old stub agent, exposes an unauthenticated Flue route, hand-rolls stream recovery, or claims adapter/UI removal before consumer inspection. Stop at a crisp host/auth blocker rather than rebuilding the AI SDK adapter under a new name. - -## Carried evidence and rejected alternatives - -The existing AI SDK transport remains a valid adapter for an AI SDK chat consumer; it is rejected only as an inherent Voice dependency. The older Voice simplifier was a workaround for pre-Mission-4 response shape, not permanent authority. Direct Flue preserves durable submission correlation, history, observation, and abort semantics while avoiding a second projection protocol. diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md index 26a3163fc76..a52ba6c346e 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/6-resumable-workpiece-petrinaut-fixture.md @@ -5,7 +5,7 @@ ## Cold-start reads - [FE-1575](https://linear.app/hash/issue/FE-1575/resume-one-brunch-workpiece-and-petrinaut-document-across-tabs) — tracker projection for this future branch mission; the eventual branch `MISSION.md` remains execution authority. -- [`../../MISSION.md`](../../MISSION.md) — Mission 4 closure pointer and explicit absence of a full-run candidate. +- [`../../MISSION.md`](../../MISSION.md) — live Mission 5 Voice authority on this branch; it supplies no Mission 6 execution authority and does not change Mission 4's explicit absence of a full-run candidate. - [`../../MISSION.next.md`](../../MISSION.next.md) — shared workpiece, projection, evidence, and product constraints. - [`../mission-archive/3-structurally-typed-runbook-to-headless-pn.md`](../mission-archive/3-structurally-typed-runbook-to-headless-pn.md) and [`../evidence/implementations/fe-1525-headless-runbook-pn.md`](../evidence/implementations/fe-1525-headless-runbook-pn.md) — accepted workpiece viability, hermetic callback route, and failed real-model nested-schema carrier. - [`../mission-archive/4-core-plugin-elicitation-proof-of-life.md`](../mission-archive/4-core-plugin-elicitation-proof-of-life.md) — accepted core/plugin architecture and exact proof exclusions. @@ -17,7 +17,7 @@ **Release note:** Brunch edits the Petrinaut net you are looking at from the conversation, and your work survives closing the tab. -**Demo script (no engineer present):** open the stable demo fixture; the canonical Brunch conversation, current Markdown workpiece, and associated Petrinaut document come back together. Tell Brunch one new realistic thing about the process. Watch the workpiece update and a meaningful change appear in the live net. Save. Open the same fixture in a second tab and continue the conversation from the saved state. +**Demo script (no engineer present):** open the stable demo fixture; the canonical Brunch conversation, current Markdown workpiece, and associated Petrinaut document come back together. Tell Brunch one new realistic thing about the process. Watch the workpiece update and a meaningful change appear in the live net. Save. Open the same fixture in a second tab and continue the conversation from the saved state. Confirm that messages retain their typed/Voice provenance and that an aborted assistant entry still presents as stopped rather than as an ordinary truncated answer. **Previously impossible:** Brunch only produced off-canvas net JSON for manual load; nothing it did touched the live document or survived a reload. @@ -65,7 +65,7 @@ For one deliberately prepared fixture: 2. one new realistic conversation turn produces an inspectable workpiece revision without erasing the unknown or unsupported meaning; 3. through the real browser client-tool boundary, Brunch reads the associated Petrinaut document and applies one meaningful supported change derived from the current workpiece; 4. canonical Petrinaut state is non-empty and visibly corresponds to the selected meaning; and -5. after save, a second tab opens the same fixture id, observes the same settled conversation/workpiece/document revision, and successfully continues or reads it without duplicate submission or identity drift. +5. after save, a second tab opens the same fixture id, observes the same settled conversation/workpiece/document revision, preserves per-message typed/Voice provenance and stopped-turn presentation, and successfully continues or reads it without duplicate submission or identity drift. One pass through those five steps is the first internal milestone, not mission completion. The retained oracles are the stable demo URL or fixture selector plus the before/after fixture manifest, exact Flue snapshot, Markdown workpiece revisions, and canonical Petrinaut document revisions; they are evidence for the builder, not the visible advance. This proves viability, not automatic full-net projection, selected-pair provenance breadth, remote replacement durability, concurrent editing, or Mission 3/4 quality superiority. @@ -91,14 +91,14 @@ This gate is the mission's completion bar: the demo script above must work for t | Browser executes real Petrinaut read/write tools | Production-boundary integration records tool call ids, canonical parsed inputs, execution outcomes, correlated result signals, and current document state. | | Change is meaningful | Human comparison binds one workpiece meaning to a visible canonical type/parameter/place/transition/arc change appropriate to the fixture; non-empty/parser-valid alone fails. | | Save is coherent | Injected workpiece-save or document-save failure leaves the prior current bundle revision selected and exposes the partial result for recovery. | -| Second tab resumes stable state | Open the same stable fixture selector after save and compare conversation id/history, workpiece revision/hash, document id/revision, and canonical definition before continuing. | +| Second tab resumes stable state | Open the same stable fixture selector after save and compare conversation id/history, per-message typed/Voice provenance, stopped-turn presentation, workpiece revision/hash, document id/revision, and canonical definition before continuing. An aborted entry rendered as ordinary truncated content fails this check even if a global latest-settlement banner still says stopped. | | No typed domain IR was smuggled in | Public-schema and dependency inspection finds only fixture identity/revision links and canonical Petrinaut payloads, not a closed process ontology or typed capture-to-workpiece model. | ## Verification approach - **Inner:** fixture-manifest parse/version/stale checks, explicit identity separation, coherent-revision publication, idempotent client-tool result handling, and canonical Petrinaut mutation tests. - **Middle:** drive the production Brunch agent through Flue, update the Markdown artifact, execute actual browser callbacks against the fixture-bound Petrinaut instance, and retain before/after artifacts plus one injected failure. -- **Outer:** from the real demo route or equivalent product selector, perform the update/save in Tab A and reopen/continue from the same fixture in Tab B. A headless callback alone does not establish this mission. +- **Outer:** from the real demo route or equivalent product selector, perform the update/save in Tab A and reopen/continue from the same fixture in Tab B. Include one Voice-origin message and one durably stopped assistant turn in the comparison; a headless callback or content-only transcript match does not establish this mission. - **Semantic:** a cold human judges whether the workpiece remained honest and the changed net region corresponds to it. ## Inputs and joins @@ -113,7 +113,8 @@ This gate is the mission's completion bar: the demo script above must work for t - If a realistic prepared conversation/workpiece cannot support one meaningful mutation without richer typed structure, record the exact lookup, identity, or ambiguity strain before adding any schema. - If existing per-action provider schemas cannot carry the required nested mutation, reduce to the smallest meaningful supported action only if semantic correspondence survives; otherwise stop with the crisp provider/Flue schema blocker. - If coherent save cannot span existing workpiece/document stores, the least fixture-scoped commit marker may publish only after both writes; do not invent distributed transactions before a failure demonstrates the need. -- If direct browser tool servicing needs a transport abstraction, extract only tool-call/result correlation from the existing AI SDK adapter; do not require the chat UI or duplicate Flue observation. +- If direct browser tool servicing needs a transport abstraction, extract only tool-call/result correlation from the browser `ChatTransport` in `transport-aisdk` (recut Mission 5); do not require the chat UI or duplicate Flue observation. +- The 2026-09-04 Mission 5 witness established that message content and the global stopped settlement can survive while the UI loses message-source metadata and renders the stopped assistant entry as ordinary truncated content. Repair the projection at the canonical history-to-Petrinaut boundary; do not create a second transcript store. The small **Show transcript** control and the separation between local **Exit voice mode** and durable composer **Stop** are additional discoverability observations to account for in the resumed surface. ## Accepted constraints and guarded invariants diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/7-capture-backed-review.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/7-capture-backed-review.md index 8d930f60581..11a1e320719 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/7-capture-backed-review.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/7-capture-backed-review.md @@ -8,7 +8,7 @@ A fresh builder must resolve the current repository and the deployment handoff rather than treating this draft as a specification: -- [`../../MISSION.md`](../../MISSION.md) — closure pointer; Mission 4 produced no full-run conversation/workpiece candidate and this draft must not imply otherwise. +- [`../../MISSION.md`](../../MISSION.md) — live Mission 5 Voice authority on this branch; it supplies no Mission 7 execution authority or conversation/workpiece candidate, and Mission 4 produced no full-run candidate. - [`6-resumable-workpiece-petrinaut-fixture.md`](6-resumable-workpiece-petrinaut-fixture.md) — independent viability predecessor. If accepted, its deliberately prepared and honestly labelled fixture may become this mission's selected pair after a separate provenance-suitability decision; a complete persona workpiece is not intrinsically required. - [`../../MISSION.next.md`](../../MISSION.next.md) — compact future spine, FE-1476 product frame, shared proof obligations, standing locks, and any later evidence admitted after this draft was written. - [`../mission-archive/2-mechanical-capture-sweep.md`](../mission-archive/2-mechanical-capture-sweep.md) — accepted mechanical capture throughline, exact close evidence, empty-payload boundary, conversation identity, and carried flags. @@ -150,7 +150,7 @@ Do not carry selected-pair provenance breadth, capture durability, owner refusal | Claim leaf | Existing evidence or candidate oracle | | --- | --- | -| Explicit harness-owned sweep over real Flue history; no interviewer sweep tool; exact excerpt, `{}` payload, idempotent retry | Existing `apps/brunch-agent/test/petrinaut-chat.test.ts`, test `the committed /api/chat door streams a plain Flue agent through server and client tools`, driven by `apps/brunch-agent/test/petrinaut-chat.integration.ts`. Run `yarn workspace @apps/brunch-agent test:unit`. | +| Explicit harness-owned sweep over real Flue history; no interviewer sweep tool; exact excerpt, `{}` payload, idempotent retry | Existing `apps/brunch-agent/test/petrinaut-chat.test.ts`, test `the committed /api/chat door streams a plain Flue agent through server and client tools`, driven by `apps/brunch-agent/test/petrinaut-chat.integration.ts`; recut Mission 5 re-expresses this scenario through the browser Flue transport against the mounted route while preserving its assertions, so cite the successor test it leaves behind. Run `yarn workspace @apps/brunch-agent test:unit`. | | Capture command closure, source evidence, all-or-nothing refusal, supersession/conflict guards, persisted parse | Existing `libs/@hashintel/brunch-agent/packages/core/test/capture-store.test.ts`, suite `capture-store contract`. Run `yarn workspace @hashintel/brunch-agent test:unit`. These tests are evidence for internal historical mechanics, not permission to expose typed payload semantics. | | Owner refusal, tmp-and-rename persistence, serialization, invalid-format failure | Existing `libs/@hashintel/brunch-agent/packages/binding-flue/test/local-capture-store.test.ts`, suite `local capture store`. Run `yarn workspace @hashintel/brunch-agent-binding-flue test:unit`. This is local-file evidence only. | | Panel executes client tools and resumes one turn through AI SDK | Existing `apps/brunch-agent/test/petrinaut-chat.test.ts` plus `libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx`, test `adds one dynamic output and sends one automatic follow-up`. Run `yarn workspace @apps/brunch-agent test:unit` and `yarn workspace @hashintel/petrinaut test:unit --run`. | diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/9-traceable-projection.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/9-traceable-projection.md index 879211b56d3..82b84336fc9 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/9-traceable-projection.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/9-traceable-projection.md @@ -8,7 +8,7 @@ A fresh builder must resolve these authorities and evidence before choosing a mechanism: -- [`../../MISSION.md`](../../MISSION.md) — current closure pointer. Mission 9 may be cut only after Mission 7 validly closes its accepted join and a new owner-authorized mission replaces that pointer as sole execution authority. +- [`../../MISSION.md`](../../MISSION.md) — live Mission 5 Voice authority on this branch. Mission 9 may be cut only after Mission 7 validly closes its accepted join and a new owner-authorized mission replaces the then-current branch authority. - [`../../MISSION.next.md`](../../MISSION.next.md) — compact future spine, FE-1476 floor, cross-mission obligations, standing locks, and current Mission 10 handoff. - [`6-resumable-workpiece-petrinaut-fixture.md`](6-resumable-workpiece-petrinaut-fixture.md) and [`7-capture-backed-review.md`](7-capture-backed-review.md) — provisional viability and provenance predecessors. At cut time replace assumptions with their accepted evidence, exact current workpiece/derivation seam, and real browser mutation behavior. - [`../mission-archive/3-structurally-typed-runbook-to-headless-pn.md`](../mission-archive/3-structurally-typed-runbook-to-headless-pn.md) and [`../evidence/implementations/fe-1525-headless-runbook-pn.md`](../evidence/implementations/fe-1525-headless-runbook-pn.md) — accepted workpiece leg, canonical callback fixture, provider-visible nested-schema failure, vacuous empty-net result, and explicit next-boundary decision. diff --git a/libs/@hashintel/brunch-agent/docs/mission-drafts/README.md b/libs/@hashintel/brunch-agent/docs/mission-drafts/README.md index 1c66541f0f3..4d1d119658c 100644 --- a/libs/@hashintel/brunch-agent/docs/mission-drafts/README.md +++ b/libs/@hashintel/brunch-agent/docs/mission-drafts/README.md @@ -1,6 +1,6 @@ # Provisional mission drafts -Files in this directory are detailed context repositories for possible future missions. They are not execution authority, do not create live missions, and must not be implemented. [`MISSION.md`](../../MISSION.md) is currently a closure pointer and [`MISSION.next.md`](../../MISSION.next.md) is the compact future spine. A draft must be re-evaluated and converted into a new root `MISSION.md` as the sole execution authority on its own issue, branch, and PR before implementation. +Files in this directory are detailed context repositories for possible future missions. They are not execution authority, do not create live missions, and must not be implemented. [`MISSION.md`](../../MISSION.md) is the current branch's sole live Mission 5 authority and [`MISSION.next.md`](../../MISSION.next.md) is the compact future spine. A draft must be re-evaluated and converted into a new root `MISSION.md` as the sole execution authority on its own issue, branch, and PR before implementation. Each planning item has one authoritative planning home across `MISSION.next.md` and these linked drafts. A spine summary is only a pointer. Keep accepted decisions, rejected alternatives and reasons, re-entry conditions, scenario classes, evidence, constraints, fog, stop conditions, risks, assumptions, and named mechanisms in one discoverable home at the precision needed by a cold-start builder. diff --git a/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md b/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md index 8e636b7eec2..bfd1bc8a77a 100644 --- a/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md +++ b/libs/@hashintel/brunch-agent/docs/reference/architecture/topology.md @@ -1,6 +1,6 @@ # Topology: verification and specification -**Status: ratified 2026-08-17 (Lu), application layout updated 2026-08-31 — recorded as [ADR-0002](../../adr/0002-topology-and-placement-rules.md); this file remains the living reference.** Verifies the current app/package topology against the three-lane model (cheatsheet, boundary summary), spec §12.2, and Flue's project-layout guide; then specifies where upcoming work lands. Pseudo-style: tree nodes with rules; `✓` complies today, `✗` violates, `→` normative rule for what's next. +**Status: ratified 2026-08-17 (Lu), application layout updated 2026-08-31 and conversation transport updated by FE-1574 / Mission 5 on 2026-09-03 — recorded as [ADR-0002](../../adr/0002-topology-and-placement-rules.md); this file remains the living reference.** Verifies the current app/package topology against the three-lane model (cheatsheet, boundary summary), spec §12.2, and Flue's project-layout guide; then specifies where upcoming work lands. Pseudo-style: tree nodes with rules; `✓` complies today, `✗` violates, `→` normative rule for what's next. ## Verification — the tree as it stands @@ -36,12 +36,13 @@ packages/binding-flue LANE 2 (translate harness ↔ Flue dialect) archive, legacy provisioning, parse-on-read, tmp+rename, per-path queue). One per deploy target per binding. Never: business rules. -packages/transport-aisdk UI REPLY WIRE (substrate-neutral) -└─ index.ts ✓ validates Petrinaut's POST, drives an application-supplied harness turn, - and encodes - harness reply events with `ai` only. Opt-in inspection emits metadata - out-of-band. Never: binding/Flue imports, inference, conversation - rendering, or diagnostics dispatched as user evidence. +packages/transport-aisdk BROWSER FLUE → AI SDK PROJECTION +├─ index.ts ✓ adapts one caller-supplied public `FlueClient` to an AI SDK `ChatTransport`; +│ sends one user message or client-tool-result signal and follows only the +│ admitted submission. Never: `@flue/runtime`, core, plugin, or binding imports. +├─ ui-stream.ts ✓ projects Flue conversation chunks into one finite AI SDK response stream +├─ transcript.ts ✓ projects SDK-maintained canonical state into renderable UI messages +└─ identity.ts ✓ browser-safe principal + logical-conversation identity and ownership headers packages/plugin-gherkin TARGET POLICY + Flue-native contribution bundle (not yet composed) ├─ index.ts ✓ pairing identity only (YAML definition removed 2026-09-02) @@ -70,10 +71,10 @@ apps/brunch-agent LANE 1 SHELL + remote server (imported from a │ ├─ agent.ts ✓ sole directive-marked registration and composition point: generic core, │ │ selected SDCPN/Petrinaut plugin, and deployment instructions │ └─ tools/ping.ts ✓ app-only server-path diagnostic -├─ src/http/ ✓ HTTP authority: assets, route names, ownership guard, local origins, -│ and `/api/chat` composition -├─ src/conversation/ ✓ identity and projection authority: shared payload, client-tool signal, -│ Flue-history transcript, and AI SDK stream projection +├─ src/http/ ✓ HTTP authority: assets, mounted route names, ownership guard, and local origins; +│ `/agents/chat/:instanceId` is the sole Brunch conversation door +├─ src/conversation/ ✓ server identity verification, client-tool catalog, and operator transcript; +│ browser AI SDK projection lives in `transport-aisdk` ├─ src/capture/ ✓ Mission 2 application composition over binding-owned history/store ports; │ no elicitation policy ├─ src/evaluations/runbook/ ✓ runbook experiment drivers, artifact recovery, and headless client; diff --git a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md index 15aaa55b7ff..b315a764651 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md +++ b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md @@ -1,11 +1,6 @@ # Integration spec: the elicitor behind Petrinaut's chat panel -**Ticket**: FE-1433 (the integration-spec issue) · **Decision record**: ADR-0004 -(`docs/adr/0004-in-petrinaut-staging-and-the-monorepo-import.md`) · **Supersedes**: -`recommendation-demo-vehicle.md` as the September staging plan · **Evidence base**: the -Petrinaut survey (FE-1358, `research/petrinaut-survey.md`), re-verified against -`hashintel/hash` source on 2026-08-18 · **Amended**: FE-1506 (stable UI and voice attach -contract), H-6763 / ADR-0009 (generic composer submission and app-owned voice boundary). +**Ticket**: FE-1433 (the integration-spec issue) · **Decision record**: ADR-0004 (`docs/adr/0004-in-petrinaut-staging-and-the-monorepo-import.md`) · **Supersedes**: `recommendation-demo-vehicle.md` as the September staging plan · **Evidence base**: the Petrinaut survey (FE-1358, `research/petrinaut-survey.md`), re-verified against `hashintel/hash` source on 2026-08-18 · **Amended**: FE-1506 (stable UI and voice attach contract), H-6763 / ADR-0009 (generic composer submission and app-owned voice boundary), and FE-1574 / Mission 5 (one mounted Flue conversation route and browser AI SDK projection). ## Problem Statement @@ -20,26 +15,14 @@ storage. The problem is connecting the second to the first without rebuilding ei ## Solution -The brunch elicitor runs as a **remote server** built on the harness + `binding-flue`. The -demo site swaps its `aiAssistant.transport` to point at that server; everything else in the -panel — rendering, the diagnostics decorator, client-side tool execution — is reused as-is. -The elicitor drives Petrinaut's editor through the **existing UI-executed tool surface** -(schemas imported from `petrinaut-core`), riding the harness's turn-suspension protocol: a -turn ends with tool calls pending, the panel executes them, and the outputs return on the next -dispatch. Sessions, captures, and IRs persist server-side, keyed to an opaque principal the -site supplies from a localStorage UID. The wire contract is the AI SDK v6 UI-message-stream -protocol, produced by a new `transport-aisdk` package that translates harness-level parts to -stream chunks and knows nothing about Flue. +The Brunch elicitor runs as a **long-running Flue server** built on the harness + `binding-flue`; Mission 5 proves the local same-origin path, while remote deployment remains a separate gate. The demo site derives one guarded `/agents/chat/:instanceId` URL from its opaque principal and logical conversation id, creates a public `@flue/sdk` client for that conversation, and supplies Petrinaut with a browser `ChatTransport` that projects Flue conversation events into the AI SDK rendering contract. Everything else in the panel — rendering, the diagnostics decorator, and client-side tool execution — is reused as-is. The elicitor drives Petrinaut's editor through the **existing UI-executed tool surface** (schemas imported from `petrinaut-core`): a response may end with client tool calls pending, the panel executes them, and one `client-tool-result` signal resumes the owning Flue conversation. Flue history is the canonical conversation record; captures and IRs remain in their own server-side stores. ## Seams One primary seam, four supporting ones — all existing except the brunch server's front door, which the design needs anyway: -1. **The ChatTransport wire seam** (primary; the contract-test surface): the AI SDK - UI-message-stream protocol over HTTP/SSE. Everything brunch-side sits behind it; everything - Petrinaut-side sits in front of it. The panel's real behavior is verified once in the - adapter spike and frozen as golden fixtures. +1. **The browser ChatTransport projection seam** (primary; the contract-test surface): `@flue/sdk` owns send, observation, offsets, retry, settlement, and recovery over the mounted Flue route; `transport-aisdk` projects one admitted submission into the finite AI SDK `UIMessageChunk` stream Petrinaut renders. The production-path integration test drives this seam through the real app router without hand-parsing SSE. 2. **The ask/affordance protocol seam** (`core`'s ask-protocol module, per ADR-0002 N1): the external-tool round-trip protocol is tested here, substrate-free. 3. **The storage port seam** (ADR-0002 N5): the owner key is tested as store-level refusals. @@ -57,8 +40,7 @@ which the design needs anyway: The panel and the voice edge attach to Brunch through one stable surface: -1. **Chat stream**: the UI sends `POST /api/chat`; a successful response is an AI SDK v6 - UI-message stream over HTTP/SSE. +1. **Conversation transport**: the browser calls `FlueClient.send()` against the guarded `/agents/chat/:instanceId` route, then follows the admitted submission through the SDK. The host-supplied browser `ChatTransport` projects that Flue stream into the finite AI SDK v6 UI-message stream consumed by `useChat`; the stock Petrinaut `/api/chat` route is a separate fallback and never carries Brunch turns. 2. **Question affordance**: the UI-executed tool is named `brunch_ask`. Its input schema is `{ question: non-empty string }`; its submitted output schema is `{ answer: non-empty string }`. @@ -140,18 +122,8 @@ host application under ADR-0009, while reusable Petrinaut and Brunch packages st **Topology and packaging** -- The elicitor server is a thin host-authored agent (spec §13) around the harness library, - deployed remotely; the demo site's same-origin `/api/chat` route reaches that server without - routing through the stock Petrinaut assistant or its prompt. -- Implemented by FE-1436 (the durable AI SDK transport): package `transport-aisdk` is the - server end of the ui shell's reply transport. It translates - harness-level parts to AI SDK v6 UI-message-stream chunks, using the `ai` package for stream - encoding only (no provider use — inference stays on Pi's adapter layer). Depends on `core`, - `ai`, and `valibot` for external request validation; never on the binding or Flue. This adds - a `transport-*` role prefix to the §12.2 - vocabulary (the glossary's avoided terms `adapter-*`/`wrapper-*` stay avoided). The - implementation and real-panel evidence are recorded in - `transport-aisdk-implementation-2026-08-19.md`. +- The elicitor server is a thin host-authored Flue agent around the harness library. The demo site's same-origin proxy forwards `/agents/chat/*` without changing the Flue protocol; the stock Petrinaut assistant and `/api/chat` prompt remain separate. +- FE-1436 originally introduced `transport-aisdk` as a server-side AI SDK HTTP adapter. FE-1574 / Mission 5 replaced that door: the package is now the browser-side projection from the public Flue client to Petrinaut's AI SDK rendering contract. Its runtime dependencies are exactly `@flue/sdk` and `ai`; it imports neither `@flue/runtime`, core, a plugin, nor a binding. The app supplies its client-tool catalog, and the package owns the shared `client-tool-result` signal representation. - Kernel spec amendments applied with this work, not silently: §12.2 package list gains `transport-aisdk` and records the monorepo import (`@hashintel/brunch-agent`, hash toolchain replacing the Bun workspace at import time); §13's shipping shape and ADR-0002 N3 diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/.oxlintrc.json b/libs/@hashintel/brunch-agent/packages/transport-aisdk/.oxlintrc.json index d7ed5324c29..7d1a38f5c67 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/.oxlintrc.json +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/.oxlintrc.json @@ -38,8 +38,8 @@ "message": "Brunch libraries must not depend on Petrinaut implementations." }, { - "group": ["@flue/*", "@earendil-works/*"], - "message": "Brunch transports must remain substrate-independent." + "group": ["@flue/!(sdk)", "@earendil-works/*"], + "message": "Brunch transports may depend on the public Flue SDK, but not a runtime or provider substrate." }, { "group": ["@hashintel/brunch-agent/*", "@hashintel/brunch-agent-*"], diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json b/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json index 97d4b0913ca..03d6d96fe3e 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json @@ -23,8 +23,8 @@ "test:unit": "vitest run" }, "dependencies": { - "ai": "6.0.182", - "valibot": "1.4.2" + "@flue/sdk": "2.0.3", + "ai": "6.0.182" }, "devDependencies": { "@types/node": "22.18.13", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/client-tool-result.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/client-tool-result.ts new file mode 100644 index 00000000000..cce5f17bb38 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/client-tool-result.ts @@ -0,0 +1 @@ +export const CLIENT_TOOL_RESULT_SIGNAL = "client-tool-result"; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/headers.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/headers.ts index 965e5feef1c..91b33c5ae68 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/headers.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/headers.ts @@ -1,2 +1,3 @@ -/** Stable browser-to-Brunch principal header owned by the HTTP transport. */ +/** Browser-safe ownership headers for the mounted Brunch conversation route. */ export const BRUNCH_PRINCIPAL_HEADER = "x-brunch-principal"; +export const BRUNCH_CONVERSATION_HEADER = "x-brunch-conversation"; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/identity.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/identity.ts new file mode 100644 index 00000000000..75cc9628d32 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/identity.ts @@ -0,0 +1,45 @@ +import { BRUNCH_CONVERSATION_HEADER, BRUNCH_PRINCIPAL_HEADER } from "./headers"; + +export interface ConversationIdentity { + readonly conversationId: string; + readonly principalKey: string; +} + +export const identityPayload = ({ + principalKey, + conversationId, +}: ConversationIdentity): Uint8Array => { + const encoder = new TextEncoder(); + const principalBytes = encoder.encode(principalKey); + const conversationBytes = encoder.encode(conversationId); + const payload = new Uint8Array( + principalBytes.length + 1 + conversationBytes.length, + ); + payload.set(principalBytes, 0); + payload[principalBytes.length] = 0; + payload.set(conversationBytes, principalBytes.length + 1); + return payload; +}; + +const hexFromDigest = (digest: ArrayBuffer): string => + [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); + +/** Browser-safe counterpart to the server's synchronous instance-id hash. */ +export const flueConversationIdWeb = async ( + identity: ConversationIdentity, +): Promise => { + const payload = identityPayload(identity); + const bytes = new ArrayBuffer(payload.byteLength); + new Uint8Array(bytes).set(payload); + const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes); + return hexFromDigest(digest); +}; + +export const agentOwnershipHeaders = ( + identity: ConversationIdentity, +): Record => ({ + [BRUNCH_PRINCIPAL_HEADER]: identity.principalKey, + [BRUNCH_CONVERSATION_HEADER]: identity.conversationId, +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index 5540732335f..0cbd6daf63f 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -1,34 +1,26 @@ -/** - * AI SDK UI-message-stream transport for a Flue-backed chat. - * - * This package owns the HTTP door: principal, CORS, POST validation, and SSE - * encoding. An application supplies the Flue turn. The transport never imports - * Brunch core, a binding, or Flue. - */ - -import { - createUIMessageStream, - createUIMessageStreamResponse, - type UIMessageChunk, -} from "ai"; -import * as v from "valibot"; - -import { BRUNCH_PRINCIPAL_HEADER } from "./headers"; - -export { BRUNCH_PRINCIPAL_HEADER } from "./headers"; - -export interface ConversationIdentity { - readonly conversationId: string; - readonly principalKey: string; -} - -export interface ChatTurnInput extends ConversationIdentity { - readonly idempotencyKey: string; - readonly userMessage: { - readonly id: string; - readonly text: string; - }; -} +import { FlueApiError, FlueExecutionError } from "@flue/sdk"; +import { getToolName, isToolUIPart } from "ai"; + +import { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; +import { createFlueUiStream } from "./ui-stream"; + +import type { AgentSendResult, DeliveredMessage, FlueClient } from "@flue/sdk"; +import type { ChatTransport, UIMessage, UIMessageChunk } from "ai"; + +export { BRUNCH_CONVERSATION_HEADER, BRUNCH_PRINCIPAL_HEADER } from "./headers"; +export { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; +export { + agentOwnershipHeaders, + flueConversationIdWeb, + identityPayload, +} from "./identity"; +export type { ConversationIdentity } from "./identity"; +export { + snapshotToUiMessages, + type SnapshotToUiMessagesOptions, + type UiHistoryMessage, +} from "./transcript"; +export { createFlueUiStream, type FlueUiStreamOptions } from "./ui-stream"; export interface ClientToolResult { readonly toolCallId: string; @@ -36,454 +28,279 @@ export interface ClientToolResult { readonly output: unknown; } -export interface ChatResumeInput extends ConversationIdentity { - readonly assistantMessageId: string; - readonly idempotencyKey: string; - readonly toolResults: readonly ClientToolResult[]; +export interface FlueChatTransportOptions { + readonly client: FlueClient; + readonly clientToolNames: ReadonlySet; + readonly onAdmission?: (event: { + readonly admission: AgentSendResult; + readonly kind: "client-tool-result" | "user"; + readonly messageId: string; + }) => void; + readonly onResponseMessage?: (event: { + readonly messageId: string; + readonly submissionId: AgentSendResult["submissionId"]; + }) => void; } -export type ChatChunkWriter = (chunk: UIMessageChunk) => void; - -export type ChatTurnRunner = ( - input: ChatTurnInput, - write: ChatChunkWriter, -) => Promise; - -export type ChatResumeRunner = ( - input: ChatResumeInput, - write: ChatChunkWriter, -) => Promise; - -export type TransportInspectionEvent = - | { - readonly type: "request-start"; - readonly requestId: string; - readonly conversationId: string; - readonly userMessageId: string; - } - | { - readonly type: "resume-start"; - readonly requestId: string; - readonly conversationId: string; - readonly assistantMessageId: string; - readonly toolCallIds: readonly string[]; - } - | { - readonly type: "history-read"; - readonly requestId: string; - readonly conversationId: string; +const completedClientToolResults = ( + messages: readonly UIMessage[], + assistantMessageId: string, + clientToolNames: ReadonlySet, +): readonly ClientToolResult[] => { + const assistantMessage = messages.find( + (message) => + message.id === assistantMessageId && message.role === "assistant", + ); + if (assistantMessage === undefined) { + return []; + } + const steps = assistantMessage.parts.reduce< + (typeof assistantMessage.parts)[] + >( + (collected, part) => { + if (part.type === "step-start") { + collected.push([]); + } else { + collected.at(-1)?.push(part); + } + return collected; + }, + [[]], + ); + const latestClientToolStep = steps.findLast((step) => + step.some( + (part) => + isToolUIPart(part) && + clientToolNames.has(getToolName(part)) && + part.providerExecuted !== true, + ), + ); + return (latestClientToolStep ?? []).flatMap((part): ClientToolResult[] => { + if (!isToolUIPart(part)) return []; + const toolName = getToolName(part); + if ( + !clientToolNames.has(toolName) || + part.providerExecuted === true || + part.state !== "output-available" || + part.toolCallId.length === 0 + ) { + return []; } - | { - readonly type: "request-finish"; - readonly requestId: string; - readonly terminal: "completed" | "failed"; - }; - -export interface AiSdkChatHandlerOptions { - readonly runTurn: ChatTurnRunner; - /** - * Client-tool return. Absent, a tool-result follow-up is refused. Present, - * completed client-tool parts on the referenced assistant message resume - * the same conversation. - */ - readonly resumeTurn?: ChatResumeRunner; - /** Snapshot used to hydrate the panel from Flue history after reload. */ - readonly loadHistory?: ( - input: ConversationIdentity, - ) => Promise<{ readonly messages: readonly unknown[] }>; - readonly allowedOrigins?: readonly string[]; - readonly inspect?: (event: TransportInspectionEvent) => void; -} - -const panelPartSchema = v.looseObject({ - type: v.optional(v.unknown()), - text: v.optional(v.unknown()), - toolName: v.optional(v.unknown()), - toolCallId: v.optional(v.unknown()), - state: v.optional(v.unknown()), - output: v.optional(v.unknown()), - providerExecuted: v.optional(v.unknown()), -}); - -const panelMessageSchema = v.looseObject({ - id: v.optional(v.unknown()), - role: v.optional(v.unknown()), - parts: v.optional(v.array(panelPartSchema)), -}); - -const panelPostBodySchema = v.looseObject({ - id: v.optional(v.unknown()), - messageId: v.optional(v.unknown()), - messages: v.optional(v.array(panelMessageSchema)), - trigger: v.optional(v.unknown()), -}); - -type PanelMessage = v.InferOutput; -type PanelPostBody = v.InferOutput; -type PanelPart = NonNullable[number]; - -const transportRequestRefusals = { - invalidChatRequest: { - status: 400, - error: "invalid_chat_request", - }, - invalidPrincipal: { - status: 400, - error: "invalid_principal", - }, - toolResultFollowUpNotSupported: { - status: 422, - error: "tool_result_follow_up_not_supported", - }, -} as const; - -type TransportRequestRefusal = - (typeof transportRequestRefusals)[keyof typeof transportRequestRefusals]; - -const jsonResponse = ( - body: unknown, - status: number, - headers?: Headers, -): Response => Response.json(body, { status, headers }); - -const corsHeaders = (origin: string): Headers => - new Headers({ - "access-control-allow-origin": origin, - "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": `content-type, x-request-id, ${BRUNCH_PRINCIPAL_HEADER}`, - vary: "Origin", + return [ + { + toolCallId: part.toolCallId, + toolName, + output: part.output, + }, + ]; }); - -const withHeaders = (response: Response, headers: Headers): Response => { - for (const [name, value] of headers) response.headers.set(name, value); - return response; }; -const userTextFrom = (message: PanelMessage): string | undefined => { - if (!Array.isArray(message.parts)) return undefined; +const finalUserMessage = ( + messages: readonly UIMessage[], +): { readonly id: string; readonly text: string } | undefined => { + const message = messages.at(-1); + if ( + message === undefined || + message.role !== "user" || + message.id.length === 0 || + message.id === "petrinaut-diagnostics-context" + ) { + return undefined; + } const text = message.parts - .filter( - (part): part is { readonly type: "text"; readonly text: string } => - typeof part === "object" && - "type" in part && - part.type === "text" && - "text" in part && - typeof part.text === "string", - ) + .filter((part) => part.type === "text") .map((part) => part.text) .join(""); - return text.length > 0 ? text : undefined; + return text.length > 0 ? { id: message.id, text } : undefined; }; -const toolNameFrom = (part: PanelPart): string | undefined => { - if (part.type === "dynamic-tool" && typeof part.toolName === "string") { - return part.toolName.length > 0 ? part.toolName : undefined; +const isAbortError = (error: unknown): boolean => + error instanceof Error && error.name === "AbortError"; + +const admissionError = (error: unknown): Error => { + if (isAbortError(error)) { + return error as Error; } - if (typeof part.type === "string" && part.type.startsWith("tool-")) { - const toolName = part.type.slice("tool-".length); - return toolName.length > 0 ? toolName : undefined; + if (error instanceof FlueApiError) { + return new Error( + `Brunch rejected the message before admission (HTTP ${error.status}).`, + { cause: error }, + ); } - return undefined; + return new Error( + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + { cause: error }, + ); }; -/** Panel parts the browser executed. Server tools set `providerExecuted: true`. */ -const isProviderExecutedPanelPart = (part: PanelPart): boolean => - part.providerExecuted === true; - -const isCompletedClientToolPart = (part: PanelPart): boolean => - part.state === "output-available" && - !isProviderExecutedPanelPart(part) && - typeof part.toolCallId === "string" && - part.toolCallId.length > 0 && - toolNameFrom(part) !== undefined; - -type ParsedTransportRequest = - | { readonly kind: "initial"; readonly value: ChatTurnInput } - | { readonly kind: "resume"; readonly value: ChatResumeInput } - | { readonly kind: "refused"; readonly refusal: TransportRequestRefusal }; - -const parseResumeTurn = ( - body: PanelPostBody, - principalKey: string, -): ParsedTransportRequest => { +const streamFailureChunk = ( + error: unknown, + signal: AbortSignal, +): Extract => { if ( - typeof body.id !== "string" || - body.id.length === 0 || - typeof body.messageId !== "string" || - body.messageId.length === 0 || - body.trigger !== "submit-message" || - !Array.isArray(body.messages) + signal.aborted || + isAbortError(error) || + (error instanceof FlueExecutionError && error.failure === "aborted") ) { return { - kind: "refused", - refusal: transportRequestRefusals.invalidChatRequest, - }; - } - - const assistantMessage = body.messages.find( - (candidate) => - candidate.id === body.messageId && candidate.role === "assistant", - ); - const toolResults = (assistantMessage?.parts ?? []) - .filter(isCompletedClientToolPart) - .map((part) => ({ - toolCallId: part.toolCallId as string, - toolName: toolNameFrom(part)!, - output: part.output, - })); - if (toolResults.length === 0) { - return { - kind: "refused", - refusal: transportRequestRefusals.toolResultFollowUpNotSupported, + type: "abort", + reason: + error instanceof FlueExecutionError + ? "The chat turn was stopped." + : "The local chat stream was cancelled.", }; } - return { - kind: "resume", - value: { - conversationId: body.id, - assistantMessageId: body.messageId, - idempotencyKey: `${body.id}:tools:${toolResults - .map((result) => result.toolCallId) - .join(",")}`, - principalKey, - toolResults, - }, + type: "error", + errorText: + error instanceof FlueExecutionError && + error.failure === "terminal_event_missing" + ? "The chat stream ended before the turn settled." + : "The chat turn failed.", }; }; -const parseInitialTurn = ( - body: PanelPostBody, - principalKey: string, -): ParsedTransportRequest => { - if ( - typeof body.id !== "string" || - body.id.length === 0 || - body.trigger !== "submit-message" - ) { - return { - kind: "refused", - refusal: transportRequestRefusals.invalidChatRequest, - }; - } - if (!Array.isArray(body.messages)) { - return { - kind: "refused", - refusal: transportRequestRefusals.invalidChatRequest, - }; - } - - const message = body.messages.at(-1) as PanelMessage | undefined; - if ( - message?.role !== "user" || - typeof message.id !== "string" || - message.id.length === 0 || - message.id === "petrinaut-diagnostics-context" - ) { - return { - kind: "refused", - refusal: transportRequestRefusals.invalidChatRequest, - }; - } - const text = userTextFrom(message); - if (text === undefined) { - return { - kind: "refused", - refusal: transportRequestRefusals.invalidChatRequest, - }; - } +const streamSubmission = ( + options: FlueChatTransportOptions, + admission: AgentSendResult, + continuationMessageId: string | undefined, + abortSignal: AbortSignal | undefined, +): ReadableStream => { + const localAbort = new AbortController(); + const signal = + abortSignal === undefined + ? localAbort.signal + : AbortSignal.any([abortSignal, localAbort.signal]); + // Shared with `cancel()`: a consumer that cancels the stream closes its + // controller immediately, so the detached `wait()` settlement below must not + // write or close again afterwards. + let closed = false; + + return new ReadableStream({ + start(controller) { + let terminalEmitted = false; + const close = (): void => { + if (closed) return; + closed = true; + controller.close(); + }; + const write = (chunk: UIMessageChunk): void => { + if (closed) return; + const projected = + chunk.type === "start" && continuationMessageId !== undefined + ? { ...chunk, messageId: continuationMessageId } + : chunk; + controller.enqueue(projected); + if ( + projected.type === "finish" || + projected.type === "error" || + projected.type === "abort" + ) { + terminalEmitted = true; + } + }; + const projector = createFlueUiStream({ + submissionId: admission.submissionId, + clientToolNames: options.clientToolNames, + write, + }); - return { - kind: "initial", - value: { - conversationId: body.id, - idempotencyKey: `${body.id}:${message.id}`, - principalKey, - userMessage: { id: message.id, text }, + void options.client + .wait(admission, { + signal, + onEvent: (event) => { + if ( + event.type === "message-started" && + event.submissionId === admission.submissionId + ) { + // Report the id the consumer sees: a client-tool continuation is + // projected onto the assistant message it resumes. + options.onResponseMessage?.({ + messageId: continuationMessageId ?? event.messageId, + submissionId: admission.submissionId, + }); + } + projector.accept(event); + }, + }) + .then(close) + .catch((error: unknown) => { + if (!terminalEmitted) { + write(streamFailureChunk(error, signal)); + } + close(); + }); }, - }; -}; - -const readPrincipal = (request: Request): string | TransportRequestRefusal => { - const principalKey = request.headers.get(BRUNCH_PRINCIPAL_HEADER)?.trim(); - if ( - principalKey === undefined || - principalKey.length === 0 || - principalKey.length > 256 - ) { - return transportRequestRefusals.invalidPrincipal; - } - return principalKey; + cancel(reason) { + closed = true; + localAbort.abort(reason); + }, + }); }; -export const createAiSdkChatHandler = - (options: AiSdkChatHandlerOptions) => - async (request: Request): Promise => { - const origin = request.headers.get("origin"); - const crossOriginHeaders = - origin !== null && options.allowedOrigins?.includes(origin) === true - ? corsHeaders(origin) - : undefined; - if (origin !== null && crossOriginHeaders === undefined) { - return jsonResponse({ error: "origin_not_allowed" }, 403); - } - if (request.method === "OPTIONS") { - return new Response(null, { status: 204, headers: crossOriginHeaders }); +export const createFlueChatTransport = < + UiMessage extends UIMessage = UIMessage, +>( + options: FlueChatTransportOptions, +): ChatTransport => ({ + reconnectToStream: async () => null, + sendMessages: async ({ trigger, messageId, messages, abortSignal }) => { + if (trigger !== "submit-message") { + throw new Error("Regenerating a Flue conversation is not supported."); } - const principal = readPrincipal(request); - if (typeof principal !== "string") { - return jsonResponse( - { error: principal.error }, - principal.status, - crossOriginHeaders, - ); - } - - if (request.method === "GET") { - const conversationId = new URL(request.url).searchParams - .get("id") - ?.trim(); - if (conversationId === undefined || conversationId.length === 0) { - return jsonResponse( - { error: transportRequestRefusals.invalidChatRequest.error }, - transportRequestRefusals.invalidChatRequest.status, - crossOriginHeaders, - ); - } - if (options.loadHistory === undefined) { - return jsonResponse( - { error: "method_not_allowed" }, - 405, - crossOriginHeaders, - ); - } - const requestId = - request.headers.get("x-request-id") || crypto.randomUUID(); - options.inspect?.({ - type: "history-read", - requestId, - conversationId, - }); - const history = await options.loadHistory({ - conversationId, - principalKey: principal, - }); - return jsonResponse(history, 200, crossOriginHeaders); - } - - if (request.method !== "POST") { - return jsonResponse( - { error: "method_not_allowed" }, - 405, - crossOriginHeaders, - ); - } - - let body: unknown; + const toolResults = + messageId === undefined + ? [] + : completedClientToolResults( + messages, + messageId, + options.clientToolNames, + ); + const userMessage = + messageId === undefined ? finalUserMessage(messages) : undefined; + const message: DeliveredMessage = + messageId === undefined + ? (() => { + if (userMessage === undefined) { + throw new Error("The submitted user message has no text."); + } + return { kind: "user", body: userMessage.text }; + })() + : (() => { + if (toolResults.length === 0) { + throw new Error( + "The client-tool follow-up has no completed result.", + ); + } + return { + kind: "signal", + type: CLIENT_TOOL_RESULT_SIGNAL, + tagName: CLIENT_TOOL_RESULT_SIGNAL, + body: JSON.stringify(toolResults), + attributes: { + toolCallIds: toolResults + .map((result) => result.toolCallId) + .join(","), + }, + }; + })(); + + let admission: AgentSendResult; try { - body = await request.json(); - } catch { - return jsonResponse( - { error: "invalid_chat_request" }, - 400, - crossOriginHeaders, - ); - } - const validatedBody = v.safeParse(panelPostBodySchema, body); - if (!validatedBody.success) { - const refusal = transportRequestRefusals.invalidChatRequest; - return jsonResponse( - { error: refusal.error }, - refusal.status, - crossOriginHeaders, - ); - } - const postBody = validatedBody.output; - const parsed = - postBody.messageId !== undefined && options.resumeTurn !== undefined - ? parseResumeTurn(postBody, principal) - : postBody.messageId !== undefined - ? ({ - kind: "refused", - refusal: transportRequestRefusals.toolResultFollowUpNotSupported, - } as const) - : parseInitialTurn(postBody, principal); - if (parsed.kind === "refused") { - return jsonResponse( - { error: parsed.refusal.error }, - parsed.refusal.status, - crossOriginHeaders, - ); - } - - const requestId = - request.headers.get("x-request-id") || crypto.randomUUID(); - const continuationMessageId = - parsed.kind === "resume" ? parsed.value.assistantMessageId : undefined; - - if (parsed.kind === "resume") { - options.inspect?.({ - type: "resume-start", - requestId, - conversationId: parsed.value.conversationId, - assistantMessageId: parsed.value.assistantMessageId, - toolCallIds: parsed.value.toolResults.map( - (result) => result.toolCallId, - ), - }); - } else { - options.inspect?.({ - type: "request-start", - requestId, - conversationId: parsed.value.conversationId, - userMessageId: parsed.value.userMessage.id, + admission = await options.client.send({ + message, + signal: abortSignal, }); + } catch (error) { + throw admissionError(error); } - - const run = - parsed.kind === "resume" - ? (write: ChatChunkWriter) => options.resumeTurn!(parsed.value, write) - : (write: ChatChunkWriter) => options.runTurn(parsed.value, write); - - let terminalEmitted = false; - const stream = createUIMessageStream({ - execute: async ({ writer }) => { - try { - await run((chunk) => { - if (chunk.type === "start" && continuationMessageId !== undefined) { - writer.write({ ...chunk, messageId: continuationMessageId }); - } else { - writer.write(chunk); - } - if ( - chunk.type === "finish" || - chunk.type === "error" || - chunk.type === "abort" - ) { - terminalEmitted = true; - } - }); - options.inspect?.({ - type: "request-finish", - requestId, - terminal: "completed", - }); - } catch (error) { - if (terminalEmitted) return; - options.inspect?.({ - type: "request-finish", - requestId, - terminal: "failed", - }); - throw error; - } - }, - onError: () => "The chat turn failed.", + options.onAdmission?.({ + admission, + kind: messageId === undefined ? "user" : "client-tool-result", + messageId: messageId ?? userMessage!.id, }); - - const response = createUIMessageStreamResponse({ stream }); - return crossOriginHeaders === undefined - ? response - : withHeaders(response, crossOriginHeaders); - }; + return streamSubmission(options, admission, messageId, abortSignal); + }, +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts new file mode 100644 index 00000000000..089fc3dd4bc --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts @@ -0,0 +1,209 @@ +import { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; + +import type { + FlueConversationMessage, + FlueConversationPart, + FlueConversationState, +} from "@flue/sdk"; +import type { UIMessage } from "ai"; + +type UiMessagePart = UIMessage["parts"][number]; + +export type UiHistoryMessage = Omit< + UIMessage, + "metadata" | "parts" | "role" +> & { + role: Extract; + parts: UiMessagePart[]; +}; + +export interface SnapshotToUiMessagesOptions { + readonly clientToolNames: ReadonlySet; +} + +const unhandledConversationPart = (part: never): never => { + throw new Error(`Unhandled Flue conversation part: ${JSON.stringify(part)}`); +}; + +const isFlueDataPart = ( + part: FlueConversationPart, +): part is Extract => + part.type.startsWith("data-"); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null; + +const clientToolResultsFrom = ( + snapshot: Pick, + signalName: string, +): ReadonlyMap => { + const outputsByCallId = new Map(); + for (const message of snapshot.messages) { + if (message.purpose !== "dispatch") continue; + if (message.signal?.tagName !== signalName) continue; + const text = message.parts + .filter( + (part): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + let parsed: unknown; + try { + parsed = JSON.parse(text) as unknown; + } catch { + continue; + } + if (!Array.isArray(parsed)) continue; + for (const result of parsed) { + if ( + !isRecord(result) || + typeof result.toolCallId !== "string" || + !("output" in result) + ) { + continue; + } + outputsByCallId.set(result.toolCallId, result.output); + } + } + return outputsByCallId; +}; + +const toolPartFrom = ( + part: Extract, + clientToolNames: ReadonlySet, + clientOutputs: ReadonlyMap, +): UiMessagePart => { + const isClientTool = clientToolNames.has(part.toolName); + const hasClientOutput = clientOutputs.has(part.toolCallId); + if (part.state === "output-error") { + return { + type: `tool-${part.toolName}`, + toolCallId: part.toolCallId, + state: "output-error", + input: part.input, + errorText: part.errorText, + ...(isClientTool ? {} : { providerExecuted: true }), + }; + } + if (isClientTool && !hasClientOutput) { + return { + type: `tool-${part.toolName}`, + toolCallId: part.toolCallId, + state: "input-available", + input: part.input, + }; + } + const output = isClientTool + ? clientOutputs.get(part.toolCallId) + : part.state === "output-available" + ? part.output + : undefined; + if (output !== undefined || hasClientOutput) { + return { + type: `tool-${part.toolName}`, + toolCallId: part.toolCallId, + state: "output-available", + input: part.input, + output, + ...(isClientTool ? {} : { providerExecuted: true }), + }; + } + return { + type: `tool-${part.toolName}`, + toolCallId: part.toolCallId, + state: "input-available", + input: part.input, + ...(isClientTool ? {} : { providerExecuted: true }), + }; +}; + +const partsFrom = ( + message: FlueConversationMessage, + options: SnapshotToUiMessagesOptions, + clientOutputs: ReadonlyMap, +): UiMessagePart[] => { + const parts: UiMessagePart[] = []; + for (const part of message.parts) { + if (part.type === "text") { + parts.push({ type: "text", text: part.text, state: "done" }); + continue; + } + if (part.type === "reasoning") { + parts.push({ type: "reasoning", text: part.text, state: "done" }); + continue; + } + if (part.type === "dynamic-tool") { + parts.push(toolPartFrom(part, options.clientToolNames, clientOutputs)); + continue; + } + if (part.type === "file") { + parts.push({ + type: "file", + mediaType: part.mediaType, + url: part.url ?? "", + ...(part.filename === undefined ? {} : { filename: part.filename }), + }); + continue; + } + if (isFlueDataPart(part)) { + parts.push({ type: part.type, data: part.data }); + continue; + } + unhandledConversationPart(part); + } + return parts; +}; + +export const snapshotToUiMessages = ( + snapshot: Pick, + options: SnapshotToUiMessagesOptions, +): UiHistoryMessage[] => { + const clientOutputs = clientToolResultsFrom( + snapshot, + CLIENT_TOOL_RESULT_SIGNAL, + ); + const messages: UiHistoryMessage[] = []; + // The live stream projects a client-tool continuation onto the assistant + // message it resumes; the snapshot records that continuation as a separate + // Flue message behind the `client-tool-result` dispatch, so fold it back. + let resumableAssistant: UiHistoryMessage | undefined; + let continuationPending = false; + for (const message of snapshot.messages) { + if ( + message.purpose === "dispatch" && + message.signal?.tagName === CLIENT_TOOL_RESULT_SIGNAL + ) { + continuationPending = resumableAssistant !== undefined; + continue; + } + if (message.display !== "visible") continue; + if (message.purpose !== "user" && message.purpose !== "assistant") continue; + if (message.role !== "user" && message.role !== "assistant") continue; + const parts = partsFrom(message, options, clientOutputs); + if (message.role === "user") { + resumableAssistant = undefined; + continuationPending = false; + } + if (parts.length === 0) continue; + if ( + message.role === "assistant" && + continuationPending && + resumableAssistant !== undefined + ) { + resumableAssistant.parts.push(...parts); + continuationPending = false; + continue; + } + const projected: UiHistoryMessage = { + id: message.id, + role: message.role, + parts, + }; + messages.push(projected); + if (message.role === "assistant") { + resumableAssistant = projected; + } + } + return messages; +}; diff --git a/apps/brunch-agent/src/conversation/ui-stream.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts similarity index 90% rename from apps/brunch-agent/src/conversation/ui-stream.ts rename to libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts index da9cf93a524..7270be320b2 100644 --- a/apps/brunch-agent/src/conversation/ui-stream.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts @@ -1,13 +1,8 @@ -/** Project Flue live conversation chunks into AI SDK UI-message-stream chunks. */ - -import { type ConversationStreamChunk } from "@flue/sdk"; - -import { providerExecutedFor } from "./client-tools.ts"; - +import type { AgentSendResult, ConversationStreamChunk } from "@flue/sdk"; import type { UIMessageChunk } from "ai"; export interface FlueUiStreamOptions { - readonly submissionId: string; + readonly submissionId: AgentSendResult["submissionId"]; readonly clientToolNames: ReadonlySet; readonly write: (chunk: UIMessageChunk) => void; } @@ -109,9 +104,6 @@ export const createFlueUiStream = ( case "conversation-reset": case "message-appended": case "stream-checkpoint": - // Observe/reconnect machinery, not assistant-message content. This - // projector emits one AI SDK assistant message for one Flue - // submission; these chunks are not parts of that message. return; case "message-delta": { if (!accepting || messageId === undefined) return; @@ -133,13 +125,12 @@ export const createFlueUiStream = ( finishPart(); const isClientTool = options.clientToolNames.has(chunk.toolName); if (isClientTool) pendingClientToolCallIds.add(chunk.toolCallId); - const providerExecuted = providerExecutedFor(isClientTool); options.write({ type: "tool-input-available", toolCallId: chunk.toolCallId, toolName: chunk.toolName, input: chunk.input, - ...(providerExecuted === undefined ? {} : { providerExecuted }), + ...(isClientTool ? {} : { providerExecuted: true }), }); return; } diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-handler.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-handler.test.ts deleted file mode 100644 index 4274cec5132..00000000000 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-handler.test.ts +++ /dev/null @@ -1,246 +0,0 @@ -/** - * Wire contracts for the Flue ↔ AI SDK chat door: principal, initial turns, - * and correlated client-tool resume. - */ - -import { describe, expect, test } from "vitest"; - -import { - createAiSdkChatHandler, - type ChatResumeInput, - type TransportInspectionEvent, -} from "../src/index"; - -import type { UIMessageChunk } from "ai"; - -const responseChunks = async ( - response: Response, -): Promise => - (await response.text()) - .trim() - .split("\n\n") - .slice(0, -1) - .map((frame) => JSON.parse(frame.slice("data: ".length)) as UIMessageChunk); - -const post = (body: unknown): Request => - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { - "content-type": "application/json", - "x-brunch-principal": "principal-mission-1", - "x-request-id": "request-mission-1", - }, - body: typeof body === "string" ? body : JSON.stringify(body), - }); - -const initialPost = { - id: "conversation-mission-1", - trigger: "submit-message", - messages: [ - { - id: "user-mission-1", - role: "user", - parts: [{ type: "text", text: "Is the server in the loop?" }], - }, - ], -}; - -const clientToolResumePost = { - id: "conversation-mission-1", - trigger: "submit-message", - messageId: "assistant-mission-1", - messages: [ - { - id: "user-mission-1", - role: "user", - parts: [{ type: "text", text: "How does simulation view work?" }], - }, - { - id: "assistant-mission-1", - role: "assistant", - parts: [ - { type: "step-start" }, - { - type: "tool-readPetrinautDoc", - toolCallId: "tool-doc-1", - state: "output-available", - input: { doc: "simulate-view" }, - output: "# Simulate view\nRun the net.", - }, - ], - }, - ], -}; - -test("refuses a valid turn without the UI shell principal", async () => { - let dispatched = false; - const handler = createAiSdkChatHandler({ - async runTurn() { - dispatched = true; - }, - }); - - const response = await handler( - new Request("http://brunch.test/api/chat", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(initialPost), - }), - ); - - expect(response.status).toBe(400); - expect(await response.json()).toEqual({ error: "invalid_principal" }); - expect(dispatched).toBe(false); -}); - -test("streams application-written chunks for an initial user turn", async () => { - const inspections: TransportInspectionEvent[] = []; - const handler = createAiSdkChatHandler({ - inspect: (event) => inspections.push(event), - async runTurn(input, write) { - expect(input.userMessage.text).toBe("Is the server in the loop?"); - write({ type: "start", messageId: "assistant-mission-1" }); - write({ type: "text-start", id: "text-1" }); - write({ type: "text-delta", id: "text-1", delta: "pong" }); - write({ type: "text-end", id: "text-1" }); - write({ type: "finish", finishReason: "stop" }); - }, - }); - - const response = await handler(post(initialPost)); - expect(response.status).toBe(200); - const chunks = await responseChunks(response); - expect(chunks.find((chunk) => chunk.type === "text-delta")).toMatchObject({ - delta: "pong", - }); - expect(chunks.at(-1)).toMatchObject({ type: "finish", finishReason: "stop" }); - expect(inspections[0]).toMatchObject({ type: "request-start" }); - expect(inspections.at(-1)).toMatchObject({ - type: "request-finish", - terminal: "completed", - }); -}); - -describe("client-tool resume", () => { - test("resumes with correlated client-tool outputs", async () => { - const resumed: ChatResumeInput[] = []; - const handler = createAiSdkChatHandler({ - async runTurn() { - throw new Error("the initial-turn path must not run for a follow-up"); - }, - async resumeTurn(input, write) { - resumed.push(input); - write({ type: "start", messageId: "assistant-next" }); - write({ type: "text-start", id: "text-2" }); - write({ - type: "text-delta", - id: "text-2", - delta: "Simulate view runs the net.", - }); - write({ type: "text-end", id: "text-2" }); - write({ type: "finish", finishReason: "stop" }); - }, - }); - - const response = await handler(post(clientToolResumePost)); - expect(response.status).toBe(200); - expect(resumed).toEqual([ - { - conversationId: "conversation-mission-1", - assistantMessageId: "assistant-mission-1", - idempotencyKey: "conversation-mission-1:tools:tool-doc-1", - principalKey: "principal-mission-1", - toolResults: [ - { - toolCallId: "tool-doc-1", - toolName: "readPetrinautDoc", - output: "# Simulate view\nRun the net.", - }, - ], - }, - ]); - const chunks = await responseChunks(response); - expect(chunks[0]).toEqual({ - type: "start", - messageId: "assistant-mission-1", - }); - }); - - test("refuses every follow-up when no resume seam is configured", async () => { - const handler = createAiSdkChatHandler({ async runTurn() {} }); - const response = await handler(post(clientToolResumePost)); - expect(response.status).toBe(422); - expect(await response.json()).toEqual({ - error: "tool_result_follow_up_not_supported", - }); - }); - - test("skips provider-executed tool parts when looking for a client resume", async () => { - let resumed = false; - const handler = createAiSdkChatHandler({ - async runTurn() {}, - async resumeTurn() { - resumed = true; - }, - }); - const response = await handler( - post({ - ...clientToolResumePost, - messages: [ - clientToolResumePost.messages[0], - { - id: "assistant-mission-1", - role: "assistant", - parts: [ - { - type: "tool-ping", - toolCallId: "tool-ping-1", - state: "output-available", - providerExecuted: true, - input: {}, - output: { ok: true }, - }, - ], - }, - ], - }), - ); - expect(response.status).toBe(422); - expect(resumed).toBe(false); - }); -}); - -test("refuses history without the UI shell principal", async () => { - const handler = createAiSdkChatHandler({ async runTurn() {} }); - const response = await handler( - new Request("http://brunch.test/api/chat?id=conversation-mission-1", { - method: "GET", - }), - ); - expect(response.status).toBe(400); - expect(await response.json()).toEqual({ error: "invalid_principal" }); -}); - -test("loads Flue history on GET for the same principal and conversation id", async () => { - const handler = createAiSdkChatHandler({ - async runTurn() {}, - async loadHistory(input) { - expect(input).toEqual({ - conversationId: "conversation-mission-1", - principalKey: "principal-mission-1", - }); - return { messages: [{ id: "user-1", role: "user", parts: [] }] }; - }, - }); - - const response = await handler( - new Request("http://brunch.test/api/chat?id=conversation-mission-1", { - method: "GET", - headers: { "x-brunch-principal": "principal-mission-1" }, - }), - ); - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ - messages: [{ id: "user-1", role: "user", parts: [] }], - }); -}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts new file mode 100644 index 00000000000..9885ebfd2d3 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -0,0 +1,509 @@ +import { FlueApiError, FlueExecutionError } from "@flue/sdk"; +import { expect, test, vi } from "vitest"; + +import { createFlueChatTransport } from "../src"; + +import type { FlueChatTransportOptions } from "../src"; +import type { + AgentSendResult, + ConversationStreamChunk, + FlueClient, +} from "@flue/sdk"; +import type { ChatTransport, UIMessage, UIMessageChunk } from "ai"; + +const admission: AgentSendResult = { + streamUrl: "http://brunch.test/stream", + offset: "offset-1", + submissionId: "submission-1", + uid: "uid-1", +}; + +const position = (index: number) => ({ batch: 1, index }); + +const completedEvents: readonly ConversationStreamChunk[] = [ + { + type: "message-started", + conversationId: "conversation-1", + messageId: "assistant-1", + submissionId: admission.submissionId, + turnId: "turn-1", + position: position(0), + }, + { + type: "message-delta", + conversationId: "conversation-1", + messageId: "assistant-1", + kind: "text", + delta: "Canonical reply.", + position: position(1), + }, + { + type: "message-completed", + conversationId: "conversation-1", + messageId: "assistant-1", + position: position(2), + }, + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: admission.submissionId, + outcome: "completed", + position: position(3), + }, +]; + +const clientWith = ( + events: readonly ConversationStreamChunk[], +): { + readonly client: FlueClient; + readonly send: ReturnType>; +} => { + const send = vi.fn(async () => admission); + const wait = vi.fn(async (_admission, options) => { + // Preserve protocol order while exercising the stateful projector. + // eslint-disable-next-line no-await-in-loop + for (const event of events) await options?.onEvent?.(event); + }); + return { + client: { send, wait } as Pick as FlueClient, + send, + }; +}; + +const readChunks = async ( + stream: ReadableStream, +): Promise => { + const chunks: UIMessageChunk[] = []; + const reader = stream.getReader(); + for (;;) { + // A stream reader is necessarily consumed in sequence. + // eslint-disable-next-line no-await-in-loop + const result = await reader.read(); + if (result.done) return chunks; + chunks.push(result.value); + } +}; + +const sendOptions = ( + messages: UIMessage[], + messageId?: string, +): Parameters["sendMessages"]>[0] => ({ + trigger: "submit-message", + chatId: "conversation-1", + messageId, + messages, + abortSignal: undefined, +}); + +test("submits results from the latest assistant step with completed client tools", async () => { + const { client, send } = clientWith(completedEvents); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(["getLatestNetDefinition", "addArc"]), + }); + + await readChunks( + await transport.sendMessages( + sendOptions( + [ + { + id: "assistant-original", + role: "assistant", + parts: [ + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "getLatestNetDefinition", + toolCallId: "read-before-1", + state: "output-available", + input: {}, + output: { revision: 0 }, + }, + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "getLatestNetDefinition", + toolCallId: "read-before-2", + state: "output-available", + input: {}, + output: { revision: 0 }, + }, + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "addArc", + toolCallId: "mutation-latest", + state: "output-available", + input: {}, + output: { applied: true }, + }, + { type: "step-start" }, + { + type: "dynamic-tool", + toolName: "activate_skill", + toolCallId: "server-tool-later", + state: "output-available", + input: {}, + output: { activated: true }, + providerExecuted: true, + }, + ], + }, + ], + "assistant-original", + ), + ), + ); + + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ + message: { + kind: "signal", + type: "client-tool-result", + tagName: "client-tool-result", + body: JSON.stringify([ + { + toolCallId: "mutation-latest", + toolName: "addArc", + output: { applied: true }, + }, + ]), + attributes: { toolCallIds: "mutation-latest" }, + }, + signal: undefined, + }), + ); +}); + +test("admits one user message and projects a finite per-turn stream", async () => { + const { client, send } = clientWith(completedEvents); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(["readPetrinautDoc"]), + }); + + const stream = await transport.sendMessages( + sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Run the transport tracer." }], + }, + ]), + ); + + expect(send).toHaveBeenCalledOnce(); + expect(send).toHaveBeenCalledWith({ + message: { kind: "user", body: "Run the transport tracer." }, + signal: undefined, + }); + expect((await readChunks(stream)).map((chunk) => chunk.type)).toEqual([ + "start", + "start-step", + "text-start", + "text-delta", + "text-end", + "finish-step", + "finish", + ]); +}); + +test("admits one client-tool result signal and resumes its assistant id", async () => { + const { client, send } = clientWith(completedEvents); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(["readPetrinautDoc"]), + }); + + const stream = await transport.sendMessages( + sendOptions( + [ + { + id: "assistant-original", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "readPetrinautDoc", + toolCallId: "tool-1", + state: "output-available", + input: { doc: "ai-assistant" }, + output: "The guide.", + }, + ], + }, + ], + "assistant-original", + ), + ); + + expect(send).toHaveBeenCalledWith({ + message: { + kind: "signal", + type: "client-tool-result", + tagName: "client-tool-result", + body: JSON.stringify([ + { + toolCallId: "tool-1", + toolName: "readPetrinautDoc", + output: "The guide.", + }, + ]), + attributes: { toolCallIds: "tool-1" }, + }, + signal: undefined, + }); + expect((await readChunks(stream))[0]).toEqual({ + type: "start", + messageId: "assistant-original", + }); +}); + +test("starts with history-only reconnection", async () => { + const { client } = clientWith([]); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(), + }); + + await expect( + transport.reconnectToStream({ chatId: "conversation-1" }), + ).resolves.toBeNull(); +}); + +test.each([ + [ + "failed", + new FlueExecutionError({ + target: "agent_submission", + targetId: admission.submissionId, + failure: "failed", + }), + { type: "error", errorText: "The chat turn failed." }, + ], + [ + "aborted", + new FlueExecutionError({ + target: "agent_submission", + targetId: admission.submissionId, + failure: "aborted", + }), + { type: "abort", reason: "The chat turn was stopped." }, + ], + [ + "missing terminal event", + new FlueExecutionError({ + target: "agent_submission", + targetId: admission.submissionId, + failure: "terminal_event_missing", + }), + { + type: "error", + errorText: "The chat stream ended before the turn settled.", + }, + ], +])( + "maps a %s wait rejection into the finite UI stream", + async (_label, waitError, expected) => { + const send = vi.fn(async () => admission); + const wait = vi.fn(async () => { + throw waitError; + }); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + const stream = await transport.sendMessages( + sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Map the outcome." }], + }, + ]), + ); + + expect(await readChunks(stream)).toEqual([expected]); + }, +); + +test("keeps caller cancellation distinct from durable abort", async () => { + const abortController = new AbortController(); + const send = vi.fn(async () => admission); + const wait = vi.fn( + async (_admission, options) => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener( + "abort", + () => reject(new DOMException("cancelled", "AbortError")), + { once: true }, + ); + }), + ); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + }); + const stream = await transport.sendMessages({ + ...sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Cancel only this observer." }], + }, + ]), + abortSignal: abortController.signal, + }); + + abortController.abort(); + + await expect(readChunks(stream)).resolves.toEqual([ + { type: "abort", reason: "The local chat stream was cancelled." }, + ]); +}); + +test("surfaces rejected and ambiguous admission without retrying", async () => { + const rejectedSend = vi.fn(async () => { + throw new FlueApiError(403, ""); + }); + const ambiguousSend = vi.fn(async () => { + throw new TypeError("connection lost after request write"); + }); + const createTransport = (send: FlueClient["send"]) => + createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + const options = sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Admit once." }], + }, + ]); + + await expect( + createTransport(rejectedSend).sendMessages(options), + ).rejects.toThrow("rejected the message before admission (HTTP 403)"); + await expect( + createTransport(ambiguousSend).sendMessages(options), + ).rejects.toThrow("may have accepted the message"); + expect(rejectedSend).toHaveBeenCalledOnce(); + expect(ambiguousSend).toHaveBeenCalledOnce(); +}); + +test("reports one admission and its correlated response message", async () => { + const { client } = clientWith(completedEvents); + const onAdmission = + vi.fn>(); + const onResponseMessage = + vi.fn>(); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(), + onAdmission, + onResponseMessage, + }); + + const stream = await transport.sendMessages( + sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Track this response." }], + }, + ]), + ); + await readChunks(stream); + + expect(onAdmission).toHaveBeenCalledOnce(); + expect(onAdmission).toHaveBeenCalledWith({ + admission, + kind: "user", + messageId: "user-1", + }); + expect(onResponseMessage).toHaveBeenCalledOnce(); + expect(onResponseMessage).toHaveBeenCalledWith({ + messageId: "assistant-1", + submissionId: admission.submissionId, + }); +}); + +test("stays silent after the consumer cancels the per-turn stream", async () => { + let waitSignal: AbortSignal | undefined; + const send = vi.fn(async () => admission); + const wait = vi.fn( + async (_admission, options) => + new Promise((_resolve, reject) => { + waitSignal = options?.signal; + options?.signal?.addEventListener( + "abort", + () => reject(new DOMException("cancelled", "AbortError")), + { once: true }, + ); + }), + ); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + }); + const stream = await transport.sendMessages( + sendOptions([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Cancel from the reader." }], + }, + ]), + ); + + const reader = stream.getReader(); + await reader.cancel(); + // Let the rejected `wait()` settle; an enqueue on the cancelled controller + // would surface here as an unhandled rejection. + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(waitSignal?.aborted).toBe(true); + await expect(reader.closed).resolves.toBeUndefined(); +}); + +test("reports a client-tool continuation against the resumed assistant id", async () => { + const { client } = clientWith(completedEvents); + const onResponseMessage = + vi.fn>(); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(["readPetrinautDoc"]), + onResponseMessage, + }); + + const stream = await transport.sendMessages( + sendOptions( + [ + { + id: "assistant-original", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "readPetrinautDoc", + toolCallId: "tool-1", + state: "output-available", + input: { doc: "ai-assistant" }, + output: "The guide.", + }, + ], + }, + ], + "assistant-original", + ), + ); + await readChunks(stream); + + expect(onResponseMessage).toHaveBeenCalledOnce(); + expect(onResponseMessage).toHaveBeenCalledWith({ + messageId: "assistant-original", + submissionId: admission.submissionId, + }); +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-client-tool.post.json b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-client-tool.post.json deleted file mode 100644 index 08568ea97d9..00000000000 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-client-tool.post.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "conversation-mission-1", - "trigger": "submit-message", - "messageId": "assistant-mission-1", - "messages": [ - { - "id": "user-mission-1", - "role": "user", - "parts": [{ "type": "text", "text": "How does the AI assistant work?" }] - }, - { - "id": "assistant-mission-1", - "role": "assistant", - "parts": [ - { "type": "step-start" }, - { - "type": "tool-readPetrinautDoc", - "toolCallId": "tool-doc-1", - "state": "output-available", - "input": { "doc": "ai-assistant" }, - "output": "# AI Assistant\nThe assistant can read its own documentation pages." - } - ] - } - ] -} diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-initial.post.json b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-initial.post.json deleted file mode 100644 index b771e13bd0f..00000000000 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/fixtures/panel-initial.post.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "m5z0GU9KJPzhOTlx", - "messages": [ - { - "parts": [ - { - "type": "text", - "text": "Run the FE-1435 transport probe." - } - ], - "id": "6ddgGkjhSxGjOtiv", - "role": "user" - } - ], - "trigger": "submit-message" -} diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/golden.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/golden.test.ts deleted file mode 100644 index 8bc0f08ea9c..00000000000 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/golden.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -import { expect, test } from "vitest"; - -import type { ChatTransport, UIMessage } from "ai"; - -type SendMessagesOptions = Parameters< - ChatTransport["sendMessages"] ->[0]; - -type PanelPostBody = { - readonly id: SendMessagesOptions["chatId"]; - readonly messageId?: SendMessagesOptions["messageId"]; - readonly messages: SendMessagesOptions["messages"]; - readonly trigger: SendMessagesOptions["trigger"]; -}; - -const FIXTURES = join(import.meta.dirname, "fixtures"); - -const readPostBody = (name: string): PanelPostBody => - JSON.parse(readFileSync(join(FIXTURES, name), "utf8")) as PanelPostBody; - -test("validates the load-bearing fields in the complete initial panel POST fixture", () => { - const body = readPostBody("panel-initial.post.json"); - expect(body.trigger).toBe("submit-message"); - expect(body.messages).toHaveLength(1); - expect(body.messages[0]?.role).toBe("user"); - expect(body.messages[0]?.parts).toEqual([ - { type: "text", text: "Run the FE-1435 transport probe." }, - ]); -}); - -test("validates the load-bearing fields in the read-only client-tool follow-up fixture", () => { - const body = readPostBody("panel-client-tool.post.json"); - expect(body.trigger).toBe("submit-message"); - expect(body.messageId).toBe("assistant-mission-1"); - const assistant = body.messages.find( - (message) => message.id === body.messageId, - ); - expect(assistant?.role).toBe("assistant"); - expect( - assistant?.parts.find((part) => part.type === "tool-readPetrinautDoc"), - ).toMatchObject({ - type: "tool-readPetrinautDoc", - toolCallId: "tool-doc-1", - state: "output-available", - }); -}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts new file mode 100644 index 00000000000..fe15a52470a --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -0,0 +1,237 @@ +import { expect, test } from "vitest"; + +import { CLIENT_TOOL_RESULT_SIGNAL, snapshotToUiMessages } from "../src"; + +import type { FlueConversationSnapshot } from "@flue/sdk"; + +const snapshotWithPendingClientTool: FlueConversationSnapshot = { + v: 1, + conversationId: "conversation-1", + offset: "0", + messages: [ + { + id: "assistant-1", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + ], + settlements: [], +}; + +const projectionOptions = { + clientToolNames: new Set(["readPetrinautDoc"]), +}; + +test("leaves an unfinished client tool available to run", () => { + expect( + snapshotToUiMessages(snapshotWithPendingClientTool, projectionOptions), + ).toEqual([ + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-readPetrinautDoc", + toolCallId: "tool-doc-1", + state: "input-available", + input: { doc: "ai-assistant" }, + }, + ], + }, + ]); +}); + +test("uses a recorded browser result even when it is null", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + ...snapshotWithPendingClientTool.messages, + { + id: "signal-1", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + text: '[{"toolCallId":"tool-doc-1","toolName":"readPetrinautDoc","output":null}]', + state: "done", + }, + ], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)[0]?.parts).toEqual([ + { + type: "tool-readPetrinautDoc", + toolCallId: "tool-doc-1", + state: "output-available", + input: { doc: "ai-assistant" }, + output: null, + }, + ]); +}); + +test("keeps Flue data parts on the AI SDK message", () => { + const snapshot: FlueConversationSnapshot = { + v: 1, + conversationId: "conversation-1", + offset: "0", + messages: [ + { + id: "assistant-1", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { type: "text", text: "Here is the order.", state: "done" }, + { + type: "data-orderCard", + data: { orderId: "42", status: "loaded" }, + }, + ], + }, + ], + settlements: [], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + { + id: "assistant-1", + role: "assistant", + parts: [ + { type: "text", text: "Here is the order.", state: "done" }, + { type: "data-orderCard", data: { orderId: "42", status: "loaded" } }, + ], + }, + ]); +}); + +test("keeps a rehydrated server tool provider-executed while it still runs", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + id: "assistant-1", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-sweep-1", + toolName: "brunch_sweep", + state: "input-available", + input: { range: "all" }, + }, + ], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)[0]?.parts).toEqual([ + { + type: "tool-brunch_sweep", + toolCallId: "tool-sweep-1", + state: "input-available", + input: { range: "all" }, + providerExecuted: true, + }, + ]); +}); + +test("folds a client-tool continuation into the assistant message it resumed", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + id: "user-1", + role: "user", + purpose: "user", + display: "visible", + parts: [{ type: "text", text: "Read the guide.", state: "done" }], + }, + ...snapshotWithPendingClientTool.messages, + { + id: "signal-1", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + text: '[{"toolCallId":"tool-doc-1","toolName":"readPetrinautDoc","output":"The guide."}]', + state: "done", + }, + ], + }, + { + id: "assistant-2", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [{ type: "text", text: "The guide says hello.", state: "done" }], + }, + { + id: "user-2", + role: "user", + purpose: "user", + display: "visible", + parts: [{ type: "text", text: "Thanks.", state: "done" }], + }, + { + id: "assistant-3", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [{ type: "text", text: "You are welcome.", state: "done" }], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + { + id: "user-1", + role: "user", + parts: [{ type: "text", text: "Read the guide.", state: "done" }], + }, + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-readPetrinautDoc", + toolCallId: "tool-doc-1", + state: "output-available", + input: { doc: "ai-assistant" }, + output: "The guide.", + }, + { type: "text", text: "The guide says hello.", state: "done" }, + ], + }, + { + id: "user-2", + role: "user", + parts: [{ type: "text", text: "Thanks.", state: "done" }], + }, + { + id: "assistant-3", + role: "assistant", + parts: [{ type: "text", text: "You are welcome.", state: "done" }], + }, + ]); +}); diff --git a/apps/brunch-agent/test/flue-ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts similarity index 78% rename from apps/brunch-agent/test/flue-ui-stream.test.ts rename to libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index c151078e40a..c567fcd9b38 100644 --- a/apps/brunch-agent/test/flue-ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "vitest"; -import { createFlueUiStream } from "../src/conversation/ui-stream.ts"; +import { createFlueUiStream } from "../src"; import type { ConversationStreamChunk } from "@flue/sdk"; import type { UIMessageChunk } from "ai"; @@ -20,7 +20,7 @@ const project = ( return written; }; -test("projects a Flue data-part onto the matching AI SDK data-* chunk", () => { +test("projects data and metadata onto the AI SDK stream", () => { const written = project([ { type: "message-started", @@ -31,51 +31,26 @@ test("projects a Flue data-part onto the matching AI SDK data-* chunk", () => { position: position(0), }, { - type: "data-part", + type: "message-metadata", conversationId: "conversation-1", messageId: "message-1", - name: "orderCard", - data: { orderId: "42", status: "loaded" }, + metadata: { elapsedMs: 17 }, position: position(1), }, { - type: "submission-settled", - conversationId: "conversation-1", - submissionId: "submission-1", - outcome: "completed", - position: position(2), - }, - ]); - - expect(written).toContainEqual({ - type: "data-orderCard", - data: { orderId: "42", status: "loaded" }, - }); -}); - -test("projects Flue message-metadata onto the AI SDK message-metadata chunk", () => { - const written = project([ - { - type: "message-started", - conversationId: "conversation-1", - messageId: "message-1", - submissionId: "submission-1", - turnId: "turn-1", - position: position(0), - }, - { - type: "message-metadata", + type: "data-part", conversationId: "conversation-1", messageId: "message-1", - metadata: { elapsedMs: 17 }, - position: position(1), + name: "orderCard", + data: { orderId: "42", status: "loaded" }, + position: position(2), }, { type: "submission-settled", conversationId: "conversation-1", submissionId: "submission-1", outcome: "completed", - position: position(2), + position: position(3), }, ]); @@ -83,9 +58,13 @@ test("projects Flue message-metadata onto the AI SDK message-metadata chunk", () type: "message-metadata", messageMetadata: { elapsedMs: 17 }, }); + expect(written).toContainEqual({ + type: "data-orderCard", + data: { orderId: "42", status: "loaded" }, + }); }); -test("does not project Flue observe/reconnect chunks onto the AI SDK stream", () => { +test("ignores observation catch-up chunks in a submission stream", () => { const written = project([ { type: "message-started", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts index a23d137f9ac..843fd00f762 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts @@ -15,7 +15,7 @@ export default defineConfig({ formats: ["es"], }, rolldownOptions: { - external: ["ai", "valibot"], + external: ["@flue/sdk", "ai"], }, sourcemap: true, }, diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 4d4d3828282..ace28fc27be 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -16,20 +16,15 @@ The assistant panel only renders in **Edit** mode. Switching to **Simulate** mod While a response is streaming you can: - Watch the model's text and reasoning appear live. The **Reasoning** block is collapsible; while it is streaming, it auto-opens, shows a shimmer effect, and (once attached timing information arrives) an elapsed timer. -- Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. +- Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. A host with durable conversation execution can record that stop before Petrinaut cancels its local stream; without that host capability, Stop is local cancellation only. A Stop pressed while the assistant is reading the net or the docs also withholds the follow-up reply that would otherwise start automatically. - Type your next message in the composer -- it is queued for after the current response ends. -The application embedding Petrinaut may place an additional control beside the message box. For -example, a host can offer another way to enter finalized text. Text submitted by that control -behaves like text sent with the keyboard: it joins the same conversation and, when an inline -question is waiting for an answer, completes that question rather than starting an unrelated -message. A host can explicitly submit a separate message instead when the text is a correction or -other follow-up that must not answer the pending question. -If the host offers voice input, a finalized spoken turn is held while an existing response -finishes and is submitted when the conversation is ready. +The application embedding Petrinaut may place an additional control beside the message box. For example, a host can offer another way to enter finalized text. Text submitted by that control behaves like text sent with the keyboard: it joins the same conversation and, when an inline question is waiting for an answer, completes that question rather than starting an unrelated message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. +If the host offers voice input, a finalized spoken turn is held while an existing response finishes and is submitted when the conversation is ready. -If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to -the conversation. Retry from the composer when the assistant is ready. +If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to the conversation. Retry from the composer when the assistant is ready. + +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. When the Brunch voice preview is enabled and available, an empty composer shows a waveform action titled **Start voice mode**. Typing non-whitespace text replaces it with **Send**. The same dynamic @@ -76,11 +71,7 @@ to the canvas toolbar. Sending non-empty typed text from the composer or first-run prompt ends Voice mode before it sends the message once through the same conversation; repeated send actions are ignored while that short handoff completes. -The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the -authority on your system. Brunch still chooses every question and interview decision; OpenAI only -delivers its words. The question and finalized response shown in the Petrinaut conversation are -authoritative. Spoken audio is generated from that Brunch text but may not be verbatim. Interrupting -audio does not undo the visible response or change the interview's saved history. +The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only delivers its words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are @@ -98,9 +89,7 @@ diagnostic record do not contain your transcript or the response being spoken. I failures use a content-free `interview-correlation`, `interview-response`, or `interview-submission` code so an operator can distinguish them without receiving your answer. -When no interview is active, **Clear AI chat** via the delete button in the top right of the panel -wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages -(if the host persists them). +When no interview is active and the host permits clearing, **Clear AI chat** via the delete button in the top right of the panel wipes the local conversation, stops any in-flight stream, and tells the host app to forget the messages if it persists them. Hosts with canonical history may disable this control. The Brunch panel disables it because clearing only the browser view would not delete Flue history and the conversation would return on rehydration. ## What the assistant can do diff --git a/libs/@hashintel/petrinaut/src/main.ts b/libs/@hashintel/petrinaut/src/main.ts index 97bac2f2784..f45791cb13d 100644 --- a/libs/@hashintel/petrinaut/src/main.ts +++ b/libs/@hashintel/petrinaut/src/main.ts @@ -98,5 +98,6 @@ export { Petrinaut } from "./ui/petrinaut"; export type { PetrinautAiAssistant, PetrinautAiChatTransport, + PetrinautAiStopResult, PetrinautProps, } from "./ui/petrinaut"; diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 911f54e83de..7c30734bff9 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -17,6 +17,7 @@ export type { export type { PetrinautAiAssistant, PetrinautAiChatTransport, + PetrinautAiStopResult, PetrinautProps, } from "./petrinaut"; export type { diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx index cf99c1bbb1d..35857da9b77 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx @@ -48,7 +48,11 @@ import type { export type PetrinautAiChatTransport = PetrinautAiTransport; +export type PetrinautAiStopResult = "already-settled" | "stop-requested"; + export type PetrinautAiAssistant = { + /** Whether the panel may clear this conversation. Defaults to true. */ + canClearMessages?: boolean; /** Optional host-owned identity; `useChat` generates one when omitted. */ conversationId?: string; /** Host-owned dynamic tools that render inline in the AI conversation. */ @@ -56,6 +60,11 @@ export type PetrinautAiAssistant = { messages?: PetrinautAiMessage[]; onClearMessages?: () => void; onMessages?: (messages: PetrinautAiMessage[]) => void; + /** + * Requests a host-owned durable stop. When omitted, Stop only cancels the + * panel's local response stream. + */ + requestStop?: () => Promise; /** Render a host-owned control inside the assistant composer. */ renderComposerControl?: PetrinautAiComposerControl; /** Render one persistent, provider-neutral Voice mode. */ diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 5369be4c639..0df936f5fc3 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -10,7 +10,11 @@ export type { PetrinautAiVoiceSessionPhase, PetrinautAiVoiceSessionState }; /** The active way a user is providing input to the AI assistant. */ export type PetrinautAiInputMode = "text" | "voice"; -/** Current lifecycle state of Petrinaut's AI SDK conversation. */ +/** + * Current lifecycle state of Petrinaut's conversation. Stays busy across the + * automatic follow-up to a step that ended in client tool calls, so hosts + * never read the SDK's momentary `ready` between them as the end of a turn. + */ export type PetrinautAiComposerStatus = | "submitted" | "streaming" @@ -99,7 +103,14 @@ export type PetrinautAiVoiceModeContext = PetrinautAiComposerControlContext & { params: Omit< Parameters[0], "source" - >, + > & { + /** + * Withdraws a retained turn that has not been submitted yet, for example + * when the Voice session ends while chat is still busy. A turn already + * handed to the composer is not cancelled. + */ + readonly signal?: AbortSignal; + }, ) => Promise; }; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 4c39bb19062..03694c12379 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -245,6 +245,86 @@ afterEach(() => { }); describe("AiAssistantPanel composer submissions", () => { + test("disables Clear when the host owns canonical conversation history", () => { + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: async () => new ReadableStream(), + }; + + renderTestPanel({ + aiAssistant: { + canClearMessages: false, + messages: [ + { + id: "assistant-canonical", + role: "assistant", + parts: [{ type: "text", text: "Canonical history" }], + }, + ], + transport, + }, + }); + + expect( + screen.getByRole("button", { + name: "Clear AI chat", + }).disabled, + ).toBe(true); + }); + + test("hydrates asynchronous host messages once for each conversation", async () => { + const sendMessages = vi.fn( + async () => new ReadableStream(), + ); + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages, + }; + const { rerenderPanel } = renderTestPanel({ + aiAssistant: { + conversationId: "conversation-1", + transport, + }, + }); + + expect(screen.queryByText("Rehydrated first conversation")).toBeNull(); + + rerenderPanel({ + conversationId: "conversation-1", + messages: [ + { + id: "assistant-history-1", + role: "assistant", + parts: [{ type: "text", text: "Rehydrated first conversation" }], + }, + ], + transport, + }); + + expect( + await screen.findByText("Rehydrated first conversation"), + ).not.toBeNull(); + expect(sendMessages).not.toHaveBeenCalled(); + + rerenderPanel({ + conversationId: "conversation-2", + messages: [ + { + id: "assistant-history-2", + role: "assistant", + parts: [{ type: "text", text: "Rehydrated second conversation" }], + }, + ], + transport, + }); + + expect( + await screen.findByText("Rehydrated second conversation"), + ).not.toBeNull(); + expect(screen.queryByText("Rehydrated first conversation")).toBeNull(); + expect(sendMessages).not.toHaveBeenCalled(); + }); + test("invalidates registered Voice controls before typed submit while active publication is pending", async () => { const events: string[] = []; let finishVoiceEnd: (() => void) | undefined; @@ -1575,6 +1655,283 @@ describe("AiAssistantPanel composer submissions", () => { expect(await screen.findByText("Response stopped")).not.toBeNull(); }); + test("records a durable Stop before cancelling the local stream", async () => { + const localCancellation = vi.fn(); + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn( + ({ + abortSignal, + }: Parameters[0]) => + Promise.resolve( + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "partial" }); + controller.enqueue({ + type: "text-delta", + id: "partial", + delta: "Durably stopping", + }); + abortSignal?.addEventListener("abort", () => { + localCancellation(); + controller.error(new DOMException("Aborted", "AbortError")); + }); + }, + }), + ), + ), + }; + const requestStop = vi.fn(async () => "stop-requested" as const); + + renderTestPanel({ + aiAssistant: { requestStop, transport }, + initialMessage: "Start durable work", + }); + await screen.findByText("Durably stopping"); + + fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); + + await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); + await waitFor(() => expect(localCancellation).toHaveBeenCalledOnce()); + expect(requestStop.mock.invocationCallOrder[0]).toBeLessThan( + localCancellation.mock.invocationCallOrder[0]!, + ); + expect(await screen.findByText("Response stopped")).not.toBeNull(); + }); + + test("keeps a response completed before the durable Stop race", async () => { + const localCancellation = vi.fn(); + let streamController: + | ReadableStreamDefaultController + | undefined; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn( + ({ + abortSignal, + }: Parameters[0]) => + Promise.resolve( + new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "answer" }); + controller.enqueue({ + type: "text-delta", + id: "answer", + delta: "Completed response", + }); + abortSignal?.addEventListener("abort", localCancellation); + }, + }), + ), + ), + }; + const requestStop = vi.fn(async () => { + streamController?.enqueue({ type: "text-end", id: "answer" }); + streamController?.enqueue({ type: "finish-step" }); + streamController?.enqueue({ type: "finish", finishReason: "stop" }); + streamController?.close(); + return "already-settled" as const; + }); + + renderTestPanel({ + aiAssistant: { requestStop, transport }, + initialMessage: "Race a completed response", + }); + await screen.findByText("Completed response"); + + fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); + + await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Send message" }), + ).toHaveProperty("disabled", true), + ); + expect(localCancellation).not.toHaveBeenCalled(); + expect(screen.queryByText("Response stopped")).toBeNull(); + }); + + test("withholds the client-tool follow-up when a durable Stop lands after a tool-calls step", async () => { + let streamController: + | ReadableStreamDefaultController + | undefined; + const sendMessages = vi.fn(() => + Promise.resolve( + new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "preamble" }); + controller.enqueue({ + type: "text-delta", + id: "preamble", + delta: "Checking the net", + }); + }, + }), + ), + ); + const requestStop = vi.fn(async () => { + // The step settles as a tool-calls step while the durable Stop is in + // flight, so Flue has nothing left to abort and reports already-settled. + streamController?.enqueue({ type: "text-end", id: "preamble" }); + streamController?.enqueue({ + type: "tool-input-available", + toolCallId: "net-read-1", + toolName: "getLatestNetDefinition", + input: {}, + }); + streamController?.enqueue({ type: "finish-step" }); + streamController?.enqueue({ type: "finish", finishReason: "tool-calls" }); + streamController?.close(); + return "already-settled" as const; + }); + + renderTestPanel({ + aiAssistant: { + requestStop, + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages, + }, + }, + initialMessage: "Stop me mid-tool", + petriNetDefinition: nonEmptySDCPN, + }); + await screen.findByText("Checking the net"); + + fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); + + await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); + expect(await screen.findByText("Response stopped")).not.toBeNull(); + // Let any automatic follow-up the SDK might schedule drain first. + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + expect(sendMessages).toHaveBeenCalledOnce(); + expect(screen.getByRole("button", { name: "Send message" })).toHaveProperty( + "disabled", + true, + ); + }); + + test("keeps hosts seeing a busy conversation between a tool-calls step and its follow-up", async () => { + const observedStatuses: PetrinautAiComposerControlContext["status"][] = []; + let requestCount = 0; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => { + requestCount += 1; + return Promise.resolve( + streamChunks( + requestCount === 1 + ? [ + ...textChunks("preamble", "Checking the net"), + { + type: "tool-input-available", + toolCallId: "net-read-1", + toolName: "getLatestNetDefinition", + input: {}, + }, + { type: "finish-step" }, + { type: "finish", finishReason: "tool-calls" }, + ] + : [ + ...textChunks("reply", "The net has one place"), + { type: "finish-step" }, + { type: "finish", finishReason: "stop" }, + ], + ), + ); + }), + }; + + renderTestPanel({ + aiAssistant: { + renderComposerControl: (context) => { + observedStatuses.push(context.status); + return null; + }, + transport, + }, + initialMessage: "Read the net", + petriNetDefinition: nonEmptySDCPN, + }); + await screen.findByText("The net has one place"); + + // The SDK reports `ready` between the tool-calls step and the follow-up it + // sends automatically; a host must not read that gap as the turn's end. + const firstBusy = observedStatuses.findIndex( + (status) => status === "submitted" || status === "streaming", + ); + const lastBusy = observedStatuses.findLastIndex( + (status) => status === "submitted" || status === "streaming", + ); + expect(firstBusy).toBeGreaterThanOrEqual(0); + expect( + observedStatuses + .slice(firstBusy, lastBusy + 1) + .filter((status) => status === "ready"), + ).toEqual([]); + expect(observedStatuses.at(-1)).toBe("ready"); + }); + + test("replaces conversation-owned busy state when the identity changes", async () => { + let requestCount = 0; + const statuses: string[] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => { + requestCount += 1; + if (requestCount > 1) { + return new Promise>(() => {}); + } + return Promise.resolve( + streamChunks([ + ...textChunks("preamble", "Checking the net"), + { + type: "tool-input-available", + toolCallId: "net-read-1", + toolName: "getLatestNetDefinition", + input: {}, + }, + { type: "finish-step" }, + { type: "finish", finishReason: "tool-calls" }, + ]), + ); + }), + }; + const createAiAssistant = ( + conversationId: string, + ): PetrinautAiAssistant => ({ + conversationId, + renderComposerControl: (context) => { + statuses.push(`${context.conversationId}:${context.status}`); + return ( + + ); + }, + transport, + }); + const rendered = renderTestPanel({ + aiAssistant: createAiAssistant("conversation-1"), + petriNetDefinition: nonEmptySDCPN, + }); + + fireEvent.click(screen.getByRole("button", { name: "Read net" })); + await waitFor(() => expect(statuses).toContain("conversation-1:submitted")); + + rendered.rerenderPanel(createAiAssistant("conversation-2")); + + await waitFor(() => expect(statuses.at(-1)).toBe("conversation-2:ready")); + }); + test("does not carry an idle host stop into a later incidental abort", async () => { let requestCount = 0; const transport: PetrinautAiTransport = { @@ -1675,6 +2032,345 @@ describe("AiAssistantPanel composer submissions", () => { expect(screen.queryByText("Response stopped")).toBeNull(); }); + test("does not let a late durable Stop cancel a newer turn", async () => { + let firstStreamController: + | ReadableStreamDefaultController + | undefined; + let secondStreamController: + | ReadableStreamDefaultController + | undefined; + const secondCancellation = vi.fn(); + let requestCount = 0; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn( + ({ + abortSignal, + }: Parameters[0]) => { + requestCount += 1; + if (requestCount === 1) { + return Promise.resolve( + new ReadableStream({ + start(controller) { + firstStreamController = controller; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "first" }); + controller.enqueue({ + type: "text-delta", + id: "first", + delta: "First partial", + }); + }, + }), + ); + } + return Promise.resolve( + new ReadableStream({ + start(controller) { + secondStreamController = controller; + abortSignal?.addEventListener("abort", secondCancellation); + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "second" }); + controller.enqueue({ + type: "text-delta", + id: "second", + delta: "Second turn", + }); + }, + }), + ); + }, + ), + }; + let resolveStop: ((result: "stop-requested") => void) | undefined; + const requestStop = vi.fn( + () => + new Promise<"stop-requested">((resolve) => { + resolveStop = resolve; + }), + ); + + renderTestPanel({ + aiAssistant: { + renderComposerControl: ({ status: hostStatus, submitText }) => ( + <> + + {hostStatus} + + ), + requestStop, + transport, + }, + initialMessage: "First", + }); + await screen.findByText("First partial"); + + fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); + await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); + + // The first response completes on its own while the durable stop is + // still in flight, and the user starts another turn. + await act(async () => { + firstStreamController?.enqueue({ type: "text-end", id: "first" }); + firstStreamController?.enqueue({ type: "finish-step" }); + firstStreamController?.enqueue({ type: "finish", finishReason: "stop" }); + firstStreamController?.close(); + }); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + fireEvent.click(screen.getByRole("button", { name: "Send second" })); + await screen.findByText("Second turn"); + + // The stale Stop result lands while the second turn is still streaming. + await act(async () => { + resolveStop?.("stop-requested"); + }); + expect(screen.getByTestId("host-status").textContent).toBe("streaming"); + expect(secondCancellation).not.toHaveBeenCalled(); + + await act(async () => { + secondStreamController?.enqueue({ type: "text-end", id: "second" }); + secondStreamController?.close(); + }); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + expect(screen.queryByText("Response stopped")).toBeNull(); + }); + + test("does not surface a late durable Stop failure on a newer turn", async () => { + let firstStreamController: + | ReadableStreamDefaultController + | undefined; + let secondStreamController: + | ReadableStreamDefaultController + | undefined; + const secondCancellation = vi.fn(); + let requestCount = 0; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn( + ({ + abortSignal, + }: Parameters[0]) => { + requestCount += 1; + const isFirst = requestCount === 1; + return Promise.resolve( + new ReadableStream({ + start(controller) { + if (isFirst) { + firstStreamController = controller; + } else { + secondStreamController = controller; + abortSignal?.addEventListener("abort", secondCancellation); + } + const id = isFirst ? "first" : "second"; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id }); + controller.enqueue({ + type: "text-delta", + id, + delta: isFirst ? "First partial" : "Second turn", + }); + }, + }), + ); + }, + ), + }; + let rejectStop: ((reason: Error) => void) | undefined; + const requestStop = vi.fn( + () => + new Promise<"stop-requested">((_resolve, reject) => { + rejectStop = reject; + }), + ); + + renderTestPanel({ + aiAssistant: { + renderComposerControl: ({ status: hostStatus, submitText }) => ( + <> + + {hostStatus} + + ), + requestStop, + transport, + }, + initialMessage: "First", + }); + await screen.findByText("First partial"); + + fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })); + await waitFor(() => expect(requestStop).toHaveBeenCalledOnce()); + await act(async () => { + firstStreamController?.enqueue({ type: "text-end", id: "first" }); + firstStreamController?.close(); + }); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + fireEvent.click(screen.getByRole("button", { name: "Send second" })); + await screen.findByText("Second turn"); + + await act(async () => { + rejectStop?.(new Error("Durable stop failed")); + }); + expect(screen.getByTestId("host-status").textContent).toBe("streaming"); + expect(secondCancellation).not.toHaveBeenCalled(); + expect(screen.queryByText(/Durable stop failed/u)).toBeNull(); + + await act(async () => { + secondStreamController?.enqueue({ type: "text-end", id: "second" }); + secondStreamController?.close(); + }); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + expect(screen.queryByText(/Durable stop failed/u)).toBeNull(); + }); + + test("withdraws a retained voice input when its signal aborts", async () => { + let streamController: + | ReadableStreamDefaultController + | undefined; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => + Promise.resolve( + new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "busy" }); + controller.enqueue({ + type: "text-delta", + id: "busy", + delta: "Still answering", + }); + }, + }), + ), + ), + }; + let latestVoiceContext: PetrinautAiVoiceModeContext | undefined; + + renderTestPanel({ + aiAssistant: { + renderVoiceMode: (context) => { + latestVoiceContext = context; + return null; + }, + transport, + }, + initialMessage: "Begin", + }); + await screen.findByText("Still answering"); + await waitFor(() => expect(latestVoiceContext?.status).toBe("streaming")); + + const withdrawal = new AbortController(); + const retained = latestVoiceContext!.submitVoiceInput({ + signal: withdrawal.signal, + text: "Stale voice input", + }); + await waitFor(() => + expect(latestVoiceContext?.canAcceptVoiceInput).toBe(false), + ); + + withdrawal.abort(); + + await expect(retained).rejects.toMatchObject({ name: "AbortError" }); + await waitFor(() => + expect(latestVoiceContext?.canAcceptVoiceInput).toBe(true), + ); + + await act(async () => { + streamController?.enqueue({ type: "text-end", id: "busy" }); + streamController?.close(); + }); + await waitFor(() => expect(latestVoiceContext?.status).toBe("ready")); + expect(transport.sendMessages).toHaveBeenCalledTimes(1); + expect(screen.queryByText("Stale voice input")).toBeNull(); + }); + + test("waits to hydrate until the host snapshot carries a turn submitted first", async () => { + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => + Promise.resolve( + streamChunks([ + { type: "start", messageId: "assistant-live-1" }, + ...textChunks("live", "Live answer"), + { type: "finish-step" }, + { type: "finish", finishReason: "stop" }, + ]), + ), + ), + }; + const { rerenderPanel } = renderTestPanel({ + aiAssistant: { conversationId: "conversation-1", transport }, + initialMessage: "Ask before history loads", + }); + await screen.findByText("Live answer"); + + // The observation publishes a snapshot that predates the live turn. + rerenderPanel({ + conversationId: "conversation-1", + messages: [ + { + id: "assistant-history-1", + role: "assistant", + parts: [{ type: "text", text: "Older history" }], + }, + ], + transport, + }); + await act(async () => {}); + expect(screen.getByText("Live answer")).not.toBeNull(); + expect(screen.queryByText("Older history")).toBeNull(); + + // Once the snapshot carries the live reply too, canonical history wins. + rerenderPanel({ + conversationId: "conversation-1", + messages: [ + { + id: "assistant-history-1", + role: "assistant", + parts: [{ type: "text", text: "Older history" }], + }, + { + id: "user-live-1", + role: "user", + parts: [{ type: "text", text: "Ask before history loads" }], + }, + { + id: "assistant-live-1", + role: "assistant", + parts: [{ type: "text", text: "Live answer" }], + }, + ], + transport, + }); + expect(await screen.findByText("Older history")).not.toBeNull(); + expect(screen.getByText("Live answer")).not.toBeNull(); + expect(transport.sendMessages).toHaveBeenCalledTimes(1); + }); + test("maps keyboard text to one unresolved host tool before sending another message", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index df83042ed1b..e3c70c88686 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -58,6 +58,7 @@ import { import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { PetrinautAiComposerControlContext, + PetrinautAiComposerStatus, PetrinautAiComposerSubmitTextResult, PetrinautAiInputMode, PetrinautAiVoiceModeContext, @@ -100,9 +101,18 @@ type QueuedVoiceInput = { PetrinautAiVoiceModeContext["submitVoiceInput"] >[0]; readonly reject: (reason?: unknown) => void; + /** Detaches the input's abort listener once the queue no longer owns it. */ + readonly release: () => void; readonly resolve: (result: PetrinautAiComposerSubmitTextResult) => void; }; +const voiceInputWithdrawn = (signal: AbortSignal | undefined): unknown => + signal?.reason ?? + new DOMException( + "The voice input was withdrawn before submission.", + "AbortError", + ); + const markVoiceToolOrigin = ( messages: PetrinautAiMessage[], messageId: string, @@ -284,19 +294,21 @@ const applyPetrinautAiCommand = async ({ } }; -export const AiAssistantPanel = ({ - aiAssistant, - initialInteractionMode, - initialMessage, - onInitialInteractionModeConsumed, - onInitialMessageConsumed, -}: { +interface AiAssistantPanelProps { aiAssistant: PetrinautAiAssistant; initialInteractionMode?: PetrinautAiInputMode | null; initialMessage?: string | null; onInitialInteractionModeConsumed?: () => void; onInitialMessageConsumed?: () => void; -}) => { +} + +const ConversationAiAssistantPanel = ({ + aiAssistant, + initialInteractionMode, + initialMessage, + onInitialInteractionModeConsumed, + onInitialMessageConsumed, +}: AiAssistantPanelProps) => { // The wrapped AI transport closes over several refs (diagnostics version, // pending mutation version, diagnostics context) so the transport's // `sendMessages` can read the latest values when it eventually runs. React @@ -436,6 +448,11 @@ export const AiAssistantPanel = ({ // response. Cleared whenever a new turn begins so it never lingers across // sends or a fresh conversation. const [stopped, setStopped] = useState(false); + // The SDK reports `ready` between a step that ended in client tool calls and + // the follow-up it sends automatically. That gap is not the end of the turn, + // so hosts keep seeing a busy conversation until the follow-up starts or a + // Stop withholds it. + const [continuationPending, setContinuationPending] = useState(false); const requestInputMode = useCallback( (nextMode: PetrinautAiInputMode) => { @@ -503,7 +520,11 @@ export const AiAssistantPanel = ({ ); const stopRequestedRef = useRef(false); + // Advances on every composer submission so an asynchronous Stop can tell + // whether the turn it was pressed for is still the current one. + const submissionGenerationRef = useRef(0); const pendingSubmissionRecoveryRef = useRef<(() => void) | null>(null); + const hydratedConversationIdRef = useRef(null); const { error, @@ -512,7 +533,7 @@ export const AiAssistantPanel = ({ addToolOutput, sendMessage, setMessages, - status, + status: chatStatus, stop, } = useChat({ ...(aiAssistant.conversationId === undefined @@ -520,7 +541,28 @@ export const AiAssistantPanel = ({ : { id: aiAssistant.conversationId }), messages: aiAssistant.messages, transport: diagnosticsTransportState.transport, - sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithToolCalls, + sendAutomaticallyWhen: ({ messages: currentMessages }) => { + if ( + !lastAssistantMessageIsCompleteWithToolCalls({ + messages: currentMessages, + }) + ) { + return false; + } + if (!stopRequestedRef.current) { + // Left pending until the follow-up's own status change lands, so hosts + // never observe the `ready` between this check and that request. + return true; + } + // Stop was pressed during the step that just ended in client tool + // calls. Flue had nothing left to abort once that step settled, so + // withholding the follow-up is what makes the Stop real. + stopRequestedRef.current = false; + setContinuationPending(false); + setStreamError(null); + setStopped(true); + return false; + }, // Without throttling, every reasoning-delta / text-delta chunk triggers a // full re-render of `AiAssistantContents`, and the SDK `structuredClone`s // the active message on each one. For a long markdown reply that locks @@ -535,8 +577,18 @@ export const AiAssistantPanel = ({ pendingSubmissionRecoveryRef.current = null; recoverPendingSubmission?.(); }, - onFinish: ({ messages: finishedMessages, isAbort }) => { + onFinish: ({ messages: finishedMessages, isAbort, isError }) => { pendingSubmissionRecoveryRef.current = null; + // A step that ended in client tool calls is followed automatically by + // the SDK unless it was aborted or errored; that follow-up is still part + // of this turn. + const followUpPending = + !isAbort && + !isError && + lastAssistantMessageIsCompleteWithToolCalls({ + messages: finishedMessages, + }); + setContinuationPending(followUpPending); if (isAbort) { // The SDK fires `onFinish` for every abort. Only act on a deliberate // Stop — clearing the chat or unmounting also aborts, and those paths @@ -560,13 +612,19 @@ export const AiAssistantPanel = ({ return; } + aiAssistant.onMessages?.(finishedMessages); + if (followUpPending) { + // The turn is not over: a Stop pressed during this step must still be + // able to withhold the follow-up, so its intent survives this step. + return; + } + // A response that runs to completion clears any pending Stop intent so a // later incidental abort can't replay the deliberate-stop path, and // drops a stale "Response stopped" note left over from an earlier turn. stopRequestedRef.current = false; setStreamError(null); setStopped(false); - aiAssistant.onMessages?.(finishedMessages); }, onToolCall: async ({ toolCall }) => { if (!instance) { @@ -757,6 +815,35 @@ export const AiAssistantPanel = ({ }, }); + const status: PetrinautAiComposerStatus = + continuationPending && chatStatus === "ready" ? "submitted" : chatStatus; + + useEffect(() => { + if ( + aiAssistant.messages === undefined || + status !== "ready" || + hydratedConversationIdRef.current === conversationId + ) { + return; + } + // A turn submitted before the host's history arrived is already visible + // locally. A snapshot that predates it would erase that turn and latch, so + // wait for a snapshot that carries every locally streamed reply. + const canonicalMessageIds = new Set( + aiAssistant.messages.map((message) => message.id), + ); + if ( + messages.some( + (message) => + message.role === "assistant" && !canonicalMessageIds.has(message.id), + ) + ) { + return; + } + hydratedConversationIdRef.current = conversationId; + setMessages(aiAssistant.messages); + }, [aiAssistant.messages, conversationId, messages, setMessages, status]); + const composerSubmissionStateRef = useLatest({ addToolOutput, interactiveTools: aiAssistant.interactiveTools, @@ -894,6 +981,7 @@ export const AiAssistantPanel = ({ setStreamError(null); setStopped(false); stopRequestedRef.current = false; + submissionGenerationRef.current += 1; composerToolSubmissionsRef.current.add(mappedToolCall.toolCallId); try { await addMappedToolOutput({ @@ -928,6 +1016,7 @@ export const AiAssistantPanel = ({ setStreamError(null); setStopped(false); stopRequestedRef.current = false; + submissionGenerationRef.current += 1; await submitMessage({ id: messageId, ...(source === "voice" ? { metadata: { source } } : {}), @@ -939,7 +1028,11 @@ export const AiAssistantPanel = ({ [composerSubmissionStateRef], ); - const stopStateRef = useLatest({ status, stop }); + const stopStateRef = useLatest({ + requestStop: aiAssistant.requestStop, + status, + stop, + }); const submitVoiceInput = useCallback< PetrinautAiVoiceModeContext["submitVoiceInput"] @@ -960,11 +1053,28 @@ export const AiAssistantPanel = ({ return submitText({ ...voiceInput, source: "voice" }); } + const { signal } = voiceInput; + if (signal?.aborted) { + return Promise.reject(voiceInputWithdrawn(signal)); + } + setVoiceInputQueued(true); return new Promise((resolve, reject) => { + const withdraw = (): void => { + // Only the entry still holding this input may be withdrawn; a + // dequeued input has already been handed to the composer. + if (queuedVoiceInputRef.current?.input !== voiceInput) { + return; + } + queuedVoiceInputRef.current = null; + setVoiceInputQueued(false); + reject(voiceInputWithdrawn(signal)); + }; + signal?.addEventListener("abort", withdraw, { once: true }); queuedVoiceInputRef.current = { input: voiceInput, reject, + release: () => signal?.removeEventListener("abort", withdraw), resolve, }; }); @@ -980,6 +1090,7 @@ export const AiAssistantPanel = ({ if (status === "error") { queuedVoiceInputRef.current = null; setVoiceInputQueued(false); + queued.release(); queued.reject(new Error("Voice mode could not accept that input.")); return; } @@ -989,6 +1100,7 @@ export const AiAssistantPanel = ({ queuedVoiceInputRef.current = null; setVoiceInputQueued(false); + queued.release(); void submitText({ ...queued.input, source: "voice" }).then( (result) => queued.resolve(result), (caught: unknown) => queued.reject(caught), @@ -997,9 +1109,9 @@ export const AiAssistantPanel = ({ useEffect( () => () => { - queuedVoiceInputRef.current?.reject( - new Error("The voice conversation changed."), - ); + const queued = queuedVoiceInputRef.current; + queued?.release(); + queued?.reject(new Error("The voice conversation changed.")); queuedVoiceInputRef.current = null; setVoiceInputQueued(false); }, @@ -1008,13 +1120,39 @@ export const AiAssistantPanel = ({ // Like submitText, stop is exposed to host controls and must stay stable. const stopComposer = useCallback(async () => { - const { status: currentStatus, stop: stopCurrentResponse } = - stopStateRef.current; + const { + requestStop, + status: currentStatus, + stop: stopCurrentResponse, + } = stopStateRef.current; if (currentStatus !== "submitted" && currentStatus !== "streaming") { return; } + const generation = submissionGenerationRef.current; stopRequestedRef.current = true; + if (requestStop !== undefined) { + try { + const result = await requestStop(); + if (submissionGenerationRef.current !== generation) { + // A newer turn started while the durable stop was in flight; that + // turn owns its own Stop and must not inherit this result. + return; + } + if (result === "stop-requested") { + await stopCurrentResponse(); + } + } catch (caught) { + if (submissionGenerationRef.current !== generation) { + return; + } + stopRequestedRef.current = false; + setStreamError( + caught instanceof Error ? caught : new Error(String(caught)), + ); + } + return; + } await stopCurrentResponse(); }, [stopStateRef]); @@ -1217,7 +1355,9 @@ export const AiAssistantPanel = ({ return ( ); }; + +/** Replace every conversation-owned hook and callback together when identity changes. */ +export const AiAssistantPanel = (props: AiAssistantPanelProps) => ( + +); diff --git a/yarn.config.cjs b/yarn.config.cjs index ef9317e9267..151062f51c1 100644 --- a/yarn.config.cjs +++ b/yarn.config.cjs @@ -330,6 +330,7 @@ function enforceBrunchTransportBoundary({ Yarn }) { for (const dependency of Yarn.dependencies()) { if ( !dependency.workspace.ident.startsWith(brunchAgentTransportPrefix) || + dependency.ident === "@flue/sdk" || (dependency.ident !== brunchAgentCore && !dependency.ident.startsWith(`${brunchAgentCore}-`) && !brunchSubstrateScopes.some((scope) => diff --git a/yarn.lock b/yarn.lock index 6920abe3d15..cf37dc107c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -939,6 +939,7 @@ __metadata: dependencies: "@ai-sdk/openai": "npm:3.0.63" "@fast-check/vitest": "npm:0.4.1" + "@flue/sdk": "npm:2.0.3" "@hashintel/brunch-agent": "workspace:*" "@hashintel/brunch-agent-transport-aisdk": "workspace:*" "@hashintel/ds-components": "workspace:*" @@ -7572,12 +7573,12 @@ __metadata: version: 0.0.0-use.local resolution: "@hashintel/brunch-agent-transport-aisdk@workspace:libs/@hashintel/brunch-agent/packages/transport-aisdk" dependencies: + "@flue/sdk": "npm:2.0.3" "@types/node": "npm:22.18.13" "@typescript/native-preview": "npm:7.0.0-dev.20260511.1" ai: "npm:6.0.182" oxlint: "npm:1.63.0" oxlint-tsgolint: "npm:0.22.1" - valibot: "npm:1.4.2" vite: "npm:8.2.2" vitest: "npm:4.1.10" languageName: unknown