From d1f4226330fe2866a7f38891167b77bc5fd76721 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 16:02:15 +0200 Subject: [PATCH 01/19] H-6763: Add the realtime voice experiment shell --- .../local-storage-demo-app.tsx | 6 + .../local-storage-demo/voice-experiment.tsx | 581 ++++++++++++++++++ .../voice-experiment-adapter.ts | 17 + .../voice-experiment-events.ts | 29 + .../voice-experiment-selection.test.ts | 30 + .../voice-experiment-selection.ts | 23 + libs/@hashintel/brunch-agent/docs/INDEX.md | 1 + ...763-realtime-audio-prototype-2026-08-24.md | 96 +++ 8 files changed, 783 insertions(+) create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-adapter.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts create mode 100644 libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 6f770fc9602..af64c9eb03f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -23,6 +23,8 @@ import { type SDCPNInLocalStorage, useLocalStorageSDCPNs, } from "./use-local-storage-sdcpns"; +import { VoiceExperiment } from "./voice-experiment"; +import { getVoiceExperiment } from "./voice-experiment/voice-experiment-selection"; import { walkthroughSteps } from "./walkthrough/walkthrough-steps"; const isEmptySDCPN = (sdcpn: SDCPN) => @@ -118,6 +120,7 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ */ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); + const voiceExperiment = getVoiceExperiment(window.location); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); @@ -307,6 +310,9 @@ export const LocalStorageDemoApp = () => { title={currentNet.title} viewportActions={[sentryFeedbackAction]} /> + {voiceExperiment ? ( + + ) : null} ); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx new file mode 100644 index 00000000000..93d1da320f0 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -0,0 +1,581 @@ +import { + type KeyboardEvent, + type PointerEvent, + useCallback, + useEffect, + useRef, + useState, +} from "react"; +import { MdGraphicEq, MdMic } from "react-icons/md"; + +import { css } from "@hashintel/ds-helpers/css"; + +import { + type VoiceExperiment as VoiceExperimentName, + voiceExperimentLabel, + voiceExperimentMode, +} from "./voice-experiment/voice-experiment-selection"; + +import type { VoiceExperimentAdapter } from "./voice-experiment/voice-experiment-adapter"; +import type { VoiceExperimentEvent } from "./voice-experiment/voice-experiment-events"; + +const SCENARIO_SCRIPT = + "Interview the expert about how an urgent customer support escalation moves from the initial report to resolution."; + +const panelStyle = css({ + position: "fixed", + zIndex: "popover", + bottom: "4", + left: "[50%]", + display: "flex", + width: "[calc(100vw - 32px)]", + maxWidth: "[620px]", + maxHeight: "[calc(100vh - 32px)]", + transform: "translateX(-50%)", + flexDirection: "column", + gap: "3", + padding: "4", + overflow: "auto", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a30", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: "xl", +}); + +const headerStyle = css({ + display: "flex", + alignItems: "flex-start", + justifyContent: "space-between", + gap: "4", +}); + +const headingGroupStyle = css({ + display: "flex", + flexDirection: "column", + gap: "0.5", +}); + +const eyebrowStyle = css({ + color: "blue.a85", + fontSize: "xs", + fontWeight: "semibold", + letterSpacing: "wide", + textTransform: "uppercase", +}); + +const headingStyle = css({ + color: "neutral.s100", + fontSize: "base", + fontWeight: "semibold", +}); + +const experimentBadgeStyle = css({ + display: "flex", + flexDirection: "column", + alignItems: "flex-end", + gap: "0.5", + paddingX: "2.5", + paddingY: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s90", + fontSize: "xs", + fontWeight: "medium", +}); + +const experimentModeStyle = css({ + color: "neutral.s60", + fontSize: "xs", + fontWeight: "normal", +}); + +const scenarioStyle = css({ + display: "flex", + flexDirection: "column", + gap: "1", + padding: "3", + borderRadius: "lg", + backgroundColor: "blue.a10", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + +const sectionLabelStyle = css({ + color: "blue.a85", + fontSize: "xs", + fontWeight: "semibold", + letterSpacing: "wide", + textTransform: "uppercase", +}); + +const transcriptStyle = css({ + minHeight: "16", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "lg", + backgroundColor: "neutral.a05", + color: "neutral.s70", + fontSize: "sm", + lineHeight: "relaxed", +}); + +const controlsStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + gap: "4", +}); + +const sessionControlsStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const sessionButtonStyle = css({ + paddingX: "3", + paddingY: "2", + borderRadius: "md", + backgroundColor: "neutral.a10", + color: "neutral.s90", + cursor: "pointer", + fontSize: "xs", + fontWeight: "medium", + _hover: { + backgroundColor: "neutral.a15", + }, + _disabled: { + color: "neutral.s50", + cursor: "not-allowed", + opacity: "0.65", + }, +}); + +const startSessionButtonStyle = css({ + backgroundColor: "blue.a85", + color: "white", + _hover: { + backgroundColor: "blue.a100", + }, +}); + +const statusStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", + color: "neutral.s70", + fontSize: "xs", +}); + +const statusIndicatorStyle = css({ + width: "2", + height: "2", + flexShrink: "0", + borderRadius: "full", + backgroundColor: "neutral.a50", +}); + +const connectedStatusIndicatorStyle = css({ + backgroundColor: "green.a85", + boxShadow: "[0 0 0 4px {colors.green.a15}]", +}); + +const liveStatusIndicatorStyle = css({ + backgroundColor: "red.a85", + boxShadow: "[0 0 0 4px {colors.red.a15}]", +}); + +const microphoneButtonStyle = css({ + display: "inline-flex", + alignItems: "center", + justifyContent: "center", + width: "14", + height: "14", + flexShrink: "0", + borderRadius: "full", + backgroundColor: "blue.a85", + color: "white", + cursor: "pointer", + boxShadow: "md", + touchAction: "none", + transition: "[transform 120ms ease, background-color 120ms ease]", + _hover: { + backgroundColor: "blue.a100", + }, + _focusVisible: { + outline: "3px solid", + outlineColor: "blue.a30", + outlineOffset: "[2px]", + }, + _disabled: { + backgroundColor: "neutral.a30", + color: "neutral.s50", + cursor: "not-allowed", + boxShadow: "[none]", + }, +}); + +const activeMicrophoneButtonStyle = css({ + transform: "scale(1.08)", + backgroundColor: "red.a85", +}); + +const disabledMicrophoneIconStyle = css({ + color: "neutral.s50", +}); + +const eventLogStyle = css({ + display: "flex", + maxHeight: "28", + flexDirection: "column", + gap: "1", + padding: "2", + overflowY: "auto", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "md", + backgroundColor: "neutral.a05", + color: "neutral.s60", + fontFamily: "mono", + fontSize: "xs", +}); + +const eventRowStyle = css({ + display: "flex", + justifyContent: "space-between", + gap: "3", +}); + +const emptyLogStyle = css({ + color: "neutral.s50", + fontFamily: "body", +}); + +type SessionState = + | "ready" + | "connecting" + | "connected" + | "responding" + | "ending" + | "ended" + | "error"; + +type LoggedEvent = { + event: VoiceExperimentEvent; + sequence: number; +}; + +const getLatestTranscript = (events: LoggedEvent[]) => { + for (let index = events.length - 1; index >= 0; index -= 1) { + const event = events[index]?.event; + if ( + event?.type === "partial-transcript" || + event?.type === "final-transcript" + ) { + return event.transcript; + } + } + return null; +}; + +const getEventSummary = (event: VoiceExperimentEvent) => { + if (event.type === "partial-transcript") { + return `${event.type}: ${event.transcript}`; + } + if (event.type === "final-transcript") { + return `${event.type}: ${event.transcript}`; + } + if (event.type === "tool-called") { + return `${event.type}: ${event.toolName}`; + } + if (event.type === "error") { + return `${event.type}: ${event.message}`; + } + return event.type; +}; + +const createErrorEvent = (error: unknown): VoiceExperimentEvent => ({ + message: error instanceof Error ? error.message : "Voice experiment failed.", + timestampMs: Date.now(), + type: "error", +}); + +export const VoiceExperiment = ({ + adapter, + experiment, +}: { + adapter?: VoiceExperimentAdapter; + experiment: VoiceExperimentName; +}) => { + const [conversationId] = useState(() => crypto.randomUUID()); + const [events, setEvents] = useState([]); + const [sessionState, setSessionState] = useState("ready"); + const [isTurnActive, setIsTurnActive] = useState(false); + const sequenceRef = useRef(0); + const pressedRef = useRef(false); + + const appendEvent = useCallback((event: VoiceExperimentEvent) => { + setEvents((previous) => [ + ...previous.slice(-49), + { event, sequence: ++sequenceRef.current }, + ]); + + if (event.type === "connected") { + setSessionState("connected"); + } else if (event.type === "recording-started") { + setIsTurnActive(true); + } else if (event.type === "response-started") { + setSessionState("responding"); + } else if (event.type === "response-completed") { + setSessionState("connected"); + } else if (event.type === "error") { + setSessionState("error"); + } + }, []); + + useEffect(() => adapter?.subscribe(appendEvent), [adapter, appendEvent]); + + useEffect(() => { + const dispose = () => { + pressedRef.current = false; + void adapter?.dispose(); + }; + + window.addEventListener("pagehide", dispose); + return () => { + window.removeEventListener("pagehide", dispose); + dispose(); + }; + }, [adapter]); + + const startSession = async () => { + if (!adapter || sessionState !== "ready") { + return; + } + + setSessionState("connecting"); + try { + await adapter.connect(); + setSessionState("connected"); + } catch (error) { + appendEvent(createErrorEvent(error)); + } + }; + + const endSession = async () => { + if (!adapter || sessionState === "ready" || sessionState === "ended") { + return; + } + + pressedRef.current = false; + setIsTurnActive(false); + setSessionState("ending"); + try { + await adapter.dispose(); + setSessionState("ended"); + } catch (error) { + appendEvent(createErrorEvent(error)); + } + }; + + const startTurn = async () => { + if (!adapter || sessionState !== "connected" || pressedRef.current) { + return; + } + + pressedRef.current = true; + setIsTurnActive(true); + try { + await adapter.startTurn(); + } catch (error) { + pressedRef.current = false; + setIsTurnActive(false); + appendEvent(createErrorEvent(error)); + } + }; + + const finishTurn = async () => { + if (!adapter || !pressedRef.current) { + return; + } + + pressedRef.current = false; + setIsTurnActive(false); + try { + await adapter.finishTurn(); + } catch (error) { + appendEvent(createErrorEvent(error)); + } + }; + + const handlePointerDown = (event: PointerEvent) => { + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + void startTurn(); + }; + + const handleKeyDown = (event: KeyboardEvent) => { + if ((event.key === " " || event.key === "Enter") && !event.repeat) { + event.preventDefault(); + void startTurn(); + } + }; + + const handleKeyUp = (event: KeyboardEvent) => { + if (event.key === " " || event.key === "Enter") { + event.preventDefault(); + void finishTurn(); + } + }; + + const transcript = getLatestTranscript(events); + const isConnected = + sessionState === "connected" || sessionState === "responding"; + const isAdapterPending = !adapter; + const canStartSession = Boolean(adapter) && sessionState === "ready"; + const canStartTurn = Boolean(adapter) && sessionState === "connected"; + const statusMessage = isAdapterPending + ? "Adapter pending — this shell does not own microphone access" + : sessionState === "ready" + ? "Ready to start a clean session" + : sessionState === "connecting" + ? "Connecting…" + : sessionState === "connected" + ? "Connected — hold the microphone to speak" + : sessionState === "responding" + ? "Response in progress" + : sessionState === "ending" + ? "Ending session…" + : sessionState === "ended" + ? "Session ended — reload for a new conversation" + : "Experiment error"; + + return ( + + ); +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-adapter.ts new file mode 100644 index 00000000000..1ea5d7b512a --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-adapter.ts @@ -0,0 +1,17 @@ +import type { VoiceExperimentEvent } from "./voice-experiment-events"; + +/** + * The observable contract shared by the experiment shell. + * + * Implementations own their browser audio, provider session, queued playback, + * and interruption state. `dispose` must be idempotent and must release all of + * those resources. Provider conversation and tool models stay private to the + * implementation. + */ +export type VoiceExperimentAdapter = { + connect(): Promise; + startTurn(): Promise; + finishTurn(): Promise; + dispose(): Promise; + subscribe(listener: (event: VoiceExperimentEvent) => void): () => void; +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts new file mode 100644 index 00000000000..eb1f4e5445d --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts @@ -0,0 +1,29 @@ +export type VoiceExperimentEvent = + | { timestampMs: number; type: "connected" } + | { timestampMs: number; turnId: number; type: "recording-started" } + | { + timestampMs: number; + transcript: string; + turnId: number; + type: "partial-transcript"; + } + | { + timestampMs: number; + transcript: string; + turnId: number; + type: "final-transcript"; + } + | { timestampMs: number; turnId: number; type: "response-started" } + | { + responseText?: string; + timestampMs: number; + turnId: number; + type: "response-completed"; + } + | { + timestampMs: number; + toolName: string; + turnId: number; + type: "tool-called"; + } + | { message: string; timestampMs: number; type: "error" }; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts new file mode 100644 index 00000000000..2fc625e1030 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "vitest"; + +import { getVoiceExperiment } from "./voice-experiment-selection"; + +describe("getVoiceExperiment", () => { + test.each(["openai-realtime", "elevenlabs-brunch"] as const)( + "selects the %s experiment for the lifetime of the page", + (experiment) => { + expect( + getVoiceExperiment({ search: `?voiceExperiment=${experiment}` }), + ).toBe(experiment); + }, + ); + + test("keeps the experiment shell hidden by default", () => { + expect(getVoiceExperiment({ search: "" })).toBeNull(); + }); + + test("rejects the superseded prototype parameter and aliases", () => { + expect( + getVoiceExperiment({ search: "?voicePrototype=elevenlabs" }), + ).toBeNull(); + expect( + getVoiceExperiment({ search: "?voiceExperiment=openai" }), + ).toBeNull(); + expect( + getVoiceExperiment({ search: "?voiceExperiment=elevenlabs" }), + ).toBeNull(); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts new file mode 100644 index 00000000000..56b1ee43a75 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts @@ -0,0 +1,23 @@ +export type VoiceExperiment = "openai-realtime" | "elevenlabs-brunch"; + +export const voiceExperimentLabel = { + "openai-realtime": "OpenAI Realtime", + "elevenlabs-brunch": "ElevenLabs + Brunch", +} satisfies Record; + +export const voiceExperimentMode = { + "openai-realtime": "Native voice · dummy tools", + "elevenlabs-brunch": "Speech edge · real elicitor", +} satisfies Record; + +export const getVoiceExperiment = ( + location: Pick, +): VoiceExperiment | null => { + const value = new URLSearchParams(location.search).get("voiceExperiment"); + + if (value === "openai-realtime" || value === "elevenlabs-brunch") { + return value; + } + + return null; +}; diff --git a/libs/@hashintel/brunch-agent/docs/INDEX.md b/libs/@hashintel/brunch-agent/docs/INDEX.md index ca6ea309c02..8ee60aee4de 100644 --- a/libs/@hashintel/brunch-agent/docs/INDEX.md +++ b/libs/@hashintel/brunch-agent/docs/INDEX.md @@ -42,6 +42,7 @@ _(empty — items settle out via the arc-close inbox sweep)_ | [petrinaut-integration-spec](planning/process-model-elicitation/petrinaut-integration-spec.md) | active | FE-1433 | Integration spec: elicitor as remote server behind the `aiAssistant` transport; suspension-borne client tools; `transport-aisdk`; principal + owner key; two gating spikes | | [FE-1434 suspension verdict](planning/process-model-elicitation/spikes/fe-1434-suspension-verdict-2026-08-19.md) | active | FE-1434 | Flue 2.0.3 carries a terminating client-tool batch through one durable pending slot and one non-user result signal; 3- and 100-result cases preserve ids in two dispatches | | [FE-1434 suspension evidence](planning/process-model-elicitation/spikes/fe-1434-suspension-evidence-2026-08-19.json) | active | FE-1434 | Deterministic transcript from the faux-provider runtime probe: native tool-result admission refused, signal resume succeeds, returned text is non-user and uncitable | +| [H-6763 realtime-audio prototype](planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md) | active | H-6763 | Two-experiment decision plan: OpenAI Realtime with dummy tools establishes the interaction-quality ceiling; ElevenLabs around the real text elicitor tests the preferred edge-adapter architecture | | [adapter-panel-spike-2026-08-19](planning/process-model-elicitation/adapter-panel-spike-2026-08-19.md) | settled | FE-1435 | Real-panel spike verdict: AI SDK v6 SSE drives Petrinaut text/reasoning, default server-tool summaries, two live-editor client tools in one batched follow-up, and the diagnostics decorator; full POST/SSE transcript frozen as golden fixtures | | [transport-aisdk-implementation-2026-08-19](planning/process-model-elicitation/transport-aisdk-implementation-2026-08-19.md) | settled | FE-1436 | Durable real-panel transport: application `/api/chat` endpoint, substrate-neutral harness reply events, AI SDK v6 encoding, boundary gates, opt-in protocol inspection, and a clean-checkout local Petrinaut launcher | | [ask-return-implementation-2026-08-19](planning/process-model-elicitation/ask-return-implementation-2026-08-19.md) | active | FE-1449 | Ask suspend/return over the wire: the ask leaves as an awaiting client tool, the correlated `{ answer }` submission is admitted against durable history and resumes the conversation; stale/forged/duplicate/non-ask outputs refused before dispatch | diff --git a/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md b/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md new file mode 100644 index 00000000000..6c42b59da9c --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md @@ -0,0 +1,96 @@ +# H-6763 realtime-audio prototype plan + +Date: 2026-08-24 +Status: active +Linear: H-6763 — support for realtime audio interviewing of domain experts + +## Decision to make + +Choose the voice edge for the September Petrinaut elicitation experience without moving +authoritative conversation state or elicitation policy out of Brunch. + +Two experiments answer different questions: + +1. **OpenAI Realtime with dummy tools** establishes the interaction-quality ceiling without + porting Brunch prompts, state, or tools. +2. **ElevenLabs around the real elicitor** tests whether a speech edge can preserve the accepted + architecture while providing an acceptable experience. + +ElevenLabs is the default architectural lean. OpenAI Realtime wins only if its interaction quality +is materially better and the team explicitly accepts the demonstrated porting cost. + +## Accepted placement and boundaries + +- `apps/petrinaut-website` owns the user-facing voice controls and provider adapters. +- `apps/brunch-agent` remains the Petrinaut-independent remote elicitor server. +- The existing AI SDK `/api/chat` transport remains the real-elicitor seam. +- Brunch owns authoritative session history, structured questions, captures, and provenance. +- Providers may own browser audio, ASR, TTS, endpointing, and interruption detection, but their + conversation history is never authoritative. +- Partial transcripts are display-only. Only admitted final human utterances may become evidence. + +## Prototype sequence + +### 0. Establish the base + +- Branch H-6763 from Lu's FE-1437 monorepo-import branch while it is under review. +- Verify the real Petrinaut panel can reach `apps/brunch-agent` through `/api/chat`. +- Keep the stock assistant unchanged when the prototype is disabled. + +### 1. Shared website shell + +- Add a query-enabled prototype panel to the local-storage Petrinaut application using only + `?voiceExperiment=openai-realtime` and `?voiceExperiment=elevenlabs-brunch`. +- Select an experiment for the lifetime of the page. Never switch providers inside an active + conversation; changing experiments must dispose the adapter and reload into a new conversation. +- Add shared hold-to-speak, start/end controls, scenario script, connection state, visible + transcript, and event/timing log. +- Keep the shell behind a narrow adapter contract covering connect, turn start/finish, disposal, + and normalized observable events. Provider conversation and tool models stay private. +- Keep provider credentials server-side. +- Instrument transcript, response, and tool events for side-by-side evaluation. + +### 2. OpenAI Realtime experiment + +- Connect over WebRTC. +- Use a small interview prompt and representative dummy tools only. +- Record perceived latency, transcript quality, tool-call reliability, and the Brunch behavior that + would need to be copied or bridged. +- Stop before porting the real elicitor. + +### 3. ElevenLabs experiment + +- Use ElevenLabs for the browser audio edge and speech conversion. +- Submit each final transcript through the real Brunch human-input path. +- Stream the elicitor's text response back for speech playback. +- Prove a spoken answer resumes the same `brunch_ask` and yields the next real elicitor turn. + +### 4. Team decision + +Run the same scenario at least three times with each provider and compare interaction quality, +transcription, tool fidelity, elicitor fidelity, state integrity, integration complexity, and +remaining work to satisfy H-6763. + +## Prototype exclusions + +- Open microphone and interruption semantics +- Provider abstraction intended for production reuse +- Durable transcript persistence +- Live capture extraction from partial transcripts +- Net projection or client-tool mutation from the voice adapter +- Gemini Live unless both primary experiments fail or social fluency becomes decisive + +## Prototype exit criteria + +- One draft Petrinaut website PR contains the shared shell and both experiments behind an explicit + query parameter. +- OpenAI Realtime completes the scripted interview with dummy tools. +- ElevenLabs completes at least one real `brunch_ask` suspend-and-resume cycle. +- Both experiments have short recordings and a completed comparison table. +- The team records a provider decision before production hardening begins. + +## Post-decision work + +The winning path then adds monotonic turn ids, transcript persistence, private browser sessions, +cancellation and stale-audio invalidation, live captures with provenance, open-mic/VAD, and the +end-session transition to the separate IR-to-net projection and Petrinaut draft. From d66828a12487b86b283e9022a9d1b4e3f63c0d29 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 16:19:22 +0200 Subject: [PATCH 02/19] H-6763: Add the OpenAI Realtime voice experiment --- apps/petrinaut-website/README.md | 27 +- .../openai-realtime-session.test.ts | 181 ++++++ .../openai-realtime-session.ts | 289 +++++++++ .../local-storage-demo-app.tsx | 13 +- .../local-storage-demo/voice-experiment.tsx | 161 ++++- .../openai-realtime-adapter.test.ts | 291 +++++++++ .../openai-realtime-adapter.ts | 570 ++++++++++++++++++ .../voice-experiment-events.ts | 2 + apps/petrinaut-website/vercel.json | 3 + apps/petrinaut-website/vite.config.ts | 51 +- ...763-realtime-audio-prototype-2026-08-24.md | 14 +- 11 files changed, 1541 insertions(+), 61 deletions(-) create mode 100644 apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts create mode 100644 apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index d4fc4c4140f..3e9d3f7e0e3 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -2,13 +2,13 @@ A website for demoing Petrinaut (libs/@hashintel/petrinaut). -A SPA plus a single API function that proxies AI requests to OpenAI. +A SPA plus API functions that proxy text and experimental Realtime requests to OpenAI. ## Quickstart ```sh cp .env.example .env.local -# add your OPENAI_API_KEY to .env.local, if you want to use the chat feature +# add your OPENAI_API_KEY to .env.local for chat or the OpenAI Realtime experiment turbo run dev ``` @@ -38,19 +38,23 @@ optimizer for isolated UI development. ## Environment variables -| Name | Required | Used by | Notes | -| ----------------------------- | ---------------- | ---------------- | --------------------------------------------------------- | -| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | +| Name | Required | Used by | Notes | +| ----------------------------- | ------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| `OPENAI_API_KEY` | for OpenAI features | `api/chat.ts`, `api/voice-experiment/openai-realtime-session.ts` | Server-only OpenAI key used for chat and to mint ephemeral Realtime client secrets. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | +| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | -Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the chat function. In production, set these in the Vercel project settings. +Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. The standard OpenAI key must never be exposed through a `VITE_` variable or sent to the browser. + +The OpenAI voice experiment is available at +`/?voiceExperiment=openai-realtime`. Its same-origin session endpoint returns only a short-lived +client secret; model, prompt, transcription, and dummy-tool configuration are fixed on the server. ## Testing the API against the built output -A plain `yarn build && yarn vite preview` only serves the static `dist/` assets - `/api/chat` will 404 because the dev plugin is not loaded by `vite preview`. Use one of the options below to exercise the production code path locally. +A plain `yarn build && yarn vite preview` only serves the static `dist/` assets - `/api/*` will 404 because the dev plugin is not loaded by `vite preview`. Use one of the options below to exercise the production code path locally. ### Option A: `vercel dev` (recommended) @@ -107,4 +111,5 @@ Useful when you want to serve the literal `dist/` artifact you just built and av ## Known caveats - **In-memory rate limiting.** [`api/chat.ts`](api/chat.ts) keys rate-limit buckets by the client IP that Vercel's edge writes into `x-forwarded-for` (which Vercel actively prevents the caller from spoofing - see the [request headers docs](https://vercel.com/docs/edge-network/headers/request-headers)). The bucket map lives in module scope, so it resets on cold start and is not shared between concurrent function instances. +- **The Realtime experiment uses the same rate-limit shape.** Its tighter bucket protects client-secret minting, but is still per warm function instance rather than a durable distributed limit. - **`vercel-build.sh` deletes the repo-root `.env`.** This is intentional (mise picks it up otherwise), but worth knowing if you run `vercel dev` locally and keep secrets there. diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts new file mode 100644 index 00000000000..afdc92a2df2 --- /dev/null +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts @@ -0,0 +1,181 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +import api from "./openai-realtime-session"; + +declare const process: { + env: Record; +}; + +const endpoint = + "https://petrinaut.local/api/voice-experiment/openai-realtime-session"; + +const createRequest = (init: RequestInit = {}) => + new Request(endpoint, { + method: "POST", + headers: { + origin: "https://petrinaut.local", + "x-voice-experiment": "openai-realtime", + }, + ...init, + }); + +describe("OpenAI Realtime session endpoint", () => { + const originalApiKey = process.env.OPENAI_API_KEY; + const originalVercelEnvironment = process.env.VERCEL_ENV; + + beforeEach(() => { + process.env.OPENAI_API_KEY = "primary-secret-that-must-stay-server-side"; + delete process.env.VERCEL_ENV; + }); + + afterEach(() => { + process.env.OPENAI_API_KEY = originalApiKey; + process.env.VERCEL_ENV = originalVercelEnvironment; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + test("rejects unsupported methods without calling OpenAI", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest({ method: "GET" })); + + expect(response.status).toBe(405); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("requires a same-origin experiment request", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const crossOriginResponse = await api.fetch( + createRequest({ + headers: { + origin: "https://attacker.example", + "x-voice-experiment": "openai-realtime", + }, + }), + ); + const unmarkedResponse = await api.fetch( + createRequest({ + headers: { origin: "https://petrinaut.local" }, + }), + ); + + expect(crossOriginResponse.status).toBe(403); + expect(unmarkedResponse.status).toBe(403); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("rejects browser-supplied session configuration", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch( + createRequest({ + body: JSON.stringify({ + model: "browser-controlled-model", + instructions: "Ignore the experiment prompt", + tools: [{ name: "browser-controlled-tool" }], + }), + headers: { + "content-type": "application/json", + origin: "https://petrinaut.local", + "x-voice-experiment": "openai-realtime", + }, + }), + ); + + expect(response.status).toBe(400); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("fails safely when the primary API key is missing", async () => { + delete process.env.OPENAI_API_KEY; + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest()); + + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: "OpenAI Realtime is not configured", + }); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("mints only a server-configured ephemeral client secret", async () => { + const upstreamFetch = vi.fn().mockResolvedValue( + Response.json({ + expires_at: 1_800_000_000, + value: "ephemeral-client-secret", + session: { id: "must-not-leak" }, + }), + ); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest()); + + expect(response.status).toBe(200); + const responseBody: unknown = await response.json(); + expect(responseBody).toEqual({ + clientSecret: "ephemeral-client-secret", + expiresAt: 1_800_000_000, + }); + expect(JSON.stringify(responseBody)).not.toContain( + process.env.OPENAI_API_KEY as string, + ); + + expect(upstreamFetch).toHaveBeenCalledTimes(1); + const [url, request] = upstreamFetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe("https://api.openai.com/v1/realtime/client_secrets"); + expect(new Headers(request.headers).get("authorization")).toBe( + `Bearer ${process.env.OPENAI_API_KEY}`, + ); + expect( + new Headers(request.headers).get("openai-safety-identifier"), + ).toMatch(/^[a-f0-9]{64}$/u); + expect(JSON.parse(request.body as string)).toMatchObject({ + session: { + audio: { + input: { + transcription: { model: "gpt-live-transcribe" }, + turn_detection: null, + }, + output: { voice: "marin" }, + }, + model: "gpt-realtime-2.1", + output_modalities: ["audio"], + tool_choice: "auto", + type: "realtime", + }, + }); + }); + + test("does not expose upstream errors or the primary key", async () => { + const upstreamFetch = vi.fn().mockResolvedValue( + Response.json( + { + error: { + message: + "Detailed upstream failure containing primary-secret-that-must-stay-server-side", + }, + }, + { status: 401 }, + ), + ); + vi.stubGlobal("fetch", upstreamFetch); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const response = await api.fetch(createRequest()); + const body = await response.text(); + + expect(response.status).toBe(502); + expect(body).toBe( + JSON.stringify({ error: "Could not start voice session" }), + ); + expect(body).not.toContain("Detailed upstream failure"); + expect(body).not.toContain(process.env.OPENAI_API_KEY as string); + }); +}); diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts new file mode 100644 index 00000000000..d8749a78461 --- /dev/null +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts @@ -0,0 +1,289 @@ +import { z } from "zod"; + +declare const process: { + env: Record; +}; + +const OPENAI_CLIENT_SECRETS_URL = + "https://api.openai.com/v1/realtime/client_secrets"; +const RATE_LIMIT_WINDOW_MS = 60_000; +const RATE_LIMIT_MAX_REQUESTS = 10; +const RATE_LIMIT_MAX_TRACKED_CLIENTS = 10_000; +const UPSTREAM_TIMEOUT_MS = 10_000; + +const sessionConfig = { + session: { + type: "realtime", + model: "gpt-realtime-2.1", + output_modalities: ["audio"], + instructions: [ + "You are running a short voice interview experiment with a domain expert.", + "Interview the expert about how an urgent customer support escalation moves from the initial report to resolution.", + "Ask one focused question at a time. Briefly acknowledge the answer, then ask the highest-value follow-up.", + "Use record_process_step when the expert describes a process step. Use record_process_decision when they describe a branch or decision.", + "The tools are experiment-only instrumentation. Never claim that their output was persisted to Brunch or Petrinaut.", + "Keep spoken responses concise and natural.", + ].join(" "), + max_output_tokens: 600, + audio: { + input: { + transcription: { + model: "gpt-live-transcribe", + delay: "low", + prompt: + "A process-model interview about urgent customer support escalations, incident ownership, handoffs, decisions, and resolution.", + }, + turn_detection: null, + }, + output: { + voice: "marin", + }, + }, + tools: [ + { + type: "function", + name: "record_process_step", + description: + "Record a process step mentioned by the expert for experiment instrumentation only.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + description: { + type: "string", + description: "What happens during the process step.", + }, + name: { + type: "string", + description: "A short name for the process step.", + }, + owner: { + type: "string", + description: "The role or team that owns the step, if known.", + }, + }, + required: ["name", "description"], + }, + }, + { + type: "function", + name: "record_process_decision", + description: + "Record a branch or decision mentioned by the expert for experiment instrumentation only.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + condition: { + type: "string", + description: "The condition that determines the path taken.", + }, + outcomes: { + type: "array", + description: "The possible paths after the decision.", + items: { type: "string" }, + }, + }, + required: ["condition", "outcomes"], + }, + }, + ], + tool_choice: "auto", + }, +} as const; + +const upstreamResponseSchema = z.object({ + expires_at: z.number().int().positive(), + value: z.string().min(1), +}); + +const rateLimitBuckets = new Map(); + +const jsonResponse = (body: unknown, init: ResponseInit = {}) => { + const headers = new Headers(init.headers); + headers.set("cache-control", "no-store"); + headers.set("content-type", "application/json"); + return new Response(JSON.stringify(body), { ...init, headers }); +}; + +const logSessionFailure = ( + reason: string, + context: Record = {}, +) => { + // Never add request bodies, API keys, or upstream response bodies here. + // oxlint-disable-next-line no-console + console.error(`[OpenAI Realtime experiment] ${reason}`, context); +}; + +const resolveClientIp = (request: Request): string | null => { + const forwardedFor = request.headers.get("x-forwarded-for"); + if (forwardedFor) { + const first = forwardedFor.split(",")[0]?.trim(); + if (first) { + return first; + } + } + return request.headers.get("x-vercel-forwarded-for"); +}; + +const checkRateLimit = (clientKey: string): boolean => { + const now = Date.now(); + const current = rateLimitBuckets.get(clientKey); + + if (!current || current.resetAt <= now) { + if (rateLimitBuckets.size >= RATE_LIMIT_MAX_TRACKED_CLIENTS) { + for (const [key, bucket] of rateLimitBuckets) { + if (bucket.resetAt <= now) { + rateLimitBuckets.delete(key); + } + } + if (rateLimitBuckets.size >= RATE_LIMIT_MAX_TRACKED_CLIENTS) { + return false; + } + } + rateLimitBuckets.set(clientKey, { + count: 1, + resetAt: now + RATE_LIMIT_WINDOW_MS, + }); + return true; + } + + if (current.count >= RATE_LIMIT_MAX_REQUESTS) { + return false; + } + + current.count += 1; + return true; +}; + +const isTrustedBrowserRequest = (request: Request): boolean => { + const origin = request.headers.get("origin"); + const experiment = request.headers.get("x-voice-experiment"); + + return ( + origin === new URL(request.url).origin && experiment === "openai-realtime" + ); +}; + +const createSafetyIdentifier = async (value: string): Promise => { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(value), + ); + return [...new Uint8Array(digest)] + .map((byte) => byte.toString(16).padStart(2, "0")) + .join(""); +}; + +const fetch = async (request: Request): Promise => { + if (request.method === "OPTIONS") { + return new Response(null, { status: 204 }); + } + + if (request.method !== "POST") { + logSessionFailure("Rejected unsupported method", { + method: request.method, + }); + return jsonResponse({ error: "Method not allowed" }, { status: 405 }); + } + + if (!isTrustedBrowserRequest(request)) { + logSessionFailure("Rejected untrusted browser request"); + return jsonResponse({ error: "Forbidden" }, { status: 403 }); + } + + if ((await request.text()).trim() !== "") { + logSessionFailure("Rejected browser-supplied session configuration"); + return jsonResponse( + { error: "Request body must be empty" }, + { status: 400 }, + ); + } + + const clientIp = resolveClientIp(request); + if (process.env.VERCEL_ENV === "production" && !clientIp) { + logSessionFailure("Rejected production request without a client IP"); + return jsonResponse( + { error: "Could not determine client IP" }, + { status: 400 }, + ); + } + + const clientKey = clientIp ?? "local-development"; + if (!checkRateLimit(clientKey)) { + logSessionFailure("Rejected rate-limited request"); + return jsonResponse({ error: "Rate limit exceeded" }, { status: 429 }); + } + + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + logSessionFailure("Missing OpenAI API key"); + return jsonResponse( + { error: "OpenAI Realtime is not configured" }, + { status: 500 }, + ); + } + + const safetyIdentifier = await createSafetyIdentifier( + `${apiKey}:${clientKey}`, + ); + + let upstreamResponse: Response; + try { + upstreamResponse = await globalThis.fetch(OPENAI_CLIENT_SECRETS_URL, { + method: "POST", + headers: { + authorization: `Bearer ${apiKey}`, + "content-type": "application/json", + "openai-safety-identifier": safetyIdentifier, + }, + body: JSON.stringify(sessionConfig), + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + } catch (error) { + logSessionFailure("Client-secret request failed", { + errorName: error instanceof Error ? error.name : "unknown", + }); + return jsonResponse( + { error: "Could not start voice session" }, + { status: 502 }, + ); + } + + if (!upstreamResponse.ok) { + logSessionFailure("OpenAI rejected the client-secret request", { + status: upstreamResponse.status, + }); + return jsonResponse( + { error: "Could not start voice session" }, + { status: 502 }, + ); + } + + let upstreamBody: unknown; + try { + upstreamBody = await upstreamResponse.json(); + } catch { + logSessionFailure("OpenAI returned invalid JSON"); + return jsonResponse( + { error: "Could not start voice session" }, + { status: 502 }, + ); + } + + const parsed = upstreamResponseSchema.safeParse(upstreamBody); + if (!parsed.success) { + logSessionFailure("OpenAI returned an invalid client secret"); + return jsonResponse( + { error: "Could not start voice session" }, + { status: 502 }, + ); + } + + return jsonResponse({ + clientSecret: parsed.data.value, + expiresAt: parsed.data.expires_at, + }); +}; + +export default { fetch }; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index af64c9eb03f..8de94ae9fb6 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -24,6 +24,7 @@ import { useLocalStorageSDCPNs, } from "./use-local-storage-sdcpns"; import { VoiceExperiment } from "./voice-experiment"; +import { createOpenAIRealtimeAdapter } from "./voice-experiment/openai-realtime-adapter"; import { getVoiceExperiment } from "./voice-experiment/voice-experiment-selection"; import { walkthroughSteps } from "./walkthrough/walkthrough-steps"; @@ -121,6 +122,13 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); const voiceExperiment = getVoiceExperiment(window.location); + const voiceExperimentAdapter = useMemo( + () => + voiceExperiment === "openai-realtime" + ? createOpenAIRealtimeAdapter() + : undefined, + [voiceExperiment], + ); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); @@ -311,7 +319,10 @@ export const LocalStorageDemoApp = () => { viewportActions={[sentryFeedbackAction]} /> {voiceExperiment ? ( - + ) : null} diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx index 93d1da320f0..d2d408d5a09 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -112,8 +112,13 @@ const sectionLabelStyle = css({ }); const transcriptStyle = css({ + display: "flex", minHeight: "16", + maxHeight: "52", + flexDirection: "column", + gap: "2.5", padding: "3", + overflowY: "auto", borderWidth: "thin", borderStyle: "solid", borderColor: "neutral.a20", @@ -124,6 +129,46 @@ const transcriptStyle = css({ lineHeight: "relaxed", }); +const transcriptEntryStyle = css({ + display: "flex", + width: "[88%]", + flexDirection: "column", + gap: "0.5", +}); + +const expertTranscriptEntryStyle = css({ + marginLeft: "auto", + alignItems: "flex-end", +}); + +const transcriptSpeakerStyle = css({ + color: "neutral.s55", + fontSize: "xs", + fontWeight: "medium", +}); + +const transcriptBubbleStyle = css({ + paddingX: "3", + paddingY: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s80", +}); + +const expertTranscriptBubbleStyle = css({ + backgroundColor: "blue.a15", +}); + +const partialTranscriptStyle = css({ + opacity: "0.7", +}); + +const transcriptPlaceholderStyle = css({ + margin: "auto", + color: "neutral.s55", + textAlign: "center", +}); + const controlsStyle = css({ display: "flex", alignItems: "center", @@ -271,25 +316,39 @@ type LoggedEvent = { sequence: number; }; -const getLatestTranscript = (events: LoggedEvent[]) => { - for (let index = events.length - 1; index >= 0; index -= 1) { - const event = events[index]?.event; +type TranscriptEntry = { + isPartial: boolean; + speaker: "assistant" | "expert"; + transcript: string; + turnId: number; +}; + +const getTranscriptEntries = (events: LoggedEvent[]): TranscriptEntry[] => { + const entries = new Map(); + + for (const { event } of events) { if ( - event?.type === "partial-transcript" || - event?.type === "final-transcript" + event.type === "partial-transcript" || + event.type === "final-transcript" ) { - return event.transcript; + entries.set(`${event.turnId}:${event.speaker}`, { + isPartial: event.type === "partial-transcript", + speaker: event.speaker, + transcript: event.transcript, + turnId: event.turnId, + }); } } - return null; + + return [...entries.values()]; }; const getEventSummary = (event: VoiceExperimentEvent) => { if (event.type === "partial-transcript") { - return `${event.type}: ${event.transcript}`; + return `${event.type} (${event.speaker}): ${event.transcript}`; } if (event.type === "final-transcript") { - return `${event.type}: ${event.transcript}`; + return `${event.type} (${event.speaker}): ${event.transcript}`; } if (event.type === "tool-called") { return `${event.type}: ${event.toolName}`; @@ -385,7 +444,11 @@ export const VoiceExperiment = ({ }; const startTurn = async () => { - if (!adapter || sessionState !== "connected" || pressedRef.current) { + if ( + !adapter || + (sessionState !== "connected" && sessionState !== "responding") || + pressedRef.current + ) { return; } @@ -434,27 +497,34 @@ export const VoiceExperiment = ({ } }; - const transcript = getLatestTranscript(events); + const transcriptEntries = getTranscriptEntries(events); const isConnected = sessionState === "connected" || sessionState === "responding"; const isAdapterPending = !adapter; const canStartSession = Boolean(adapter) && sessionState === "ready"; - const canStartTurn = Boolean(adapter) && sessionState === "connected"; + const canEndSession = + Boolean(adapter) && + sessionState !== "ready" && + sessionState !== "ending" && + sessionState !== "ended"; + const canStartTurn = Boolean(adapter) && isConnected; const statusMessage = isAdapterPending ? "Adapter pending — this shell does not own microphone access" - : sessionState === "ready" - ? "Ready to start a clean session" - : sessionState === "connecting" - ? "Connecting…" - : sessionState === "connected" - ? "Connected — hold the microphone to speak" - : sessionState === "responding" - ? "Response in progress" - : sessionState === "ending" - ? "Ending session…" - : sessionState === "ended" - ? "Session ended — reload for a new conversation" - : "Experiment error"; + : isTurnActive + ? "Recording — release the microphone to send" + : sessionState === "ready" + ? "Ready to start a clean session" + : sessionState === "connecting" + ? "Connecting…" + : sessionState === "connected" + ? "Connected — hold the microphone to speak" + : sessionState === "responding" + ? "Response in progress" + : sessionState === "ending" + ? "Ending session…" + : sessionState === "ended" + ? "Session ended — reload for a new conversation" + : "Experiment error"; return ( ); }; From bca938dcdee3354d85b4129b0f7598b8a47ae63a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 17:27:18 +0200 Subject: [PATCH 04/19] H-6763: Add the ElevenLabs Brunch voice experiment --- apps/brunch-agent/.env.example | 5 + apps/brunch-agent/package.json | 4 +- .../petrinaut-local.vite.config.ts | 2 +- apps/brunch-agent/src/brunch-voice-bridge.ts | 215 +++++++++++++++ .../src/elevenlabs-speech-engine-server.ts | 78 ++++++ .../src/elevenlabs-speech-engine.ts | 100 +++++++ .../test/brunch-voice-bridge.test.ts | 223 +++++++++++++++ .../test/elevenlabs-speech-engine.test.ts | 96 +++++++ apps/petrinaut-website/.env.example | 2 + apps/petrinaut-website/README.md | 51 +++- .../elevenlabs-conversation-token.test.ts | 168 ++++++++++++ .../elevenlabs-conversation-token.ts | 185 +++++++++++++ apps/petrinaut-website/package.json | 1 + .../local-storage-demo-app.tsx | 17 +- .../elevenlabs-adapter.test.ts | 173 ++++++++++++ .../voice-experiment/elevenlabs-adapter.ts | 259 ++++++++++++++++++ apps/petrinaut-website/vercel.json | 3 + apps/petrinaut-website/vite.config.ts | 42 ++- yarn.lock | 130 ++++++++- 19 files changed, 1715 insertions(+), 39 deletions(-) create mode 100644 apps/brunch-agent/.env.example create mode 100644 apps/brunch-agent/src/brunch-voice-bridge.ts create mode 100644 apps/brunch-agent/src/elevenlabs-speech-engine-server.ts create mode 100644 apps/brunch-agent/src/elevenlabs-speech-engine.ts create mode 100644 apps/brunch-agent/test/brunch-voice-bridge.test.ts create mode 100644 apps/brunch-agent/test/elevenlabs-speech-engine.test.ts create mode 100644 apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.test.ts create mode 100644 apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts diff --git a/apps/brunch-agent/.env.example b/apps/brunch-agent/.env.example new file mode 100644 index 00000000000..7a79371414e --- /dev/null +++ b/apps/brunch-agent/.env.example @@ -0,0 +1,5 @@ +ELEVENLABS_API_KEY=sk_xxxx +ELEVENLABS_SPEECH_ENGINE_ID=seng_xxxx +ELEVENLABS_SPEECH_ENGINE_HOST=127.0.0.1 +ELEVENLABS_SPEECH_ENGINE_PORT=3001 +BRUNCH_CHAT_ORIGIN=http://127.0.0.1:4321 diff --git a/apps/brunch-agent/package.json b/apps/brunch-agent/package.json index fc199cec26f..40590f9fdcd 100644 --- a/apps/brunch-agent/package.json +++ b/apps/brunch-agent/package.json @@ -12,9 +12,11 @@ "lint:eslint": "oxlint --type-aware --report-unused-disable-directives-severity=error .", "lint:tsc": "tsgo --noEmit", "petrinaut:dev": "vite dev --config petrinaut-local.vite.config.ts", - "test:unit": "vitest run --config vitest.config.ts" + "test:unit": "vitest run --config vitest.config.ts", + "voice:dev": "node --env-file-if-exists=.env.local --experimental-strip-types src/elevenlabs-speech-engine-server.ts" }, "dependencies": { + "@elevenlabs/elevenlabs-js": "2.64.0", "@flue/react": "2.0.3", "@flue/runtime": "2.0.3", "@flue/sdk": "2.0.3", diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index 249686c8498..d234b55d427 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -35,7 +35,7 @@ const withoutIncumbentChatHandler = ( ) { return true; } - return plugin.name !== "petrinaut-api-dev"; + return plugin.name !== "petrinaut-chat-api-dev"; }); export default defineConfig(async (environment) => { diff --git a/apps/brunch-agent/src/brunch-voice-bridge.ts b/apps/brunch-agent/src/brunch-voice-bridge.ts new file mode 100644 index 00000000000..07fceb79b0b --- /dev/null +++ b/apps/brunch-agent/src/brunch-voice-bridge.ts @@ -0,0 +1,215 @@ +type BrunchVoiceBridgeDependencies = { + chatEndpoint: string; + createId?: () => string; + fetch?: typeof globalThis.fetch; +}; + +type VoiceTurn = { + conversationId: string; + signal: AbortSignal; + transcript: string; +}; + +type PendingAsk = { + assistantMessageId: string; + input: unknown; + toolCallId: string; +}; + +type BridgeSessionState = { + pendingAsk?: PendingAsk; +}; + +type UiStreamChunk = Record & { type: string }; + +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null + ? (value as Record) + : null; + +const stringProperty = ( + value: Record | null, + key: string, +): string | null => { + const candidate = value?.[key]; + return typeof candidate === "string" ? candidate : null; +}; + +const chunksFromFrame = (frame: string): UiStreamChunk[] => { + const data = frame + .split(/\r?\n/u) + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice("data:".length).trimStart()) + .join("\n"); + if (!data || data === "[DONE]") { + return []; + } + + try { + const parsed = JSON.parse(data) as unknown; + const record = asRecord(parsed); + return record && typeof record.type === "string" + ? [record as UiStreamChunk] + : []; + } catch { + throw new Error("Brunch returned an invalid voice response."); + } +}; + +const readUiMessageStream = async function* ( + body: ReadableStream, +): AsyncGenerator { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + buffer += decoder.decode(value, { stream: !done }); + + const frames = buffer.split(/\r?\n\r?\n/u); + buffer = frames.pop() ?? ""; + for (const frame of frames) { + yield* chunksFromFrame(frame); + } + + if (done) { + break; + } + } + + if (buffer.trim()) { + yield* chunksFromFrame(buffer); + } + } finally { + reader.releaseLock(); + } +}; + +/** + * Translates finalized Speech Engine turns into the existing AI SDK transport. + * Brunch remains authoritative: this bridge retains only enough correlation to + * return a spoken answer to a pending `brunch_ask` affordance. + */ +export class BrunchVoiceBridge { + readonly #chatEndpoint: string; + readonly #createId: () => string; + readonly #fetch: typeof globalThis.fetch; + readonly #sessions = new Map(); + + public constructor(dependencies: BrunchVoiceBridgeDependencies) { + this.#chatEndpoint = dependencies.chatEndpoint; + this.#createId = dependencies.createId ?? (() => crypto.randomUUID()); + this.#fetch = dependencies.fetch ?? globalThis.fetch; + } + + public async *respond({ + conversationId, + signal, + transcript, + }: VoiceTurn): AsyncGenerator { + const normalizedTranscript = transcript.trim(); + if (!normalizedTranscript) { + return; + } + + const state = this.#sessions.get(conversationId) ?? {}; + this.#sessions.set(conversationId, state); + const pendingAsk = state.pendingAsk; + // The answer is consumed when dispatched. A barge-in must not replay that + // same affordance after its previous response stream is cancelled. + state.pendingAsk = undefined; + + const brunchConversationId = `voice:${conversationId}`; + const body = pendingAsk + ? { + id: brunchConversationId, + trigger: "submit-message", + messageId: pendingAsk.assistantMessageId, + messages: [ + { + id: pendingAsk.assistantMessageId, + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "brunch_ask", + toolCallId: pendingAsk.toolCallId, + state: "output-available", + input: pendingAsk.input, + output: { answer: normalizedTranscript }, + }, + ], + }, + ], + } + : { + id: brunchConversationId, + trigger: "submit-message", + messages: [ + { + id: this.#createId(), + role: "user", + parts: [{ type: "text", text: normalizedTranscript }], + }, + ], + }; + + const response = await this.#fetch(this.#chatEndpoint, { + method: "POST", + headers: { + "content-type": "application/json", + "x-request-id": this.#createId(), + }, + body: JSON.stringify(body), + signal, + }); + if (!response.ok || !response.body) { + throw new Error("Brunch could not answer the voice turn."); + } + + let assistantMessageId: string | null = + pendingAsk?.assistantMessageId ?? null; + for await (const chunk of readUiMessageStream(response.body)) { + if (chunk.type === "start") { + assistantMessageId = stringProperty(chunk, "messageId"); + continue; + } + + if (chunk.type === "text-delta") { + const delta = stringProperty(chunk, "delta"); + if (delta) { + yield delta; + } + continue; + } + + if ( + chunk.type === "tool-input-available" && + stringProperty(chunk, "toolName") === "brunch_ask" + ) { + const toolCallId = stringProperty(chunk, "toolCallId"); + const input = chunk.input; + if (!assistantMessageId || !toolCallId) { + throw new Error("Brunch returned an invalid voice response."); + } + state.pendingAsk = { assistantMessageId, input, toolCallId }; + + const question = stringProperty(asRecord(input), "question"); + if (question) { + yield question; + } + continue; + } + + if (chunk.type === "error") { + throw new Error("Brunch could not answer the voice turn."); + } + } + } + + public release(conversationId: string): void { + this.#sessions.delete(conversationId); + } +} diff --git a/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts b/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts new file mode 100644 index 00000000000..ae3d9179414 --- /dev/null +++ b/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts @@ -0,0 +1,78 @@ +import { createServer } from "node:http"; + +import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; + +import { BrunchVoiceBridge } from "./brunch-voice-bridge.ts"; +import { createElevenLabsSpeechEngineCallbacks } from "./elevenlabs-speech-engine.ts"; + +const apiKey = process.env.ELEVENLABS_API_KEY; +const speechEngineId = process.env.ELEVENLABS_SPEECH_ENGINE_ID; +if (!apiKey || !speechEngineId) { + throw new Error( + "ELEVENLABS_API_KEY and ELEVENLABS_SPEECH_ENGINE_ID are required.", + ); +} + +const port = Number(process.env.ELEVENLABS_SPEECH_ENGINE_PORT ?? "3001"); +if (!Number.isInteger(port) || port < 1 || port > 65_535) { + throw new Error("ELEVENLABS_SPEECH_ENGINE_PORT must be a valid port."); +} + +const host = process.env.ELEVENLABS_SPEECH_ENGINE_HOST ?? "127.0.0.1"; +const brunchChatOrigin = + process.env.BRUNCH_CHAT_ORIGIN ?? "http://127.0.0.1:4321"; +const chatEndpoint = new URL("/api/chat", brunchChatOrigin).toString(); + +const bridge = new BrunchVoiceBridge({ chatEndpoint }); +const callbacks = createElevenLabsSpeechEngineCallbacks({ bridge }); +const elevenLabs = new ElevenLabsClient({ apiKey }); + +const httpServer = createServer((request, response) => { + if (request.method === "GET" && request.url === "/health") { + response.writeHead(200, { + "cache-control": "no-store", + "content-type": "application/json", + }); + response.end(JSON.stringify({ status: "ok" })); + return; + } + + response.writeHead(404, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "Not found" })); +}); + +// Authentication remains enabled: the SDK verifies ElevenLabs' signed JWT on +// every WebSocket upgrade before any transcript can reach Brunch. +const attachment = elevenLabs.speechEngine.attach( + speechEngineId, + httpServer, + "/ws", + callbacks, +); + +await new Promise((resolve, reject) => { + httpServer.once("error", reject); + httpServer.listen(port, host, resolve); +}); + +console.log( + `ElevenLabs Speech Engine listening on http://${host}:${port}/ws; Brunch chat is ${chatEndpoint}`, +); + +let shuttingDown = false; +const shutdown = async () => { + if (shuttingDown) { + return; + } + shuttingDown = true; + await attachment.close(); + await new Promise((resolve, reject) => { + httpServer.close((error) => (error ? reject(error) : resolve())); + }); +}; + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + void shutdown().finally(() => process.exit(0)); + }); +} diff --git a/apps/brunch-agent/src/elevenlabs-speech-engine.ts b/apps/brunch-agent/src/elevenlabs-speech-engine.ts new file mode 100644 index 00000000000..ea63770bfb2 --- /dev/null +++ b/apps/brunch-agent/src/elevenlabs-speech-engine.ts @@ -0,0 +1,100 @@ +import type { SpeechEngineCallbacks } from "@elevenlabs/elevenlabs-js"; + +type TranscriptMessage = { + content: string; + role: "agent" | "user"; +}; + +type VoiceBridge = { + release(conversationId: string): void; + respond(input: { + conversationId: string; + signal: AbortSignal; + transcript: string; + }): AsyncIterable; +}; + +type CallbackDependencies = { + bridge: VoiceBridge; + log?: (reason: string, context?: Record) => void; +}; + +const MAX_TRANSCRIPT_CHARACTERS = 12_000; + +const normalizeTranscript = (transcript: string): string => { + let sanitized = ""; + for (const character of transcript.normalize("NFKC")) { + const codePoint = character.codePointAt(0) ?? 0; + if ( + codePoint === 9 || + codePoint === 10 || + codePoint === 13 || + codePoint >= 32 + ) { + sanitized += character; + } + } + + return sanitized + .replace(/\s+/gu, " ") + .trim() + .slice(0, MAX_TRANSCRIPT_CHARACTERS); +}; + +const latestUserTranscript = (transcript: TranscriptMessage[]): string => { + for (let index = transcript.length - 1; index >= 0; index -= 1) { + const message = transcript[index]; + if (message?.role === "user") { + return normalizeTranscript(message.content); + } + } + return ""; +}; + +const defaultLog = (reason: string, context: Record = {}) => { + // Provider transcript, response text, and secrets must never be logged here. + console.error(`[ElevenLabs Speech Engine] ${reason}`, context); +}; + +export const createElevenLabsSpeechEngineCallbacks = ({ + bridge, + log = defaultLog, +}: CallbackDependencies): SpeechEngineCallbacks => ({ + debug: process.env.NODE_ENV !== "production", + onTranscript(transcript, signal, session) { + const conversationId = session.conversationId; + if (!conversationId) { + log("Rejected transcript before session initialization"); + return; + } + const userTurn = latestUserTranscript(transcript); + const response = userTurn + ? bridge.respond({ + conversationId, + signal, + transcript: userTurn, + }) + : "I didn't catch that. Please hold the button and try again."; + + void session.sendResponse(response).catch((error: unknown) => { + if (!signal.aborted) { + log("Could not stream the Brunch voice response", { + errorName: error instanceof Error ? error.name : "unknown", + }); + } + }); + }, + onClose(session) { + if (session.conversationId) { + bridge.release(session.conversationId); + } + }, + onDisconnect(session) { + if (session.conversationId) { + bridge.release(session.conversationId); + } + }, + onError(error) { + log("Speech Engine session failed", { errorName: error.name }); + }, +}); diff --git a/apps/brunch-agent/test/brunch-voice-bridge.test.ts b/apps/brunch-agent/test/brunch-voice-bridge.test.ts new file mode 100644 index 00000000000..01cdc4f5857 --- /dev/null +++ b/apps/brunch-agent/test/brunch-voice-bridge.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, test, vi } from "vitest"; + +import { BrunchVoiceBridge } from "../src/brunch-voice-bridge.ts"; + +const encoder = new TextEncoder(); + +const sseResponse = (...chunks: Record[]) => + new Response( + new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue( + encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`), + ); + } + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + controller.close(); + }, + }), + { + headers: { + "content-type": "text/event-stream; charset=utf-8", + "x-vercel-ai-ui-message-stream": "v1", + }, + }, + ); + +const collect = async (stream: AsyncIterable) => { + let text = ""; + for await (const chunk of stream) { + text += chunk; + } + return text; +}; + +describe("BrunchVoiceBridge", () => { + test("sends the first finalized transcript through the existing chat route", async () => { + const fetch = vi + .fn() + .mockResolvedValue( + sseResponse( + { type: "start", messageId: "assistant-1" }, + { type: "text-delta", id: "text-1", delta: "Tell me more." }, + { type: "finish", finishReason: "stop" }, + ), + ); + const bridge = new BrunchVoiceBridge({ + chatEndpoint: "http://127.0.0.1:4321/api/chat", + createId: () => "generated-user-message", + fetch, + }); + const signal = new AbortController().signal; + + const reply = await collect( + bridge.respond({ + conversationId: "conv_elevenlabs", + signal, + transcript: "The support lead triages the escalation.", + }), + ); + + expect(reply).toBe("Tell me more."); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, request] = fetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe("http://127.0.0.1:4321/api/chat"); + expect(request.signal).toBe(signal); + expect(JSON.parse(request.body as string)).toEqual({ + id: "voice:conv_elevenlabs", + trigger: "submit-message", + messages: [ + { + id: "generated-user-message", + role: "user", + parts: [ + { + type: "text", + text: "The support lead triages the escalation.", + }, + ], + }, + ], + }); + }); + + test("speaks brunch_ask and submits the next transcript as its human answer", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce( + sseResponse( + { type: "start", messageId: "assistant-ask" }, + { + type: "tool-input-available", + toolCallId: "tool-call-1", + toolName: "brunch_ask", + input: { question: "Who owns the first response?" }, + }, + { type: "finish", finishReason: "tool-calls" }, + ), + ) + .mockResolvedValueOnce( + sseResponse( + { type: "start", messageId: "assistant-ask" }, + { + type: "text-delta", + id: "text-2", + delta: "The support lead owns it. What happens next?", + }, + { type: "finish", finishReason: "stop" }, + ), + ); + const bridge = new BrunchVoiceBridge({ + chatEndpoint: "http://127.0.0.1:4321/api/chat", + createId: () => "generated-user-message", + fetch, + }); + + const question = await collect( + bridge.respond({ + conversationId: "conv_elevenlabs", + signal: new AbortController().signal, + transcript: "Help me model our escalation process.", + }), + ); + const answerReply = await collect( + bridge.respond({ + conversationId: "conv_elevenlabs", + signal: new AbortController().signal, + transcript: "The support lead.", + }), + ); + + expect(question).toBe("Who owns the first response?"); + expect(answerReply).toBe("The support lead owns it. What happens next?"); + const [, secondRequest] = fetch.mock.calls[1] as [string, RequestInit]; + expect(JSON.parse(secondRequest.body as string)).toEqual({ + id: "voice:conv_elevenlabs", + trigger: "submit-message", + messageId: "assistant-ask", + messages: [ + { + id: "assistant-ask", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "brunch_ask", + toolCallId: "tool-call-1", + state: "output-available", + input: { question: "Who owns the first response?" }, + output: { answer: "The support lead." }, + }, + ], + }, + ], + }); + }); + + test("forgets provider history and keeps sessions isolated", async () => { + const fetch = vi + .fn() + .mockResolvedValue( + sseResponse( + { type: "start", messageId: "assistant-1" }, + { type: "text-delta", id: "text-1", delta: "Understood." }, + { type: "finish", finishReason: "stop" }, + ), + ); + const bridge = new BrunchVoiceBridge({ + chatEndpoint: "http://127.0.0.1:4321/api/chat", + createId: () => "message-id", + fetch, + }); + + await collect( + bridge.respond({ + conversationId: "conv_one", + signal: new AbortController().signal, + transcript: "Only this finalized turn.", + }), + ); + await collect( + bridge.respond({ + conversationId: "conv_two", + signal: new AbortController().signal, + transcript: "A different expert's turn.", + }), + ); + + const requestBodies = fetch.mock.calls.map(([, request]) => + JSON.parse((request as RequestInit).body as string), + ) as { id: string; messages: unknown[] }[]; + expect(requestBodies.map(({ id }) => id)).toEqual([ + "voice:conv_one", + "voice:conv_two", + ]); + expect(JSON.stringify(requestBodies)).not.toContain("provider history"); + }); + + test("surfaces a safe error without consuming an upstream body", async () => { + const text = vi.fn().mockResolvedValue("private upstream details"); + const fetch = vi.fn().mockResolvedValue({ + body: null, + ok: false, + status: 500, + text, + }); + const bridge = new BrunchVoiceBridge({ + chatEndpoint: "http://127.0.0.1:4321/api/chat", + fetch, + }); + + await expect( + collect( + bridge.respond({ + conversationId: "conv_error", + signal: new AbortController().signal, + transcript: "A turn that fails.", + }), + ), + ).rejects.toThrow("Brunch could not answer the voice turn."); + expect(text).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/brunch-agent/test/elevenlabs-speech-engine.test.ts b/apps/brunch-agent/test/elevenlabs-speech-engine.test.ts new file mode 100644 index 00000000000..9078ddbf905 --- /dev/null +++ b/apps/brunch-agent/test/elevenlabs-speech-engine.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test, vi } from "vitest"; + +import { createElevenLabsSpeechEngineCallbacks } from "../src/elevenlabs-speech-engine.ts"; + +const collect = async (response: string | AsyncIterable) => { + if (typeof response === "string") { + return response; + } + let text = ""; + for await (const chunk of response) { + text += String(chunk); + } + return text; +}; + +const createSession = () => { + let responsePromise: Promise | null = null; + const session = { + conversationId: "conv_speech_engine", + sendResponse: vi.fn(async (response: string | AsyncIterable) => { + responsePromise = collect(response); + await responsePromise; + }), + }; + return { + response: async () => { + await vi.waitFor(() => expect(responsePromise).not.toBeNull()); + return responsePromise!; + }, + session, + }; +}; + +describe("ElevenLabs Speech Engine callbacks", () => { + test("forwards only the latest normalized user turn and the interruption signal", async () => { + const respond = vi.fn(async function* () { + yield "Brunch response"; + }); + const bridge = { release: vi.fn(), respond }; + const callbacks = createElevenLabsSpeechEngineCallbacks({ bridge }); + const { response, session } = createSession(); + const signal = new AbortController().signal; + + callbacks.onTranscript?.( + [ + { role: "user", content: "Old provider history" }, + { role: "agent", content: "Old provider response" }, + { + role: "user", + content: " The\u0000 support\n lead owns triage. ", + }, + ], + signal, + session as never, + ); + + expect(await response()).toBe("Brunch response"); + expect(respond).toHaveBeenCalledWith({ + conversationId: "conv_speech_engine", + signal, + transcript: "The support lead owns triage.", + }); + expect(JSON.stringify(respond.mock.calls)).not.toContain( + "Old provider history", + ); + }); + + test("does not invoke Brunch for an empty transcript", async () => { + const bridge = { release: vi.fn(), respond: vi.fn() }; + const callbacks = createElevenLabsSpeechEngineCallbacks({ bridge }); + const { response, session } = createSession(); + + callbacks.onTranscript?.( + [{ role: "user", content: "\u0000 \n" }], + new AbortController().signal, + session as never, + ); + + expect(await response()).toBe( + "I didn't catch that. Please hold the button and try again.", + ); + expect(bridge.respond).not.toHaveBeenCalled(); + }); + + test("releases bridge correlation on clean and unexpected disconnects", () => { + const bridge = { release: vi.fn(), respond: vi.fn() }; + const callbacks = createElevenLabsSpeechEngineCallbacks({ bridge }); + const session = { conversationId: "conv_speech_engine" }; + + callbacks.onClose?.(session as never); + callbacks.onDisconnect?.(session as never); + + expect(bridge.release).toHaveBeenNthCalledWith(1, "conv_speech_engine"); + expect(bridge.release).toHaveBeenNthCalledWith(2, "conv_speech_engine"); + }); +}); diff --git a/apps/petrinaut-website/.env.example b/apps/petrinaut-website/.env.example index dfb9d72a1b6..0496a6b5d63 100644 --- a/apps/petrinaut-website/.env.example +++ b/apps/petrinaut-website/.env.example @@ -1 +1,3 @@ OPENAI_API_KEY=sk-xxxx +ELEVENLABS_API_KEY=sk_xxxx +ELEVENLABS_SPEECH_ENGINE_ID=seng_xxxx diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 3e9d3f7e0e3..60a54942215 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -2,13 +2,14 @@ A website for demoing Petrinaut (libs/@hashintel/petrinaut). -A SPA plus API functions that proxy text and experimental Realtime requests to OpenAI. +A SPA plus API functions for text chat and the OpenAI Realtime and ElevenLabs +Speech Engine voice experiments. ## Quickstart ```sh cp .env.example .env.local -# add your OPENAI_API_KEY to .env.local for chat or the OpenAI Realtime experiment +# add the provider values needed by the experiment you are running turbo run dev ``` @@ -38,20 +39,48 @@ optimizer for isolated UI development. ## Environment variables -| Name | Required | Used by | Notes | -| ----------------------------- | ------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------- | -| `OPENAI_API_KEY` | for OpenAI features | `api/chat.ts`, `api/voice-experiment/openai-realtime-session.ts` | Server-only OpenAI key used for chat and to mint ephemeral Realtime client secrets. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | +| Name | Required | Used by | Notes | +| ----------------------------- | -------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| `OPENAI_API_KEY` | for OpenAI features | `api/chat.ts`, `api/voice-experiment/openai-realtime-session.ts` | Server-only OpenAI key used for chat and to mint ephemeral Realtime client secrets. | +| `ELEVENLABS_API_KEY` | for ElevenLabs voice | `api/voice-experiment/elevenlabs-conversation-token.ts` | Server-only key used to mint short-lived WebRTC conversation tokens. | +| `ELEVENLABS_SPEECH_ENGINE_ID` | for ElevenLabs voice | `api/voice-experiment/elevenlabs-conversation-token.ts` | Server-owned `seng_…` resource id; the browser cannot override it. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | +| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | -Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. The standard OpenAI key must never be exposed through a `VITE_` variable or sent to the browser. +Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. Provider API keys must never be exposed through a `VITE_` variable or sent to the browser. The OpenAI voice experiment is available at `/?voiceExperiment=openai-realtime`. Its same-origin session endpoint returns only a short-lived client secret; model, prompt, transcription, and dummy-tool configuration are fixed on the server. +## ElevenLabs + Brunch voice experiment + +The real-elicitor experiment is available at +`/?voiceExperiment=elevenlabs-brunch`. ElevenLabs owns browser WebRTC, speech recognition, +turn-taking, speech synthesis, playback, and interruption detection. The Speech Engine server +forwards only the latest finalized expert transcript into Brunch's existing `/api/chat` transport; +Brunch remains authoritative for the session and `brunch_ask` state. + +For local development: + +1. Copy `.env.example` to `.env.local` in both `apps/petrinaut-website` and + `apps/brunch-agent`, then set the same `ELEVENLABS_API_KEY` and + `ELEVENLABS_SPEECH_ENGINE_ID` in both files. +2. Start the real Brunch server on `127.0.0.1:4321` as usual. +3. Run `yarn workspace @apps/brunch-agent voice:dev`. It serves the authenticated Speech Engine + WebSocket at `ws://127.0.0.1:3001/ws` and a health check at `/health`. +4. Expose port `3001` through a public HTTPS tunnel and configure the ElevenLabs Speech Engine + resource's WebSocket URL as `wss:///ws`. +5. Start the real-panel launcher with `PETRINAUT_WEBSITE_ROOT` pointing at this app and open + `http://127.0.0.1:4915/?voiceExperiment=elevenlabs-brunch`. + +The browser receives only a short-lived conversation token. The primary ElevenLabs key is used +by the website token endpoint and the authenticated Speech Engine server, never by browser code. +The first slice intentionally uses hold-to-speak; starting a new turn lets ElevenLabs interrupt +playback and aborts the in-flight Brunch request through the SDK's `AbortSignal`. + ## Testing the API against the built output A plain `yarn build && yarn vite preview` only serves the static `dist/` assets - `/api/*` will 404 because the dev plugin is not loaded by `vite preview`. Use one of the options below to exercise the production code path locally. @@ -111,5 +140,5 @@ Useful when you want to serve the literal `dist/` artifact you just built and av ## Known caveats - **In-memory rate limiting.** [`api/chat.ts`](api/chat.ts) keys rate-limit buckets by the client IP that Vercel's edge writes into `x-forwarded-for` (which Vercel actively prevents the caller from spoofing - see the [request headers docs](https://vercel.com/docs/edge-network/headers/request-headers)). The bucket map lives in module scope, so it resets on cold start and is not shared between concurrent function instances. -- **The Realtime experiment uses the same rate-limit shape.** Its tighter bucket protects client-secret minting, but is still per warm function instance rather than a durable distributed limit. +- **The voice token endpoints use the same rate-limit shape.** Their tighter buckets protect ephemeral credential minting, but are still per warm function instance rather than a durable distributed limit. - **`vercel-build.sh` deletes the repo-root `.env`.** This is intentional (mise picks it up otherwise), but worth knowing if you run `vercel dev` locally and keep secrets there. diff --git a/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.test.ts b/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.test.ts new file mode 100644 index 00000000000..d69de683889 --- /dev/null +++ b/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +import api from "./elevenlabs-conversation-token"; + +declare const process: { + env: Record; +}; + +const endpoint = + "https://petrinaut.local/api/voice-experiment/elevenlabs-conversation-token"; + +const createRequest = (init: RequestInit = {}) => + new Request(endpoint, { + method: "POST", + headers: { + origin: "https://petrinaut.local", + "x-voice-experiment": "elevenlabs-brunch", + }, + ...init, + }); + +describe("ElevenLabs conversation-token endpoint", () => { + const originalApiKey = process.env.ELEVENLABS_API_KEY; + const originalSpeechEngineId = process.env.ELEVENLABS_SPEECH_ENGINE_ID; + const originalVercelEnvironment = process.env.VERCEL_ENV; + + beforeEach(() => { + process.env.ELEVENLABS_API_KEY = + "primary-elevenlabs-secret-that-must-stay-server-side"; + process.env.ELEVENLABS_SPEECH_ENGINE_ID = "seng_server_owned"; + delete process.env.VERCEL_ENV; + }); + + afterEach(() => { + process.env.ELEVENLABS_API_KEY = originalApiKey; + process.env.ELEVENLABS_SPEECH_ENGINE_ID = originalSpeechEngineId; + process.env.VERCEL_ENV = originalVercelEnvironment; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + test("rejects unsupported methods without calling ElevenLabs", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest({ method: "GET" })); + + expect(response.status).toBe(405); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("requires a same-origin ElevenLabs experiment request", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const crossOriginResponse = await api.fetch( + createRequest({ + headers: { + origin: "https://attacker.example", + "x-voice-experiment": "elevenlabs-brunch", + }, + }), + ); + const unmarkedResponse = await api.fetch( + createRequest({ + headers: { origin: "https://petrinaut.local" }, + }), + ); + + expect(crossOriginResponse.status).toBe(403); + expect(unmarkedResponse.status).toBe(403); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test("rejects browser-supplied speech-engine configuration", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch( + createRequest({ + body: JSON.stringify({ speechEngineId: "seng_browser_controlled" }), + headers: { + "content-type": "application/json", + origin: "https://petrinaut.local", + "x-voice-experiment": "elevenlabs-brunch", + }, + }), + ); + + expect(response.status).toBe(400); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + + test.each(["ELEVENLABS_API_KEY", "ELEVENLABS_SPEECH_ENGINE_ID"])( + "fails safely when %s is missing", + async (variable) => { + delete process.env[variable]; + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest()); + + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: "ElevenLabs voice is not configured", + }); + expect(upstreamFetch).not.toHaveBeenCalled(); + }, + ); + + test("mints only a token for the server-configured speech engine", async () => { + const upstreamFetch = vi.fn().mockResolvedValue( + Response.json({ + token: "short-lived-conversation-token", + conversation_id: "conv_123", + speech_engine: { id: "must-not-leak" }, + }), + ); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch(createRequest()); + + expect(response.status).toBe(200); + const responseBody: unknown = await response.json(); + expect(responseBody).toEqual({ + conversationId: "conv_123", + conversationToken: "short-lived-conversation-token", + }); + expect(JSON.stringify(responseBody)).not.toContain( + process.env.ELEVENLABS_API_KEY as string, + ); + expect(JSON.stringify(responseBody)).not.toContain("seng_server_owned"); + + expect(upstreamFetch).toHaveBeenCalledTimes(1); + const [url, request] = upstreamFetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe( + "https://api.elevenlabs.io/v1/convai/conversation/token?agent_id=seng_server_owned", + ); + expect(request.method).toBe("GET"); + expect(new Headers(request.headers).get("xi-api-key")).toBe( + process.env.ELEVENLABS_API_KEY, + ); + }); + + test("does not expose upstream errors or the primary key", async () => { + const upstreamFetch = vi.fn().mockResolvedValue( + Response.json( + { + detail: + "Failure containing primary-elevenlabs-secret-that-must-stay-server-side", + }, + { status: 401 }, + ), + ); + vi.stubGlobal("fetch", upstreamFetch); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const response = await api.fetch(createRequest()); + const body = await response.text(); + + expect(response.status).toBe(502); + expect(body).toBe( + JSON.stringify({ error: "Could not start ElevenLabs voice session" }), + ); + expect(body).not.toContain("Failure containing"); + expect(body).not.toContain(process.env.ELEVENLABS_API_KEY as string); + }); +}); diff --git a/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.ts b/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.ts new file mode 100644 index 00000000000..bbd3dbd8f3f --- /dev/null +++ b/apps/petrinaut-website/api/voice-experiment/elevenlabs-conversation-token.ts @@ -0,0 +1,185 @@ +import { z } from "zod"; + +declare const process: { + env: Record; +}; + +const ELEVENLABS_CONVERSATION_TOKEN_URL = + "https://api.elevenlabs.io/v1/convai/conversation/token"; +const RATE_LIMIT_WINDOW_MS = 60_000; +const RATE_LIMIT_MAX_REQUESTS = 10; +const RATE_LIMIT_MAX_TRACKED_CLIENTS = 10_000; +const UPSTREAM_TIMEOUT_MS = 10_000; + +const upstreamResponseSchema = z.object({ + conversation_id: z.string().min(1), + token: z.string().min(1), +}); + +const rateLimitBuckets = new Map(); + +const jsonResponse = (body: unknown, init: ResponseInit = {}) => { + const headers = new Headers(init.headers); + headers.set("cache-control", "no-store"); + headers.set("content-type", "application/json"); + return new Response(JSON.stringify(body), { ...init, headers }); +}; + +const logTokenFailure = ( + reason: string, + context: Record = {}, +) => { + // Never add request bodies, API keys, or upstream response bodies here. + // oxlint-disable-next-line no-console + console.error(`[ElevenLabs voice experiment] ${reason}`, context); +}; + +const resolveClientIp = (request: Request): string | null => { + const forwardedFor = request.headers.get("x-forwarded-for"); + if (forwardedFor) { + const first = forwardedFor.split(",")[0]?.trim(); + if (first) { + return first; + } + } + return request.headers.get("x-vercel-forwarded-for"); +}; + +const checkRateLimit = (clientKey: string): boolean => { + const now = Date.now(); + const current = rateLimitBuckets.get(clientKey); + + if (!current || current.resetAt <= now) { + if (rateLimitBuckets.size >= RATE_LIMIT_MAX_TRACKED_CLIENTS) { + for (const [key, bucket] of rateLimitBuckets) { + if (bucket.resetAt <= now) { + rateLimitBuckets.delete(key); + } + } + if (rateLimitBuckets.size >= RATE_LIMIT_MAX_TRACKED_CLIENTS) { + return false; + } + } + rateLimitBuckets.set(clientKey, { + count: 1, + resetAt: now + RATE_LIMIT_WINDOW_MS, + }); + return true; + } + + if (current.count >= RATE_LIMIT_MAX_REQUESTS) { + return false; + } + + current.count += 1; + return true; +}; + +const isTrustedBrowserRequest = (request: Request): boolean => + request.headers.get("origin") === new URL(request.url).origin && + request.headers.get("x-voice-experiment") === "elevenlabs-brunch"; + +const fetch = async (request: Request): Promise => { + if (request.method === "OPTIONS") { + return new Response(null, { status: 204 }); + } + + if (request.method !== "POST") { + logTokenFailure("Rejected unsupported method", { method: request.method }); + return jsonResponse({ error: "Method not allowed" }, { status: 405 }); + } + + if (!isTrustedBrowserRequest(request)) { + logTokenFailure("Rejected untrusted browser request"); + return jsonResponse({ error: "Forbidden" }, { status: 403 }); + } + + if ((await request.text()).trim() !== "") { + logTokenFailure("Rejected browser-supplied speech-engine configuration"); + return jsonResponse( + { error: "Request body must be empty" }, + { status: 400 }, + ); + } + + const clientIp = resolveClientIp(request); + if (process.env.VERCEL_ENV === "production" && !clientIp) { + logTokenFailure("Rejected production request without a client IP"); + return jsonResponse( + { error: "Could not determine client IP" }, + { status: 400 }, + ); + } + + if (!checkRateLimit(clientIp ?? "local-development")) { + logTokenFailure("Rejected rate-limited request"); + return jsonResponse({ error: "Rate limit exceeded" }, { status: 429 }); + } + + const apiKey = process.env.ELEVENLABS_API_KEY; + const speechEngineId = process.env.ELEVENLABS_SPEECH_ENGINE_ID; + if (!apiKey || !speechEngineId) { + logTokenFailure("Missing ElevenLabs server configuration"); + return jsonResponse( + { error: "ElevenLabs voice is not configured" }, + { status: 500 }, + ); + } + + const upstreamUrl = new URL(ELEVENLABS_CONVERSATION_TOKEN_URL); + upstreamUrl.searchParams.set("agent_id", speechEngineId); + + let upstreamResponse: Response; + try { + upstreamResponse = await globalThis.fetch(upstreamUrl.toString(), { + method: "GET", + headers: { "xi-api-key": apiKey }, + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + } catch (error) { + logTokenFailure("Conversation-token request failed", { + errorName: error instanceof Error ? error.name : "unknown", + }); + return jsonResponse( + { error: "Could not start ElevenLabs voice session" }, + { status: 502 }, + ); + } + + if (!upstreamResponse.ok) { + logTokenFailure("ElevenLabs rejected the conversation-token request", { + status: upstreamResponse.status, + }); + return jsonResponse( + { error: "Could not start ElevenLabs voice session" }, + { status: 502 }, + ); + } + + let upstreamBody: unknown; + try { + upstreamBody = await upstreamResponse.json(); + } catch { + logTokenFailure("ElevenLabs returned invalid JSON"); + return jsonResponse( + { error: "Could not start ElevenLabs voice session" }, + { status: 502 }, + ); + } + + const parsed = upstreamResponseSchema.safeParse(upstreamBody); + if (!parsed.success) { + logTokenFailure("ElevenLabs returned an invalid conversation token"); + return jsonResponse( + { error: "Could not start ElevenLabs voice session" }, + { status: 502 }, + ); + } + + return jsonResponse({ + conversationId: parsed.data.conversation_id, + conversationToken: parsed.data.token, + }); +}; + +export default { fetch }; diff --git a/apps/petrinaut-website/package.json b/apps/petrinaut-website/package.json index 46e48eeb96f..97c24f5a918 100644 --- a/apps/petrinaut-website/package.json +++ b/apps/petrinaut-website/package.json @@ -17,6 +17,7 @@ }, "dependencies": { "@ai-sdk/openai": "3.0.63", + "@elevenlabs/client": "1.18.0", "@hashintel/brunch-agent-transport-aisdk": "workspace:*", "@hashintel/ds-components": "workspace:*", "@hashintel/ds-helpers": "workspace:*", diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 8de94ae9fb6..3b7ac3ab2c6 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -24,6 +24,7 @@ import { useLocalStorageSDCPNs, } from "./use-local-storage-sdcpns"; import { VoiceExperiment } from "./voice-experiment"; +import { createElevenLabsAdapter } from "./voice-experiment/elevenlabs-adapter"; import { createOpenAIRealtimeAdapter } from "./voice-experiment/openai-realtime-adapter"; import { getVoiceExperiment } from "./voice-experiment/voice-experiment-selection"; import { walkthroughSteps } from "./walkthrough/walkthrough-steps"; @@ -122,13 +123,15 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); const voiceExperiment = getVoiceExperiment(window.location); - const voiceExperimentAdapter = useMemo( - () => - voiceExperiment === "openai-realtime" - ? createOpenAIRealtimeAdapter() - : undefined, - [voiceExperiment], - ); + const voiceExperimentAdapter = useMemo(() => { + if (voiceExperiment === "openai-realtime") { + return createOpenAIRealtimeAdapter(); + } + if (voiceExperiment === "elevenlabs-brunch") { + return createElevenLabsAdapter(); + } + return undefined; + }, [voiceExperiment]); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts new file mode 100644 index 00000000000..b7232e211d5 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, test, vi } from "vitest"; + +import { createElevenLabsAdapter } from "./elevenlabs-adapter"; + +import type { VoiceExperimentEvent } from "./voice-experiment-events"; + +type SessionCallbacks = { + onConnect?: (event: { conversationId: string }) => void; + onDisconnect?: (details: { reason: string }) => void; + onError?: (message: string) => void; + onInterruption?: () => void; + onMessage?: (event: { + event_id?: number; + message: string; + role: "agent" | "user"; + }) => void; + onModeChange?: (event: { mode: "listening" | "speaking" }) => void; + onConversationCreated?: (conversation: FakeConversation) => void; +}; + +class FakeConversation { + public endSession = vi.fn(async () => undefined); + public setMicMuted = vi.fn(); +} + +const createHarness = () => { + const conversation = new FakeConversation(); + let callbacks: SessionCallbacks | null = null; + const startSession = vi.fn(async (options: SessionCallbacks) => { + callbacks = options; + options.onConversationCreated?.(conversation); + options.onConnect?.({ conversationId: "conv_123" }); + return conversation; + }); + const permissionTrack = { stop: vi.fn() }; + const getUserMedia = vi.fn(async () => ({ + getTracks: () => [permissionTrack], + })); + const fetch = vi.fn().mockResolvedValue( + Response.json({ + conversationId: "conv_123", + conversationToken: "short-lived-token", + }), + ); + let now = 1_000; + const adapter = createElevenLabsAdapter({ + fetch: fetch as typeof globalThis.fetch, + getUserMedia: getUserMedia as unknown as ( + constraints: MediaStreamConstraints, + ) => Promise, + now: () => ++now, + startSession, + }); + const events: VoiceExperimentEvent[] = []; + adapter.subscribe((event) => events.push(event)); + + return { + adapter, + callbacks: () => callbacks, + conversation, + events, + fetch, + getUserMedia, + permissionTrack, + startSession, + }; +}; + +describe("ElevenLabsAdapter", () => { + test("starts an authenticated WebRTC session with the microphone gated", async () => { + const harness = createHarness(); + + await harness.adapter.connect(); + + expect(harness.fetch).toHaveBeenCalledWith( + "/api/voice-experiment/elevenlabs-conversation-token", + expect.objectContaining({ + headers: { "x-voice-experiment": "elevenlabs-brunch" }, + method: "POST", + }), + ); + expect(harness.getUserMedia).toHaveBeenCalledWith({ audio: true }); + expect(harness.permissionTrack.stop).toHaveBeenCalledTimes(1); + expect(harness.startSession).toHaveBeenCalledWith( + expect.objectContaining({ + connectionType: "webrtc", + conversationToken: "short-lived-token", + }), + ); + expect(harness.conversation.setMicMuted).toHaveBeenCalledWith(true); + expect(harness.fetch.mock.calls.flat().join(" ")).not.toContain( + "/api/chat", + ); + expect(harness.events).toContainEqual({ + timestampMs: 1_001, + type: "connected", + }); + }); + + test("uses microphone mute as the hold-to-speak boundary", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + + await harness.adapter.startTurn(); + await harness.adapter.finishTurn(); + + expect(harness.conversation.setMicMuted.mock.calls).toEqual([ + [true], + [false], + [true], + ]); + expect(harness.events.at(-1)).toEqual({ + timestampMs: 1_002, + turnId: 1, + type: "recording-started", + }); + }); + + test("normalizes expert and Brunch response events", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + await harness.adapter.startTurn(); + await harness.adapter.finishTurn(); + + harness.callbacks()?.onMessage?.({ + event_id: 10, + message: "The support lead triages the escalation.", + role: "user", + }); + harness.callbacks()?.onModeChange?.({ mode: "speaking" }); + harness.callbacks()?.onMessage?.({ + event_id: 11, + message: "Who owns the next handoff?", + role: "agent", + }); + + expect(harness.events).toContainEqual({ + speaker: "expert", + timestampMs: 1_003, + transcript: "The support lead triages the escalation.", + turnId: 1, + type: "final-transcript", + }); + expect(harness.events).toContainEqual({ + timestampMs: 1_004, + turnId: 1, + type: "response-started", + }); + expect(harness.events).toContainEqual({ + speaker: "assistant", + timestampMs: 1_005, + transcript: "Who owns the next handoff?", + turnId: 1, + type: "final-transcript", + }); + expect(harness.events).toContainEqual({ + responseText: "Who owns the next handoff?", + timestampMs: 1_006, + turnId: 1, + type: "response-completed", + }); + }); + + test("releases ElevenLabs microphone, playback, and connection once", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + + await harness.adapter.dispose(); + await harness.adapter.dispose(); + + expect(harness.conversation.endSession).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts new file mode 100644 index 00000000000..22f268f2bbc --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts @@ -0,0 +1,259 @@ +import type { VoiceExperimentAdapter } from "./voice-experiment-adapter"; +import type { VoiceExperimentEvent } from "./voice-experiment-events"; + +const TOKEN_ENDPOINT = "/api/voice-experiment/elevenlabs-conversation-token"; + +type ConversationControl = { + endSession(): Promise; + setMicMuted(isMuted: boolean): void; +}; + +type SessionOptions = { + connectionType: "webrtc"; + conversationToken: string; + onConnect(event: { conversationId: string }): void; + onConversationCreated(conversation: ConversationControl): void; + onDisconnect(details: { reason: string }): void; + onError(message: string): void; + onInterruption(): void; + onMessage(event: { + event_id?: number; + message: string; + role: "agent" | "user"; + }): void; + onModeChange(event: { mode: "listening" | "speaking" }): void; +}; + +type ElevenLabsAdapterDependencies = { + fetch: typeof globalThis.fetch; + getUserMedia: (constraints: MediaStreamConstraints) => Promise; + now: () => number; + startSession: (options: SessionOptions) => Promise; +}; + +const defaultDependencies: ElevenLabsAdapterDependencies = { + fetch: (...args) => globalThis.fetch(...args), + getUserMedia: (constraints) => + navigator.mediaDevices.getUserMedia(constraints), + now: () => Date.now(), + startSession: async (options) => { + const { Conversation } = await import("@elevenlabs/client"); + return Conversation.startSession(options); + }, +}; + +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null + ? (value as Record) + : null; + +const getString = ( + value: Record | null, + key: string, +): string | null => { + const candidate = value?.[key]; + return typeof candidate === "string" ? candidate : null; +}; + +class ElevenLabsAdapter implements VoiceExperimentAdapter { + readonly #dependencies: ElevenLabsAdapterDependencies; + readonly #listeners = new Set<(event: VoiceExperimentEvent) => void>(); + + #connectPromise: Promise | null = null; + #connected = false; + #conversation: ConversationControl | null = null; + #disposed = false; + #responseInProgress = false; + #turnId = 0; + + public constructor(dependencies: ElevenLabsAdapterDependencies) { + this.#dependencies = dependencies; + } + + public subscribe(listener: (event: VoiceExperimentEvent) => void) { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + public async connect(): Promise { + if (this.#connected) { + return; + } + if (this.#connectPromise) { + return this.#connectPromise; + } + + this.#connectPromise = this.#establishConnection().finally(() => { + this.#connectPromise = null; + }); + return this.#connectPromise; + } + + public async startTurn(): Promise { + const conversation = this.#requireConversation(); + this.#turnId += 1; + this.#responseInProgress = false; + conversation.setMicMuted(false); + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId: this.#turnId, + type: "recording-started", + }); + } + + public async finishTurn(): Promise { + this.#requireConversation().setMicMuted(true); + } + + public async dispose(): Promise { + if (this.#disposed) { + return; + } + this.#disposed = true; + this.#connected = false; + const conversation = this.#conversation; + this.#conversation = null; + if (conversation) { + await conversation.endSession(); + } + } + + async #establishConnection(): Promise { + const tokenResponse = await this.#dependencies.fetch(TOKEN_ENDPOINT, { + method: "POST", + headers: { "x-voice-experiment": "elevenlabs-brunch" }, + }); + if (!tokenResponse.ok) { + throw new Error("The ElevenLabs voice session could not be started."); + } + + const tokenBody = asRecord(await tokenResponse.json()); + const conversationToken = getString(tokenBody, "conversationToken"); + if (!conversationToken) { + throw new Error( + "The ElevenLabs voice session returned an invalid token.", + ); + } + + const permissionStream = await this.#dependencies.getUserMedia({ + audio: true, + }); + for (const track of permissionStream.getTracks()) { + track.stop(); + } + + const conversation = await this.#dependencies.startSession({ + connectionType: "webrtc", + conversationToken, + onConnect: () => { + if (this.#disposed) { + return; + } + this.#connected = true; + this.#emit({ + timestampMs: this.#dependencies.now(), + type: "connected", + }); + }, + onConversationCreated: (createdConversation) => { + this.#conversation = createdConversation; + createdConversation.setMicMuted(true); + }, + onDisconnect: ({ reason }) => { + this.#connected = false; + if (!this.#disposed && reason === "error") { + this.#emitError("The ElevenLabs voice connection was lost."); + } + }, + onError: () => { + if (!this.#disposed) { + this.#emitError("The ElevenLabs voice connection failed."); + } + }, + onInterruption: () => { + this.#responseInProgress = false; + }, + onMessage: ({ message, role }) => { + if (!message.trim() || this.#disposed) { + return; + } + const turnId = Math.max(this.#turnId, 1); + if (role === "user") { + this.#emit({ + speaker: "expert", + timestampMs: this.#dependencies.now(), + transcript: message, + turnId, + type: "final-transcript", + }); + return; + } + + this.#emitResponseStarted(turnId); + this.#emit({ + speaker: "assistant", + timestampMs: this.#dependencies.now(), + transcript: message, + turnId, + type: "final-transcript", + }); + this.#emit({ + responseText: message, + timestampMs: this.#dependencies.now(), + turnId, + type: "response-completed", + }); + this.#responseInProgress = false; + }, + onModeChange: ({ mode }) => { + if (mode === "speaking" && !this.#disposed) { + this.#emitResponseStarted(Math.max(this.#turnId, 1)); + } + }, + }); + + this.#conversation = conversation; + if (this.#disposed) { + await conversation.endSession(); + this.#conversation = null; + } + } + + #emitResponseStarted(turnId: number): void { + if (this.#responseInProgress) { + return; + } + this.#responseInProgress = true; + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId, + type: "response-started", + }); + } + + #requireConversation(): ConversationControl { + if (!this.#connected || !this.#conversation || this.#disposed) { + throw new Error("The ElevenLabs voice session is not connected."); + } + return this.#conversation; + } + + #emit(event: VoiceExperimentEvent): void { + for (const listener of this.#listeners) { + listener(event); + } + } + + #emitError(message: string): void { + this.#emit({ + message, + timestampMs: this.#dependencies.now(), + type: "error", + }); + } +} + +export const createElevenLabsAdapter = ( + dependencies: Partial = {}, +): VoiceExperimentAdapter => + new ElevenLabsAdapter({ ...defaultDependencies, ...dependencies }); diff --git a/apps/petrinaut-website/vercel.json b/apps/petrinaut-website/vercel.json index 5fa17c2c8c4..1e294677728 100644 --- a/apps/petrinaut-website/vercel.json +++ b/apps/petrinaut-website/vercel.json @@ -23,6 +23,9 @@ }, "api/voice-experiment/openai-realtime-session.ts": { "maxDuration": 15 + }, + "api/voice-experiment/elevenlabs-conversation-token.ts": { + "maxDuration": 15 } } } diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 83d211480e8..56d3c1c37ec 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -37,20 +37,13 @@ const createApiAdapter = (server: ViteDevServer, modulePath: string) => } }); -// Plugin required to serve the API endpoints in dev. -// In production, Vercel deploys the files in `api` as functions. -const petrinautApiDevPlugin = (): Plugin => ({ - name: "petrinaut-api-dev", +// Split chat from voice so the Brunch launcher can replace `/api/chat` while +// retaining the same-origin provider-token endpoints. +const petrinautChatApiDevPlugin = (): Plugin => ({ + name: "petrinaut-chat-api-dev", apply: "serve", configureServer(server) { - // Each endpoint ships a default `{ fetch }` so Vercel's Node.js - // runtime treats it as a Web fetch handler in production. We mirror the - // same shape here so dev and prod hit the same code path. const chatAdapter = createApiAdapter(server, "/api/chat.ts"); - const openAIRealtimeAdapter = createApiAdapter( - server, - "/api/voice-experiment/openai-realtime-session.ts", - ); server.middlewares.use( "/api/chat", @@ -58,12 +51,36 @@ const petrinautApiDevPlugin = (): Plugin => ({ void chatAdapter(request, response); }, ); + }, +}); + +// Each endpoint ships a default `{ fetch }` so Vercel's Node.js runtime treats +// it as a Web fetch handler in production. Dev mirrors that same code path. +const petrinautVoiceApiDevPlugin = (): Plugin => ({ + name: "petrinaut-voice-api-dev", + apply: "serve", + configureServer(server) { + const openAIRealtimeAdapter = createApiAdapter( + server, + "/api/voice-experiment/openai-realtime-session.ts", + ); + const elevenLabsAdapter = createApiAdapter( + server, + "/api/voice-experiment/elevenlabs-conversation-token.ts", + ); + server.middlewares.use( "/api/voice-experiment/openai-realtime-session", (request: IncomingMessage, response: ServerResponse) => { void openAIRealtimeAdapter(request, response); }, ); + server.middlewares.use( + "/api/voice-experiment/elevenlabs-conversation-token", + (request: IncomingMessage, response: ServerResponse) => { + void elevenLabsAdapter(request, response); + }, + ); }, }); @@ -100,7 +117,8 @@ export default defineConfig(({ mode }) => { }, plugins: [ - petrinautApiDevPlugin(), + petrinautChatApiDevPlugin(), + petrinautVoiceApiDevPlugin(), react(), babel({ presets: [ diff --git a/yarn.lock b/yarn.lock index df8f3cfdc8b..c30d9b9d90e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -435,6 +435,7 @@ __metadata: resolution: "@apps/brunch-agent@workspace:apps/brunch-agent" dependencies: "@earendil-works/pi-ai": "npm:0.83.0" + "@elevenlabs/elevenlabs-js": "npm:2.64.0" "@flue/react": "npm:2.0.3" "@flue/runtime": "npm:2.0.3" "@flue/sdk": "npm:2.0.3" @@ -914,6 +915,7 @@ __metadata: resolution: "@apps/petrinaut-website@workspace:apps/petrinaut-website" dependencies: "@ai-sdk/openai": "npm:3.0.63" + "@elevenlabs/client": "npm:1.18.0" "@hashintel/brunch-agent-transport-aisdk": "workspace:*" "@hashintel/ds-components": "workspace:*" "@hashintel/ds-helpers": "workspace:*" @@ -4351,6 +4353,13 @@ __metadata: languageName: node linkType: hard +"@bufbuild/protobuf@npm:^1.10.0": + version: 1.10.1 + resolution: "@bufbuild/protobuf@npm:1.10.1" + checksum: 10c0/a89572ae99aa193dd232fca0cdc9ece1dfe2f3d8b061be1f966a4f88fb63410aeb0fe7de927037e970aefcb52036eec58a7f89a40fe1286eed1448ea1bd2634e + languageName: node + linkType: hard + "@bufbuild/protobuf@npm:^2.6.2": version: 2.12.1 resolution: "@bufbuild/protobuf@npm:2.12.1" @@ -5283,6 +5292,34 @@ __metadata: languageName: node linkType: hard +"@elevenlabs/client@npm:1.18.0": + version: 1.18.0 + resolution: "@elevenlabs/client@npm:1.18.0" + dependencies: + "@elevenlabs/types": "npm:0.20.0" + livekit-client: "npm:^2.21.0" + checksum: 10c0/331fe90bffe89005fadece646ef72dfdc6858a1ae87089d6fa9c33c25ce7df01f0d8fe9f5e2cd39c4822de9edc9ffcedfc3982d416abc800e7bfe16cca0b6fc7 + languageName: node + linkType: hard + +"@elevenlabs/elevenlabs-js@npm:2.64.0": + version: 2.64.0 + resolution: "@elevenlabs/elevenlabs-js@npm:2.64.0" + dependencies: + command-exists: "npm:^1.2.9" + node-fetch: "npm:^2.7.0" + ws: "npm:^8.18.3" + checksum: 10c0/6cb8f9bcf70e237bc15d664ec3569ade1d72591c775d45e599b44c46bb08d306685c6d5be77791547bb11877169802a2fb9065b18b285386bf0ba5d6af477e97 + languageName: node + linkType: hard + +"@elevenlabs/types@npm:0.20.0": + version: 0.20.0 + resolution: "@elevenlabs/types@npm:0.20.0" + checksum: 10c0/726454bdfbf8c4272858a8e2bac31d6e190bdff6b228a9428674b3bf7685effcf67010293252aa2358dfb629d5cfdd30574724db756c3246573ac12de3a763b3 + languageName: node + linkType: hard + "@emmetio/abbreviation@npm:^2.3.3": version: 2.3.3 resolution: "@emmetio/abbreviation@npm:2.3.3" @@ -9112,6 +9149,22 @@ __metadata: languageName: node linkType: hard +"@livekit/mutex@npm:1.1.1": + version: 1.1.1 + resolution: "@livekit/mutex@npm:1.1.1" + checksum: 10c0/d4bb1bd34e20939dfc8af0ae10b86918f3944336d0236d219e80a8c554207e8bfaf21e86794f0c56d2c28b43d74ca966111172a95eacb0e12b72133dd184d49a + languageName: node + linkType: hard + +"@livekit/protocol@npm:1.50.4": + version: 1.50.4 + resolution: "@livekit/protocol@npm:1.50.4" + dependencies: + "@bufbuild/protobuf": "npm:^1.10.0" + checksum: 10c0/6e986854bdae38991d60f6dca204b1ad90d8f033d699fb5a317abbfe97df9da8f8a5f905f94de0e25300cede03462bfcc0802232660e599daa97205302cc7384 + languageName: node + linkType: hard + "@llamaindex/core@npm:0.6.22": version: 0.6.22 resolution: "@llamaindex/core@npm:0.6.22" @@ -24135,6 +24188,13 @@ __metadata: languageName: node linkType: hard +"command-exists@npm:^1.2.9": + version: 1.2.9 + resolution: "command-exists@npm:1.2.9" + checksum: 10c0/75040240062de46cd6cd43e6b3032a8b0494525c89d3962e280dde665103f8cc304a8b313a5aa541b91da2f5a9af75c5959dc3a77893a2726407a5e9a0234c16 + languageName: node + linkType: hard + "command-line-args@npm:^4.0.6": version: 4.0.7 resolution: "command-line-args@npm:4.0.7" @@ -32346,10 +32406,10 @@ __metadata: languageName: node linkType: hard -"jose@npm:^6.1.3": - version: 6.1.3 - resolution: "jose@npm:6.1.3" - checksum: 10c0/b9577b4a7a5e84131011c23823db9f5951eae3ba796771a6a2401ae5dd50daf71104febc8ded9c38146aa5ebe94a92ac09c725e699e613ef26949b9f5a8bc30f +"jose@npm:^6.1.0, jose@npm:^6.1.3": + version: 6.2.9 + resolution: "jose@npm:6.2.9" + checksum: 10c0/fc6d79b11fdd5cc1393bccd644533b3e2445fd8eb2468066eb23c01ed7f6b41deca710e8aabe9379a382cca7920740900c7f610bdf0be4ff5e7ee44412730d4c languageName: node linkType: hard @@ -33495,6 +33555,25 @@ __metadata: languageName: node linkType: hard +"livekit-client@npm:^2.21.0": + version: 2.22.0 + resolution: "livekit-client@npm:2.22.0" + dependencies: + "@livekit/mutex": "npm:1.1.1" + "@livekit/protocol": "npm:1.50.4" + events: "npm:^3.3.0" + jose: "npm:^6.1.0" + loglevel: "npm:^1.9.2" + sdp-transform: "npm:^2.15.0" + tslib: "npm:2.8.1" + typed-emitter: "npm:^2.1.0" + webrtc-adapter: "npm:9.0.6" + peerDependencies: + "@types/dom-mediacapture-record": ^1 + checksum: 10c0/a881f7477f6d675a291e146f27e748137ca0d37a9601eb4488818c29817417412588d78a52eff08ff7a523926e334ddc94b85f3f9da50ef43c2f1266eda49276 + languageName: node + linkType: hard + "llamaindex@npm:0.12.1": version: 0.12.1 resolution: "llamaindex@npm:0.12.1" @@ -33767,7 +33846,7 @@ __metadata: languageName: node linkType: hard -"loglevel@npm:^1.6.8": +"loglevel@npm:^1.6.8, loglevel@npm:^1.9.2": version: 1.9.2 resolution: "loglevel@npm:1.9.2" checksum: 10c0/1e317fa4648fe0b4a4cffef6de037340592cee8547b07d4ce97a487abe9153e704b98451100c799b032c72bb89c9366d71c9fb8192ada8703269263ae77acdc7 @@ -36001,7 +36080,7 @@ __metadata: languageName: node linkType: hard -"node-fetch@npm:2.7.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:^2.6.9": +"node-fetch@npm:2.7.0, node-fetch@npm:^2.6.1, node-fetch@npm:^2.6.7, node-fetch@npm:^2.6.9, node-fetch@npm:^2.7.0": version: 2.7.0 resolution: "node-fetch@npm:2.7.0" dependencies: @@ -41000,7 +41079,7 @@ __metadata: languageName: node linkType: hard -"rxjs@npm:7.8.2, rxjs@npm:^7.8.1, rxjs@npm:^7.8.2": +"rxjs@npm:*, rxjs@npm:7.8.2, rxjs@npm:^7.8.1, rxjs@npm:^7.8.2": version: 7.8.2 resolution: "rxjs@npm:7.8.2" dependencies: @@ -41240,6 +41319,22 @@ __metadata: languageName: node linkType: hard +"sdp-transform@npm:^2.15.0": + version: 2.15.0 + resolution: "sdp-transform@npm:2.15.0" + bin: + sdp-verify: checker.js + checksum: 10c0/96c060f113a3d5418defa168db609f7e23e5bd7954fa1cf7784f103dbe702e24d667e5310d2ac6d88abdb32322af83d6ebd0df08e07f4f172d5ed5888f921386 + languageName: node + linkType: hard + +"sdp@npm:^3.2.0": + version: 3.2.2 + resolution: "sdp@npm:3.2.2" + checksum: 10c0/62913cbd92b0cca8feb17850dee9e331d6e38402385d4985aac80bde6b35032bfb7453c00096c4d7e6a91a9d73a7ba6235c2368998a54a8edd1c0dd4ec94d9e3 + languageName: node + linkType: hard + "selderee@npm:^0.11.0": version: 0.11.0 resolution: "selderee@npm:0.11.0" @@ -44085,6 +44180,18 @@ __metadata: languageName: node linkType: hard +"typed-emitter@npm:^2.1.0": + version: 2.1.0 + resolution: "typed-emitter@npm:2.1.0" + dependencies: + rxjs: "npm:*" + dependenciesMeta: + rxjs: + optional: true + checksum: 10c0/01fc354ba8e87bd39b1bf4fe1c96fe7ecff7fde83161003b0f8c7f4b285a368052e185ba655dd8c102c4445301b7a1e032c8972f181b440fc95bd810450f1314 + languageName: node + linkType: hard + "typedarray@npm:^0.0.6": version: 0.0.6 resolution: "typedarray@npm:0.0.6" @@ -46142,6 +46249,15 @@ __metadata: languageName: node linkType: hard +"webrtc-adapter@npm:9.0.6": + version: 9.0.6 + resolution: "webrtc-adapter@npm:9.0.6" + dependencies: + sdp: "npm:^3.2.0" + checksum: 10c0/19b44f507d4583df300ce338a24fe343473edf433228978666eacfc7eee27d2603cb8683da8f1bb380031ac5e827e224824d2921395e008fea85d14ad59b1aba + languageName: node + linkType: hard + "websocket-driver@npm:>=0.5.1, websocket-driver@npm:^0.7.4": version: 0.7.5 resolution: "websocket-driver@npm:0.7.5" From ec86f1b550b9f990342e53b5dbc8e8d1ed433738 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 20:08:15 +0200 Subject: [PATCH 05/19] H-6763: Refine voice experiments and add tool diagnostics --- apps/brunch-agent/src/app.ts | 10 +- apps/brunch-agent/src/petrinaut-chat.ts | 13 +- apps/brunch-agent/src/routes.ts | 4 + .../src/voice-experiment-diagnostics.ts | 190 +++ .../test/voice-experiment-diagnostics.test.ts | 117 ++ .../openai-realtime-session.test.ts | 34 +- .../openai-realtime-session.ts | 9 +- apps/petrinaut-website/src/app.css | 65 + .../local-storage-demo/voice-experiment.tsx | 1089 +++++++++++------ .../elevenlabs-adapter.test.ts | 123 +- .../voice-experiment/elevenlabs-adapter.ts | 200 ++- .../openai-realtime-adapter.test.ts | 102 +- .../openai-realtime-adapter.ts | 86 +- .../voice-experiment-events.ts | 2 + apps/petrinaut-website/vite.config.ts | 4 + 15 files changed, 1659 insertions(+), 389 deletions(-) create mode 100644 apps/brunch-agent/src/voice-experiment-diagnostics.ts create mode 100644 apps/brunch-agent/test/voice-experiment-diagnostics.test.ts diff --git a/apps/brunch-agent/src/app.ts b/apps/brunch-agent/src/app.ts index 6ce50876433..7292c86e29f 100644 --- a/apps/brunch-agent/src/app.ts +++ b/apps/brunch-agent/src/app.ts @@ -15,7 +15,12 @@ import { Hono } from "hono"; import { GherkinElicitor } from "./agents/gherkin-elicitor.ts"; import { assetHandler } from "./assets.ts"; import { petrinautChatHandler } from "./petrinaut-chat.ts"; -import { GHERKIN_AGENT_ROUTE, PETRINAUT_CHAT_ROUTE } from "./routes.ts"; +import { + GHERKIN_AGENT_ROUTE, + PETRINAUT_CHAT_ROUTE, + VOICE_EXPERIMENT_DIAGNOSTICS_ROUTE, +} from "./routes.ts"; +import { voiceExperimentDiagnosticsHandler } from "./voice-experiment-diagnostics.ts"; const app = new Hono(); @@ -30,6 +35,9 @@ app.route(`/agents/${GHERKIN_AGENT_ROUTE}`, createAgentRouter(GherkinElicitor)); app.on(["POST", "OPTIONS"], PETRINAUT_CHAT_ROUTE, (c) => petrinautChatHandler(c.req.raw), ); +app.get(VOICE_EXPERIMENT_DIAGNOSTICS_ROUTE, (c) => + voiceExperimentDiagnosticsHandler(c.req.raw), +); // The flue dev controller owns the whole request space — no fall-through to // vite's html serving — so the ui is app-served, in dev and in production diff --git a/apps/brunch-agent/src/petrinaut-chat.ts b/apps/brunch-agent/src/petrinaut-chat.ts index 9480653f56d..0ad8c7bbe05 100644 --- a/apps/brunch-agent/src/petrinaut-chat.ts +++ b/apps/brunch-agent/src/petrinaut-chat.ts @@ -19,6 +19,7 @@ import { import { GherkinElicitor } from "./agents/gherkin-elicitor.ts"; import { createGherkinElicitationSession } from "./elicitation-session.ts"; import { defaultPanelOrigins } from "./local-dev-origins.ts"; +import { voiceExperimentDiagnostics } from "./voice-experiment-diagnostics.ts"; const inspect = process.env.BRUNCH_TRANSPORT_AISDK_INSPECT === "1" @@ -39,6 +40,7 @@ const streamElicitorTurn = async ( dispatch: { readonly message: string; readonly idempotencyKey: string }, emit: (event: HarnessReplyEvent) => void, ): Promise => { + const voiceTurnId = voiceExperimentDiagnostics.beginTurn(conversationId); const agent = init(GherkinElicitor, { id: conversationId }); const receipt = await agent.dispatch({ ...dispatch, @@ -46,7 +48,16 @@ const streamElicitorTurn = async ( }); const projector = createFlueReplyProjector({ submissionId: receipt.submissionId, - emit, + emit: (event) => { + if (event.type === "tool-input") { + voiceExperimentDiagnostics.recordToolCall( + conversationId, + voiceTurnId, + event, + ); + } + emit(event); + }, }); await agent.read(receipt, { onEvent: (chunk) => projector.accept(chunk) }); }; diff --git a/apps/brunch-agent/src/routes.ts b/apps/brunch-agent/src/routes.ts index 739c6323023..cddd6374cae 100644 --- a/apps/brunch-agent/src/routes.ts +++ b/apps/brunch-agent/src/routes.ts @@ -3,3 +3,7 @@ export const GHERKIN_AGENT_ROUTE = "gherkin"; /** Stock `DefaultChatTransport` endpoint used by Petrinaut's local panel. */ export const PETRINAUT_CHAT_ROUTE = "/api/chat"; + +/** Read-only metadata for comparing the two local voice experiments. */ +export const VOICE_EXPERIMENT_DIAGNOSTICS_ROUTE = + "/api/voice-experiment/elevenlabs-brunch-diagnostics"; diff --git a/apps/brunch-agent/src/voice-experiment-diagnostics.ts b/apps/brunch-agent/src/voice-experiment-diagnostics.ts new file mode 100644 index 00000000000..76d3ffce39e --- /dev/null +++ b/apps/brunch-agent/src/voice-experiment-diagnostics.ts @@ -0,0 +1,190 @@ +import type { HarnessReplyEvent } from "@hashintel/brunch-agent"; + +const VOICE_CONVERSATION_PREFIX = "voice:"; +const TRUSTED_EXPERIMENT = "elevenlabs-brunch"; +const MAX_SESSIONS = 100; +const MAX_EVENTS_PER_SESSION = 50; +const MAX_IDENTIFIER_CHARACTERS = 96; +const MAX_SUMMARY_CHARACTERS = 240; +const providerConversationIdPattern = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u; + +export type VoiceToolDiagnostic = { + readonly argumentSummary: string; + readonly callId: string; + readonly sequence: number; + readonly timestampMs: number; + readonly toolName: string; + readonly turnId: number; +}; + +type DiagnosticSession = { + events: VoiceToolDiagnostic[]; + nextSequence: number; + nextTurnId: number; +}; + +type VoiceExperimentDiagnosticsDependencies = { + now: () => number; +}; + +const defaultDependencies: VoiceExperimentDiagnosticsDependencies = { + now: () => Date.now(), +}; + +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null + ? (value as Record) + : null; + +const boundedText = (value: unknown, limit: number): string => { + if (typeof value !== "string") { + return ""; + } + + let sanitized = ""; + for (const character of value.normalize("NFKC")) { + const codePoint = character.codePointAt(0) ?? 0; + if (codePoint >= 32 && codePoint !== 127) { + sanitized += character; + } + } + + return sanitized.replace(/\s+/gu, " ").trim().slice(0, limit); +}; + +const safeIdentifier = (value: string, fallback: string): string => + boundedText(value, MAX_IDENTIFIER_CHARACTERS) || fallback; + +const summarizeToolInput = (toolName: string, input: unknown): string => { + if (toolName === "brunch_ask") { + const question = boundedText( + asRecord(input)?.question, + MAX_SUMMARY_CHARACTERS - "Question: ".length, + ); + return question ? `Question: ${question}` : "Question unavailable"; + } + + if (toolName === "brunch_sweep") { + return "Settlement requested"; + } + + return "Arguments hidden"; +}; + +export class VoiceExperimentDiagnostics { + readonly #dependencies: VoiceExperimentDiagnosticsDependencies; + readonly #sessions = new Map(); + + public constructor( + dependencies: Partial = {}, + ) { + this.#dependencies = { ...defaultDependencies, ...dependencies }; + } + + public beginTurn(conversationId: string): number { + if (!conversationId.startsWith(VOICE_CONVERSATION_PREFIX)) { + return 0; + } + + const session = this.#sessionFor(conversationId); + session.nextTurnId += 1; + return session.nextTurnId; + } + + public recordToolCall( + conversationId: string, + turnId: number, + event: Pick< + Extract, + "input" | "toolCallId" | "toolName" + >, + ): void { + if ( + !conversationId.startsWith(VOICE_CONVERSATION_PREFIX) || + !Number.isSafeInteger(turnId) || + turnId < 1 + ) { + return; + } + + const session = this.#sessionFor(conversationId); + session.nextSequence += 1; + const toolName = safeIdentifier(event.toolName, "unknown-tool"); + session.events.push({ + argumentSummary: summarizeToolInput(toolName, event.input), + callId: safeIdentifier(event.toolCallId, "unknown-call"), + sequence: session.nextSequence, + timestampMs: this.#dependencies.now(), + toolName, + turnId, + }); + session.events = session.events.slice(-MAX_EVENTS_PER_SESSION); + } + + public read(providerConversationId: string, afterSequence: number) { + const session = this.#sessions.get( + `${VOICE_CONVERSATION_PREFIX}${providerConversationId}`, + ); + return ( + session?.events.filter(({ sequence }) => sequence > afterSequence) ?? [] + ); + } + + #sessionFor(conversationId: string): DiagnosticSession { + const existing = this.#sessions.get(conversationId); + if (existing) { + return existing; + } + + if (this.#sessions.size >= MAX_SESSIONS) { + const oldest = this.#sessions.keys().next().value as string | undefined; + if (oldest) { + this.#sessions.delete(oldest); + } + } + + const session: DiagnosticSession = { + events: [], + nextSequence: 0, + nextTurnId: 0, + }; + this.#sessions.set(conversationId, session); + return session; + } +} + +const jsonResponse = (body: unknown, status = 200): Response => + Response.json(body, { + status, + headers: { "cache-control": "no-store" }, + }); + +export const createVoiceExperimentDiagnosticsHandler = + (diagnostics: VoiceExperimentDiagnostics) => + async (request: Request): Promise => { + if (request.method !== "GET") { + return jsonResponse({ error: "Method not allowed" }, 405); + } + if (request.headers.get("x-voice-experiment") !== TRUSTED_EXPERIMENT) { + return jsonResponse({ error: "Forbidden" }, 403); + } + + const url = new URL(request.url); + const conversationId = url.searchParams.get("conversationId") ?? ""; + const afterText = url.searchParams.get("after") ?? "0"; + const after = Number(afterText); + if ( + !providerConversationIdPattern.test(conversationId) || + !/^\d+$/u.test(afterText) || + !Number.isSafeInteger(after) || + after < 0 + ) { + return jsonResponse({ error: "Invalid diagnostic query" }, 400); + } + + return jsonResponse({ events: diagnostics.read(conversationId, after) }); + }; + +export const voiceExperimentDiagnostics = new VoiceExperimentDiagnostics(); +export const voiceExperimentDiagnosticsHandler = + createVoiceExperimentDiagnosticsHandler(voiceExperimentDiagnostics); diff --git a/apps/brunch-agent/test/voice-experiment-diagnostics.test.ts b/apps/brunch-agent/test/voice-experiment-diagnostics.test.ts new file mode 100644 index 00000000000..4f217939945 --- /dev/null +++ b/apps/brunch-agent/test/voice-experiment-diagnostics.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "vitest"; + +import { + createVoiceExperimentDiagnosticsHandler, + VoiceExperimentDiagnostics, +} from "../src/voice-experiment-diagnostics.ts"; + +const DIAGNOSTICS_URL = + "http://brunch.test/api/voice-experiment/elevenlabs-brunch-diagnostics"; + +describe("voice experiment diagnostics", () => { + test("exposes only allowlisted, bounded tool metadata", async () => { + const diagnostics = new VoiceExperimentDiagnostics({ + now: () => 1_234, + }); + const turnId = diagnostics.beginTurn("voice:conv_safe"); + + diagnostics.recordToolCall("voice:conv_safe", turnId, { + input: { + question: `Who owns triage?\u0000${"x".repeat(400)}`, + secret: "must-not-leak", + }, + toolCallId: "call_ask", + toolName: "brunch_ask", + }); + diagnostics.recordToolCall("voice:conv_safe", turnId, { + input: { apiKey: "sk-private", transcript: "private interview" }, + toolCallId: "call_unknown", + toolName: "unrecognized_tool", + }); + + const handler = createVoiceExperimentDiagnosticsHandler(diagnostics); + const response = await handler( + new Request(`${DIAGNOSTICS_URL}?conversationId=conv_safe`, { + headers: { "x-voice-experiment": "elevenlabs-brunch" }, + }), + ); + const body = (await response.json()) as { events: unknown[] }; + const serialized = JSON.stringify(body); + + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(body.events).toEqual([ + { + argumentSummary: expect.stringMatching(/^Question: Who owns triage\?/u), + callId: "call_ask", + sequence: 1, + timestampMs: 1_234, + toolName: "brunch_ask", + turnId: 1, + }, + { + argumentSummary: "Arguments hidden", + callId: "call_unknown", + sequence: 2, + timestampMs: 1_234, + toolName: "unrecognized_tool", + turnId: 1, + }, + ]); + expect(serialized).not.toContain("must-not-leak"); + expect(serialized).not.toContain("sk-private"); + expect(serialized).not.toContain("private interview"); + expect(serialized).not.toContain("\\u0000"); + expect(serialized.length).toBeLessThan(900); + }); + + test("isolates sessions, supports cursors, and rejects untrusted queries", async () => { + const diagnostics = new VoiceExperimentDiagnostics({ now: () => 2_000 }); + diagnostics.recordToolCall( + "voice:conv_one", + diagnostics.beginTurn("voice:conv_one"), + { + input: {}, + toolCallId: "call_one", + toolName: "brunch_sweep", + }, + ); + diagnostics.recordToolCall( + "voice:conv_two", + diagnostics.beginTurn("voice:conv_two"), + { + input: {}, + toolCallId: "call_two", + toolName: "brunch_sweep", + }, + ); + + const handler = createVoiceExperimentDiagnosticsHandler(diagnostics); + const trusted = { "x-voice-experiment": "elevenlabs-brunch" }; + const response = await handler( + new Request(`${DIAGNOSTICS_URL}?conversationId=conv_one&after=0`, { + headers: trusted, + }), + ); + const afterResponse = await handler( + new Request(`${DIAGNOSTICS_URL}?conversationId=conv_one&after=1`, { + headers: trusted, + }), + ); + const forbidden = await handler( + new Request(`${DIAGNOSTICS_URL}?conversationId=conv_one`), + ); + const invalid = await handler( + new Request(`${DIAGNOSTICS_URL}?conversationId=../conv_one`, { + headers: trusted, + }), + ); + + expect(await response.json()).toEqual({ + events: [expect.objectContaining({ callId: "call_one", sequence: 1 })], + }); + expect(await afterResponse.json()).toEqual({ events: [] }); + expect(forbidden.status).toBe(403); + expect(invalid.status).toBe(400); + }); +}); diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts index afdc92a2df2..a450f0504bc 100644 --- a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts @@ -136,12 +136,25 @@ describe("OpenAI Realtime session endpoint", () => { expect( new Headers(request.headers).get("openai-safety-identifier"), ).toMatch(/^[a-f0-9]{64}$/u); - expect(JSON.parse(request.body as string)).toMatchObject({ + const sessionRequest = JSON.parse(request.body as string) as { + session: { + instructions: string; + tools: { name: string }[]; + }; + }; + expect(sessionRequest).toMatchObject({ session: { audio: { input: { transcription: { model: "gpt-live-transcribe" }, - turn_detection: null, + turn_detection: { + create_response: true, + interrupt_response: true, + prefix_padding_ms: 300, + silence_duration_ms: 500, + threshold: 0.5, + type: "server_vad", + }, }, output: { voice: "marin" }, }, @@ -151,6 +164,23 @@ describe("OpenAI Realtime session endpoint", () => { type: "realtime", }, }); + expect(sessionRequest.session.instructions).toContain( + "Stochastic Dynamic Coloured Petri Net", + ); + expect(sessionRequest.session.instructions).toContain( + "wait_for_user", + ); + expect(sessionRequest.session.instructions).not.toContain( + "urgent customer support escalation", + ); + expect(sessionRequest.session.tools.map(({ name }) => name)).toEqual([ + "record_process_state", + "record_process_step", + "record_process_decision", + "record_process_flow", + "record_model_requirement", + "wait_for_user", + ]); }); test("does not expose upstream errors or the primary key", async () => { diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts index d8749a78461..1dfd96d4b17 100644 --- a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts @@ -33,7 +33,14 @@ const sessionConfig = { prompt: "A process-model interview about urgent customer support escalations, incident ownership, handoffs, decisions, and resolution.", }, - turn_detection: null, + turn_detection: { + type: "server_vad", + create_response: true, + interrupt_response: true, + prefix_padding_ms: 300, + silence_duration_ms: 500, + threshold: 0.5, + }, }, output: { voice: "marin", diff --git a/apps/petrinaut-website/src/app.css b/apps/petrinaut-website/src/app.css index e27a23b7745..a7e956baff4 100644 --- a/apps/petrinaut-website/src/app.css +++ b/apps/petrinaut-website/src/app.css @@ -1 +1,66 @@ @layer reset, base, tokens, recipes, utilities; + +@keyframes voice-recording-ring { + 0% { + opacity: 0.5; + transform: scale(0.96); + } + + 75%, + 100% { + opacity: 0; + transform: scale(1.42); + } +} + +@keyframes voice-recording-glow { + 0%, + 100% { + box-shadow: + 0 0 0 0 rgb(211 47 47 / 0.12), + 0 10px 28px rgb(211 47 47 / 0.34), + inset 0 1px 0 rgb(255 255 255 / 0.2); + } + + 50% { + box-shadow: + 0 0 0 12px rgb(211 47 47 / 0.1), + 0 14px 32px rgb(211 47 47 / 0.4), + inset 0 1px 0 rgb(255 255 255 / 0.22); + } +} + +@media (prefers-reduced-motion: no-preference) { + .voice-conversation-active { + animation: voice-recording-glow 1.35s ease-in-out infinite; + } + + .voice-conversation-active::before, + .voice-conversation-active::after { + position: absolute; + inset: -2px; + z-index: 0; + border: 2px solid rgb(211 47 47 / 0.55); + border-radius: inherit; + content: ""; + pointer-events: none; + animation: voice-recording-ring 1.8s ease-out infinite; + will-change: opacity, transform; + } + + .voice-conversation-active::after { + animation-delay: 0.6s; + } + + .voice-conversation-active > svg { + position: relative; + z-index: 1; + } +} + +@media (prefers-reduced-motion: reduce) { + .voice-experiment-launcher, + .voice-experiment-panel { + transition: none !important; + } +} diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx index 4e606a02102..5a2a210038f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -1,12 +1,12 @@ +import { useCallback, useEffect, useRef, useState } from "react"; import { - type KeyboardEvent, - type PointerEvent, - useCallback, - useEffect, - useRef, - useState, -} from "react"; -import { MdGraphicEq, MdMic } from "react-icons/md"; + FiActivity, + FiChevronDown, + FiMessageSquare, + FiMic, + FiSquare, + FiTool, +} from "react-icons/fi"; import { css } from "@hashintel/ds-helpers/css"; @@ -18,29 +18,122 @@ import { import type { VoiceExperimentAdapter } from "./voice-experiment/voice-experiment-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment/voice-experiment-events"; -const SCENARIO_SCRIPT = - "Trace an urgent customer support escalation from first report to resolution."; - -const panelStyle = css({ +const dockStyle = css({ position: "fixed", zIndex: "popover", - bottom: "4", + bottom: "[76px]", left: "[50%]", - display: "flex", width: "[calc(100vw - 32px)]", - maxWidth: "[620px]", - maxHeight: "[calc(100vh - 32px)]", + maxWidth: "[600px]", transform: "translateX(-50%)", + pointerEvents: "none", +}); + +const panelStyle = css({ + position: "relative", + display: "flex", + width: "full", + maxHeight: "[calc(100vh - 108px)]", flexDirection: "column", - gap: "3", - padding: "4", + gap: "3.5", + padding: "[18px]", overflow: "auto", borderWidth: "thin", borderStyle: "solid", borderColor: "neutral.a30", - borderRadius: "xl", - backgroundColor: "neutral.s00", - boxShadow: "xl", + borderRadius: "2xl", + backgroundColor: "neutral.s05", + boxShadow: + "[0 24px 72px rgb(15 23 42 / 0.22), 0 2px 8px rgb(15 23 42 / 0.08)]", + transformOrigin: "bottom center", + transition: + "[opacity 180ms ease, transform 220ms cubic-bezier(0.22, 1, 0.36, 1), visibility 0s linear 180ms]", +}); + +const collapsedPanelStyle = css({ + visibility: "hidden", + opacity: "0", + pointerEvents: "none", + transform: "translateY(14px) scale(0.965)", +}); + +const expandedPanelStyle = css({ + visibility: "visible", + opacity: "1", + pointerEvents: "auto", + transform: "translateY(0) scale(1)", + transition: + "[opacity 180ms ease, transform 220ms cubic-bezier(0.22, 1, 0.36, 1), visibility 0s]", +}); + +const launcherButtonStyle = css({ + position: "absolute", + bottom: "0", + left: "[50%]", + display: "inline-flex", + width: "12", + height: "12", + transform: "translateX(-50%)", + alignItems: "center", + justifyContent: "center", + padding: "0", + pointerEvents: "auto", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "blue.a100", + borderRadius: "full", + backgroundColor: "blue.s100", + color: "white", + cursor: "pointer", + isolation: "isolate", + boxShadow: + "[0 8px 22px rgb(42 128 200 / 0.28), inset 0 1px 0 rgb(255 255 255 / 0.25)]", + transition: + "[opacity 150ms ease, transform 200ms cubic-bezier(0.22, 1, 0.36, 1), background-color 150ms ease, box-shadow 150ms ease]", + _hover: { + transform: "translateX(-50%) translateY(-2px) scale(1.04)", + boxShadow: + "[0 11px 26px rgb(42 128 200 / 0.34), inset 0 1px 0 rgb(255 255 255 / 0.28)]", + }, + _focusVisible: { + outline: "3px solid", + outlineColor: "blue.a40", + outlineOffset: "[3px]", + }, +}); + +const hiddenLauncherButtonStyle = css({ + visibility: "hidden", + opacity: "0", + pointerEvents: "none", + transform: "translateX(-50%) translateY(8px) scale(0.78)", + transition: + "[opacity 150ms ease, transform 200ms cubic-bezier(0.22, 1, 0.36, 1), background-color 150ms ease, box-shadow 150ms ease, visibility 0s linear 150ms]", +}); + +const activeLauncherButtonStyle = css({ + borderColor: "red.s100", + backgroundColor: "red.s100", + boxShadow: + "[0 8px 24px rgb(211 47 47 / 0.34), inset 0 1px 0 rgb(255 255 255 / 0.22)]", + _hover: { + backgroundColor: "red.s110", + boxShadow: + "[0 11px 28px rgb(211 47 47 / 0.40), inset 0 1px 0 rgb(255 255 255 / 0.24)]", + }, +}); + +const launcherStatusStyle = css({ + position: "absolute", + top: "[-1px]", + right: "[-1px]", + width: "2.5", + height: "2.5", + borderWidth: "[2px]", + borderStyle: "solid", + borderColor: "neutral.s00", + borderRadius: "full", + backgroundColor: "neutral.s55", }); const headerStyle = css({ @@ -48,38 +141,82 @@ const headerStyle = css({ alignItems: "center", justifyContent: "space-between", gap: "4", + paddingBottom: "3", + borderBottomWidth: "thin", + borderBottomStyle: "solid", + borderBottomColor: "neutral.a20", +}); + +const headerIdentityStyle = css({ + display: "flex", + minWidth: "0", + alignItems: "center", +}); + +const headerActionsStyle = css({ + display: "flex", + alignItems: "center", + gap: "3", +}); + +const minimizeButtonStyle = css({ + display: "inline-flex", + width: "8", + height: "8", + alignItems: "center", + justifyContent: "center", + padding: "0", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "lg", + backgroundColor: "white", + color: "neutral.s75", + cursor: "pointer", + transition: + "[background-color 140ms ease, color 140ms ease, transform 140ms ease]", + _hover: { + backgroundColor: "neutral.a10", + color: "neutral.s100", + transform: "translateY(1px)", + }, + _focusVisible: { + outline: "2px solid", + outlineColor: "blue.a35", + outlineOffset: "[2px]", + }, +}); + +const titleCopyStyle = css({ + display: "flex", + minWidth: "0", + alignItems: "center", + gap: "2", + flexWrap: "wrap", }); const headingStyle = css({ - color: "neutral.s100", + color: "neutral.s115", fontSize: "lg", fontWeight: "semibold", + lineHeight: "tight", }); const experimentBadgeStyle = css({ - paddingX: "3", - paddingY: "1.5", + paddingX: "2", + paddingY: "0.5", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a30", borderRadius: "full", - backgroundColor: "blue.a10", - color: "blue.a85", - fontSize: "xs", - fontWeight: "semibold", -}); - -const scenarioStyle = css({ - display: "flex", - flexDirection: "column", - gap: "1", - padding: "3", - borderRadius: "lg", - backgroundColor: "neutral.a10", + backgroundColor: "white", color: "neutral.s80", fontSize: "xs", - lineHeight: "relaxed", + fontWeight: "medium", }); const sectionLabelStyle = css({ - color: "blue.a85", + color: "neutral.s90", fontSize: "xs", fontWeight: "semibold", letterSpacing: "wide", @@ -88,20 +225,57 @@ const sectionLabelStyle = css({ const transcriptStyle = css({ display: "flex", - minHeight: "16", - maxHeight: "52", + minHeight: "28", + maxHeight: "64", flexDirection: "column", gap: "2.5", padding: "3", overflowY: "auto", borderWidth: "thin", borderStyle: "solid", - borderColor: "neutral.a20", - borderRadius: "lg", - backgroundColor: "neutral.a05", - color: "neutral.s70", + borderColor: "neutral.a25", + borderRadius: "xl", + backgroundColor: "white", + color: "neutral.s80", + boxShadow: "[inset 0 1px 0 rgb(255 255 255 / 0.85)]", fontSize: "sm", lineHeight: "relaxed", + scrollBehavior: "smooth", + _focusVisible: { + outline: "2px solid", + outlineColor: "blue.a30", + outlineOffset: "[2px]", + }, +}); + +const transcriptSectionStyle = css({ + display: "flex", + minHeight: "0", + flexDirection: "column", + gap: "2", +}); + +const transcriptHeaderStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + paddingX: "1", +}); + +const sectionHeadingStyle = css({ + display: "inline-flex", + alignItems: "center", + gap: "1.5", + color: "neutral.s80", +}); + +const transcriptCountStyle = css({ + paddingX: "2", + paddingY: "0.5", + borderRadius: "full", + backgroundColor: "neutral.a15", + color: "neutral.s70", + fontSize: "xs", }); const transcriptEntryStyle = css({ @@ -117,7 +291,7 @@ const expertTranscriptEntryStyle = css({ }); const transcriptSpeakerStyle = css({ - color: "neutral.s55", + color: "neutral.s65", fontSize: "xs", fontWeight: "medium", }); @@ -125,13 +299,13 @@ const transcriptSpeakerStyle = css({ const transcriptBubbleStyle = css({ paddingX: "3", paddingY: "2", - borderRadius: "lg", - backgroundColor: "neutral.a10", - color: "neutral.s80", + borderRadius: "xl", + backgroundColor: "neutral.a15", + color: "neutral.s95", }); const expertTranscriptBubbleStyle = css({ - backgroundColor: "blue.a15", + backgroundColor: "blue.a20", }); const partialTranscriptStyle = css({ @@ -140,85 +314,16 @@ const partialTranscriptStyle = css({ const transcriptPlaceholderStyle = css({ margin: "auto", - color: "neutral.s55", - textAlign: "center", -}); - -const controlsStyle = css({ display: "flex", alignItems: "center", - justifyContent: "space-between", - gap: "3", -}); - -const sessionAreaStyle = css({ - display: "flex", - minWidth: "0", - flex: "1", - flexDirection: "column", gap: "2", -}); - -const sessionControlsStyle = css({ - display: "flex", - width: "full", - alignItems: "center", - gap: "2", -}); - -const sessionButtonStyle = css({ - display: "inline-flex", - minHeight: "12", - alignItems: "center", - justifyContent: "center", - paddingX: "4", - paddingY: "3", - borderWidth: "thin", - borderStyle: "solid", - borderRadius: "lg", - cursor: "pointer", - fontSize: "sm", - fontWeight: "semibold", - transition: "[transform 120ms ease, box-shadow 120ms ease]", - _disabled: { - cursor: "not-allowed", - opacity: "0.4", - }, -}); - -const startSessionButtonStyle = css({ - flex: "[2 1 0%]", - borderColor: "blue.a100", - backgroundColor: "blue.a100", - color: "white", - boxShadow: "md", - _hover: { - transform: "translateY(-1px)", - boxShadow: "lg", - }, -}); - -const endSessionButtonStyle = css({ - flex: "1", - borderColor: "red.a35", - backgroundColor: "neutral.s00", - color: "red.a85", - _hover: { - backgroundColor: "red.a10", - }, -}); - -const statusStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", - color: "neutral.s70", - fontSize: "xs", + color: "neutral.s65", + textAlign: "center", }); const statusIndicatorStyle = css({ - width: "2", - height: "2", + width: "2.5", + height: "2.5", flexShrink: "0", borderRadius: "full", backgroundColor: "neutral.a50", @@ -226,51 +331,77 @@ const statusIndicatorStyle = css({ const connectedStatusIndicatorStyle = css({ backgroundColor: "green.a85", - boxShadow: "[0 0 0 4px {colors.green.a15}]", + boxShadow: "[0 0 0 4px {colors.green.a10}]", }); -const liveStatusIndicatorStyle = css({ - backgroundColor: "red.a85", - boxShadow: "[0 0 0 4px {colors.red.a15}]", +const pendingStatusIndicatorStyle = css({ + backgroundColor: "yellow.a85", + boxShadow: "[0 0 0 4px {colors.yellow.a10}]", }); const errorStatusIndicatorStyle = css({ backgroundColor: "red.a85", + boxShadow: "[0 0 0 4px {colors.red.a10}]", +}); + +const conversationControlStyle = css({ + display: "flex", + width: "full", + minHeight: "24", + alignItems: "center", + justifyContent: "center", + paddingY: "1", }); const microphoneButtonStyle = css({ + position: "relative", display: "inline-flex", + width: "[68px]", + height: "[68px]", + flexShrink: "0", alignItems: "center", justifyContent: "center", - width: "14", - height: "14", - flexShrink: "0", + padding: "0", + borderWidth: "[2px]", + borderStyle: "solid", + borderColor: "blue.a100", borderRadius: "full", - backgroundColor: "blue.a85", + backgroundColor: "blue.a100", color: "white", cursor: "pointer", - boxShadow: "md", - touchAction: "none", - transition: "[transform 120ms ease, background-color 120ms ease]", + isolation: "isolate", + boxShadow: + "[0 10px 26px rgb(42 128 200 / 0.30), inset 0 1px 0 rgb(255 255 255 / 0.22)]", + transition: + "[transform 160ms ease, background-color 160ms ease, border-color 160ms ease, box-shadow 160ms ease, opacity 160ms ease]", _hover: { - backgroundColor: "blue.a100", + backgroundColor: "blue.a110", + transform: "translateY(-2px) scale(1.03)", + boxShadow: + "[0 14px 30px rgb(42 128 200 / 0.34), inset 0 1px 0 rgb(255 255 255 / 0.24)]", }, _focusVisible: { outline: "3px solid", - outlineColor: "blue.a30", - outlineOffset: "[2px]", + outlineColor: "blue.a40", + outlineOffset: "[4px]", }, _disabled: { - backgroundColor: "neutral.a30", - color: "neutral.s50", cursor: "not-allowed", - boxShadow: "[none]", + opacity: "0.52", + transform: "none", }, }); const activeMicrophoneButtonStyle = css({ - transform: "scale(1.08)", - backgroundColor: "red.a85", + borderColor: "red.s100", + backgroundColor: "red.s100", + boxShadow: + "[0 10px 28px rgb(211 47 47 / 0.34), inset 0 1px 0 rgb(255 255 255 / 0.20)]", + _hover: { + backgroundColor: "red.s110", + boxShadow: + "[0 14px 32px rgb(211 47 47 / 0.38), inset 0 1px 0 rgb(255 255 255 / 0.22)]", + }, }); const eventLogStyle = css({ @@ -283,10 +414,10 @@ const eventLogStyle = css({ overflowY: "auto", borderWidth: "thin", borderStyle: "solid", - borderColor: "neutral.a20", + borderColor: "neutral.a15", borderRadius: "md", backgroundColor: "neutral.a05", - color: "neutral.s60", + color: "neutral.s70", fontFamily: "mono", fontSize: "xs", }); @@ -294,7 +425,7 @@ const eventLogStyle = css({ const technicalDetailsStyle = css({ borderTopWidth: "thin", borderTopStyle: "solid", - borderTopColor: "neutral.a15", + borderTopColor: "neutral.a20", paddingTop: "2", }); @@ -302,14 +433,20 @@ const technicalSummaryStyle = css({ display: "flex", alignItems: "center", justifyContent: "space-between", - color: "neutral.s60", + color: "neutral.s75", cursor: "pointer", fontSize: "xs", fontWeight: "medium", }); +const technicalSummaryLabelStyle = css({ + display: "inline-flex", + alignItems: "center", + gap: "1.5", +}); + const eventCountStyle = css({ - color: "neutral.s50", + color: "neutral.s65", fontWeight: "normal", }); @@ -324,6 +461,91 @@ const emptyLogStyle = css({ fontFamily: "body", }); +const toolDiagnosticsSectionStyle = css({ + display: "flex", + minHeight: "0", + flexDirection: "column", + gap: "2", +}); + +const toolDiagnosticsHeaderStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + paddingX: "1", +}); + +const toolDiagnosticsLogStyle = css({ + display: "flex", + maxHeight: "36", + flexDirection: "column", + gap: "2", + padding: "2", + overflowY: "auto", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a15", + borderRadius: "xl", + backgroundColor: "neutral.a10", +}); + +const toolDiagnosticEmptyStyle = css({ + display: "flex", + minHeight: "14", + alignItems: "center", + justifyContent: "center", + gap: "2", + color: "neutral.s65", + fontSize: "sm", +}); + +const toolDiagnosticCardStyle = css({ + display: "grid", + gridTemplateColumns: "[minmax(0, 1fr) auto]", + gap: "1.5", + paddingX: "2.5", + paddingY: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "lg", + backgroundColor: "white", +}); + +const toolDiagnosticNameStyle = css({ + minWidth: "0", + overflow: "hidden", + color: "blue.a85", + fontFamily: "mono", + fontSize: "xs", + fontWeight: "semibold", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}); + +const toolDiagnosticTurnStyle = css({ + color: "neutral.s50", + fontFamily: "mono", + fontSize: "xs", + whiteSpace: "nowrap", +}); + +const toolDiagnosticSummaryStyle = css({ + gridColumn: "[1 / -1]", + color: "neutral.s75", + fontSize: "sm", + lineHeight: "relaxed", + overflowWrap: "anywhere", +}); + +const toolDiagnosticCallStyle = css({ + gridColumn: "[1 / -1]", + color: "neutral.s45", + fontFamily: "mono", + fontSize: "xs", + overflowWrap: "anywhere", +}); + type SessionState = | "ready" | "connecting" @@ -339,6 +561,7 @@ type LoggedEvent = { }; type TranscriptEntry = { + id: number; isPartial: boolean; speaker: "assistant" | "expert"; transcript: string; @@ -346,23 +569,42 @@ type TranscriptEntry = { }; const getTranscriptEntries = (events: LoggedEvent[]): TranscriptEntry[] => { - const entries = new Map(); + const entries: TranscriptEntry[] = []; + const partialEntryIndexes = new Map(); - for (const { event } of events) { + for (const { event, sequence } of events) { if ( event.type === "partial-transcript" || event.type === "final-transcript" ) { - entries.set(`${event.turnId}:${event.speaker}`, { + const partialKey = `${event.turnId}:${event.speaker}`; + const partialEntryIndex = partialEntryIndexes.get(partialKey); + const entry: TranscriptEntry = { + id: + partialEntryIndex === undefined + ? sequence + : (entries[partialEntryIndex]?.id ?? sequence), isPartial: event.type === "partial-transcript", speaker: event.speaker, transcript: event.transcript, turnId: event.turnId, - }); + }; + + if (partialEntryIndex === undefined) { + if (event.type === "partial-transcript") { + partialEntryIndexes.set(partialKey, entries.length); + } + entries.push(entry); + } else { + entries[partialEntryIndex] = entry; + if (event.type === "final-transcript") { + partialEntryIndexes.delete(partialKey); + } + } } } - return [...entries.values()]; + return entries; }; const getEventSummary = (event: VoiceExperimentEvent) => { @@ -373,7 +615,7 @@ const getEventSummary = (event: VoiceExperimentEvent) => { return `${event.type} (${event.speaker}): ${event.transcript}`; } if (event.type === "tool-called") { - return `${event.type}: ${event.toolName}`; + return `${event.type}: ${event.toolName} · turn ${event.turnId} · call ${event.callId}`; } if (event.type === "error") { return `${event.type}: ${event.message}`; @@ -396,10 +638,15 @@ export const VoiceExperiment = ({ }) => { const [conversationId] = useState(() => crypto.randomUUID()); const [events, setEvents] = useState([]); + const [isExpanded, setIsExpanded] = useState(false); const [sessionState, setSessionState] = useState("ready"); - const [isTurnActive, setIsTurnActive] = useState(false); + const [isConversationActive, setIsConversationActive] = useState(false); + const hasToggledPanelRef = useRef(false); + const launcherButtonRef = useRef(null); + const minimizeButtonRef = useRef(null); const sequenceRef = useRef(0); - const pressedRef = useRef(false); + const transcriptRef = useRef(null); + const shouldAutoScrollTranscriptRef = useRef(true); const appendEvent = useCallback((event: VoiceExperimentEvent) => { setEvents((previous) => [ @@ -410,7 +657,7 @@ export const VoiceExperiment = ({ if (event.type === "connected") { setSessionState("connected"); } else if (event.type === "recording-started") { - setIsTurnActive(true); + setIsConversationActive(true); } else if (event.type === "response-started") { setSessionState("responding"); } else if (event.type === "response-completed") { @@ -422,9 +669,20 @@ export const VoiceExperiment = ({ useEffect(() => adapter?.subscribe(appendEvent), [adapter, appendEvent]); + useEffect(() => { + if (!hasToggledPanelRef.current) { + return; + } + + if (isExpanded) { + minimizeButtonRef.current?.focus(); + } else { + launcherButtonRef.current?.focus(); + } + }, [isExpanded]); + useEffect(() => { const dispose = () => { - pressedRef.current = false; void adapter?.dispose(); }; @@ -435,7 +693,7 @@ export const VoiceExperiment = ({ }; }, [adapter]); - const startSession = async () => { + const startConversation = async () => { if (!adapter || sessionState !== "ready") { return; } @@ -443,19 +701,22 @@ export const VoiceExperiment = ({ setSessionState("connecting"); try { await adapter.connect(); + await adapter.startTurn(); + setIsConversationActive(true); setSessionState("connected"); } catch (error) { + setIsConversationActive(false); + await adapter.dispose().catch(() => undefined); appendEvent(createErrorEvent(error)); } }; - const endSession = async () => { + const stopConversation = async () => { if (!adapter || sessionState === "ready" || sessionState === "ended") { return; } - pressedRef.current = false; - setIsTurnActive(false); + setIsConversationActive(false); setSessionState("ending"); try { await adapter.dispose(); @@ -465,236 +726,344 @@ export const VoiceExperiment = ({ } }; - const startTurn = async () => { - if ( - !adapter || - (sessionState !== "connected" && sessionState !== "responding") || - pressedRef.current - ) { - return; - } - - pressedRef.current = true; - setIsTurnActive(true); - try { - await adapter.startTurn(); - } catch (error) { - pressedRef.current = false; - setIsTurnActive(false); - appendEvent(createErrorEvent(error)); - } - }; - - const finishTurn = async () => { - if (!adapter || !pressedRef.current) { - return; - } - - pressedRef.current = false; - setIsTurnActive(false); - try { - await adapter.finishTurn(); - } catch (error) { - appendEvent(createErrorEvent(error)); - } - }; - - const handlePointerDown = (event: PointerEvent) => { - event.preventDefault(); - event.currentTarget.setPointerCapture(event.pointerId); - void startTurn(); - }; - - const handleKeyDown = (event: KeyboardEvent) => { - if ((event.key === " " || event.key === "Enter") && !event.repeat) { - event.preventDefault(); - void startTurn(); + useEffect(() => { + const transcript = transcriptRef.current; + if (transcript && shouldAutoScrollTranscriptRef.current) { + transcript.scrollTo({ top: transcript.scrollHeight }); } - }; - - const handleKeyUp = (event: KeyboardEvent) => { - if (event.key === " " || event.key === "Enter") { - event.preventDefault(); - void finishTurn(); + }, [events]); + + const handleTranscriptScroll = () => { + const transcript = transcriptRef.current; + if (transcript) { + const distanceFromBottom = + transcript.scrollHeight - + transcript.scrollTop - + transcript.clientHeight; + shouldAutoScrollTranscriptRef.current = distanceFromBottom < 24; } }; const transcriptEntries = getTranscriptEntries(events); + const toolDiagnostics = events.filter( + ( + entry, + ): entry is LoggedEvent & { + event: Extract; + } => entry.event.type === "tool-called", + ); const isConnected = sessionState === "connected" || sessionState === "responding"; - const isAdapterPending = !adapter; - const canStartSession = Boolean(adapter) && sessionState === "ready"; - const canEndSession = + const isPending = sessionState === "connecting" || sessionState === "ending"; + const canToggleConversation = Boolean(adapter) && - sessionState !== "ready" && - sessionState !== "ending" && - sessionState !== "ended"; - const canStartTurn = Boolean(adapter) && isConnected; + !isPending && + (sessionState === "ready" || isConversationActive); const latestEvent = events.at(-1)?.event; - const statusMessage = isAdapterPending - ? "Voice connection unavailable" - : isTurnActive - ? "Listening — release to send" - : sessionState === "ready" - ? "Ready" - : sessionState === "connecting" - ? "Connecting…" - : sessionState === "connected" - ? "Connected — hold the microphone to speak" - : sessionState === "responding" - ? "Interviewer is responding…" - : sessionState === "ending" - ? "Ending…" - : sessionState === "ended" - ? "Session ended" - : latestEvent?.type === "error" - ? latestEvent.message - : "Could not start the voice session"; + const statusLabel = !adapter + ? "Unavailable" + : sessionState === "ready" + ? "Ready" + : sessionState === "connecting" + ? "Connecting" + : sessionState === "connected" || sessionState === "responding" + ? "Conversation active" + : sessionState === "ending" + ? "Stopping" + : sessionState === "ended" + ? "Conversation ended" + : latestEvent?.type === "error" + ? latestEvent.message + : "Connection error"; + const controlLabel = isConversationActive + ? "Stop conversation" + : sessionState === "connecting" + ? "Starting…" + : sessionState === "ending" + ? "Stopping…" + : sessionState === "ended" + ? "Conversation ended" + : sessionState === "error" + ? "Unavailable" + : "Start conversation"; return ( - + + {events.length} {events.length === 1 ? "event" : "events"} + + +
+ {events.length === 0 ? ( + + No events · conversation {conversationId.slice(0, 8)} + + ) : ( + events.map(({ event, sequence }) => ( +
+ + {String(sequence).padStart(2, "0")} ·{" "} + {getEventSummary(event)} + + +
+ )) + )} +
+ + + ); }; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts index b7232e211d5..975f51c6b92 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.test.ts @@ -23,32 +23,50 @@ class FakeConversation { public setMicMuted = vi.fn(); } -const createHarness = () => { +const createHarness = ({ autoConnect = true } = {}) => { const conversation = new FakeConversation(); let callbacks: SessionCallbacks | null = null; + let diagnosticPoll: (() => void) | null = null; + let diagnosticResponse: unknown = { events: [] }; const startSession = vi.fn(async (options: SessionCallbacks) => { callbacks = options; options.onConversationCreated?.(conversation); - options.onConnect?.({ conversationId: "conv_123" }); + if (autoConnect) { + options.onConnect?.({ conversationId: "conv_123" }); + } return conversation; }); const permissionTrack = { stop: vi.fn() }; const getUserMedia = vi.fn(async () => ({ getTracks: () => [permissionTrack], })); - const fetch = vi.fn().mockResolvedValue( - Response.json({ - conversationId: "conv_123", - conversationToken: "short-lived-token", - }), + const fetch = vi.fn(async (input: RequestInfo | URL) => + (input instanceof Request + ? input.url + : input instanceof URL + ? input.href + : input + ).includes("elevenlabs-brunch-diagnostics") + ? Response.json(diagnosticResponse) + : Response.json({ + conversationId: "conv_123", + conversationToken: "short-lived-token", + }), ); + const clearInterval = vi.fn(); + const setInterval = vi.fn((callback: () => void) => { + diagnosticPoll = callback; + return 17 as unknown as ReturnType; + }); let now = 1_000; const adapter = createElevenLabsAdapter({ + clearInterval, fetch: fetch as typeof globalThis.fetch, getUserMedia: getUserMedia as unknown as ( constraints: MediaStreamConstraints, ) => Promise, now: () => ++now, + setInterval, startSession, }); const events: VoiceExperimentEvent[] = []; @@ -57,16 +75,57 @@ const createHarness = () => { return { adapter, callbacks: () => callbacks, + clearInterval, conversation, events, fetch, getUserMedia, permissionTrack, + pollDiagnostics: async (response: unknown) => { + diagnosticResponse = response; + diagnosticPoll?.(); + await new Promise((resolve) => setTimeout(resolve, 0)); + }, + setInterval, startSession, }; }; describe("ElevenLabsAdapter", () => { + test("can connect after an unused adapter is disposed by a Strict Mode cleanup", async () => { + const harness = createHarness(); + + await harness.adapter.dispose(); + await harness.adapter.connect(); + + await expect(harness.adapter.startTurn()).resolves.toBeUndefined(); + expect(harness.conversation.endSession).not.toHaveBeenCalled(); + expect(harness.conversation.setMicMuted.mock.calls).toEqual([ + [true], + [false], + ]); + }); + + test("does not resolve connect until ElevenLabs reports the session connected", async () => { + const harness = createHarness({ autoConnect: false }); + let connectionResolved = false; + + const connection = harness.adapter.connect().then(() => { + connectionResolved = true; + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(harness.startSession).toHaveBeenCalledTimes(1); + expect(connectionResolved).toBe(false); + + harness.callbacks()?.onConnect?.({ conversationId: "conv_123" }); + await connection; + + expect(connectionResolved).toBe(true); + await expect(harness.adapter.startTurn()).resolves.toBeUndefined(); + }); + test("starts an authenticated WebRTC session with the microphone gated", async () => { const harness = createHarness(); @@ -88,9 +147,17 @@ describe("ElevenLabsAdapter", () => { }), ); expect(harness.conversation.setMicMuted).toHaveBeenCalledWith(true); - expect(harness.fetch.mock.calls.flat().join(" ")).not.toContain( - "/api/chat", - ); + expect( + harness.fetch.mock.calls + .map(([input]) => + input instanceof Request + ? input.url + : input instanceof URL + ? input.href + : input, + ) + .join(" "), + ).not.toContain("/api/chat"); expect(harness.events).toContainEqual({ timestampMs: 1_001, type: "connected", @@ -161,6 +228,42 @@ describe("ElevenLabsAdapter", () => { }); }); + test("polls normalized Brunch tool diagnostics for the provider conversation", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + + await harness.pollDiagnostics({ + events: [ + { + argumentSummary: "Question: Who owns triage?", + callId: "call_ask", + privateInput: "must-not-appear", + sequence: 1, + timestampMs: 2_000, + toolName: "brunch_ask", + turnId: 1, + }, + ], + }); + + expect(harness.fetch).toHaveBeenCalledWith( + "/api/voice-experiment/elevenlabs-brunch-diagnostics?conversationId=conv_123&after=0", + { headers: { "x-voice-experiment": "elevenlabs-brunch" } }, + ); + expect(harness.events).toContainEqual({ + argumentSummary: "Question: Who owns triage?", + callId: "call_ask", + timestampMs: 2_000, + toolName: "brunch_ask", + turnId: 1, + type: "tool-called", + }); + expect(JSON.stringify(harness.events)).not.toContain("must-not-appear"); + + await harness.adapter.dispose(); + expect(harness.clearInterval).toHaveBeenCalledWith(17); + }); + test("releases ElevenLabs microphone, playback, and connection once", async () => { const harness = createHarness(); await harness.adapter.connect(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts index 22f268f2bbc..4b6d3a877b3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/elevenlabs-adapter.ts @@ -2,6 +2,9 @@ import type { VoiceExperimentAdapter } from "./voice-experiment-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment-events"; const TOKEN_ENDPOINT = "/api/voice-experiment/elevenlabs-conversation-token"; +const DIAGNOSTICS_ENDPOINT = + "/api/voice-experiment/elevenlabs-brunch-diagnostics"; +const DIAGNOSTICS_POLL_INTERVAL_MS = 500; type ConversationControl = { endSession(): Promise; @@ -25,17 +28,25 @@ type SessionOptions = { }; type ElevenLabsAdapterDependencies = { + clearInterval: (handle: ReturnType) => void; fetch: typeof globalThis.fetch; getUserMedia: (constraints: MediaStreamConstraints) => Promise; now: () => number; + setInterval: ( + callback: () => void, + intervalMs: number, + ) => ReturnType; startSession: (options: SessionOptions) => Promise; }; const defaultDependencies: ElevenLabsAdapterDependencies = { + clearInterval: (handle) => globalThis.clearInterval(handle), fetch: (...args) => globalThis.fetch(...args), getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints), now: () => Date.now(), + setInterval: (callback, intervalMs) => + globalThis.setInterval(callback, intervalMs), startSession: async (options) => { const { Conversation } = await import("@elevenlabs/client"); return Conversation.startSession(options); @@ -55,6 +66,60 @@ const getString = ( return typeof candidate === "string" ? candidate : null; }; +const boundedDiagnosticText = (value: unknown, limit: number): string => { + if (typeof value !== "string") { + return ""; + } + let sanitized = ""; + for (const character of value.normalize("NFKC")) { + const codePoint = character.codePointAt(0) ?? 0; + if (codePoint >= 32 && codePoint !== 127) { + sanitized += character; + } + } + return sanitized.replace(/\s+/gu, " ").trim().slice(0, limit); +}; + +const parseToolDiagnostic = ( + value: unknown, +): + | ({ sequence: number } & Extract< + VoiceExperimentEvent, + { type: "tool-called" } + >) + | null => { + const record = asRecord(value); + const sequence = record?.sequence; + const timestampMs = record?.timestampMs; + const turnId = record?.turnId; + const toolName = boundedDiagnosticText(record?.toolName, 96); + const callId = boundedDiagnosticText(record?.callId, 96); + const argumentSummary = boundedDiagnosticText(record?.argumentSummary, 240); + if ( + !Number.isSafeInteger(sequence) || + Number(sequence) < 1 || + typeof timestampMs !== "number" || + !Number.isFinite(timestampMs) || + !Number.isSafeInteger(turnId) || + Number(turnId) < 1 || + !toolName || + !callId || + !argumentSummary + ) { + return null; + } + + return { + argumentSummary, + callId, + sequence: Number(sequence), + timestampMs, + toolName, + turnId: Number(turnId), + type: "tool-called", + }; +}; + class ElevenLabsAdapter implements VoiceExperimentAdapter { readonly #dependencies: ElevenLabsAdapterDependencies; readonly #listeners = new Set<(event: VoiceExperimentEvent) => void>(); @@ -62,7 +127,13 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { #connectPromise: Promise | null = null; #connected = false; #conversation: ConversationControl | null = null; + #diagnosticConversationId: string | null = null; + #diagnosticCursor = 0; + #diagnosticPollHandle: ReturnType | null = + null; + #diagnosticPollInFlight = false; #disposed = false; + #hasConnected = false; #responseInProgress = false; #turnId = 0; @@ -76,6 +147,11 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { } public async connect(): Promise { + // React Strict Mode probes effect cleanup before the first real mount. + // Re-arm an adapter that was disposed before it ever owned a session. + if (this.#disposed && !this.#hasConnected) { + this.#disposed = false; + } if (this.#connected) { return; } @@ -111,6 +187,7 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { } this.#disposed = true; this.#connected = false; + this.#stopDiagnosticsPolling(); const conversation = this.#conversation; this.#conversation = null; if (conversation) { @@ -142,31 +219,73 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { track.stop(); } + let providerConnected = false; + let readinessSettled = false; + let resolveReady: () => void = () => undefined; + let rejectReady: (error: Error) => void = () => undefined; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + const markReadyIfConnected = () => { + if ( + readinessSettled || + !providerConnected || + !this.#conversation || + this.#disposed + ) { + return; + } + readinessSettled = true; + this.#connected = true; + this.#hasConnected = true; + this.#emit({ + timestampMs: this.#dependencies.now(), + type: "connected", + }); + resolveReady(); + }; + const failBeforeConnected = (message: string) => { + if (!readinessSettled) { + readinessSettled = true; + rejectReady(new Error(message)); + } + }; + const setConversation = (createdConversation: ConversationControl) => { + if (this.#conversation !== createdConversation) { + this.#conversation = createdConversation; + createdConversation.setMicMuted(true); + } + markReadyIfConnected(); + }; + const conversation = await this.#dependencies.startSession({ connectionType: "webrtc", conversationToken, - onConnect: () => { + onConnect: ({ conversationId }) => { if (this.#disposed) { return; } - this.#connected = true; - this.#emit({ - timestampMs: this.#dependencies.now(), - type: "connected", - }); + providerConnected = true; + this.#startDiagnosticsPolling(conversationId); + markReadyIfConnected(); }, onConversationCreated: (createdConversation) => { - this.#conversation = createdConversation; - createdConversation.setMicMuted(true); + setConversation(createdConversation); }, onDisconnect: ({ reason }) => { this.#connected = false; + this.#stopDiagnosticsPolling(); + failBeforeConnected( + "The ElevenLabs voice connection closed before it was ready.", + ); if (!this.#disposed && reason === "error") { this.#emitError("The ElevenLabs voice connection was lost."); } }, onError: () => { if (!this.#disposed) { + failBeforeConnected("The ElevenLabs voice connection failed."); this.#emitError("The ElevenLabs voice connection failed."); } }, @@ -212,11 +331,16 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { }, }); - this.#conversation = conversation; + setConversation(conversation); if (this.#disposed) { await conversation.endSession(); this.#conversation = null; + readinessSettled = true; + resolveReady(); + return; } + + await ready; } #emitResponseStarted(turnId: number): void { @@ -231,6 +355,64 @@ class ElevenLabsAdapter implements VoiceExperimentAdapter { }); } + #startDiagnosticsPolling(conversationId: string): void { + this.#stopDiagnosticsPolling(); + this.#diagnosticConversationId = conversationId; + this.#diagnosticCursor = 0; + this.#diagnosticPollHandle = this.#dependencies.setInterval(() => { + void this.#pollDiagnostics(); + }, DIAGNOSTICS_POLL_INTERVAL_MS); + } + + #stopDiagnosticsPolling(): void { + if (this.#diagnosticPollHandle !== null) { + this.#dependencies.clearInterval(this.#diagnosticPollHandle); + } + this.#diagnosticPollHandle = null; + this.#diagnosticConversationId = null; + this.#diagnosticPollInFlight = false; + } + + async #pollDiagnostics(): Promise { + const conversationId = this.#diagnosticConversationId; + if (!conversationId || this.#diagnosticPollInFlight || this.#disposed) { + return; + } + + this.#diagnosticPollInFlight = true; + try { + const query = new URLSearchParams({ + conversationId, + after: String(this.#diagnosticCursor), + }); + const response = await this.#dependencies.fetch( + `${DIAGNOSTICS_ENDPOINT}?${query}`, + { headers: { "x-voice-experiment": "elevenlabs-brunch" } }, + ); + if (!response.ok || this.#diagnosticConversationId !== conversationId) { + return; + } + + const body = asRecord(await response.json()); + if (!Array.isArray(body?.events)) { + return; + } + for (const value of body.events) { + const diagnostic = parseToolDiagnostic(value); + if (!diagnostic || diagnostic.sequence <= this.#diagnosticCursor) { + continue; + } + this.#diagnosticCursor = diagnostic.sequence; + const { sequence: _sequence, ...event } = diagnostic; + this.#emit(event); + } + } catch { + // Diagnostics are non-authoritative and must never disrupt voice turns. + } finally { + this.#diagnosticPollInFlight = false; + } + } + #requireConversation(): ConversationControl { if (!this.#connected || !this.#conversation || this.#disposed) { throw new Error("The ElevenLabs voice session is not connected."); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts index d00e88ff84b..084b9df494e 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts @@ -212,6 +212,55 @@ describe("OpenAIRealtimeAdapter", () => { }); }); + test("keeps the microphone open while server VAD creates distinct turns", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + await harness.adapter.startTurn(); + + harness.dataChannel.receive({ + type: "input_audio_buffer.committed", + item_id: "expert-item-1", + }); + harness.dataChannel.receive({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "expert-item-1", + transcript: "The support lead triages it.", + }); + harness.dataChannel.receive({ type: "response.created" }); + harness.dataChannel.receive({ + type: "response.done", + response: { output: [], status: "completed" }, + }); + harness.dataChannel.receive({ + type: "input_audio_buffer.committed", + item_id: "expert-item-2", + }); + harness.dataChannel.receive({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "expert-item-2", + transcript: "Then the incident owner takes over.", + }); + + expect(harness.microphoneTrack.enabled).toBe(true); + expect(harness.dataChannel.sent).toEqual([ + { type: "input_audio_buffer.clear" }, + ]); + expect(harness.events).toContainEqual({ + speaker: "expert", + timestampMs: 1_003, + transcript: "The support lead triages it.", + turnId: 1, + type: "final-transcript", + }); + expect(harness.events).toContainEqual({ + speaker: "expert", + timestampMs: 1_006, + transcript: "Then the incident owner takes over.", + turnId: 2, + type: "final-transcript", + }); + }); + test("executes only dummy tools and continues the same turn", async () => { const harness = createHarness(); await harness.adapter.connect(); @@ -224,7 +273,8 @@ describe("OpenAIRealtimeAdapter", () => { response: { output: [ { - arguments: '{"name":"Triage","description":"Assess severity"}', + arguments: + '{"name":"Triage","description":"Assess severity","owner":"Support lead","secret":"must-not-leak"}', call_id: "call-1", name: "record_process_step", type: "function_call", @@ -235,11 +285,14 @@ describe("OpenAIRealtimeAdapter", () => { }); expect(harness.events).toContainEqual({ + argumentSummary: "Triage · Assess severity · Owner: Support lead", + callId: "call-1", timestampMs: 1_004, toolName: "record_process_step", turnId: 1, type: "tool-called", }); + expect(JSON.stringify(harness.events)).not.toContain("must-not-leak"); expect(harness.dataChannel.sent).toContainEqual({ type: "conversation.item.create", item: { @@ -259,6 +312,53 @@ describe("OpenAIRealtimeAdapter", () => { ).toBe(false); }); + test("ends a background-audio turn silently after wait_for_user", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + await harness.adapter.startTurn(); + harness.dataChannel.receive({ type: "response.created" }); + + harness.dataChannel.receive({ + type: "response.done", + response: { + output: [ + { + arguments: "{}", + call_id: "call-wait", + name: "wait_for_user", + type: "function_call", + }, + ], + status: "completed", + }, + }); + + expect(harness.events).toContainEqual({ + argumentSummary: "Silent no-op", + callId: "call-wait", + timestampMs: 1_004, + toolName: "wait_for_user", + turnId: 1, + type: "tool-called", + }); + expect(harness.dataChannel.sent.at(-1)).toEqual({ + type: "conversation.item.create", + item: { + call_id: "call-wait", + output: JSON.stringify({ mode: "experiment-only", waited: true }), + type: "function_call_output", + }, + }); + expect(harness.dataChannel.sent).not.toContainEqual({ + type: "response.create", + }); + expect(harness.events.at(-1)).toEqual({ + timestampMs: 1_005, + turnId: 1, + type: "response-completed", + }); + }); + test("cancels response audio when the expert barges in", async () => { const harness = createHarness(); await harness.adapter.connect(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts index e3e1b34e986..dfc25967445 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts @@ -23,6 +23,7 @@ type RealtimeEvent = { }; type FunctionCall = { + arguments: string | null; callId: string; name: string; }; @@ -65,6 +66,70 @@ const parseServerEvent = (data: unknown): RealtimeEvent | null => { } }; +const boundedSummaryText = (value: unknown, limit = 120): string => { + if (typeof value !== "string") { + return ""; + } + let sanitized = ""; + for (const character of value.normalize("NFKC")) { + const codePoint = character.codePointAt(0) ?? 0; + if (codePoint >= 32 && codePoint !== 127) { + sanitized += character; + } + } + return sanitized.replace(/\s+/gu, " ").trim().slice(0, limit); +}; + +const summarizeFunctionCall = ({ + arguments: serializedArguments, + name, +}: FunctionCall): string => { + if (!serializedArguments) { + return "Arguments unavailable"; + } + + let input: Record | null = null; + try { + input = asRecord(JSON.parse(serializedArguments)); + } catch { + return "Arguments unavailable"; + } + + if (name === "record_process_step") { + const parts = [ + boundedSummaryText(input?.name), + boundedSummaryText(input?.description), + ]; + const owner = boundedSummaryText(input?.owner); + if (owner) { + parts.push(`Owner: ${owner}`); + } + return ( + parts.filter(Boolean).join(" · ").slice(0, 240) || "Arguments unavailable" + ); + } + + if (name === "record_process_decision") { + const condition = boundedSummaryText(input?.condition); + const outcomes = Array.isArray(input?.outcomes) + ? input.outcomes + .slice(0, 4) + .map((outcome) => boundedSummaryText(outcome, 80)) + .filter(Boolean) + .join(" / ") + : ""; + const parts = [ + condition ? `Condition: ${condition}` : "", + outcomes ? `Outcomes: ${outcomes}` : "", + ]; + return ( + parts.filter(Boolean).join(" · ").slice(0, 240) || "Arguments unavailable" + ); + } + + return "Arguments hidden"; +}; + class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { readonly #dependencies: OpenAIRealtimeAdapterDependencies; readonly #listeners = new Set<(event: VoiceExperimentEvent) => void>(); @@ -309,9 +374,18 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { if (event.type === "input_audio_buffer.committed") { const itemId = getString(event, "item_id"); - if (itemId && this.#pendingCommittedTurnId !== null) { - this.#turnByItemId.set(itemId, this.#pendingCommittedTurnId); - this.#pendingCommittedTurnId = null; + if (itemId) { + if (this.#pendingCommittedTurnId !== null) { + this.#turnByItemId.set(itemId, this.#pendingCommittedTurnId); + this.#pendingCommittedTurnId = null; + } else { + if (this.#turnByItemId.size > 0) { + this.#turnId += 1; + this.#latestTurnId = this.#turnId; + this.#latestAssistantTranscript = ""; + } + this.#turnByItemId.set(itemId, this.#latestTurnId); + } } return; } @@ -432,6 +506,8 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { for (const functionCall of functionCalls) { this.#emit({ + argumentSummary: summarizeFunctionCall(functionCall), + callId: functionCall.callId, timestampMs: this.#dependencies.now(), toolName: functionCall.name, turnId: this.#latestTurnId, @@ -481,7 +557,9 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { } const callId = getString(record, "call_id"); const name = getString(record, "name"); - return callId && name ? [{ callId, name }] : []; + return callId && name + ? [{ arguments: getString(record, "arguments"), callId, name }] + : []; }); } diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts index 38aa0790096..f8ca2b1c77d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-events.ts @@ -23,6 +23,8 @@ export type VoiceExperimentEvent = type: "response-completed"; } | { + argumentSummary: string; + callId: string; timestampMs: number; toolName: string; turnId: number; diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 56d3c1c37ec..7ac9d56767b 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -108,6 +108,10 @@ export default defineConfig(({ mode }) => { }, server: { proxy: { + "/api/voice-experiment/elevenlabs-brunch-diagnostics": { + target: process.env.BRUNCH_CHAT_ORIGIN ?? "http://127.0.0.1:4321", + changeOrigin: true, + }, "/api/petrinaut-opt": { target: process.env.PETRINAUT_OPT_ORIGIN ?? "http://127.0.0.1:4004", changeOrigin: true, From e527e30c5d55423fe6c9774e33e9a0160b257326 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 20:11:22 +0200 Subject: [PATCH 06/19] H-6763: Elicit Petri net models in the OpenAI voice experiment --- .../openai-realtime-session.ts | 138 ++++++++++++++++-- .../openai-realtime-adapter.ts | 75 +++++++++- 2 files changed, 198 insertions(+), 15 deletions(-) diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts index 1dfd96d4b17..d6961484ea8 100644 --- a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts @@ -17,13 +17,34 @@ const sessionConfig = { model: "gpt-realtime-2.1", output_modalities: ["audio"], instructions: [ - "You are running a short voice interview experiment with a domain expert.", - "Interview the expert about how an urgent customer support escalation moves from the initial report to resolution.", - "Ask one focused question at a time. Briefly acknowledge the answer, then ask the highest-value follow-up.", - "Use record_process_step when the expert describes a process step. Use record_process_decision when they describe a branch or decision.", - "The tools are experiment-only instrumentation. Never claim that their output was persisted to Brunch or Petrinaut.", - "Keep spoken responses concise and natural.", - ].join(" "), + "# Role and Objective", + "You are a voice elicitation agent helping a domain expert describe a process well enough to draft a Stochastic Dynamic Coloured Petri Net in Petrinaut.", + "Do not assume a particular domain. Let the expert's first substantive statement establish the process being modeled.", + "# Conversation Flow", + "Work through these phases in order, while following the expert when they reveal important details early:", + "1. Scope: establish the process goal, the entity or token being modeled, and the start and end boundaries.", + "2. Structure: identify stable states, queues, and resources as candidate places; identify activities, events, and handoffs as candidate transitions; then establish their order.", + "3. Logic: identify branches and their conditions, loops and retries, concurrency, failure paths, and the flows that enable or consume each step.", + "4. Dynamics and evaluation: identify timing or rates, capacity constraints, useful metrics, and scenarios the model should support.", + "5. Confirmation: give a brief recap of the understood model and ask for one correction or missing detail at a time.", + "# Turn Discipline", + "Ask exactly one short, focused question in each spoken response.", + "Briefly acknowledge the answer, then ask only the highest-value missing fact.", + "Do not advance until the current question has been meaningfully answered.", + "If the expert gives a terse reply such as yes or no without the requested detail, clarify the same gap instead of repeating the question verbatim or moving on.", + "Do not speak again unless there is new, meaningful expert input or you are continuing immediately after recording an explicit fact with a tool.", + "Avoid compound questions and keep each spoken response to one or two concise sentences.", + "# Tools", + "All tools are dummy, experiment-only instrumentation. Never claim that their output was persisted to Brunch, Petrinaut, or any authoritative model.", + "Record only facts explicitly supplied or confirmed by the expert. Do not invent missing model elements.", + "Use record_process_state for a candidate place, record_process_step for a candidate transition, record_process_decision for branching logic, record_process_flow for a candidate arc, and record_model_requirement for timing, capacity, metrics, scenarios, or assumptions.", + "Do not narrate tool use. After a recording tool returns, continue with exactly one short next question.", + "# Silence and Background Audio", + "If the input is silence, background noise, television or speaker audio, a side conversation, your own playback, or speech not addressed to you, call wait_for_user and do not respond conversationally afterward.", + "If speech addressed to you is unclear, ask one brief clarification question.", + "# Voice Style", + "Sound natural and attentive. Avoid filler preambles and keep questions ideally under twenty words.", + ].join("\n"), max_output_tokens: 600, audio: { input: { @@ -31,7 +52,7 @@ const sessionConfig = { model: "gpt-live-transcribe", delay: "low", prompt: - "A process-model interview about urgent customer support escalations, incident ownership, handoffs, decisions, and resolution.", + "A domain-expert interview to elicit processes, states, transitions, flows, decisions, timing, constraints, metrics, and scenarios for Petri-net modeling.", }, turn_detection: { type: "server_vad", @@ -47,11 +68,41 @@ const sessionConfig = { }, }, tools: [ + { + type: "function", + name: "record_process_state", + description: + "Record an explicitly stated stable state, queue, resource, source, or sink as a candidate Petri-net place. Experiment instrumentation only.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + name: { + type: "string", + description: "A short name for the candidate place.", + }, + description: { + type: "string", + description: "What it represents in the expert's process.", + }, + category: { + type: "string", + description: "The kind of process state being recorded.", + enum: ["state", "queue", "resource", "source", "sink"], + }, + tokenDescription: { + type: "string", + description: "What a token at this place represents, if known.", + }, + }, + required: ["name", "description", "category"], + }, + }, { type: "function", name: "record_process_step", description: - "Record a process step mentioned by the expert for experiment instrumentation only.", + "Record an explicitly stated activity, event, or handoff as a candidate Petri-net transition. Experiment instrumentation only.", parameters: { type: "object", additionalProperties: false, @@ -68,6 +119,15 @@ const sessionConfig = { type: "string", description: "The role or team that owns the step, if known.", }, + trigger: { + type: "string", + description: "What enables or triggers the step, if known.", + }, + timing: { + type: "string", + description: "The known timing behavior of the step.", + enum: ["immediate", "deterministic", "stochastic", "unknown"], + }, }, required: ["name", "description"], }, @@ -94,6 +154,66 @@ const sessionConfig = { required: ["condition", "outcomes"], }, }, + { + type: "function", + name: "record_process_flow", + description: + "Record an explicitly stated flow between candidate places and transitions as a candidate Petri-net arc. Experiment instrumentation only.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + from: { + type: "string", + description: "The source state or step named by the expert.", + }, + to: { + type: "string", + description: "The destination state or step named by the expert.", + }, + condition: { + type: "string", + description: "A condition on this flow, if one was stated.", + }, + }, + required: ["from", "to"], + }, + }, + { + type: "function", + name: "record_model_requirement", + description: + "Record an explicitly stated timing, capacity, metric, scenario, or assumption for the candidate model. Experiment instrumentation only.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + category: { + type: "string", + description: "The kind of model requirement being recorded.", + enum: ["timing", "capacity", "metric", "scenario", "assumption"], + }, + description: { + type: "string", + description: + "The requirement as stated or confirmed by the expert.", + }, + }, + required: ["category", "description"], + }, + }, + { + type: "function", + name: "wait_for_user", + description: + "Use when the input is silence, background noise, playback, a side conversation, or speech not addressed to the interviewer. This is a silent no-op and must not be followed by a spoken response.", + parameters: { + type: "object", + additionalProperties: false, + properties: {}, + required: [], + }, + }, ], tool_choice: "auto", }, diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts index dfc25967445..0da8fa5641d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts @@ -4,9 +4,14 @@ import type { VoiceExperimentEvent } from "./voice-experiment-events"; const SESSION_ENDPOINT = "/api/voice-experiment/openai-realtime-session"; const REALTIME_CALLS_ENDPOINT = "https://api.openai.com/v1/realtime/calls"; const CONNECTION_TIMEOUT_MS = 20_000; +const WAIT_FOR_USER_TOOL_NAME = "wait_for_user"; const DUMMY_TOOL_NAMES = new Set([ "record_process_decision", + "record_process_flow", "record_process_step", + "record_process_state", + "record_model_requirement", + WAIT_FOR_USER_TOOL_NAME, ]); type OpenAIRealtimeAdapterDependencies = { @@ -84,6 +89,10 @@ const summarizeFunctionCall = ({ arguments: serializedArguments, name, }: FunctionCall): string => { + if (name === WAIT_FOR_USER_TOOL_NAME) { + return "Silent no-op"; + } + if (!serializedArguments) { return "Arguments unavailable"; } @@ -95,6 +104,20 @@ const summarizeFunctionCall = ({ return "Arguments unavailable"; } + if (name === "record_process_state") { + const nameSummary = boundedSummaryText(input?.name); + const category = boundedSummaryText(input?.category); + const description = boundedSummaryText(input?.description); + const parts = [ + nameSummary, + category ? `Type: ${category}` : "", + description, + ]; + return ( + parts.filter(Boolean).join(" · ").slice(0, 240) || "Arguments unavailable" + ); + } + if (name === "record_process_step") { const parts = [ boundedSummaryText(input?.name), @@ -127,6 +150,28 @@ const summarizeFunctionCall = ({ ); } + if (name === "record_process_flow") { + const from = boundedSummaryText(input?.from); + const to = boundedSummaryText(input?.to); + const condition = boundedSummaryText(input?.condition); + const parts = [ + from && to ? `${from} → ${to}` : from || to, + condition ? `Condition: ${condition}` : "", + ]; + return ( + parts.filter(Boolean).join(" · ").slice(0, 240) || "Arguments unavailable" + ); + } + + if (name === "record_model_requirement") { + const category = boundedSummaryText(input?.category); + const description = boundedSummaryText(input?.description); + const parts = [category ? `Type: ${category}` : "", description]; + return ( + parts.filter(Boolean).join(" · ").slice(0, 240) || "Arguments unavailable" + ); + } + return "Arguments hidden"; }; @@ -505,6 +550,7 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { } for (const functionCall of functionCalls) { + const isWaitForUser = functionCall.name === WAIT_FOR_USER_TOOL_NAME; this.#emit({ argumentSummary: summarizeFunctionCall(functionCall), callId: functionCall.callId, @@ -518,15 +564,32 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { item: { type: "function_call_output", call_id: functionCall.callId, - output: JSON.stringify({ - mode: "experiment-only", - recorded: DUMMY_TOOL_NAMES.has(functionCall.name), - }), + output: JSON.stringify( + isWaitForUser + ? { mode: "experiment-only", waited: true } + : { + mode: "experiment-only", + recorded: DUMMY_TOOL_NAMES.has(functionCall.name), + }, + ), }, }); } - this.#send(dataChannel, { type: "response.create" }); - this.#responseInProgress = true; + + if ( + functionCalls.some( + (functionCall) => functionCall.name !== WAIT_FOR_USER_TOOL_NAME, + ) + ) { + this.#send(dataChannel, { type: "response.create" }); + this.#responseInProgress = true; + } else { + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId: this.#latestTurnId, + type: "response-completed", + }); + } return; } From ef4c975a5b559c3e01c523b7391ba47209dcde6b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 24 Aug 2026 20:14:57 +0200 Subject: [PATCH 07/19] H-6763: Integrate voice sessions with the AI prompt --- .../local-storage-demo/voice-experiment.tsx | 68 ++++++++++++++++--- .../views/Editor/components/ai-cta-modal.tsx | 6 ++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx index 5a2a210038f..2726bd40da2 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -555,6 +555,8 @@ type SessionState = | "ended" | "error"; +const panelTransitionDurationMs = 220; + type LoggedEvent = { event: VoiceExperimentEvent; sequence: number; @@ -639,11 +641,14 @@ export const VoiceExperiment = ({ const [conversationId] = useState(() => crypto.randomUUID()); const [events, setEvents] = useState([]); const [isExpanded, setIsExpanded] = useState(false); + const [isPanelMounted, setIsPanelMounted] = useState(false); const [sessionState, setSessionState] = useState("ready"); const [isConversationActive, setIsConversationActive] = useState(false); const hasToggledPanelRef = useRef(false); const launcherButtonRef = useRef(null); const minimizeButtonRef = useRef(null); + const panelCloseTimeoutRef = useRef(undefined); + const panelOpenFrameRef = useRef(undefined); const sequenceRef = useRef(0); const transcriptRef = useRef(null); const shouldAutoScrollTranscriptRef = useRef(true); @@ -681,6 +686,18 @@ export const VoiceExperiment = ({ } }, [isExpanded]); + useEffect( + () => () => { + if (panelCloseTimeoutRef.current !== undefined) { + window.clearTimeout(panelCloseTimeoutRef.current); + } + if (panelOpenFrameRef.current !== undefined) { + window.cancelAnimationFrame(panelOpenFrameRef.current); + } + }, + [], + ); + useEffect(() => { const dispose = () => { void adapter?.dispose(); @@ -726,6 +743,41 @@ export const VoiceExperiment = ({ } }; + const openVoiceInterview = () => { + hasToggledPanelRef.current = true; + + if (panelCloseTimeoutRef.current !== undefined) { + window.clearTimeout(panelCloseTimeoutRef.current); + panelCloseTimeoutRef.current = undefined; + } + if (panelOpenFrameRef.current !== undefined) { + window.cancelAnimationFrame(panelOpenFrameRef.current); + } + + setIsPanelMounted(true); + panelOpenFrameRef.current = window.requestAnimationFrame(() => { + setIsExpanded(true); + panelOpenFrameRef.current = undefined; + }); + + if (sessionState === "ready") { + void startConversation(); + } + }; + + const minimizeVoiceInterview = () => { + hasToggledPanelRef.current = true; + setIsExpanded(false); + + if (panelCloseTimeoutRef.current !== undefined) { + window.clearTimeout(panelCloseTimeoutRef.current); + } + panelCloseTimeoutRef.current = window.setTimeout(() => { + setIsPanelMounted(false); + panelCloseTimeoutRef.current = undefined; + }, panelTransitionDurationMs); + }; + useEffect(() => { const transcript = transcriptRef.current; if (transcript && shouldAutoScrollTranscriptRef.current) { @@ -788,7 +840,10 @@ export const VoiceExperiment = ({ : "Start conversation"; return ( -
+
- + )}
); }; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts index 49707f0663b..bce590972fd 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts @@ -127,7 +127,7 @@ describe("OpenAIRealtimeAdapter", () => { await harness.adapter.startTurn(); - expect(harness.microphoneTrack.enabled).toBe(true); + expect(harness.microphoneTrack.enabled).toBe(false); expect(harness.dataChannel.sent).toEqual([ { type: "input_audio_buffer.clear" }, { @@ -138,16 +138,12 @@ describe("OpenAIRealtimeAdapter", () => { }, }, ]); - expect(harness.events.at(-1)).toEqual({ - timestampMs: 1_002, - turnId: 1, - type: "recording-started", - }); + expect(harness.events).toEqual([{ timestampMs: 1_001, type: "connected" }]); await harness.adapter.finishTurn(); await harness.adapter.startTurn(); - expect(harness.microphoneTrack.enabled).toBe(true); + expect(harness.microphoneTrack.enabled).toBe(false); expect(harness.dataChannel.sent).toEqual([ { type: "input_audio_buffer.clear" }, { @@ -197,45 +193,65 @@ describe("OpenAIRealtimeAdapter", () => { expect(harness.events).toContainEqual({ speaker: "expert", - timestampMs: 1_003, + timestampMs: 1_002, transcript: "The support lead", turnId: 1, type: "partial-transcript", }); expect(harness.events).toContainEqual({ speaker: "expert", - timestampMs: 1_004, + timestampMs: 1_003, transcript: "The support lead owns the escalation.", turnId: 1, type: "final-transcript", }); expect(harness.events).toContainEqual({ speaker: "assistant", - timestampMs: 1_006, + timestampMs: 1_005, transcript: "What happens next?", turnId: 1, type: "partial-transcript", }); expect(harness.events).toContainEqual({ responseText: "What happens next?", - timestampMs: 1_008, + timestampMs: 1_007, turnId: 1, type: "response-completed", }); }); - test("listens only between interviewer responses", async () => { + test("does not render an empty finalized expert transcript", async () => { const harness = createHarness(); await harness.adapter.connect(); await harness.adapter.startTurn(); - expect(harness.microphoneTrack.enabled).toBe(true); harness.dataChannel.receive({ type: "input_audio_buffer.committed", - item_id: "expert-answer", + item_id: "empty-expert-item", }); - expect(harness.microphoneTrack.enabled).toBe(false); + harness.dataChannel.receive({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "empty-expert-item", + transcript: " ", + }); + + expect( + harness.events.some( + (event) => + event.type === "final-transcript" && event.speaker === "expert", + ), + ).toBe(false); + expect(harness.dataChannel.sent).not.toContainEqual({ + type: "response.create", + }); + }); + test("listens only between interviewer responses", async () => { + const harness = createHarness(); + await harness.adapter.connect(); + await harness.adapter.startTurn(); + + expect(harness.microphoneTrack.enabled).toBe(false); harness.dataChannel.receive({ type: "response.created", response: { id: "opening-response" }, @@ -244,7 +260,50 @@ describe("OpenAIRealtimeAdapter", () => { harness.dataChannel.receive({ type: "response.done", - response: { id: "opening-response", output: [], status: "completed" }, + response: { + id: "opening-response", + output: [ + { + type: "message", + content: [{ type: "audio", transcript: "Opening question" }], + }, + ], + status: "completed", + }, + }); + expect(harness.microphoneTrack.enabled).toBe(false); + harness.dataChannel.receive({ + type: "output_audio_buffer.stopped", + response_id: "opening-response", + }); + expect(harness.microphoneTrack.enabled).toBe(true); + + harness.dataChannel.receive({ + type: "input_audio_buffer.committed", + item_id: "expert-answer", + }); + expect(harness.microphoneTrack.enabled).toBe(false); + harness.dataChannel.receive({ + type: "response.created", + response: { id: "answer-response" }, + }); + harness.dataChannel.receive({ + type: "response.done", + response: { + id: "answer-response", + output: [ + { + type: "message", + content: [{ type: "audio", transcript: "Next question" }], + }, + ], + status: "completed", + }, + }); + expect(harness.microphoneTrack.enabled).toBe(false); + harness.dataChannel.receive({ + type: "output_audio_buffer.stopped", + response_id: "answer-response", }); expect(harness.microphoneTrack.enabled).toBe(true); }); @@ -288,10 +347,13 @@ describe("OpenAIRealtimeAdapter", () => { 'Say exactly: "Hi—what process would you like us to model today?" Do not call a tool.', }, }, + { type: "response.create" }, + { type: "input_audio_buffer.clear" }, + { type: "response.create" }, ]); expect(harness.events).toContainEqual({ speaker: "expert", - timestampMs: 1_003, + timestampMs: 1_002, transcript: "The support lead triages it.", turnId: 1, type: "final-transcript", @@ -327,11 +389,16 @@ describe("OpenAIRealtimeAdapter", () => { response: { id: "response-1", status: "cancelled" }, }); - expect(harness.events.at(-1)).toEqual({ - timestampMs: 1_004, + expect(harness.events).toContainEqual({ + timestampMs: 1_003, turnId: 1, type: "response-completed", }); + expect(harness.events.at(-1)).toEqual({ + timestampMs: 1_004, + turnId: 3, + type: "recording-started", + }); }); test("keeps late assistant transcript events on their response turn", async () => { @@ -365,7 +432,7 @@ describe("OpenAIRealtimeAdapter", () => { expect(harness.events.at(-1)).toEqual({ speaker: "assistant", - timestampMs: 1_004, + timestampMs: 1_003, transcript: "Interrupted question", turnId: 1, type: "partial-transcript", @@ -397,7 +464,7 @@ describe("OpenAIRealtimeAdapter", () => { expect(harness.events).toContainEqual({ argumentSummary: "Triage · Assess severity · Owner: Support lead", callId: "call-1", - timestampMs: 1_004, + timestampMs: 1_003, toolName: "record_process_step", turnId: 1, type: "tool-called", @@ -446,12 +513,12 @@ describe("OpenAIRealtimeAdapter", () => { expect(harness.events).toContainEqual({ argumentSummary: "Silent no-op", callId: "call-wait", - timestampMs: 1_004, + timestampMs: 1_003, toolName: "wait_for_user", turnId: 1, type: "tool-called", }); - expect(harness.dataChannel.sent.at(-1)).toEqual({ + expect(harness.dataChannel.sent).toContainEqual({ type: "conversation.item.create", item: { call_id: "call-wait", @@ -462,10 +529,15 @@ describe("OpenAIRealtimeAdapter", () => { expect(harness.dataChannel.sent).not.toContainEqual({ type: "response.create", }); + expect(harness.events).toContainEqual({ + timestampMs: 1_004, + turnId: 1, + type: "response-completed", + }); expect(harness.events.at(-1)).toEqual({ timestampMs: 1_005, turnId: 1, - type: "response-completed", + type: "recording-started", }); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts index 8320739d6bc..89fedccec85 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts @@ -179,6 +179,8 @@ const summarizeFunctionCall = ({ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { readonly #dependencies: OpenAIRealtimeAdapterDependencies; readonly #listeners = new Set<(event: VoiceExperimentEvent) => void>(); + readonly #pendingListeningTurnByResponseId = new Map(); + readonly #responsesWithAudio = new Set(); readonly #transcriptByItemId = new Map(); readonly #turnByItemId = new Map(); readonly #turnByResponseId = new Map(); @@ -238,12 +240,7 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { this.#latestAssistantTranscript = ""; this.#send(dataChannel, { type: "input_audio_buffer.clear" }); - microphoneTrack.enabled = true; - this.#emit({ - timestampMs: this.#dependencies.now(), - turnId: this.#turnId, - type: "recording-started", - }); + microphoneTrack.enabled = false; this.#send(dataChannel, { type: "response.create", response: { @@ -435,7 +432,27 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { if ( event.type === "conversation.item.input_audio_transcription.completed" ) { - this.#handleTranscriptCompleted(event, "expert"); + const itemId = getString(event, "item_id"); + const turnId = itemId ? this.#getTurnId(itemId) : this.#latestTurnId; + const transcript = this.#handleTranscriptCompleted(event, "expert"); + if (transcript) { + const dataChannel = this.#dataChannel; + if (dataChannel?.readyState === "open") { + this.#send(dataChannel, { type: "response.create" }); + } + } else { + this.#openListeningWindow(turnId); + } + return; + } + + if (event.type === "conversation.item.input_audio_transcription.failed") { + const itemId = getString(event, "item_id"); + const turnId = itemId ? this.#getTurnId(itemId) : this.#latestTurnId; + if (itemId) { + this.#transcriptByItemId.delete(itemId); + } + this.#openListeningWindow(turnId); return; } @@ -467,11 +484,19 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { } if (event.type === "response.output_audio_transcript.delta") { + const responseId = getString(event, "response_id"); + if (responseId) { + this.#responsesWithAudio.add(responseId); + } this.#handleTranscriptDelta(event, "assistant"); return; } if (event.type === "response.output_audio_transcript.done") { + const responseId = getString(event, "response_id"); + if (responseId) { + this.#responsesWithAudio.add(responseId); + } this.#handleTranscriptCompleted(event, "assistant"); return; } @@ -481,6 +506,22 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { return; } + if ( + event.type === "output_audio_buffer.stopped" || + event.type === "output_audio_buffer.cleared" + ) { + const responseId = getString(event, "response_id"); + const turnId = responseId + ? this.#pendingListeningTurnByResponseId.get(responseId) + : undefined; + if (responseId && turnId !== undefined) { + this.#pendingListeningTurnByResponseId.delete(responseId); + this.#responsesWithAudio.delete(responseId); + this.#openListeningWindow(turnId); + } + return; + } + if (event.type === "error") { const error = asRecord(event.error); this.#emitError( @@ -518,16 +559,20 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { #handleTranscriptCompleted( event: RealtimeEvent, speaker: "assistant" | "expert", - ) { + ): string | null { const itemId = getString(event, "item_id"); if (!itemId) { - return; + return null; } const transcript = getString(event, "transcript") ?? this.#transcriptByItemId.get(itemId) ?? ""; + if (!transcript.trim()) { + this.#transcriptByItemId.delete(itemId); + return null; + } this.#transcriptByItemId.set(itemId, transcript); if (speaker === "assistant") { this.#latestAssistantTranscript = transcript; @@ -540,6 +585,7 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { turnId: this.#getTurnId(itemId, getString(event, "response_id")), type: "final-transcript", }); + return transcript; } #handleResponseDone(event: RealtimeEvent) { @@ -547,12 +593,12 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { const responseTurnId = this.#getResponseTurnId(response); const status = getString(response, "status"); if (status === "cancelled") { - this.#setMicrophoneEnabled(true); this.#emit({ timestampMs: this.#dependencies.now(), turnId: responseTurnId, type: "response-completed", }); + this.#openListeningWindow(responseTurnId); return; } @@ -598,12 +644,12 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { ) { this.#send(dataChannel, { type: "response.create" }); } else { - this.#setMicrophoneEnabled(true); this.#emit({ timestampMs: this.#dependencies.now(), turnId: responseTurnId, type: "response-completed", }); + this.#openListeningWindow(responseTurnId); } return; } @@ -614,7 +660,6 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { return; } - this.#setMicrophoneEnabled(true); this.#emit({ ...(this.#latestAssistantTranscript ? { responseText: this.#latestAssistantTranscript } @@ -623,6 +668,19 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { turnId: responseTurnId, type: "response-completed", }); + const responseId = getString(response, "id"); + if ( + responseId && + (this.#responsesWithAudio.has(responseId) || + this.#responseHasAudioOutput(response)) + ) { + this.#pendingListeningTurnByResponseId.set( + responseId, + responseTurnId, + ); + } else { + this.#openListeningWindow(responseTurnId); + } } #getFunctionCalls(output: unknown): FunctionCall[] { @@ -643,6 +701,21 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { }); } + #responseHasAudioOutput(response: Record | null): boolean { + if (!Array.isArray(response?.output)) { + return false; + } + return response.output.some((outputItem) => { + const content = asRecord(outputItem)?.content; + return ( + Array.isArray(content) && + content.some( + (contentPart) => getString(asRecord(contentPart), "type") === "audio", + ) + ); + }); + } + #getTurnId(itemId: string, responseId: string | null = null): number { return ( this.#turnByItemId.get(itemId) ?? @@ -665,6 +738,22 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { } } + #openListeningWindow(responseTurnId: number): void { + const dataChannel = this.#dataChannel; + if (dataChannel?.readyState === "open") { + this.#send(dataChannel, { type: "input_audio_buffer.clear" }); + } + this.#setMicrophoneEnabled(true); + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId: + this.#turnByItemId.size === 0 + ? responseTurnId + : Math.max(this.#latestTurnId + 1, responseTurnId + 1), + type: "recording-started", + }); + } + #requireOpenDataChannel(): RTCDataChannel { const dataChannel = this.#dataChannel; if (!this.#connected || !dataChannel || dataChannel.readyState !== "open") { @@ -733,6 +822,8 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { } this.#transcriptByItemId.clear(); + this.#pendingListeningTurnByResponseId.clear(); + this.#responsesWithAudio.clear(); this.#turnByItemId.clear(); this.#turnByResponseId.clear(); this.#interviewStarted = false; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.test.ts index fbcf547f190..72ca8fbb54f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.test.ts @@ -3,6 +3,70 @@ import { describe, expect, test } from "vitest"; import { getTranscriptEntries } from "./transcript-entries"; describe("getTranscriptEntries", () => { + test("reconciles a late expert final around an interviewer partial", () => { + expect( + getTranscriptEntries([ + { + sequence: 1, + event: { + speaker: "expert", + timestampMs: 1, + transcript: "Single", + turnId: 1, + type: "partial-transcript", + }, + }, + { + sequence: 2, + event: { + speaker: "assistant", + timestampMs: 2, + transcript: "Okay, thanks for that detail—let’s", + turnId: 1, + type: "partial-transcript", + }, + }, + { + sequence: 3, + event: { + speaker: "expert", + timestampMs: 3, + transcript: "Single battery", + turnId: 1, + type: "final-transcript", + }, + }, + { + sequence: 4, + event: { + speaker: "assistant", + timestampMs: 4, + transcript: + "What’s the starting condition for that battery when the process begins?", + turnId: 1, + type: "final-transcript", + }, + }, + ]), + ).toEqual([ + { + id: 1, + isPartial: false, + speaker: "expert", + transcript: "Single battery", + turnId: 1, + }, + { + id: 2, + isPartial: false, + speaker: "assistant", + transcript: + "What’s the starting condition for that battery when the process begins?", + turnId: 1, + }, + ]); + }); + test("collapses consecutive expert revisions into one bubble", () => { expect( getTranscriptEntries([ diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.ts index e0f1d427dbb..e19b4f0a3d9 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/transcript-entries.ts @@ -17,6 +17,7 @@ export const getTranscriptEntries = ( events: readonly LoggedTranscriptSource[], ): TranscriptEntry[] => { const entries: TranscriptEntry[] = []; + const partialEntryIndexes = new Map(); for (const { event, sequence } of events) { if ( @@ -33,12 +34,31 @@ export const getTranscriptEntries = ( transcript: event.transcript, turnId: event.turnId, }; + const partialKey = `${event.turnId}:${event.speaker}`; + const partialEntryIndex = partialEntryIndexes.get(partialKey); + if (partialEntryIndex !== undefined) { + entries[partialEntryIndex] = { + ...entry, + id: entries[partialEntryIndex]?.id ?? sequence, + }; + if (event.type === "final-transcript") { + partialEntryIndexes.delete(partialKey); + } + continue; + } + const lastEntry = entries.at(-1); if (lastEntry?.speaker === event.speaker) { entries[entries.length - 1] = { ...entry, id: lastEntry.id }; + if (event.type === "partial-transcript") { + partialEntryIndexes.set(partialKey, entries.length - 1); + } continue; } + if (event.type === "partial-transcript") { + partialEntryIndexes.set(partialKey, entries.length); + } entries.push(entry); } diff --git a/apps/petrinaut-website/vite.config.test.ts b/apps/petrinaut-website/vite.config.test.ts new file mode 100644 index 00000000000..c30e8948c9b --- /dev/null +++ b/apps/petrinaut-website/vite.config.test.ts @@ -0,0 +1,32 @@ +import { afterEach, describe, expect, test } from "vitest"; + +import viteConfig from "./vite.config"; + +const originalOpenAiApiKey = process.env.OPENAI_API_KEY; + +afterEach(() => { + if (originalOpenAiApiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = originalOpenAiApiKey; + } +}); + +describe("server environment loading", () => { + test("treats the repository OpenAI placeholder as unset", () => { + process.env.OPENAI_API_KEY = "dummy"; + + if (typeof viteConfig !== "function") { + throw new TypeError("Expected a Vite config function"); + } + + void viteConfig({ + command: "serve", + isPreview: false, + isSsrBuild: false, + mode: "test", + }); + + expect(process.env.OPENAI_API_KEY).not.toBe("dummy"); + }); +}); diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 7ac9d56767b..5843db4a3ec 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -10,6 +10,12 @@ import type { IncomingMessage, ServerResponse } from "node:http"; const appRoot = fileURLToPath(new URL(".", import.meta.url)); const loadServerEnv = (mode: string) => { + // mise injects the repository-wide Compose placeholder before Vite can load + // the website's real local key. + if (process.env.OPENAI_API_KEY === "dummy") { + delete process.env.OPENAI_API_KEY; + } + const env = loadEnv(mode, appRoot, ""); for (const [key, value] of Object.entries(env)) { diff --git a/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md b/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md index 993cef47323..27f4d7e7c77 100644 --- a/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md +++ b/libs/@hashintel/brunch-agent/docs/planning/process-model-elicitation/spikes/h-6763-realtime-audio-prototype-2026-08-24.md @@ -22,11 +22,14 @@ is materially better and the team explicitly accepts the demonstrated porting co ## Implementation progress - The shared, URL-selected shell is committed as `2903ed77bc98eeddc3c4fa3df95da7f9c11bcf2f`. -- The OpenAI path now has a server-owned client-secret endpoint, WebRTC adapter, semantic - `semantic_vad` (`eagerness: "low"`), input/output transcripts, dummy tool handling, and +- The OpenAI path now has a server-owned client-secret endpoint, WebRTC adapter, + `server_vad` (500ms silence, 300ms prefix padding), input/output transcripts, dummy tool handling, and idempotent resource cleanup. Start speaks the shared opening question, then each cycle opens the - microphone for one expert answer and mutes it while the interviewer responds. Semantic VAD, not - a browser commit, closes the expert side of each cycle. + microphone for one expert answer and mutes it while the interviewer responds. The next listening + window waits for `output_audio_buffer.stopped`, not merely `response.done`, so buffered playback + cannot become an automatic expert turn. Server VAD, not a browser commit, closes the expert side + of each cycle; `create_response` is disabled and the adapter admits a model response only after a + non-empty finalized expert transcript. - The OpenAI code path does not call `/api/chat` and does not import or mutate Lu's Brunch session. - The ElevenLabs path is connected: Speech Engine owns ASR, TTS, and expert endpointing; finalized transcripts go through `BrunchVoiceBridge` into `/api/chat`. It speaks the same opening @@ -99,7 +102,7 @@ remaining work to satisfy H-6763. ## Prototype exclusions -- Custom open-microphone VAD of our own; providers own endpointing (`semantic_vad` / Speech Engine +- Custom open-microphone VAD of our own; providers own endpointing (`server_vad` / Speech Engine `turn_v3`). Tuning beyond the checked-in patient/low settings waits on the comparison recordings. - Provider abstraction intended for production reuse - Durable transcript persistence From f7e25f8e854de17fa3348182f8427d009f071a67 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 25 Aug 2026 14:50:27 +0200 Subject: [PATCH 11/19] H-6763: Connect OpenAI realtime to Brunch elicitation Keep Brunch authoritative while OpenAI handles transcription and speech, and separate voice-provider selection from elicitor selection for comparable experiments. Co-authored-by: Cursor --- .../src/elevenlabs-speech-engine-server.ts | 3 +- .../test/brunch-voice-bridge.test.ts | 2 +- apps/petrinaut-website/README.md | 32 +- .../openai-realtime-session.test.ts | 60 ++- .../openai-realtime-session.ts | 31 +- .../local-storage-demo-app.tsx | 19 +- .../local-storage-demo/voice-experiment.tsx | 392 +++++++++--------- .../openai-realtime-adapter.test.ts | 169 +++++++- .../openai-realtime-adapter.ts | 225 +++++++++- .../voice-experiment-selection.test.ts | 51 ++- .../voice-experiment-selection.ts | 57 ++- apps/petrinaut-website/vite.config.ts | 5 + .../packages/transport-aisdk/package.json | 4 + .../transport-aisdk/src/voice-bridge.ts | 35 +- .../packages/transport-aisdk/vite.config.ts | 3 + 15 files changed, 809 insertions(+), 279 deletions(-) rename apps/brunch-agent/src/brunch-voice-bridge.ts => libs/@hashintel/brunch-agent/packages/transport-aisdk/src/voice-bridge.ts (87%) diff --git a/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts b/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts index 7459c4a5aa3..ceb9425266e 100644 --- a/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts +++ b/apps/brunch-agent/src/elevenlabs-speech-engine-server.ts @@ -2,7 +2,8 @@ import { createServer } from "node:http"; import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; -import { BrunchVoiceBridge } from "./brunch-voice-bridge.ts"; +import { BrunchVoiceBridge } from "@hashintel/brunch-agent-transport-aisdk/voice-bridge"; + import { applySpeechEngineInterviewConfig, createElevenLabsSpeechEngineCallbacks, diff --git a/apps/brunch-agent/test/brunch-voice-bridge.test.ts b/apps/brunch-agent/test/brunch-voice-bridge.test.ts index ac0952ac4db..1154e0f7e13 100644 --- a/apps/brunch-agent/test/brunch-voice-bridge.test.ts +++ b/apps/brunch-agent/test/brunch-voice-bridge.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test, vi } from "vitest"; -import { BrunchVoiceBridge } from "../src/brunch-voice-bridge.ts"; +import { BrunchVoiceBridge } from "@hashintel/brunch-agent-transport-aisdk/voice-bridge"; const encoder = new TextEncoder(); diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 6ae9c070c99..354b3f9f85c 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -51,19 +51,26 @@ optimizer for isolated UI development. Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. Provider API keys must never be exposed through a `VITE_` variable or sent to the browser. -The OpenAI voice experiment is available at -`/?voiceExperiment=openai-realtime`. Its same-origin session endpoint returns only a short-lived -client secret; model, prompt, transcription, and dummy-tool configuration are fixed on the server. -Start opens with one short interviewer question, then opens the microphone for the answer. OpenAI -server VAD owns the end of each expert answer; the browser mutes while the interviewer responds and -reopens only after OpenAI reports that the WebRTC output-audio buffer has drained. The browser does -not manually commit turns. Server VAD does not auto-create model responses: as on the ElevenLabs -path, only a non-empty finalized expert transcript admits the next interviewer response. +The OpenAI voice experiment supports two server-owned elicitor modes: + +- `/?voiceProvider=openai&elicitor=mock` uses the Realtime model and experiment-only dummy tools. +- `/?voiceProvider=openai&elicitor=brunch` sends finalized expert transcripts through the real + Brunch `/api/chat` transport. Brunch owns the interview response and pending `brunch_ask` state; + Realtime renders the returned text as speech. Brunch's source text remains the visible, + authoritative transcript because generated speech is not guaranteed to be verbatim. + +The same-origin session endpoint returns only a short-lived client secret; each mode maps to a fixed +server-side configuration. Start opens with one short interviewer question, then opens the +microphone for the answer. OpenAI server VAD owns the end of each expert answer; the browser mutes +while the interviewer responds and reopens only after OpenAI reports that the WebRTC output-audio +buffer has drained. The browser does not manually commit turns. Server VAD does not auto-create +model responses: only a non-empty finalized expert transcript admits the next interviewer response. +The legacy `/?voiceExperiment=openai-realtime` URL continues to select mock mode. ## ElevenLabs + Brunch voice experiment The real-elicitor experiment is available at -`/?voiceExperiment=elevenlabs-brunch`. ElevenLabs owns browser WebRTC, speech recognition, +`/?voiceProvider=elevenlabs&elicitor=brunch`. ElevenLabs owns browser WebRTC, speech recognition, turn-taking, speech synthesis, playback, and interruption detection. The Speech Engine server forwards only the latest finalized expert transcript into Brunch's existing `/api/chat` transport; Brunch remains authoritative for the session and `brunch_ask` state. The conversation panel shows @@ -81,7 +88,9 @@ For local development: 4. Expose port `3001` through a public HTTPS tunnel and configure the ElevenLabs Speech Engine resource's WebSocket URL as `wss:///ws`. 5. Start the real-panel launcher with `PETRINAUT_WEBSITE_ROOT` pointing at this app and open - `http://127.0.0.1:4915/?voiceExperiment=elevenlabs-brunch`. + `http://127.0.0.1:4915/?voiceProvider=elevenlabs&elicitor=brunch`. Use + `?voiceProvider=openai&elicitor=brunch` on the same launcher to test OpenAI against the real + elicitor. The browser receives only a short-lived conversation token. The primary ElevenLabs key is used by the website token endpoint and the authenticated Speech Engine server, never by browser code. @@ -91,6 +100,9 @@ alternation: it mutes after one finalized expert answer and reopens only after i ends. `voice:dev` enables the client-supplied opening message and applies the turn config to the Speech Engine resource on startup. The server serializes revised provider turns and retains pending `brunch_ask` correlation when interruption happens before Brunch admits the answer. +The legacy `/?voiceExperiment=elevenlabs-brunch` URL remains available. ElevenLabs with the mock +elicitor is intentionally unsupported because a Speech Engine conversation is bound to its +server-side callback. ## Testing the API against the built output diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts index 79ff8c85df5..25b5a0ce762 100644 --- a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.test.ts @@ -14,6 +14,7 @@ const createRequest = (init: RequestInit = {}) => method: "POST", headers: { origin: "https://petrinaut.local", + "x-voice-elicitor": "mock", "x-voice-experiment": "openai-realtime", }, ...init, @@ -91,6 +92,27 @@ describe("OpenAI Realtime session endpoint", () => { expect(upstreamFetch).not.toHaveBeenCalled(); }); + test("rejects unsupported elicitor modes", async () => { + const upstreamFetch = vi.fn(); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch( + createRequest({ + headers: { + origin: "https://petrinaut.local", + "x-voice-elicitor": "browser-controlled", + "x-voice-experiment": "openai-realtime", + }, + }), + ); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "Unsupported elicitor mode", + }); + expect(upstreamFetch).not.toHaveBeenCalled(); + }); + test("fails safely when the primary API key is missing", async () => { delete process.env.OPENAI_API_KEY; const upstreamFetch = vi.fn(); @@ -167,9 +189,7 @@ describe("OpenAI Realtime session endpoint", () => { expect(sessionRequest.session.instructions).toContain( "Stochastic Dynamic Coloured Petri Net", ); - expect(sessionRequest.session.instructions).toContain( - "wait_for_user", - ); + expect(sessionRequest.session.instructions).toContain("wait_for_user"); expect(sessionRequest.session.instructions).toContain( "before emitting spoken audio", ); @@ -186,6 +206,40 @@ describe("OpenAI Realtime session endpoint", () => { ]); }); + test("mints a fixed speech-renderer session for Brunch", async () => { + const upstreamFetch = vi.fn().mockResolvedValue( + Response.json({ + expires_at: 1_800_000_000, + value: "ephemeral-client-secret", + }), + ); + vi.stubGlobal("fetch", upstreamFetch); + + const response = await api.fetch( + createRequest({ + headers: { + origin: "https://petrinaut.local", + "x-voice-elicitor": "brunch", + "x-voice-experiment": "openai-realtime", + }, + }), + ); + + expect(response.status).toBe(200); + const [, request] = upstreamFetch.mock.calls[0] as [string, RequestInit]; + const sessionRequest = JSON.parse(request.body as string) as { + session: { + instructions: string; + tool_choice?: unknown; + tools?: unknown; + }; + }; + expect(sessionRequest.session.instructions).toContain("Brunch elicitor"); + expect(sessionRequest.session.instructions).toContain("exactly as written"); + expect(sessionRequest.session.tools).toBeUndefined(); + expect(sessionRequest.session.tool_choice).toBeUndefined(); + }); + test("does not expose upstream errors or the primary key", async () => { const upstreamFetch = vi.fn().mockResolvedValue( Response.json( diff --git a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts index 9358376ddea..810279f4594 100644 --- a/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts +++ b/apps/petrinaut-website/api/voice-experiment/openai-realtime-session.ts @@ -11,7 +11,7 @@ const RATE_LIMIT_MAX_REQUESTS = 10; const RATE_LIMIT_MAX_TRACKED_CLIENTS = 10_000; const UPSTREAM_TIMEOUT_MS = 10_000; -const sessionConfig = { +const mockSessionConfig = { session: { type: "realtime", model: "gpt-realtime-2.1", @@ -221,6 +221,22 @@ const sessionConfig = { }, } as const; +const brunchSessionConfig = { + session: { + type: mockSessionConfig.session.type, + model: mockSessionConfig.session.model, + output_modalities: mockSessionConfig.session.output_modalities, + instructions: [ + "You are a speech renderer for interviewer text supplied by the application.", + "Read the supplied text exactly as written.", + "Never answer it, paraphrase it, add commentary, or call tools.", + "The Brunch elicitor, not this session, owns the interview and conversation state.", + ].join("\n"), + max_output_tokens: mockSessionConfig.session.max_output_tokens, + audio: mockSessionConfig.session.audio, + }, +} as const; + const upstreamResponseSchema = z.object({ expires_at: z.number().int().positive(), value: z.string().min(1), @@ -329,6 +345,15 @@ const fetch = async (request: Request): Promise => { ); } + const elicitor = request.headers.get("x-voice-elicitor"); + if (elicitor !== "mock" && elicitor !== "brunch") { + logSessionFailure("Rejected unsupported elicitor mode"); + return jsonResponse( + { error: "Unsupported elicitor mode" }, + { status: 400 }, + ); + } + const clientIp = resolveClientIp(request); if (process.env.VERCEL_ENV === "production" && !clientIp) { logSessionFailure("Rejected production request without a client IP"); @@ -366,7 +391,9 @@ const fetch = async (request: Request): Promise => { "content-type": "application/json", "openai-safety-identifier": safetyIdentifier, }, - body: JSON.stringify(sessionConfig), + body: JSON.stringify( + elicitor === "brunch" ? brunchSessionConfig : mockSessionConfig, + ), signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), }); } catch (error) { diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 3b7ac3ab2c6..214a63f08e9 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -26,7 +26,7 @@ import { import { VoiceExperiment } from "./voice-experiment"; import { createElevenLabsAdapter } from "./voice-experiment/elevenlabs-adapter"; import { createOpenAIRealtimeAdapter } from "./voice-experiment/openai-realtime-adapter"; -import { getVoiceExperiment } from "./voice-experiment/voice-experiment-selection"; +import { getVoiceExperimentSelection } from "./voice-experiment/voice-experiment-selection"; import { walkthroughSteps } from "./walkthrough/walkthrough-steps"; const isEmptySDCPN = (sdcpn: SDCPN) => @@ -122,16 +122,22 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ */ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); - const voiceExperiment = getVoiceExperiment(window.location); + const voiceExperiment = getVoiceExperimentSelection(window.location); + const [voiceConversationId] = useState(() => crypto.randomUUID()); + const voiceProvider = voiceExperiment?.provider; + const voiceElicitor = voiceExperiment?.elicitor; const voiceExperimentAdapter = useMemo(() => { - if (voiceExperiment === "openai-realtime") { - return createOpenAIRealtimeAdapter(); + if (voiceProvider === "openai" && voiceElicitor) { + return createOpenAIRealtimeAdapter({ + conversationId: voiceConversationId, + elicitor: voiceElicitor, + }); } - if (voiceExperiment === "elevenlabs-brunch") { + if (voiceProvider === "elevenlabs" && voiceElicitor === "brunch") { return createElevenLabsAdapter(); } return undefined; - }, [voiceExperiment]); + }, [voiceConversationId, voiceElicitor, voiceProvider]); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); @@ -324,6 +330,7 @@ export const LocalStorageDemoApp = () => { {voiceExperiment ? ( ) : null} diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx index 78fe3564d3d..2d88e647450 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -10,12 +10,12 @@ import { import { css } from "@hashintel/ds-helpers/css"; +import { getTranscriptEntries } from "./voice-experiment/transcript-entries"; import { - type VoiceExperiment as VoiceExperimentName, - voiceExperimentLabel, + getVoiceExperimentLabel, + type VoiceExperimentSelection, } from "./voice-experiment/voice-experiment-selection"; -import { getTranscriptEntries } from "./voice-experiment/transcript-entries"; import type { VoiceExperimentAdapter } from "./voice-experiment/voice-experiment-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment/voice-experiment-events"; @@ -544,7 +544,6 @@ type LoggedEvent = { sequence: number; }; - const getEventSummary = (event: VoiceExperimentEvent) => { if (event.type === "partial-transcript") { return `${event.type} (${event.speaker}): ${event.transcript}`; @@ -569,12 +568,13 @@ const createErrorEvent = (error: unknown): VoiceExperimentEvent => ({ export const VoiceExperiment = ({ adapter, + conversationId, experiment, }: { adapter?: VoiceExperimentAdapter; - experiment: VoiceExperimentName; + conversationId: string; + experiment: VoiceExperimentSelection; }) => { - const [conversationId] = useState(() => crypto.randomUUID()); const [events, setEvents] = useState([]); const [isExpanded, setIsExpanded] = useState(false); const [sessionState, setSessionState] = useState("ready"); @@ -796,216 +796,216 @@ export const VoiceExperiment = ({ className={`${panelStyle} voice-experiment-panel`} id="voice-experiment-panel" > -
-
-
- Voice interview - - {voiceExperimentLabel[experiment]} - +
+
+
+ Voice interview + + {getVoiceExperimentLabel(experiment)} + +
-
-
- + + +
+
+ +
+
- - -
- -
- -
-
- - - - {transcriptEntries.length}{" "} - {transcriptEntries.length === 1 ? "message" : "messages"} - -
-
- {transcriptEntries.length === 0 ? ( -

-

- ) : ( - transcriptEntries.map((entry) => ( -
- - {entry.speaker === "expert" ? "Expert" : "Interviewer"} - -

+

+ + + + {transcriptEntries.length}{" "} + {transcriptEntries.length === 1 ? "message" : "messages"} + +
+
+ {transcriptEntries.length === 0 ? ( +

+

+ ) : ( + transcriptEntries.map((entry) => ( +
- {entry.transcript} -

-
- )) - )} -
-
+ + {entry.speaker === "expert" ? "Expert" : "Interviewer"} + +

+ {entry.transcript} +

+
+ )) + )} + + -
-
- - - - {toolDiagnostics.length} - -
-
- {toolDiagnostics.length === 0 ? ( -

+

+
-
- -
- - - - - {events.length} {events.length === 1 ? "event" : "events"} - - -
- {events.length === 0 ? ( - - No events · conversation {conversationId.slice(0, 8)} + + Tool calls + - ) : ( - events.map(({ event, sequence }) => ( -
- - {String(sequence).padStart(2, "0")} ·{" "} - {getEventSummary(event)} - - -
- )) - )} -
-
+ + {toolDiagnostics.length} + + +
+ {toolDiagnostics.length === 0 ? ( +

+

+ ) : ( + toolDiagnostics.map(({ event, sequence }) => ( +
+ + {event.toolName} + + + Turn {event.turnId} + +

+ {event.argumentSummary} +

+ + Call {event.callId} + +
+ )) + )} +
+ + +
+ + + + + {events.length} {events.length === 1 ? "event" : "events"} + + +
+ {events.length === 0 ? ( + + No events · conversation {conversationId.slice(0, 8)} + + ) : ( + events.map(({ event, sequence }) => ( +
+ + {String(sequence).padStart(2, "0")} ·{" "} + {getEventSummary(event)} + + +
+ )) + )} +
+
)} diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts index bce590972fd..f421ad7115e 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.test.ts @@ -4,6 +4,23 @@ import { createOpenAIRealtimeAdapter } from "./openai-realtime-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment-events"; +const encoder = new TextEncoder(); + +const sseResponse = (...chunks: Record[]) => + new Response( + new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue( + encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`), + ); + } + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + controller.close(); + }, + }), + ); + class FakeDataChannel extends EventTarget { public readyState: RTCDataChannelState = "connecting"; public readonly sent: unknown[] = []; @@ -28,7 +45,13 @@ class FakeDataChannel extends EventTarget { } } -const createHarness = () => { +const createHarness = ({ + brunchResponse, + elicitor = "mock", +}: { + brunchResponse?: Response; + elicitor?: "brunch" | "mock"; +} = {}) => { const dataChannel = new FakeDataChannel(); const microphoneTrack = { enabled: true, @@ -64,11 +87,16 @@ const createHarness = () => { }), ) .mockResolvedValueOnce(new Response("answer-sdp")); + if (brunchResponse) { + fetch.mockResolvedValueOnce(brunchResponse); + } let now = 1_000; const adapter = createOpenAIRealtimeAdapter({ + conversationId: "voice-conversation", createAudioElement: () => audioElement as unknown as HTMLAudioElement, createPeerConnection: () => peerConnection as unknown as RTCPeerConnection, + elicitor, fetch: fetch as typeof globalThis.fetch, getUserMedia: vi.fn(async () => mediaStream as unknown as MediaStream), now: () => ++now, @@ -98,7 +126,10 @@ describe("OpenAIRealtimeAdapter", () => { 1, "/api/voice-experiment/openai-realtime-session", expect.objectContaining({ - headers: { "x-voice-experiment": "openai-realtime" }, + headers: { + "x-voice-elicitor": "mock", + "x-voice-experiment": "openai-realtime", + }, method: "POST", }), ); @@ -246,6 +277,140 @@ describe("OpenAIRealtimeAdapter", () => { }); }); + test("uses Brunch as the authoritative elicitor and Realtime only for speech", async () => { + const harness = createHarness({ + elicitor: "brunch", + brunchResponse: sseResponse( + { type: "start", messageId: "assistant-ask" }, + { type: "text-delta", id: "text-1", delta: "Thanks. " }, + { + type: "tool-input-available", + toolCallId: "ask-1", + toolName: "brunch_ask", + input: { question: "What happens next?" }, + }, + { type: "finish", finishReason: "tool-calls" }, + ), + }); + await harness.adapter.connect(); + await harness.adapter.startTurn(); + + harness.dataChannel.receive({ + type: "input_audio_buffer.committed", + item_id: "expert-item", + }); + harness.dataChannel.receive({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "expert-item", + transcript: "The support lead triages it.", + }); + + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledTimes(3)); + const [chatUrl, chatRequest] = harness.fetch.mock.calls[2] as [ + string, + RequestInit, + ]; + expect(chatUrl).toBe("/api/voice-experiment/brunch-chat"); + expect(JSON.parse(chatRequest.body as string)).toMatchObject({ + id: "voice:voice-conversation", + messages: [ + { + role: "user", + parts: [{ type: "text", text: "The support lead triages it." }], + }, + ], + }); + type SentResponseCreate = { + response?: { + conversation?: string; + input?: unknown[]; + metadata?: Record; + output_modalities?: string[]; + }; + type?: string; + }; + let brunchSpeechRequest: SentResponseCreate | undefined; + await vi.waitFor(() => { + brunchSpeechRequest = ( + harness.dataChannel.sent as SentResponseCreate[] + ).find( + (event) => + event.type === "response.create" && + event.response?.metadata?.source === "brunch", + ); + expect(brunchSpeechRequest).toBeDefined(); + }); + expect(brunchSpeechRequest).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + input: [], + metadata: { source: "brunch", turnId: "1" }, + output_modalities: ["audio"], + }, + }); + + expect(harness.events).toContainEqual( + expect.objectContaining({ + speaker: "assistant", + transcript: "Thanks. What happens next?", + turnId: 1, + type: "final-transcript", + }), + ); + expect(harness.events).toContainEqual( + expect.objectContaining({ + argumentSummary: "Question: What happens next?", + callId: "ask-1", + toolName: "brunch_ask", + type: "tool-called", + }), + ); + expect(harness.microphoneTrack.enabled).toBe(false); + + harness.dataChannel.receive({ + type: "response.created", + response: { id: "brunch-speech-response" }, + }); + harness.dataChannel.receive({ + type: "response.output_audio_transcript.done", + response_id: "brunch-speech-response", + item_id: "generated-speech-item", + transcript: "A paraphrase that is not authoritative.", + }); + harness.dataChannel.receive({ + type: "response.done", + response: { + id: "brunch-speech-response", + output: [ + { + type: "message", + content: [{ type: "audio", transcript: "Generated speech" }], + }, + ], + status: "completed", + }, + }); + + expect(JSON.stringify(harness.events)).not.toContain( + "A paraphrase that is not authoritative.", + ); + expect(harness.events).toContainEqual( + expect.objectContaining({ + responseText: "Thanks. What happens next?", + turnId: 1, + type: "response-completed", + }), + ); + expect(harness.microphoneTrack.enabled).toBe(false); + + harness.dataChannel.receive({ + type: "output_audio_buffer.stopped", + response_id: "brunch-speech-response", + }); + expect(harness.microphoneTrack.enabled).toBe(true); + }); + test("listens only between interviewer responses", async () => { const harness = createHarness(); await harness.adapter.connect(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts index 89fedccec85..9b276263c95 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/openai-realtime-adapter.ts @@ -1,7 +1,15 @@ +import { + BrunchVoiceBridge, + type BrunchVoiceToolCall, +} from "@hashintel/brunch-agent-transport-aisdk/voice-bridge"; + +import { interviewOpeningQuestion } from "./interview-opening"; + import type { VoiceExperimentAdapter } from "./voice-experiment-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment-events"; -import { interviewOpeningQuestion } from "./interview-opening"; +import type { VoiceElicitor } from "./voice-experiment-selection"; +const BRUNCH_CHAT_ENDPOINT = "/api/voice-experiment/brunch-chat"; const SESSION_ENDPOINT = "/api/voice-experiment/openai-realtime-session"; const REALTIME_CALLS_ENDPOINT = "https://api.openai.com/v1/realtime/calls"; const CONNECTION_TIMEOUT_MS = 20_000; @@ -16,8 +24,10 @@ const DUMMY_TOOL_NAMES = new Set([ ]); type OpenAIRealtimeAdapterDependencies = { + conversationId: string; createAudioElement: () => HTMLAudioElement; createPeerConnection: () => RTCPeerConnection; + elicitor: VoiceElicitor; fetch: typeof globalThis.fetch; getUserMedia: (constraints: MediaStreamConstraints) => Promise; now: () => number; @@ -34,14 +44,17 @@ type FunctionCall = { name: string; }; -const defaultDependencies: OpenAIRealtimeAdapterDependencies = { +const defaultDependencies = { createAudioElement: () => document.createElement("audio"), createPeerConnection: () => new RTCPeerConnection(), fetch: (...args) => globalThis.fetch(...args), getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints), now: () => Date.now(), -}; +} satisfies Omit< + OpenAIRealtimeAdapterDependencies, + "conversationId" | "elicitor" +>; const asRecord = (value: unknown): Record | null => typeof value === "object" && value !== null @@ -177,6 +190,7 @@ const summarizeFunctionCall = ({ }; class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { + readonly #brunchBridge: BrunchVoiceBridge | null; readonly #dependencies: OpenAIRealtimeAdapterDependencies; readonly #listeners = new Set<(event: VoiceExperimentEvent) => void>(); readonly #pendingListeningTurnByResponseId = new Map(); @@ -186,6 +200,7 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { readonly #turnByResponseId = new Map(); #audioElement: HTMLAudioElement | null = null; + #brunchAbortController: AbortController | null = null; #connectAbortController: AbortController | null = null; #connectPromise: Promise | null = null; #connected = false; @@ -201,6 +216,14 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { public constructor(dependencies: OpenAIRealtimeAdapterDependencies) { this.#dependencies = dependencies; + this.#brunchBridge = + dependencies.elicitor === "brunch" + ? new BrunchVoiceBridge({ + chatEndpoint: BRUNCH_CHAT_ENDPOINT, + fetch: dependencies.fetch, + onToolCall: (toolCall) => this.#emitBrunchToolCall(toolCall), + }) + : null; } public subscribe(listener: (event: VoiceExperimentEvent) => void) { @@ -237,10 +260,25 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { this.#interviewStarted = true; this.#turnId += 1; this.#latestTurnId = this.#turnId; - this.#latestAssistantTranscript = ""; + this.#latestAssistantTranscript = + this.#dependencies.elicitor === "brunch" ? interviewOpeningQuestion : ""; this.#send(dataChannel, { type: "input_audio_buffer.clear" }); microphoneTrack.enabled = false; + if (this.#dependencies.elicitor === "brunch") { + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId: this.#latestTurnId, + type: "response-started", + }); + this.#emit({ + speaker: "assistant", + timestampMs: this.#dependencies.now(), + transcript: interviewOpeningQuestion, + turnId: this.#latestTurnId, + type: "final-transcript", + }); + } this.#send(dataChannel, { type: "response.create", response: { @@ -270,7 +308,10 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { try { const tokenResponse = await this.#dependencies.fetch(SESSION_ENDPOINT, { method: "POST", - headers: { "x-voice-experiment": "openai-realtime" }, + headers: { + "x-voice-elicitor": this.#dependencies.elicitor, + "x-voice-experiment": "openai-realtime", + }, signal: abortController.signal, }); if (!tokenResponse.ok) { @@ -436,9 +477,13 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { const turnId = itemId ? this.#getTurnId(itemId) : this.#latestTurnId; const transcript = this.#handleTranscriptCompleted(event, "expert"); if (transcript) { - const dataChannel = this.#dataChannel; - if (dataChannel?.readyState === "open") { - this.#send(dataChannel, { type: "response.create" }); + if (this.#brunchBridge) { + void this.#requestBrunchTurn(transcript, turnId); + } else { + const dataChannel = this.#dataChannel; + if (dataChannel?.readyState === "open") { + this.#send(dataChannel, { type: "response.create" }); + } } } else { this.#openListeningWindow(turnId); @@ -463,11 +508,13 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { this.#turnByResponseId.set(responseId, this.#latestTurnId); } this.#setMicrophoneEnabled(false); - this.#emit({ - timestampMs: this.#dependencies.now(), - turnId: this.#latestTurnId, - type: "response-started", - }); + if (!this.#brunchBridge) { + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId: this.#latestTurnId, + type: "response-started", + }); + } return; } @@ -488,7 +535,9 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { if (responseId) { this.#responsesWithAudio.add(responseId); } - this.#handleTranscriptDelta(event, "assistant"); + if (!this.#brunchBridge) { + this.#handleTranscriptDelta(event, "assistant"); + } return; } @@ -497,7 +546,9 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { if (responseId) { this.#responsesWithAudio.add(responseId); } - this.#handleTranscriptCompleted(event, "assistant"); + if (!this.#brunchBridge) { + this.#handleTranscriptCompleted(event, "assistant"); + } return; } @@ -588,6 +639,135 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { return transcript; } + async #requestBrunchTurn(transcript: string, turnId: number): Promise { + const bridge = this.#brunchBridge; + if (!bridge) { + return; + } + + this.#brunchAbortController?.abort(); + const abortController = new AbortController(); + this.#brunchAbortController = abortController; + let responseStarted = false; + let responseText = ""; + + try { + for await (const delta of bridge.respond({ + conversationId: this.#dependencies.conversationId, + signal: abortController.signal, + transcript, + })) { + if (abortController.signal.aborted || !this.#connected) { + return; + } + responseText += delta; + if (!responseStarted) { + responseStarted = true; + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId, + type: "response-started", + }); + } + this.#emit({ + speaker: "assistant", + timestampMs: this.#dependencies.now(), + transcript: responseText, + turnId, + type: "partial-transcript", + }); + } + + if (abortController.signal.aborted || !this.#connected) { + return; + } + if (!responseText.trim()) { + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId, + type: "response-completed", + }); + this.#openListeningWindow(turnId); + return; + } + + if (!responseStarted) { + this.#emit({ + timestampMs: this.#dependencies.now(), + turnId, + type: "response-started", + }); + } + this.#latestAssistantTranscript = responseText; + this.#emit({ + speaker: "assistant", + timestampMs: this.#dependencies.now(), + transcript: responseText, + turnId, + type: "final-transcript", + }); + + const dataChannel = this.#requireOpenDataChannel(); + this.#send(dataChannel, { + type: "response.create", + response: { + conversation: "none", + input: [], + instructions: [ + "Read the supplied interviewer text exactly as written.", + "Do not add, remove, paraphrase, answer, or comment on any words.", + `Interviewer text: ${JSON.stringify(responseText)}`, + ].join("\n"), + metadata: { + source: "brunch", + turnId: String(turnId), + }, + output_modalities: ["audio"], + }, + }); + } catch (error) { + if (abortController.signal.aborted) { + return; + } + this.#emitError( + error instanceof Error + ? error.message + : "Brunch could not answer the voice turn.", + ); + this.#openListeningWindow(turnId); + } finally { + if (this.#brunchAbortController === abortController) { + this.#brunchAbortController = null; + } + } + } + + #emitBrunchToolCall({ + input, + toolCallId, + toolName, + }: BrunchVoiceToolCall): void { + const question = + toolName === "brunch_ask" + ? boundedSummaryText(asRecord(input)?.question, 220) + : ""; + this.#emit({ + argumentSummary: + toolName === "brunch_ask" + ? question + ? `Question: ${question}` + : "Question unavailable" + : toolName === "brunch_sweep" + ? "Settlement requested" + : "Arguments hidden", + callId: boundedSummaryText(toolCallId, 96) || "unknown-call", + timestampMs: this.#dependencies.now(), + toolName: boundedSummaryText(toolName, 96) || "unknown-tool", + turnId: this.#latestTurnId, + type: "tool-called", + }); + } + #handleResponseDone(event: RealtimeEvent) { const response = asRecord(event.response); const responseTurnId = this.#getResponseTurnId(response); @@ -674,10 +854,7 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { (this.#responsesWithAudio.has(responseId) || this.#responseHasAudioOutput(response)) ) { - this.#pendingListeningTurnByResponseId.set( - responseId, - responseTurnId, - ); + this.#pendingListeningTurnByResponseId.set(responseId, responseTurnId); } else { this.#openListeningWindow(responseTurnId); } @@ -790,6 +967,9 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { #releaseResources() { this.#isReleasingResources = true; this.#connected = false; + this.#brunchAbortController?.abort(); + this.#brunchAbortController = null; + this.#brunchBridge?.release(this.#dependencies.conversationId); const dataChannel = this.#dataChannel; this.#dataChannel = null; @@ -834,4 +1014,9 @@ class OpenAIRealtimeAdapter implements VoiceExperimentAdapter { export const createOpenAIRealtimeAdapter = ( dependencies: Partial = {}, ): VoiceExperimentAdapter => - new OpenAIRealtimeAdapter({ ...defaultDependencies, ...dependencies }); + new OpenAIRealtimeAdapter({ + ...defaultDependencies, + conversationId: crypto.randomUUID(), + elicitor: "mock", + ...dependencies, + }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts index 2fc625e1030..d5fc4a3249d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.test.ts @@ -1,30 +1,51 @@ import { describe, expect, test } from "vitest"; -import { getVoiceExperiment } from "./voice-experiment-selection"; +import { getVoiceExperimentSelection } from "./voice-experiment-selection"; -describe("getVoiceExperiment", () => { - test.each(["openai-realtime", "elevenlabs-brunch"] as const)( - "selects the %s experiment for the lifetime of the page", - (experiment) => { - expect( - getVoiceExperiment({ search: `?voiceExperiment=${experiment}` }), - ).toBe(experiment); - }, - ); +describe("getVoiceExperimentSelection", () => { + test.each([ + ["openai", "mock"], + ["openai", "brunch"], + ["elevenlabs", "brunch"], + ] as const)("selects %s voice with the %s elicitor", (provider, elicitor) => { + expect( + getVoiceExperimentSelection({ + search: `?voiceProvider=${provider}&elicitor=${elicitor}`, + }), + ).toEqual({ elicitor, provider }); + }); test("keeps the experiment shell hidden by default", () => { - expect(getVoiceExperiment({ search: "" })).toBeNull(); + expect(getVoiceExperimentSelection({ search: "" })).toBeNull(); }); - test("rejects the superseded prototype parameter and aliases", () => { + test("rejects partial, invalid, and unsupported combinations", () => { + expect( + getVoiceExperimentSelection({ search: "?voiceProvider=openai" }), + ).toBeNull(); expect( - getVoiceExperiment({ search: "?voicePrototype=elevenlabs" }), + getVoiceExperimentSelection({ search: "?elicitor=brunch" }), ).toBeNull(); expect( - getVoiceExperiment({ search: "?voiceExperiment=openai" }), + getVoiceExperimentSelection({ + search: "?voiceProvider=elevenlabs&elicitor=mock", + }), ).toBeNull(); expect( - getVoiceExperiment({ search: "?voiceExperiment=elevenlabs" }), + getVoiceExperimentSelection({ + search: "?voiceProvider=other&elicitor=brunch", + }), ).toBeNull(); }); + + test.each([ + ["openai-realtime", { elicitor: "mock", provider: "openai" }], + ["elevenlabs-brunch", { elicitor: "brunch", provider: "elevenlabs" }], + ] as const)("keeps the legacy %s link working", (experiment, selection) => { + expect( + getVoiceExperimentSelection({ + search: `?voiceExperiment=${experiment}`, + }), + ).toEqual(selection); + }); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts index 56b1ee43a75..380148765b3 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/voice-experiment-selection.ts @@ -1,23 +1,50 @@ -export type VoiceExperiment = "openai-realtime" | "elevenlabs-brunch"; +export type VoiceProvider = "elevenlabs" | "openai"; +export type VoiceElicitor = "brunch" | "mock"; -export const voiceExperimentLabel = { - "openai-realtime": "OpenAI Realtime", - "elevenlabs-brunch": "ElevenLabs + Brunch", -} satisfies Record; +export type VoiceExperimentSelection = { + elicitor: VoiceElicitor; + provider: VoiceProvider; +}; + +const legacySelections = { + "elevenlabs-brunch": { + elicitor: "brunch", + provider: "elevenlabs", + }, + "openai-realtime": { + elicitor: "mock", + provider: "openai", + }, +} as const satisfies Record; -export const voiceExperimentMode = { - "openai-realtime": "Native voice · dummy tools", - "elevenlabs-brunch": "Speech edge · real elicitor", -} satisfies Record; +export const getVoiceExperimentLabel = ({ + elicitor, + provider, +}: VoiceExperimentSelection): string => + `${provider === "openai" ? "OpenAI Realtime" : "ElevenLabs"} · ${ + elicitor === "brunch" ? "Brunch" : "mock tools" + }`; -export const getVoiceExperiment = ( +export const getVoiceExperimentSelection = ( location: Pick, -): VoiceExperiment | null => { - const value = new URLSearchParams(location.search).get("voiceExperiment"); +): VoiceExperimentSelection | null => { + const searchParams = new URLSearchParams(location.search); + const provider = searchParams.get("voiceProvider"); + const elicitor = searchParams.get("elicitor"); - if (value === "openai-realtime" || value === "elevenlabs-brunch") { - return value; + if (provider !== null || elicitor !== null) { + if ( + (provider !== "openai" && provider !== "elevenlabs") || + (elicitor !== "mock" && elicitor !== "brunch") || + (provider === "elevenlabs" && elicitor === "mock") + ) { + return null; + } + return { elicitor, provider }; } - return null; + const legacyExperiment = searchParams.get("voiceExperiment"); + return legacyExperiment && Object.hasOwn(legacySelections, legacyExperiment) + ? legacySelections[legacyExperiment as keyof typeof legacySelections] + : null; }; diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 5843db4a3ec..fae751066c1 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -114,6 +114,11 @@ export default defineConfig(({ mode }) => { }, server: { proxy: { + "/api/voice-experiment/brunch-chat": { + target: process.env.BRUNCH_CHAT_ORIGIN ?? "http://127.0.0.1:4321", + changeOrigin: true, + rewrite: () => "/api/chat", + }, "/api/voice-experiment/elevenlabs-brunch-diagnostics": { target: process.env.BRUNCH_CHAT_ORIGIN ?? "http://127.0.0.1:4321", changeOrigin: true, diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json b/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json index 6095c8f6a15..80314cd2d95 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json @@ -13,6 +13,10 @@ "./client-tools": { "types": "./src/client-tools.ts", "import": "./dist/client-tools.js" + }, + "./voice-bridge": { + "types": "./src/voice-bridge.ts", + "import": "./dist/voice-bridge.js" } }, "scripts": { diff --git a/apps/brunch-agent/src/brunch-voice-bridge.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/voice-bridge.ts similarity index 87% rename from apps/brunch-agent/src/brunch-voice-bridge.ts rename to libs/@hashintel/brunch-agent/packages/transport-aisdk/src/voice-bridge.ts index 099d9a697e3..6ff26d9a4dc 100644 --- a/apps/brunch-agent/src/brunch-voice-bridge.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/voice-bridge.ts @@ -1,7 +1,14 @@ +export type BrunchVoiceToolCall = { + input: unknown; + toolCallId: string; + toolName: string; +}; + type BrunchVoiceBridgeDependencies = { chatEndpoint: string; createId?: () => string; fetch?: typeof globalThis.fetch; + onToolCall?: (toolCall: BrunchVoiceToolCall) => void; }; type VoiceTurn = { @@ -88,7 +95,7 @@ const readUiMessageStream = async function* ( }; /** - * Translates finalized Speech Engine turns into the existing AI SDK transport. + * Translates finalized voice turns into the existing AI SDK transport. * Brunch remains authoritative: this bridge retains only enough correlation to * return a spoken answer to a pending `brunch_ask` affordance. */ @@ -96,12 +103,14 @@ export class BrunchVoiceBridge { readonly #chatEndpoint: string; readonly #createId: () => string; readonly #fetch: typeof globalThis.fetch; + readonly #onToolCall: ((toolCall: BrunchVoiceToolCall) => void) | undefined; readonly #sessions = new Map(); public constructor(dependencies: BrunchVoiceBridgeDependencies) { this.#chatEndpoint = dependencies.chatEndpoint; this.#createId = dependencies.createId ?? (() => crypto.randomUUID()); this.#fetch = dependencies.fetch ?? globalThis.fetch; + this.#onToolCall = dependencies.onToolCall; } public async *respond({ @@ -189,11 +198,20 @@ export class BrunchVoiceBridge { continue; } - if ( - chunk.type === "tool-input-available" && - stringProperty(chunk, "toolName") === "brunch_ask" - ) { + if (chunk.type === "tool-input-available") { const toolCallId = stringProperty(chunk, "toolCallId"); + const toolName = stringProperty(chunk, "toolName"); + if (toolCallId && toolName) { + this.#onToolCall?.({ + input: chunk.input, + toolCallId, + toolName, + }); + } + if (toolName !== "brunch_ask") { + continue; + } + const input = chunk.input; if (!assistantMessageId || !toolCallId) { throw new Error("Brunch returned an invalid voice response."); @@ -203,9 +221,10 @@ export class BrunchVoiceBridge { const question = stringProperty(asRecord(input), "question"); if ( question && - !spokenText.trim().toLocaleLowerCase().endsWith( - question.trim().toLocaleLowerCase(), - ) + !spokenText + .trim() + .toLocaleLowerCase() + .endsWith(question.trim().toLocaleLowerCase()) ) { yield question; } diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts index 77a7a9fe479..27cee69b122 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/vite.config.ts @@ -12,6 +12,9 @@ export default defineConfig({ new URL("src/client-tools.ts", import.meta.url), ), index: fileURLToPath(new URL("src/index.ts", import.meta.url)), + "voice-bridge": fileURLToPath( + new URL("src/voice-bridge.ts", import.meta.url), + ), }, fileName: (_format, entryName) => `${entryName}.js`, formats: ["es"], From 81e2f256e4885d3f5df33d3ce178954f75c62a65 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 25 Aug 2026 15:02:05 +0200 Subject: [PATCH 12/19] H-6763: Allow Brunch from the website dev server Permit the regular Petrinaut development origins so OpenAI voice sessions can reach the real Brunch elicitor through the local proxy. Co-authored-by: Cursor --- apps/brunch-agent/src/local-dev-origins.ts | 2 ++ apps/brunch-agent/test/local-dev-origins.test.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/apps/brunch-agent/src/local-dev-origins.ts b/apps/brunch-agent/src/local-dev-origins.ts index 797fc52b346..9993c63b1b9 100644 --- a/apps/brunch-agent/src/local-dev-origins.ts +++ b/apps/brunch-agent/src/local-dev-origins.ts @@ -17,6 +17,8 @@ export const defaultChatOrigin = `http://${localChatListen.host}:${localChatList export const defaultPanelOrigins = [ `http://${localPanelListen.host}:${localPanelListen.port}`, `http://localhost:${localPanelListen.port}`, + "http://127.0.0.1:5173", + "http://localhost:5173", ] as const; export const petrinautLocalServer = (chatOrigin: string) => ({ diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index 26eec099d35..307eed156fe 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -27,6 +27,8 @@ test("petrinaut:dev listens on the panel origin chat CORS already assumes", () = expect(defaultPanelOrigins).toEqual([ "http://127.0.0.1:4915", "http://localhost:4915", + "http://127.0.0.1:5173", + "http://localhost:5173", ]); expect(localPanelListen).toEqual({ host: "127.0.0.1", From b1da72eb971183274433285a93154583a0ff9cbe Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 25 Aug 2026 15:17:14 +0200 Subject: [PATCH 13/19] H-6763: Close the voice interview draft loop Create and persist a mock Petrinaut net from completed voice interviews so provider and elicitor combinations can be validated end to end. Co-authored-by: Cursor --- apps/petrinaut-website/README.md | 18 ++ .../local-storage-demo-app.tsx | 25 ++ .../use-local-storage-sdcpns.ts | 10 + .../local-storage-demo/voice-experiment.tsx | 87 +++-- .../elevenlabs-adapter.test.ts | 10 +- .../voice-experiment/elevenlabs-adapter.ts | 8 +- .../voice-experiment/interview-draft.test.ts | 115 +++++++ .../voice-experiment/interview-draft.ts | 297 ++++++++++++++++++ .../openai-realtime-adapter.test.ts | 15 +- .../openai-realtime-adapter.ts | 74 ++++- .../voice-experiment-events.ts | 5 +- .../voice-experiment-selection.test.ts | 26 ++ .../voice-experiment-selection.ts | 21 +- 13 files changed, 674 insertions(+), 37 deletions(-) create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/interview-draft.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment/interview-draft.ts diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 354b3f9f85c..6e1c7d548ba 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -67,6 +67,24 @@ buffer has drained. The browser does not manually commit turns. Server VAD does model responses: only a non-empty finalized expert transcript admits the next interviewer response. The legacy `/?voiceExperiment=openai-realtime` URL continues to select mock mode. +### Mock draft completion + +Append `&draft=mock` to any supported voice URL to exercise the complete handoff from interview +to an opened Petrinaut net: + +- `/?voiceProvider=openai&elicitor=mock&draft=mock` +- `/?voiceProvider=openai&elicitor=brunch&draft=mock` +- `/?voiceProvider=elevenlabs&elicitor=brunch&draft=mock` + +While this mode is active, the stop control reads **Finish and create net**. Finishing disposes the +voice session, collects the final transcript and any experiment capture calls, passes them through +the provider-independent draft contract, saves the result in local storage, and opens it in the +editor. OpenAI mock capture calls can produce a state-step-flow graph. Other incomplete interviews +produce a visibly labelled two-place placeholder so the end-to-end loop remains testable while the +Brunch projector evolves. The saved record includes the transcript, source conversation id, and +projector warnings. `draft=mock` is deliberately explicit: URLs without it keep the current +stop-without-projection behavior. + ## ElevenLabs + Brunch voice experiment The real-elicitor experiment is available at diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 214a63f08e9..5e2c33b34b4 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -25,6 +25,10 @@ import { } from "./use-local-storage-sdcpns"; import { VoiceExperiment } from "./voice-experiment"; import { createElevenLabsAdapter } from "./voice-experiment/elevenlabs-adapter"; +import { + createMockInterviewDraft, + type FinalizeInterviewInput, +} from "./voice-experiment/interview-draft"; import { createOpenAIRealtimeAdapter } from "./voice-experiment/openai-realtime-adapter"; import { getVoiceExperimentSelection } from "./voice-experiment/voice-experiment-selection"; import { walkthroughSteps } from "./walkthrough/walkthrough-steps"; @@ -62,6 +66,7 @@ const createDefaultStoredSDCPN = (): SDCPNInLocalStorage => ({ const createLocalStorageNetRecord = (params: { petriNetDefinition: SDCPN; title: string; + voiceInterview?: SDCPNInLocalStorage["voiceInterview"]; }): SDCPNInLocalStorage => { const now = new Date(); @@ -70,6 +75,7 @@ const createLocalStorageNetRecord = (params: { title: params.title, sdcpn: params.petriNetDefinition, lastUpdated: now.toISOString(), + ...(params.voiceInterview ? { voiceInterview: params.voiceInterview } : {}), }; }; @@ -205,6 +211,7 @@ export const LocalStorageDemoApp = () => { const createNewNet = (params: { petriNetDefinition: SDCPN; title: string; + voiceInterview?: SDCPNInLocalStorage["voiceInterview"]; }) => { const newNet = createLocalStorageNetRecord(params); const previousNet = @@ -229,6 +236,23 @@ export const LocalStorageDemoApp = () => { setCurrentNetId(newNet.id); }; + const finalizeVoiceInterview = + voiceExperiment?.draft === "mock" + ? (input: FinalizeInterviewInput) => { + const result = createMockInterviewDraft(input); + createNewNet({ + petriNetDefinition: result.petriNetDefinition, + title: result.title, + voiceInterview: { + conversationId: result.conversationId, + source: result.source, + transcript: result.transcript, + warnings: result.warnings, + }, + }); + } + : undefined; + const loadPetriNet = (petriNetId: string) => { const netToLoad = storedSDCPNsForDisplay[petriNetId]; if (!netToLoad) { @@ -332,6 +356,7 @@ export const LocalStorageDemoApp = () => { adapter={voiceExperimentAdapter} conversationId={voiceConversationId} experiment={voiceExperiment} + onFinalize={finalizeVoiceInterview} /> ) : null} diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-local-storage-sdcpns.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-local-storage-sdcpns.ts index 796a72af1d6..376b58c573c 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-local-storage-sdcpns.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-local-storage-sdcpns.ts @@ -9,6 +9,16 @@ export type SDCPNInLocalStorage = { lastUpdated: string; // ISO timestamp sdcpn: SDCPN; title: string; + voiceInterview?: { + conversationId: string; + source: "brunch" | "mock"; + transcript: { + speaker: "assistant" | "expert"; + transcript: string; + turnId: number; + }[]; + warnings: string[]; + }; }; type LocalStorageSDCPNsStore = Record; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx index 2d88e647450..178adc9af40 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-experiment.tsx @@ -16,6 +16,7 @@ import { type VoiceExperimentSelection, } from "./voice-experiment/voice-experiment-selection"; +import type { FinalizeInterviewInput } from "./voice-experiment/interview-draft"; import type { VoiceExperimentAdapter } from "./voice-experiment/voice-experiment-adapter"; import type { VoiceExperimentEvent } from "./voice-experiment/voice-experiment-events"; @@ -333,10 +334,18 @@ const conversationControlStyle = css({ width: "full", minHeight: "24", alignItems: "center", + flexDirection: "column", + gap: "2", justifyContent: "center", paddingY: "1", }); +const conversationControlLabelStyle = css({ + color: "neutral.s80", + fontSize: "xs", + fontWeight: "semibold", +}); + const microphoneButtonStyle = css({ position: "relative", display: "inline-flex", @@ -570,10 +579,12 @@ export const VoiceExperiment = ({ adapter, conversationId, experiment, + onFinalize, }: { adapter?: VoiceExperimentAdapter; conversationId: string; experiment: VoiceExperimentSelection; + onFinalize?: (input: FinalizeInterviewInput) => Promise | void; }) => { const [events, setEvents] = useState([]); const [isExpanded, setIsExpanded] = useState(false); @@ -583,27 +594,39 @@ export const VoiceExperiment = ({ const launcherButtonRef = useRef(null); const minimizeButtonRef = useRef(null); const sequenceRef = useRef(0); + const finalizationConversationIdRef = useRef(conversationId); + const finalizationEventsRef = useRef([]); const transcriptRef = useRef(null); const shouldAutoScrollTranscriptRef = useRef(true); - const appendEvent = useCallback((event: VoiceExperimentEvent) => { - setEvents((previous) => [ - ...previous.slice(-49), - { event, sequence: ++sequenceRef.current }, - ]); - - if (event.type === "connected") { - setSessionState("connected"); - } else if (event.type === "recording-started") { - setIsConversationActive(true); - } else if (event.type === "response-started") { - setSessionState("responding"); - } else if (event.type === "response-completed") { - setSessionState("connected"); - } else if (event.type === "error") { - setSessionState("error"); - } - }, []); + const appendEvent = useCallback( + (event: VoiceExperimentEvent) => { + const loggedEvent = { event, sequence: ++sequenceRef.current }; + if ( + event.type === "partial-transcript" || + event.type === "final-transcript" || + event.type === "tool-called" + ) { + finalizationEventsRef.current.push(loggedEvent); + } + setEvents((previous) => [...previous.slice(-49), loggedEvent]); + + if (event.type === "connected") { + finalizationConversationIdRef.current = + event.conversationId ?? conversationId; + setSessionState("connected"); + } else if (event.type === "recording-started") { + setIsConversationActive(true); + } else if (event.type === "response-started") { + setSessionState("responding"); + } else if (event.type === "response-completed") { + setSessionState("connected"); + } else if (event.type === "error") { + setSessionState("error"); + } + }, + [conversationId], + ); useEffect(() => adapter?.subscribe(appendEvent), [adapter, appendEvent]); @@ -658,6 +681,23 @@ export const VoiceExperiment = ({ setSessionState("ending"); try { await adapter.dispose(); + if (onFinalize) { + const transcript = getTranscriptEntries(finalizationEventsRef.current) + .filter((entry) => !entry.isPartial) + .map(({ speaker, transcript: text, turnId }) => ({ + speaker, + transcript: text, + turnId, + })); + const captures = finalizationEventsRef.current.flatMap(({ event }) => + event.type === "tool-called" && event.capture ? [event.capture] : [], + ); + await onFinalize({ + captures, + conversationId: finalizationConversationIdRef.current, + transcript, + }); + } setSessionState("ended"); } catch (error) { appendEvent(createErrorEvent(error)); @@ -728,11 +768,15 @@ export const VoiceExperiment = ({ ? latestEvent.message : "Connection error"; const controlLabel = isConversationActive - ? "Stop conversation" + ? onFinalize + ? "Finish and create net" + : "Stop conversation" : sessionState === "connecting" ? "Starting…" : sessionState === "ending" - ? "Stopping…" + ? onFinalize + ? "Creating draft…" + : "Stopping…" : sessionState === "ended" ? "Conversation ended" : sessionState === "error" @@ -862,6 +906,9 @@ export const VoiceExperiment = ({