From 06ee9e466cf6f01e520c8540d8b0f0086a309142 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Nov 2026 14:37:22 -0800 Subject: [PATCH 1/3] fix: prepare Session publications while joining independently of capture --- .changeset/session-prepared-publications.md | 4 + .../src/hooks/sessionPublication.test.ts | 186 ++++++++++++++++++ .../react/src/hooks/sessionPublication.ts | 132 +++++++++++++ packages/react/src/hooks/useSession.ts | 111 +++++++---- scripts/README.md | 15 ++ scripts/test-session-publication.mjs | 184 +++++++++++++++++ 6 files changed, 590 insertions(+), 42 deletions(-) create mode 100644 .changeset/session-prepared-publications.md create mode 100644 packages/react/src/hooks/sessionPublication.test.ts create mode 100644 packages/react/src/hooks/sessionPublication.ts create mode 100644 scripts/README.md create mode 100644 scripts/test-session-publication.mjs diff --git a/.changeset/session-prepared-publications.md b/.changeset/session-prepared-publications.md new file mode 100644 index 000000000..1c8813f72 --- /dev/null +++ b/.changeset/session-prepared-publications.md @@ -0,0 +1,4 @@ +--- +'@livekit/components-react': patch +--- +Start session connection alongside device capture and use prepared microphone and camera publications when the SDK supports them, with cleanup of late captures after cancellation. diff --git a/packages/react/src/hooks/sessionPublication.test.ts b/packages/react/src/hooks/sessionPublication.test.ts new file mode 100644 index 000000000..220b8ccbd --- /dev/null +++ b/packages/react/src/hooks/sessionPublication.test.ts @@ -0,0 +1,186 @@ +import { Room, Track, type LocalTrack, type LocalTrackPublication } from 'livekit-client'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { connectSessionWithTracks } from './sessionPublication'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((a, b) => { + resolve = a; + reject = b; + }); + return { promise, resolve, reject }; +} +function setup() { + const room = new Room(); + const controller = new AbortController(); + const track = { kind: Track.Kind.Audio, stop: vi.fn() } as unknown as LocalTrack; + const captured = deferred(); + const attachTrack = vi.fn(async () => ({}) as LocalTrackPublication); + const cancel = vi.fn(async () => {}); + const reservation = { attachTrack, cancel }; + const prepare = vi.fn(() => reservation); + Object.defineProperty(room, 'preparePublication', { value: prepare, configurable: true }); + const capture = vi.spyOn(room.localParticipant, 'createTracks').mockReturnValue(captured.promise); + vi.spyOn(room, 'disconnect').mockResolvedValue(undefined); + const connect = vi.fn(async () => {}); + return { + room, + controller, + track, + captured, + capture, + prepare, + attachTrack, + cancel, + reservation, + connect, + }; +} +afterEach(() => vi.restoreAllMocks()); + +describe('Session publication startup', () => { + it('starts JOIN before capture finishes and forwards all publication options', async () => { + const f = setup(); + const options = { forceStereo: true, red: false, audioPreset: { maxBitrate: 96000 } }; + const starting = connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true, publishOptions: options } }, + f.controller.signal, + ); + expect(f.connect).toHaveBeenCalledWith([f.reservation]); + expect(f.connect.mock.invocationCallOrder[0]).toBeLessThan( + f.capture.mock.invocationCallOrder[0], + ); + expect(f.prepare).toHaveBeenCalledWith({ + source: Track.Source.Microphone, + publishOptions: options, + }); + expect(f.attachTrack).not.toHaveBeenCalled(); + f.captured.resolve([f.track]); + await starting; + expect(f.attachTrack).toHaveBeenCalledWith(f.track); + expect(f.track.stop).not.toHaveBeenCalled(); + expect(f.cancel).toHaveBeenCalledOnce(); + }); + it('prepares camera encodings from configured resolution without requesting the camera', async () => { + const f = setup(); + f.room.options.videoCaptureDefaults = { + resolution: { width: 1280, height: 720, frameRate: 30 }, + }; + const options = { + videoCodec: 'h264' as const, + simulcast: false, + videoEncoding: { maxBitrate: 700000 }, + }; + const starting = connectSessionWithTracks( + f.room, + f.connect, + { camera: { enabled: true, publishOptions: options } }, + f.controller.signal, + ); + expect(f.prepare).toHaveBeenCalledWith({ + source: Track.Source.Camera, + format: { width: 1280, height: 720, frameRate: 30 }, + publishOptions: options, + }); + expect(f.capture).toHaveBeenCalledWith({ audio: false, video: true }); + f.captured.resolve([f.track]); + await starting; + }); + it('aborts startup promptly and stops a capture that resolves after end', async () => { + const f = setup(); + const starting = connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true } }, + f.controller.signal, + ); + const rejected = expect(starting).rejects.toThrow(); + f.controller.abort(); + await rejected; + f.captured.resolve([f.track]); + await Promise.resolve(); + expect(f.track.stop).toHaveBeenCalledOnce(); + expect(f.attachTrack).not.toHaveBeenCalled(); + expect(f.cancel).toHaveBeenCalled(); + expect(f.room.disconnect).toHaveBeenCalledOnce(); + }); + it('cleans up late capture when JOIN fails', async () => { + const f = setup(); + f.connect.mockRejectedValue(new Error('join rejected')); + await expect( + connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true } }, + f.controller.signal, + ), + ).rejects.toThrow('join rejected'); + f.captured.resolve([f.track]); + await Promise.resolve(); + expect(f.track.stop).toHaveBeenCalledOnce(); + expect(f.attachTrack).not.toHaveBeenCalled(); + expect(f.cancel).toHaveBeenCalledOnce(); + }); + it('cancels unused reservations and disconnects on permission denial', async () => { + const f = setup(); + const starting = connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true } }, + f.controller.signal, + ); + f.captured.reject(new DOMException('denied', 'NotAllowedError')); + await expect(starting).rejects.toThrow('denied'); + expect(f.cancel).toHaveBeenCalledOnce(); + expect(f.room.disconnect).toHaveBeenCalledOnce(); + }); + it('stops captured media when attachment fails', async () => { + const f = setup(); + f.attachTrack.mockRejectedValue(new Error('publication rejected')); + const starting = connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true } }, + f.controller.signal, + ); + f.captured.resolve([f.track]); + await expect(starting).rejects.toThrow('publication rejected'); + expect(f.track.stop).toHaveBeenCalledOnce(); + expect(f.cancel).toHaveBeenCalledOnce(); + }); + it('does not capture disabled sources', async () => { + const f = setup(); + await connectSessionWithTracks(f.room, f.connect, {}, f.controller.signal); + expect(f.connect).toHaveBeenCalledWith([]); + expect(f.prepare).not.toHaveBeenCalled(); + expect(f.capture).not.toHaveBeenCalled(); + }); + it('retains ordinary publication on older SDKs', async () => { + const f = setup(); + Object.defineProperty(f.room, 'preparePublication', { value: undefined }); + const microphone = vi + .spyOn(f.room.localParticipant, 'setMicrophoneEnabled') + .mockResolvedValue(undefined); + await connectSessionWithTracks( + f.room, + f.connect, + { microphone: { enabled: true } }, + f.controller.signal, + ); + expect(f.connect).toHaveBeenCalledWith([]); + expect(microphone).toHaveBeenCalledWith(true, undefined, {}); + expect(f.capture).not.toHaveBeenCalled(); + }); + it('rejects an already aborted start without joining or capturing', async () => { + const f = setup(); + f.controller.abort(); + await expect( + connectSessionWithTracks(f.room, f.connect, {}, f.controller.signal), + ).rejects.toThrow(); + expect(f.connect).not.toHaveBeenCalled(); + expect(f.capture).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/react/src/hooks/sessionPublication.ts b/packages/react/src/hooks/sessionPublication.ts new file mode 100644 index 000000000..5602157b7 --- /dev/null +++ b/packages/react/src/hooks/sessionPublication.ts @@ -0,0 +1,132 @@ +import { + type Room, + Track, + type LocalTrack, + type LocalTrackPublication, + type TrackPublishOptions, +} from 'livekit-client'; +import type { SessionConnectOptions } from './useSession'; + +// Optional capability bridge: components continues to support SDK releases without reservations. +// Keep these structural types internal until the SDK API is released as stable. +interface Reservation { + attachTrack(track: LocalTrack): Promise; + cancel(): Promise; +} +interface PublicationIntent { + source: Track.Source.Microphone | Track.Source.Camera; + format?: { channelCount: 1 | 2 } | { width: number; height: number; frameRate?: number }; + publishOptions?: TrackPublishOptions; +} +type PreparingRoom = Room & { preparePublication?: (intent: PublicationIntent) => Reservation }; + +/** Connects immediately while capture and prepared publication proceed independently. */ +export async function connectSessionWithTracks( + room: Room, + connect: (reservations: Reservation[]) => Promise, + tracks: NonNullable, + signal: AbortSignal, +): Promise { + const reservations: Reservation[] = []; + const captures = new Set(); + let cancelled = signal.aborted; + const abortError = () => + signal.reason ?? new DOMException('Session startup aborted', 'AbortError'); + let rejectAbort!: (reason: unknown) => void; + const aborted = new Promise((_, reject) => { + rejectAbort = reject; + }); + const onAbort = () => { + cancelled = true; + rejectAbort(abortError()); + for (const reservation of reservations) void reservation.cancel().catch(() => {}); + for (const track of captures) track.stop(); + }; + signal.addEventListener('abort', onAbort, { once: true }); + + try { + if (cancelled) throw abortError(); + const prepare = (intent: PublicationIntent) => { + const reservation = (room as PreparingRoom).preparePublication?.(intent); + if (reservation) reservations.push(reservation); + return reservation; + }; + const microphone = tracks.microphone?.enabled + ? prepare({ + source: Track.Source.Microphone, + publishOptions: tracks.microphone.publishOptions, + }) + : undefined; + // Resolution is a declared envelope; never query a device to discover it before JOIN. + const resolution = room.options.videoCaptureDefaults?.resolution; + const camera = + tracks.camera?.enabled && resolution + ? prepare({ + source: Track.Source.Camera, + format: resolution, + publishOptions: tracks.camera.publishOptions, + }) + : undefined; + + const capture = async ( + source: Track.Source.Microphone | Track.Source.Camera, + reservation: Reservation, + ) => { + const localTracks = await room.localParticipant.createTracks({ + audio: source === Track.Source.Microphone, + video: source === Track.Source.Camera, + }); + for (const track of localTracks) captures.add(track); + if (cancelled || signal.aborted) { + localTracks.forEach((track) => track.stop()); + throw abortError(); + } + const track = localTracks[0]; + if (!track) throw new Error(`No ${source} track was captured`); + await reservation.attachTrack(track); + }; + + // Invoke connection first. Neither credential acquisition nor JOIN waits for getUserMedia. + const connecting = connect(reservations); + await Promise.race([ + Promise.all([ + connecting, + tracks.microphone?.enabled + ? microphone + ? capture(Track.Source.Microphone, microphone) + : room.localParticipant.setMicrophoneEnabled( + true, + undefined, + tracks.microphone.publishOptions ?? {}, + ) + : undefined, + tracks.camera?.enabled + ? camera + ? capture(Track.Source.Camera, camera) + : room.localParticipant.setCameraEnabled( + true, + undefined, + tracks.camera.publishOptions ?? {}, + ) + : undefined, + tracks.screenShare?.enabled + ? room.localParticipant.setScreenShareEnabled( + true, + undefined, + tracks.screenShare.publishOptions ?? {}, + ) + : undefined, + ]), + aborted, + ]); + } catch (error) { + cancelled = true; + for (const track of captures) track.stop(); + // Disconnect settles pending JOIN before cancelling senders from its deferred local offer. + await room.disconnect(); + throw error; + } finally { + signal.removeEventListener('abort', onAbort); + await Promise.all(reservations.map((reservation) => reservation.cancel())); + } +} diff --git a/packages/react/src/hooks/useSession.ts b/packages/react/src/hooks/useSession.ts index 422541199..9de66e7da 100644 --- a/packages/react/src/hooks/useSession.ts +++ b/packages/react/src/hooks/useSession.ts @@ -23,6 +23,7 @@ import { useMaybeRoomContext } from '../context'; import { AgentState, useAgent, useAgentTimeoutIdStore } from './useAgent'; import { TrackReference, log } from '@livekit/components-core'; import { useLocalParticipant } from './useLocalParticipant'; +import { connectSessionWithTracks } from './sessionPublication'; /** @beta */ export enum SessionEvent { @@ -138,7 +139,10 @@ type SessionActions = { prepareConnection: () => Promise; - /** Connect to the underlying room and dispatch any agents */ + /** + * Connect to the room and dispatch agents while acquiring requested media in parallel. + * Supported SDKs reserve initial publications without waiting for device access. + */ start: (options?: SessionConnectOptions) => Promise; /** Disconnect from the underlying room */ @@ -648,6 +652,19 @@ export function useSession( const tokenSourceFetch = useSessionTokenSourceFetch(tokenSource, unstableRestOptions); const wasSessionEndCalledRef = React.useRef(false); + const startupAbortRef = React.useRef(null); + const disconnectListenerRef = React.useRef<(() => void) | null>(null); + React.useEffect( + () => () => { + if (startupAbortRef.current) { + startupAbortRef.current.abort(); + void room.disconnect(); + } + if (disconnectListenerRef.current) + room.off(RoomEvent.Disconnected, disconnectListenerRef.current); + }, + [room], + ); const start = React.useCallback( async (connectOptions: SessionConnectOptions = {}) => { @@ -660,12 +677,19 @@ export function useSession( await waitUntilDisconnected(signal); wasSessionEndCalledRef.current = false; + const startup = new AbortController(); + startupAbortRef.current?.abort(); + startupAbortRef.current = startup; const onSignalAbort = () => { - room.disconnect(); + startup.abort(signal?.reason); + void room.disconnect(); }; - signal?.addEventListener('abort', onSignalAbort); + signal?.addEventListener('abort', onSignalAbort, { once: true }); + if (signal?.aborted) onSignalAbort(); const onDisconnected = () => { + if (disconnectListenerRef.current === onDisconnected) disconnectListenerRef.current = null; + startup.abort(); // on disconnection force a new token to be fetched in order to avoid reusing the same room right after // this works around the fact that agents won't rejoin a room that existed previously // and depends on the assumption that the endpoint will return a token for a different room @@ -673,55 +697,58 @@ export function useSession( tokenSourceFetch(true); } }; + if (disconnectListenerRef.current) + room.off(RoomEvent.Disconnected, disconnectListenerRef.current); + disconnectListenerRef.current = onDisconnected; room.once(RoomEvent.Disconnected, onDisconnected); let tokenDispatchesAgent = false; - await Promise.all([ - tokenSourceFetch().then(({ serverUrl, participantToken }) => { - const participantTokenPayload = decodeTokenPayload(participantToken); - const participantTokenAgentDispatchCount = - participantTokenPayload.roomConfig?.agents?.length ?? 0; - tokenDispatchesAgent = participantTokenAgentDispatchCount > 0; - - return room.connect(serverUrl, participantToken, roomConnectOptions); - }), - - // Start microphone (with preconnect buffer) by default - tracks.microphone?.enabled - ? room.localParticipant.setMicrophoneEnabled( - true, - undefined, - tracks.microphone?.publishOptions ?? {}, - ) - : Promise.resolve(), - tracks.camera?.enabled - ? room.localParticipant.setCameraEnabled( - true, - undefined, - tracks.camera?.publishOptions ?? {}, - ) - : Promise.resolve(), - tracks.screenShare?.enabled - ? room.localParticipant.setScreenShareEnabled( - true, - undefined, - tracks.screenShare?.publishOptions ?? {}, - ) - : Promise.resolve(), - ]); - - await waitUntilConnected(signal); - if (tokenDispatchesAgent) { - await agent.waitUntilConnected(signal); - } + try { + await connectSessionWithTracks( + room, + async (initialPublications) => { + const { serverUrl, participantToken } = await tokenSourceFetch(); + if (startup.signal.aborted) throw startup.signal.reason; + const participantTokenPayload = decodeTokenPayload(participantToken); + tokenDispatchesAgent = (participantTokenPayload.roomConfig?.agents?.length ?? 0) > 0; + await room.connect(serverUrl, participantToken, { + ...roomConnectOptions, + ...(initialPublications.length > 0 + ? { + initialPublications: [ + ...(( + roomConnectOptions as { initialPublications?: typeof initialPublications } + )?.initialPublications ?? []), + ...initialPublications, + ], + } + : {}), + }); + }, + tracks, + startup.signal, + ); - signal?.removeEventListener('abort', onSignalAbort); + await waitUntilConnected(startup.signal); + if (tokenDispatchesAgent) { + await agent.waitUntilConnected(startup.signal); + } + } finally { + if (room.state === ConnectionState.Disconnected) { + room.off(RoomEvent.Disconnected, onDisconnected); + if (disconnectListenerRef.current === onDisconnected) + disconnectListenerRef.current = null; + } + signal?.removeEventListener('abort', onSignalAbort); + if (startupAbortRef.current === startup) startupAbortRef.current = null; + } }, [room, waitUntilDisconnected, tokenSourceFetch, waitUntilConnected, agent.waitUntilConnected], ); const end = React.useCallback(async () => { wasSessionEndCalledRef.current = true; + startupAbortRef.current?.abort(); tokenSourceFetch(true); await room.disconnect(); }, [room, tokenSourceFetch]); diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..d55bd8c99 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,15 @@ +# Session publication browser check + +Build the companion JS SDK branch that supplies `Room.preparePublication`, install this workspace, and start a local `livekit-server --dev`. Then run from this repo: + +```sh +LIVEKIT_SDK_DIR=../client-sdk-js \ +LIVEKIT_URL=ws://127.0.0.1:7880 \ +node scripts/test-session-publication.mjs +``` + +The runner uses Playwright and `livekit-server-sdk` from the companion SDK's development dependencies, and serves the production Session startup helper through this workspace's Vite. Set `CHROMIUM_EXECUTABLE_PATH` to use an installed Chrome. Keys default to the local dev server's `devkey`/`secret`; another server requires `LIVEKIT_API_KEY` and `LIVEKIT_API_SECRET`. + +It checks both delayed capture (JOIN finishes first) and early capture (microphone samples are buffered while JOIN proceeds). An agent peer must receive microphone audio and decoded camera video, and its preconnect stream must finish, carrying samples in the early-capture case. Attachment may not emit AddTrack or an offer after connection and capture are ready. Initial transport setup may independently emit an offer before that point. Both browser contexts and the temporary Vite server are closed by the runner. + +Ordinary `useSession().start()` options remain unchanged. Components detects the SDK preparation capability at runtime; SDKs without it retain the existing enable-track methods. The SDK decides whether a prepared handle can use JOIN or needs ordinary publication. Screen sharing keeps its existing user-gesture path. diff --git a/scripts/test-session-publication.mjs b/scripts/test-session-publication.mjs new file mode 100644 index 000000000..02c5c3b25 --- /dev/null +++ b/scripts/test-session-publication.mjs @@ -0,0 +1,184 @@ +// Build the companion SDK first. Requires a local real SFU (`livekit-server --dev`). +// LIVEKIT_SDK_DIR=../client-sdk-js node scripts/test-session-publication.mjs +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +const root = fileURLToPath(new URL('..', import.meta.url)); +const sdk = resolve(root, process.env.LIVEKIT_SDK_DIR ?? '../client-sdk-js'); +const sdkRequire = createRequire(resolve(sdk, 'package.json')); +const reactRequire = createRequire(resolve(root, 'packages/react/package.json')); +const { chromium } = sdkRequire('playwright'); +const { AccessToken } = sdkRequire('livekit-server-sdk'); +const { createServer } = await import(pathToFileURL(reactRequire.resolve('vite')).href); +const url = process.env.LIVEKIT_URL ?? 'ws://127.0.0.1:7880'; +const vite = await createServer({ + configFile: false, + root: resolve(root, 'packages/react'), + resolve: { alias: { 'livekit-client': resolve(sdk, 'dist/livekit-client.esm.mjs') } }, + server: { host: '127.0.0.1', port: 0, hmr: false, fs: { allow: [root, sdk] } }, + plugins: [ + { + name: 'test-page', + configureServer(server) { + server.middlewares.use((request, response, next) => { + if (request.url === '/') { + response.setHeader('Content-Type', 'text/html'); + response.end(''); + } else if (request.url === '/favicon.ico') { + response.statusCode = 204; + response.end(); + } else next(); + }); + }, + }, + ], +}); +await vite.listen(); +let browser; +try { + browser = await chromium.launch({ + executablePath: process.env.CHROMIUM_EXECUTABLE_PATH, + headless: true, + args: [ + '--autoplay-policy=no-user-gesture-required', + '--use-fake-ui-for-media-stream', + '--use-fake-device-for-media-stream', + ], + }); + for (const early of [false, true]) { + const room = `session-publication-${Date.now()}`; + const tokens = await Promise.all( + ['publisher', 'agent'].map(async (identity) => { + const token = new AccessToken( + process.env.LIVEKIT_API_KEY ?? 'devkey', + process.env.LIVEKIT_API_SECRET ?? 'secret', + { identity }, + ); + if (identity === 'agent') token.kind = 'agent'; + token.addGrant({ roomJoin: true, room, agent: identity === 'agent' }); + return token.toJwt(); + }), + ); + const page = await browser.newPage(); + try { + await page.goto(`http://127.0.0.1:${vite.httpServer.address().port}`); + const result = await page.evaluate( + async ({ tokens, early, url, sdkModule }) => { + const { Room, RoomEvent, Track } = await import(sdkModule); + const { connectSessionWithTracks } = await import('/src/hooks/sessionPublication.ts'); + const delay = (ms) => new Promise((r) => setTimeout(r, ms)); + const room = new Room({ + videoCaptureDefaults: { resolution: { width: 640, height: 480 } }, + }); + const agent = new Room(); + let bufferBytes = 0, + bufferEnded = false; + agent.registerByteStreamHandler('lk.agent.pre-connect-audio-buffer', async (reader) => { + try { + for await (const chunk of reader) bufferBytes += chunk.byteLength; + bufferEnded = true; + } catch (e) { + console.error(e); + } + }); + const remotes = new Map(); + agent.on(RoomEvent.TrackSubscribed, (track) => { + remotes.set(track.kind, track); + const e = track.attach(); + e.muted = true; + document.body.appendChild(e); + void e.play(); + }); + await agent.connect(url, tokens[1]); + let connectedAt; + room.on(RoomEvent.Connected, () => (connectedAt = performance.now())); + let captureAt; + const gum = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); + navigator.mediaDevices.getUserMedia = async (c) => { + await delay(early ? 0 : 750); + const stream = await gum(c); + captureAt = performance.now(); + return stream; + }; + const trace = []; + const client = room.engine.client; + const send = client.sendRequest.bind(client); + client.sendRequest = (m, ...a) => { + trace.push({ type: m.case, at: performance.now() }); + return send(m, ...a); + }; + try { + const controller = new AbortController(); + await connectSessionWithTracks( + room, + async (plans) => { + if (early) await delay(900); + return room.connect(url, tokens[0], { initialPublications: plans }); + }, + { + microphone: { enabled: true, publishOptions: { preConnectBuffer: true } }, + camera: { enabled: true }, + }, + controller.signal, + ); + const deadline = performance.now() + 5000; + let decoded = false; + while (performance.now() < deadline) { + const results = await Promise.all( + [...remotes.values()].map(async (t) => { + if (!t.receiver) return false; + const s = await t.receiver.getStats(); + return [...s.values()].some( + (s) => + s.type === 'inbound-rtp' && + (t.kind === Track.Kind.Video + ? s.framesDecoded > 0 + : s.totalSamplesReceived > 0), + ); + }), + ); + decoded = results.length === 2 && results.every(Boolean); + if (decoded && bufferEnded) break; + await delay(30); + } + return { + joinedBeforeCapture: connectedAt < captureAt, + decoded, + bufferEnded, + bufferBytes, + signals: trace + .filter((m) => m.type === 'addTrack' || m.type === 'offer') + .map((m) => ({ + type: m.type, + afterReady: m.at >= Math.max(captureAt, connectedAt), + })), + agentKind: [...room.remoteParticipants.values()][0]?.kind, + }; + } finally { + navigator.mediaDevices.getUserMedia = gum; + await room.disconnect(); + await agent.disconnect(); + await room.dispose(); + await agent.dispose(); + } + }, + { tokens, early, url, sdkModule: `/@fs/${resolve(sdk, 'dist/livekit-client.esm.mjs')}` }, + ); + console.log(JSON.stringify({ capture: early ? 'before-join' : 'after-join', ...result })); + assert.equal(result.joinedBeforeCapture, !early); + assert.ok(result.decoded, 'agent receives both microphone and camera media'); + assert.ok(result.bufferEnded, 'preconnect stream closes when the agent subscribes'); + if (early) assert.ok(result.bufferBytes > 0, 'early microphone samples reach the agent'); + assert.ok( + result.signals.every((signal) => !signal.afterReady), + 'attachment must not register or renegotiate', + ); + } finally { + await page.close(); + } + } +} finally { + await browser?.close(); + await vite.close(); +} From c1ce6a87064b39f5e9f713ce02fe9214ee15e543 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Nov 2026 14:41:09 -0800 Subject: [PATCH 2/3] fix: abort late credential continuations after Session startup failure --- packages/react/src/hooks/useSession.test.ts | 30 +++++++++++++++++++++ packages/react/src/hooks/useSession.ts | 5 ++++ 2 files changed, 35 insertions(+) diff --git a/packages/react/src/hooks/useSession.test.ts b/packages/react/src/hooks/useSession.test.ts index 4f65e600c..9eca25455 100644 --- a/packages/react/src/hooks/useSession.test.ts +++ b/packages/react/src/hooks/useSession.test.ts @@ -5,6 +5,7 @@ import { LocalTrackPublication, ParticipantEvent, Room, + RoomEvent, TokenSource, Track, } from 'livekit-client'; @@ -69,3 +70,32 @@ describe('useSession local screen share', () => { }, ); }); + +describe('useSession startup cancellation', () => { + test('late credentials cannot connect after microphone permission failure', async () => { + const room = new Room(); + Object.defineProperty(room, 'preparePublication', { value: undefined }); + vi.spyOn(room, 'prepareConnection').mockResolvedValue(undefined); + vi.spyOn(room, 'disconnect').mockResolvedValue(undefined); + const connect = vi.spyOn(room, 'connect').mockResolvedValue(undefined); + vi.spyOn(room.localParticipant, 'setMicrophoneEnabled').mockRejectedValue( + new Error('permission denied'), + ); + const credentials = { serverUrl: 'wss://example.invalid', participantToken: 'test-token' }; + let resolve!: (value: typeof credentials) => void; + const pending = new Promise((r) => { + resolve = r; + }); + const tokenSource = TokenSource.literal(credentials); + vi.spyOn(tokenSource, 'fetch').mockReturnValue(pending); + const { result } = renderHook(() => useSession(tokenSource, { room })); + const listeners = room.listenerCount(RoomEvent.Disconnected); + await act(async () => { + await expect(result.current.start()).rejects.toThrow('permission denied'); + resolve(credentials); + await Promise.resolve(); + }); + expect(connect).not.toHaveBeenCalled(); + expect(room.listenerCount(RoomEvent.Disconnected)).toBe(listeners); + }); +}); diff --git a/packages/react/src/hooks/useSession.ts b/packages/react/src/hooks/useSession.ts index 9de66e7da..681aeec8f 100644 --- a/packages/react/src/hooks/useSession.ts +++ b/packages/react/src/hooks/useSession.ts @@ -243,6 +243,7 @@ function useSessionWaitUntilConnectionState( const waitUntilConnectionState = React.useCallback( async (state: UseSessionReturn['connectionState'], signal?: AbortSignal) => { + if (signal?.aborted) throw signal.reason ?? new Error('Session wait aborted'); if (connectionStateRef.current === state) { return; } @@ -733,6 +734,10 @@ export function useSession( if (tokenDispatchesAgent) { await agent.waitUntilConnected(startup.signal); } + } catch (error) { + // Late credentials must not start a connection after capture/startup has failed. + startup.abort(error); + throw error; } finally { if (room.state === ConnectionState.Disconnected) { room.off(RoomEvent.Disconnected, onDisconnected); From 3ca94eba04359f1f584121427720276fe6bd8fac Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 20 Nov 2026 16:35:47 -0800 Subject: [PATCH 3/3] test: benchmark Session UI readiness and agent audio with full RTC latency --- benchmarks/session-experience/README.md | 89 +++ benchmarks/session-experience/results.json | 763 +++++++++++++++++++++ benchmarks/session-experience/samples.csv | 101 +++ scripts/README.md | 6 + scripts/benchmark-session-experience.mjs | 341 +++++++++ scripts/session-experience-browser.ts | 347 ++++++++++ 6 files changed, 1647 insertions(+) create mode 100644 benchmarks/session-experience/README.md create mode 100644 benchmarks/session-experience/results.json create mode 100644 benchmarks/session-experience/samples.csv create mode 100644 scripts/benchmark-session-experience.mjs create mode 100644 scripts/session-experience-browser.ts diff --git a/benchmarks/session-experience/README.md b/benchmarks/session-experience/README.md new file mode 100644 index 000000000..f16af6cfe --- /dev/null +++ b/benchmarks/session-experience/README.md @@ -0,0 +1,89 @@ +# Session startup experience benchmark + +The useful UX gain is early feedback while the microphone is starting. Authentication and the local Connected indicator barely change. When capture is late, prepared publications make the microphone-ready state faster and let a join-triggered agent greet the user much earlier. Fast capture shows little consistent improvement once ICE/DTLS and media latency are included. + +This is a controlled **feature A/B**, not a production login, page-load, or AI model benchmark. The baseline is the original `useSession` implementation from `87c7326ebf320efbccdcb29655d1ca8d7f1b79a9`, while the candidate is the implementation in this PR. Both use the same companion SDK build to isolate prepared publication behavior. A minimal React UI renders immediately, then displays Connecting, Connected / microphone starting, and Ready to talk from actual Session state. + +## Results + +Every value below is milliseconds from the Connect click; **existing flow → prepared flow**. P50 is nearest rank over ten runs per variant per scenario. An extra pair per scenario is excluded as warmup; A/B order alternates. The final sweep completed 110 runs without failure: 10 warmups and 100 measured samples. No measured samples were retried or discarded. + +P50: + +| Scenario | Connected UI | Microphone-ready UI | First greeting audio | First response audio | +| --------------------------- | -----------: | ------------------: | -------------------: | -------------------: | +| Nearby, fast capture | 249 → 255 | 249 → 261 | 372 → 346 | 487 → 459 | +| 150 ms RTT, fast capture | 446 → 445 | 468 → 453 | 523 → 530 | 646 → 648 | +| 150 ms RTT, +800 ms capture | 443 → 442 | 1114 → 908 | 1161 → 538 | 1265 → 1138 | +| 150 ms RTT, 1 s credentials | 1378 → 1387 | 1401 → 1390 | 1469 → 1471 | 1583 → 1591 | +| 150 ms RTT, +2 s capture | 451 → 451 | 2320 → 2112 | 2368 → 528 | 2472 → 2310 | + +P95 (with only ten samples, this is each group's observed maximum; no production tail-latency claim): + +| Scenario | Connected UI | Microphone-ready UI | First greeting audio | First response audio | +| --------------------------- | -----------: | ------------------: | -------------------: | -------------------: | +| Nearby, fast capture | 486 → 266 | 486 → 279 | 590 → 370 | 703 → 484 | +| 150 ms RTT, fast capture | 684 → 688 | 699 → 705 | 760 → 785 | 885 → 901 | +| 150 ms RTT, +800 ms capture | 666 → 668 | 1126 → 925 | 1193 → 731 | 1332 → 1160 | +| 150 ms RTT, 1 s credentials | 1734 → 1610 | 1734 → 1610 | 1803 → 1700 | 1926 → 1822 | +| 150 ms RTT, +2 s capture | 462 → 706 | 2330 → 2124 | 2396 → 790 | 2531 → 2345 | + +The credential HTTP response took about 82–84 ms in the 80 ms scenarios and **1,004 ms for both variants** in the one-second scenario. Initial UI feedback took about 24–81 ms in both variants; this is a frame-opportunity proxy, not an app bundle-load measurement. + +For the 150 ms RTT cases: + +- Fast capture: the Connected UI is 446 → 445 ms, microphone-ready UI 468 → 453 ms, and first response 646 → 648 ms. Publication overlaps transport establishment, so this is little observable UX gain. The greeting is also essentially unchanged at 523 → 530 ms. +- An added 800 ms capture delay: microphone-ready UI improves **1,114 → 908 ms**, the greeting **1,161 → 538 ms**, and the response **1,265 → 1,138 ms**. +- An added two-second capture/permission delay: microphone-ready UI improves **2,320 → 2,112 ms**, the greeting **2,368 → 528 ms**, and the response **2,472 → 2,310 ms**. The local Connected indicator remains **451 ms in both flows**. +- Slow credentials remain slow: the one-second credential stage is unchanged, with no meaningful median improvement to connection or first response. + +These results support a UI that displays room connection separately from microphone readiness. Applications that hide the UI until `await session.start()` completes will still wait for microphone permission: its p50 is **2,295 → 2,100 ms** in the two-second case. The existing `session.isConnected` state can drive an earlier Connected UI at about **451 ms** in either flow; this does not require another public API. + +## What the greeting improvement means + +The warm fixture agent starts its greeting 50 ms after the user's `ParticipantConnected` event reaches the agent. Prepared publication advertises publisher intent during JOIN, so the remote side can discover the user while device access is pending. Ordinary publication can leave that notification waiting for publication or a batched participant update. The observed benefit is consistent with the SFU's publisher-update / subscriber-only batching policy; it is **not all a two-RTT saving**. A different server update policy or an agent that deliberately waits for microphone input before greeting can reduce this benefit. + +The response tone starts 50 ms after the agent decodes the first live microphone samples. It measures transport readiness and a deterministic response, not speech recognition, reasoning, synthesis, or semantic conversation quality. Preconnect buffering remains enabled and its stream is drained by the fixture agent. + +## Method + +- Chrome 158.0.8127.36, macOS arm64, Node v22.22.0; LiveKit server 1.13.10. Single PC, microphone-only, mono capture, default Session preconnect buffering. +- Actual production `useSession` hook for both variants, a small React UI, a real local SFU, native ICE/DTLS/SRTP/RTCP/SCTP and Opus, and two running browser AudioContexts. +- The user's signaling TCP connection and **all RTC UDP packets** pass through delay relays with half the scenario's RTT in each direction. The warm browser-agent peer connects directly to the SFU, modeling an agent near the server. Every run verifies that the nominated ICE candidate uses the relay and checks the measured ICE RTT: approximately 21–22 ms or 152–159 ms. +- The SFU uses a fixed UDP mux port and loopback candidates. The relay rewrites the client's SFU candidates, routes STUN transactions back to their originating socket, and forwards DTLS and media over the selected socket. No packet loss, bandwidth cap, jitter, TURN, DNS or WSS/TLS handshake is modeled. +- Device permission/startup is modeled by delaying `getUserMedia`, then opening Chrome's fake input device. This does not measure an actual OS permission dialog, real device selection, camera initialization, or Bluetooth startup. +- Credential retrieval is a local HTTP endpoint with the declared delay; real OAuth/password authentication and token-service execution are not measured. The app and JS modules are already loaded before the click. +- The agent emits distinct real audio waveforms through the SFU. The client records the first non-silent decoded waveform reaching its WebAudio graph. The sink is muted; physical speaker / audio-driver latency and autoplay-policy failures are not measured. Agent cold start and STT/LLM/TTS are excluded. +- Ten runs per condition are useful for finding this critical-path difference, not for production p95 or statistical-significance claims. Raw data retains all measured samples, including outliers and cases with no improvement. + +The SDK size CI reports the ESM bundle at 120.97 KB compressed, +2.16% (about 2.6 KB). These PRs should not be described as reducing initial JS download or login time. + +## Reproduce + +Build [SDK PR #2292](https://github.com/livekit/client-sdk-js/pull/2292), install this workspace and run `pnpm build:core`. Point `LIVEKIT_SDK_DIR` at that SDK checkout. Start a local SFU with the following configuration (dev keys only): + +```yaml +port: 17880 +rtc: + tcp_port: 17881 + udp_port: 17882 + use_external_ip: false + enable_loopback_candidate: true +keys: + devkey: secret +``` + +```sh +livekit-server --dev --node-ip 127.0.0.1 --config /tmp/session-ux-server.yaml +LIVEKIT_SDK_DIR=../client-sdk-js \ +LIVEKIT_URL=ws://127.0.0.1:17880 \ +LIVEKIT_UDP_PORT=17882 \ +BASELINE_REF=87c7326ebf320efbccdcb29655d1ca8d7f1b79a9 \ +REPETITIONS=10 \ +OUTPUT=/tmp/session-experience-results.json \ +node scripts/benchmark-session-experience.mjs +``` + +Set `CHROMIUM_EXECUTABLE_PATH` for an installed Chrome, `BASELINE_REF` for another baseline Session revision, or `SCENARIO` to run one named condition. The default baseline is the merge base with `origin/main`. The default SFU keys are local `devkey` / `secret`. The runner owns and closes browser contexts, TCP/UDP relays, credential responses, timers and the temporary Vite server; the caller owns the SFU. + +Sources: [runner](../../scripts/benchmark-session-experience.mjs), [browser fixture](../../scripts/session-experience-browser.ts). Data: [metadata and summaries](results.json), [all 100 samples](samples.csv). Source revisions and percentile definitions are recorded in the metadata. diff --git a/benchmarks/session-experience/results.json b/benchmarks/session-experience/results.json new file mode 100644 index 000000000..9a9ccfcf0 --- /dev/null +++ b/benchmarks/session-experience/results.json @@ -0,0 +1,763 @@ +{ + "baselineRef": "87c7326ebf320efbccdcb29655d1ca8d7f1b79a9", + "browserVersion": "158.0.8127.36", + "repetitions": 10, + "methodology": "Actual React useSession, same companion SDK build; delayed credential HTTP response and getUserMedia, client TCP signaling + UDP ICE/DTLS/SRTP/RTCP/SCTP RTT relays; agent directly connected to local SFU. Warm agent returns deterministic tones, no STT/LLM/TTS; loaded app, no production authentication or bundle-load measurement.", + "summary": [ + { + "name": "nearby-fast-device", + "networkRttMs": 20, + "tokenDelayMs": 80, + "captureDelayMs": 0, + "prepared": false, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 10, + "p95": 11 + }, + "feedbackUiMs": { + "p50": 81, + "p95": 82 + }, + "credentialsMs": { + "p50": 82, + "p95": 83 + }, + "micPublishedMs": { + "p50": 180, + "p95": 409 + }, + "roomConnectedMs": { + "p50": 226, + "p95": 456 + }, + "userTrackSubscribedMs": { + "p50": 226, + "p95": 457 + }, + "sessionStartResolvedMs": { + "p50": 230, + "p95": 462 + }, + "connectedUiMs": { + "p50": 249, + "p95": 486 + }, + "readyToTalkUiMs": { + "p50": 249, + "p95": 486 + }, + "agentTrackSubscribedMs": { + "p50": 266, + "p95": 496 + }, + "greetingSentMs": { + "p50": 279, + "p95": 508 + }, + "micDecodedAtAgentMs": { + "p50": 323, + "p95": 551 + }, + "greetingAudioMs": { + "p50": 372, + "p95": 590 + }, + "responseSentMs": { + "p50": 378, + "p95": 607 + }, + "responseAudioMs": { + "p50": 487, + "p95": 703 + }, + "observedMediaRttMs": { + "p50": 21, + "p95": 22 + } + } + }, + { + "name": "nearby-fast-device", + "networkRttMs": 20, + "tokenDelayMs": 80, + "captureDelayMs": 0, + "prepared": true, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 10, + "p95": 12 + }, + "feedbackUiMs": { + "p50": 80, + "p95": 84 + }, + "credentialsMs": { + "p50": 83, + "p95": 84 + }, + "roomConnectedMs": { + "p50": 225, + "p95": 239 + }, + "userTrackSubscribedMs": { + "p50": 226, + "p95": 240 + }, + "micPublishedMs": { + "p50": 232, + "p95": 246 + }, + "sessionStartResolvedMs": { + "p50": 233, + "p95": 248 + }, + "agentTrackSubscribedMs": { + "p50": 246, + "p95": 256 + }, + "connectedUiMs": { + "p50": 255, + "p95": 266 + }, + "readyToTalkUiMs": { + "p50": 261, + "p95": 279 + }, + "greetingSentMs": { + "p50": 254, + "p95": 268 + }, + "micDecodedAtAgentMs": { + "p50": 303, + "p95": 313 + }, + "greetingAudioMs": { + "p50": 346, + "p95": 370 + }, + "responseSentMs": { + "p50": 359, + "p95": 368 + }, + "responseAudioMs": { + "p50": 459, + "p95": 484 + }, + "observedMediaRttMs": { + "p50": 21, + "p95": 22 + } + } + }, + { + "name": "remote-fast-device", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 0, + "prepared": false, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 10, + "p95": 11 + }, + "feedbackUiMs": { + "p50": 78, + "p95": 82 + }, + "credentialsMs": { + "p50": 82, + "p95": 83 + }, + "roomConnectedMs": { + "p50": 414, + "p95": 655 + }, + "userTrackSubscribedMs": { + "p50": 415, + "p95": 656 + }, + "agentTrackSubscribedMs": { + "p50": 431, + "p95": 670 + }, + "greetingSentMs": { + "p50": 434, + "p95": 669 + }, + "micPublishedMs": { + "p50": 443, + "p95": 682 + }, + "sessionStartResolvedMs": { + "p50": 445, + "p95": 684 + }, + "connectedUiMs": { + "p50": 446, + "p95": 684 + }, + "readyToTalkUiMs": { + "p50": 468, + "p95": 699 + }, + "micDecodedAtAgentMs": { + "p50": 489, + "p95": 729 + }, + "greetingAudioMs": { + "p50": 523, + "p95": 760 + }, + "responseSentMs": { + "p50": 545, + "p95": 783 + }, + "responseAudioMs": { + "p50": 646, + "p95": 885 + }, + "observedMediaRttMs": { + "p50": 152, + "p95": 153 + } + } + }, + { + "name": "remote-fast-device", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 0, + "prepared": true, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 10, + "p95": 13 + }, + "feedbackUiMs": { + "p50": 78, + "p95": 83 + }, + "credentialsMs": { + "p50": 83, + "p95": 84 + }, + "roomConnectedMs": { + "p50": 414, + "p95": 666 + }, + "userTrackSubscribedMs": { + "p50": 415, + "p95": 667 + }, + "micPublishedMs": { + "p50": 424, + "p95": 676 + }, + "sessionStartResolvedMs": { + "p50": 426, + "p95": 679 + }, + "greetingSentMs": { + "p50": 436, + "p95": 685 + }, + "agentTrackSubscribedMs": { + "p50": 438, + "p95": 686 + }, + "connectedUiMs": { + "p50": 445, + "p95": 688 + }, + "readyToTalkUiMs": { + "p50": 453, + "p95": 705 + }, + "micDecodedAtAgentMs": { + "p50": 492, + "p95": 741 + }, + "greetingAudioMs": { + "p50": 530, + "p95": 785 + }, + "responseSentMs": { + "p50": 548, + "p95": 792 + }, + "responseAudioMs": { + "p50": 648, + "p95": 901 + }, + "observedMediaRttMs": { + "p50": 153, + "p95": 154 + } + } + }, + { + "name": "remote-slow-device", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 800, + "prepared": false, + "count": 10, + "metrics": { + "feedbackUiMs": { + "p50": 26, + "p95": 33 + }, + "credentialsMs": { + "p50": 83, + "p95": 84 + }, + "roomConnectedMs": { + "p50": 413, + "p95": 638 + }, + "userTrackSubscribedMs": { + "p50": 415, + "p95": 640 + }, + "connectedUiMs": { + "p50": 443, + "p95": 666 + }, + "captureReadyMs": { + "p50": 824, + "p95": 825 + }, + "agentTrackSubscribedMs": { + "p50": 1052, + "p95": 1061 + }, + "greetingSentMs": { + "p50": 1064, + "p95": 1074 + }, + "micPublishedMs": { + "p50": 1087, + "p95": 1096 + }, + "sessionStartResolvedMs": { + "p50": 1089, + "p95": 1099 + }, + "micDecodedAtAgentMs": { + "p50": 1114, + "p95": 1166 + }, + "readyToTalkUiMs": { + "p50": 1114, + "p95": 1126 + }, + "responseSentMs": { + "p50": 1168, + "p95": 1224 + }, + "greetingAudioMs": { + "p50": 1161, + "p95": 1193 + }, + "responseAudioMs": { + "p50": 1265, + "p95": 1332 + }, + "observedMediaRttMs": { + "p50": 152, + "p95": 153 + } + } + }, + { + "name": "remote-slow-device", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 800, + "prepared": true, + "count": 10, + "metrics": { + "feedbackUiMs": { + "p50": 25, + "p95": 29 + }, + "credentialsMs": { + "p50": 83, + "p95": 83 + }, + "roomConnectedMs": { + "p50": 419, + "p95": 643 + }, + "userTrackSubscribedMs": { + "p50": 420, + "p95": 644 + }, + "greetingSentMs": { + "p50": 441, + "p95": 651 + }, + "agentTrackSubscribedMs": { + "p50": 434, + "p95": 650 + }, + "connectedUiMs": { + "p50": 442, + "p95": 668 + }, + "greetingAudioMs": { + "p50": 538, + "p95": 731 + }, + "captureReadyMs": { + "p50": 818, + "p95": 838 + }, + "micPublishedMs": { + "p50": 892, + "p95": 916 + }, + "sessionStartResolvedMs": { + "p50": 894, + "p95": 918 + }, + "readyToTalkUiMs": { + "p50": 908, + "p95": 925 + }, + "micDecodedAtAgentMs": { + "p50": 952, + "p95": 973 + }, + "responseSentMs": { + "p50": 1009, + "p95": 1029 + }, + "responseAudioMs": { + "p50": 1138, + "p95": 1160 + }, + "observedMediaRttMs": { + "p50": 152, + "p95": 153 + } + } + }, + { + "name": "slow-credentials", + "networkRttMs": 150, + "tokenDelayMs": 1000, + "captureDelayMs": 0, + "prepared": false, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 10, + "p95": 13 + }, + "feedbackUiMs": { + "p50": 79, + "p95": 84 + }, + "credentialsMs": { + "p50": 1004, + "p95": 1005 + }, + "roomConnectedMs": { + "p50": 1350, + "p95": 1698 + }, + "userTrackSubscribedMs": { + "p50": 1351, + "p95": 1699 + }, + "greetingSentMs": { + "p50": 1365, + "p95": 1712 + }, + "agentTrackSubscribedMs": { + "p50": 1366, + "p95": 1711 + }, + "micPublishedMs": { + "p50": 1378, + "p95": 1717 + }, + "sessionStartResolvedMs": { + "p50": 1380, + "p95": 1719 + }, + "connectedUiMs": { + "p50": 1378, + "p95": 1734 + }, + "readyToTalkUiMs": { + "p50": 1401, + "p95": 1734 + }, + "micDecodedAtAgentMs": { + "p50": 1424, + "p95": 1766 + }, + "greetingAudioMs": { + "p50": 1469, + "p95": 1803 + }, + "responseSentMs": { + "p50": 1480, + "p95": 1821 + }, + "responseAudioMs": { + "p50": 1583, + "p95": 1926 + }, + "observedMediaRttMs": { + "p50": 153, + "p95": 159 + } + } + }, + { + "name": "slow-credentials", + "networkRttMs": 150, + "tokenDelayMs": 1000, + "captureDelayMs": 0, + "prepared": true, + "count": 10, + "metrics": { + "captureReadyMs": { + "p50": 11, + "p95": 14 + }, + "feedbackUiMs": { + "p50": 79, + "p95": 85 + }, + "credentialsMs": { + "p50": 1004, + "p95": 1005 + }, + "roomConnectedMs": { + "p50": 1355, + "p95": 1583 + }, + "userTrackSubscribedMs": { + "p50": 1356, + "p95": 1584 + }, + "micPublishedMs": { + "p50": 1364, + "p95": 1592 + }, + "sessionStartResolvedMs": { + "p50": 1367, + "p95": 1595 + }, + "greetingSentMs": { + "p50": 1373, + "p95": 1610 + }, + "connectedUiMs": { + "p50": 1387, + "p95": 1610 + }, + "agentTrackSubscribedMs": { + "p50": 1376, + "p95": 1611 + }, + "readyToTalkUiMs": { + "p50": 1390, + "p95": 1610 + }, + "micDecodedAtAgentMs": { + "p50": 1431, + "p95": 1667 + }, + "greetingAudioMs": { + "p50": 1471, + "p95": 1700 + }, + "responseSentMs": { + "p50": 1488, + "p95": 1724 + }, + "responseAudioMs": { + "p50": 1591, + "p95": 1822 + }, + "observedMediaRttMs": { + "p50": 154, + "p95": 156 + } + } + }, + { + "name": "permission-prompt", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 2000, + "prepared": false, + "count": 10, + "metrics": { + "feedbackUiMs": { + "p50": 29, + "p95": 33 + }, + "credentialsMs": { + "p50": 83, + "p95": 85 + }, + "roomConnectedMs": { + "p50": 423, + "p95": 431 + }, + "userTrackSubscribedMs": { + "p50": 424, + "p95": 432 + }, + "connectedUiMs": { + "p50": 451, + "p95": 462 + }, + "captureReadyMs": { + "p50": 2024, + "p95": 2029 + }, + "agentTrackSubscribedMs": { + "p50": 2261, + "p95": 2267 + }, + "greetingSentMs": { + "p50": 2270, + "p95": 2276 + }, + "micPublishedMs": { + "p50": 2292, + "p95": 2298 + }, + "sessionStartResolvedMs": { + "p50": 2295, + "p95": 2300 + }, + "readyToTalkUiMs": { + "p50": 2320, + "p95": 2330 + }, + "micDecodedAtAgentMs": { + "p50": 2319, + "p95": 2370 + }, + "greetingAudioMs": { + "p50": 2368, + "p95": 2396 + }, + "responseSentMs": { + "p50": 2374, + "p95": 2426 + }, + "responseAudioMs": { + "p50": 2472, + "p95": 2531 + }, + "observedMediaRttMs": { + "p50": 154, + "p95": 155 + } + } + }, + { + "name": "permission-prompt", + "networkRttMs": 150, + "tokenDelayMs": 80, + "captureDelayMs": 2000, + "prepared": true, + "count": 10, + "metrics": { + "feedbackUiMs": { + "p50": 24, + "p95": 31 + }, + "credentialsMs": { + "p50": 83, + "p95": 84 + }, + "roomConnectedMs": { + "p50": 426, + "p95": 681 + }, + "userTrackSubscribedMs": { + "p50": 427, + "p95": 682 + }, + "greetingSentMs": { + "p50": 446, + "p95": 698 + }, + "agentTrackSubscribedMs": { + "p50": 440, + "p95": 700 + }, + "connectedUiMs": { + "p50": 451, + "p95": 706 + }, + "greetingAudioMs": { + "p50": 528, + "p95": 790 + }, + "captureReadyMs": { + "p50": 2024, + "p95": 2028 + }, + "micPublishedMs": { + "p50": 2099, + "p95": 2104 + }, + "sessionStartResolvedMs": { + "p50": 2100, + "p95": 2107 + }, + "readyToTalkUiMs": { + "p50": 2112, + "p95": 2124 + }, + "micDecodedAtAgentMs": { + "p50": 2159, + "p95": 2166 + }, + "responseSentMs": { + "p50": 2212, + "p95": 2222 + }, + "responseAudioMs": { + "p50": 2310, + "p95": 2345 + }, + "observedMediaRttMs": { + "p50": 154, + "p95": 155 + } + } + } + ], + "collectedAt": "2026-11-20", + "sdkCommit": "13d5dd8eb88257fd93595abbf8678dcaedf8ec94", + "componentsSourceCommit": "c1ce6a87064b39f5e9f713ce02fe9214ee15e543", + "nodeVersion": "v22.22.0", + "platform": "Darwin/arm64", + "sfuVersion": "1.13.10", + "agentReactionMs": 50, + "agentPlacement": "Direct to local SFU; only the user connection passes through the TCP/UDP latency relays.", + "rawSamples": "samples.csv", + "sampleCount": 100, + "warmupsExcluded": 10, + "p50Definition": "nearest-rank: sorted[ceil(n * 0.50) - 1]", + "p95Definition": "nearest-rank: sorted[ceil(n * 0.95) - 1]; with ten samples this is the observed maximum", + "audioMeasurement": "Decoded 440 Hz greeting / 1170 Hz response reaching a running WebAudio analyser. Muted sink; physical speaker output not measured.", + "uiMeasurement": "Actual useSession hook and React status changes, observed after two requestAnimationFrame callbacks; already-loaded app." +} diff --git a/benchmarks/session-experience/samples.csv b/benchmarks/session-experience/samples.csv new file mode 100644 index 000000000..4a23506c0 --- /dev/null +++ b/benchmarks/session-experience/samples.csv @@ -0,0 +1,101 @@ +scenario,prepared,repetition,agentTrackSubscribedMs,captureReadyMs,connectedUiMs,credentialsMs,feedbackUiMs,greetingAudioMs,greetingSentMs,micDecodedAtAgentMs,micPublishedMs,observedMediaRttMs,readyToTalkUiMs,responseAudioMs,responseSentMs,roomConnectedMs,sessionStartResolvedMs,userTrackSubscribedMs +nearby-fast-device,True,0,256.09999990463257,10,265.59999990463257,83.79999995231628,83.59999990463257,345.59999990463257,268,313.19999980926514,241.19999980926514,21,265.59999990463257,458.69999980926514,368.39999985694885,234.39999985694885,242.69999980926514,235.29999995231628 +nearby-fast-device,False,0,266,10.299999952316284,245.09999990463257,82.59999990463257,82.40000009536743,372.2000000476837,277.5,322.2999999523163,180.5,21,245.09999990463257,491.7999999523163,377.7999999523163,225.79999995231628,230.90000009536743,226.5 +nearby-fast-device,False,1,262.2000000476837,10.5,260.7000000476837,81.60000014305115,81,365.90000009536743,274,320.5,178.40000009536743,21,260.7000000476837,473.40000009536743,377.40000009536743,225.5,230.30000019073486,226.30000019073486 +nearby-fast-device,True,1,241.20000004768372,9.5,242,81.59999990463257,79.09999990463257,329.39999985694885,252.79999995231628,297.7999999523163,227.39999985694885,21,258.2999999523163,442.5,352.2999999523163,221.20000004768372,228.79999995231628,221.89999985694885 +nearby-fast-device,True,2,245.90000009536743,10.100000143051147,249.20000004768372,82.60000014305115,79.20000004768372,370.10000014305115,257.7999999523163,305,233.29999995231628,20,265.7000000476837,483.90000009536743,358.7000000476837,225.79999995231628,234.60000014305115,226.5 +nearby-fast-device,False,2,268.2000000476837,10,255.59999990463257,81.59999990463257,76.40000009536743,368.5,279.2999999523163,323.09999990463257,183.70000004768372,22,255.59999990463257,470.7999999523163,374.2000000476837,231.79999995231628,236,232.40000009536743 +nearby-fast-device,False,3,273.2000000476837,10.600000143051147,268.2999999523163,82.60000014305115,2,367.90000009536743,284.90000009536743,328.2999999523163,180,20,268.2999999523163,486.5,378.7999999523163,235.60000014305115,240.20000004768372,236.20000004768372 +nearby-fast-device,True,3,240.20000004768372,10.400000095367432,243.79999995231628,82.70000004768372,80,360.90000009536743,251.40000009536743,298.40000009536743,228,21,260.5,462.09999990463257,355.40000009536743,220.79999995231628,229.5,221.59999990463257 +nearby-fast-device,True,4,252.79999995231628,11.399999856948853,255.59999990463257,82.69999980926514,77.19999980926514,337.09999990463257,252.19999980926514,309.2999999523163,243.69999980926514,20,272.19999980926514,449.2999999523163,365,235.39999985694885,245.59999990463257,236.09999990463257 +nearby-fast-device,False,4,265.2999999523163,10.099999904632568,248.89999985694885,81.79999995231628,81.59999990463257,386.39999985694885,281.5,325,180.89999985694885,21,248.89999985694885,485.39999985694885,380.69999980926514,222.29999995231628,226.79999995231628,223 +nearby-fast-device,False,5,495.59999990463257,10,485.69999980926514,81.39999985694885,79.5,589.8999998569489,507.5,551.1999998092651,409.19999980926514,21,485.69999980926514,702.5,607,455.69999980926514,461.89999985694885,456.5 +nearby-fast-device,True,5,253.90000009536743,12.099999904632568,261.40000009536743,82.5,79.40000009536743,350.40000009536743,259.90000009536743,310.90000009536743,246.09999990463257,21,278.90000009536743,462.7000000476837,367.5,239.09999990463257,247.79999995231628,239.70000004768372 +nearby-fast-device,True,6,246.40000009536743,12.400000095367432,257.40000009536743,83,80.90000009536743,365,257.5,301.90000009536743,231.40000009536743,20,257.40000009536743,467,359.30000019073486,225.40000009536743,232.60000014305115,226.10000014305115 +nearby-fast-device,False,6,261.19999980926514,9.799999952316284,248.79999995231628,82,81.89999985694885,350.5,272.19999980926514,317.5,178.39999985694885,22,248.79999995231628,442,367.59999990463257,223.19999980926514,227.89999985694885,223.79999995231628 +nearby-fast-device,False,7,274.90000009536743,9.799999952316284,265.2999999523163,81.90000009536743,77.40000009536743,379,278.7999999523163,330.10000014305115,179.90000009536743,21,265.40000009536743,497.90000009536743,384.7999999523163,232,236.70000004768372,232.79999995231628 +nearby-fast-device,True,7,248.59999990463257,11.899999856948853,253.29999995231628,82.5,79.79999995231628,330.59999990463257,251,302.89999985694885,237.59999990463257,21,270.5,444.5,358.89999985694885,228.70000004768372,239.09999990463257,229.39999985694885 +nearby-fast-device,True,8,245.89999985694885,11.699999809265137,258.69999980926514,83.19999980926514,82.5,334.5,257.7999999523163,303.59999990463257,231.69999980926514,22,258.69999980926514,430.7999999523163,356.7999999523163,224.89999985694885,233,225.5 +nearby-fast-device,False,8,264.7999999523163,10.600000143051147,242.40000009536743,82.5,82,386,277.7999999523163,322.40000009536743,178.70000004768372,22,242.40000009536743,493.60000014305115,380.5,221.40000009536743,226.5,222.20000004768372 +nearby-fast-device,False,9,267.2000000476837,9.400000095367432,248.70000004768372,81.90000009536743,80.80000019073486,385.90000009536743,287.60000014305115,323.7000000476837,181.80000019073486,20,248.70000004768372,488.7000000476837,375.10000014305115,223.30000019073486,227.70000004768372,224 +nearby-fast-device,True,9,243,10.399999856948853,255.29999995231628,81.39999985694885,80.69999980926514,354.7999999523163,254.29999995231628,299,227.79999995231628,21,255.29999995231628,463.09999990463257,354.7999999523163,221.39999985694885,229.5,222.09999990463257 +remote-fast-device,True,0,680.7999999523163,10.299999952316284,684,82.5,80.40000009536743,785.4000000953674,679.7000000476837,737.2000000476837,670.7999999523163,154,699.6000001430511,893.4000000953674,791.1000001430511,658.2000000476837,673.1000001430511,659.2999999523163 +remote-fast-device,False,0,432.7999999523163,10.5,445.09999990463257,81.79999995231628,77.70000004768372,518.7999999523163,441.40000009536743,491.2999999523163,442.7000000476837,152,460.2999999523163,632.5,547.2000000476837,408.2999999523163,444.59999990463257,409.2000000476837 +remote-fast-device,False,1,434.90000009536743,11.099999904632568,443.7999999523163,82.79999995231628,82.20000004768372,536.2999999523163,435.7999999523163,492.09999990463257,446.09999990463257,151,475.59999990463257,650.2000000476837,547.5999999046326,412.09999990463257,448,413.2000000476837 +remote-fast-device,True,1,437.90000009536743,10.5,441.2999999523163,82.59999990463257,77.40000009536743,521.0999999046326,436,439,427.40000009536743,154,457.90000009536743,577.5999999046326,496.40000009536743,417,429.90000009536743,418 +remote-fast-device,True,2,438.5,9.799999952316284,445.2000000476837,81.70000004768372,78.29999995231628,527.5,439.5,495.2999999523163,422.10000014305115,153,445.2000000476837,646.1000001430511,550.2000000476837,412,424.60000014305115,413 +remote-fast-device,False,2,424.60000014305115,10.200000047683716,448.60000014305115,82.30000019073486,79.40000009536743,532.7000000476837,428.90000009536743,480.7000000476837,443.80000019073486,151,465.30000019073486,645.8000001907349,538.4000000953674,414.2000000476837,445.80000019073486,414.90000009536743 +remote-fast-device,False,3,669.5,8.900000095367432,684.2000000476837,81.40000009536743,78.40000009536743,760.0999999046326,668.5999999046326,728.5999999046326,681.5999999046326,152,699.2999999523163,885,782.7000000476837,654.7000000476837,683.5999999046326,655.7999999523163 +remote-fast-device,True,3,431.2999999523163,10.299999952316284,433.09999990463257,83.09999990463257,83,516.1999998092651,430.5,486.59999990463257,422.19999980926514,153,449.7999999523163,634.8999998569489,539,411.7999999523163,424.5,412.59999990463257 +remote-fast-device,True,4,432.60000014305115,11.200000047683716,434.60000014305115,81.80000019073486,2.700000047683716,569,428.7000000476837,488.60000014305115,423.30000019073486,154,451.90000009536743,648.4000000953674,546.3000001907349,411.90000009536743,425.5,413.10000014305115 +remote-fast-device,False,4,423.30000019073486,10.300000190734863,438.10000014305115,81.5,80.90000009536743,523.2000000476837,434.30000019073486,477.7000000476837,439.7000000476837,152,470.60000014305115,614,528.9000000953674,412.90000009536743,441.40000009536743,413.5 +remote-fast-device,False,5,431.2999999523163,10.600000143051147,443.5,80.60000014305115,80,571.2000000476837,430.40000009536743,489.10000014305115,441,151,459.7000000476837,651.2000000476837,542.7999999523163,414.10000014305115,442.90000009536743,415.10000014305115 +remote-fast-device,True,5,427.5,13.200000047683716,439.80000019073486,83.70000004768372,83.10000014305115,574.1000001430511,426.2000000476837,484,423.60000014305115,154,456.80000019073486,653.3000001907349,540.6000001430511,413.90000009536743,425.7000000476837,414.7000000476837 +remote-fast-device,True,6,686.3999998569489,9.899999856948853,688,82,78.19999980926514,780.7999999523163,684.6999998092651,741.0999999046326,676.1999998092651,152,704.5999999046326,901.0999999046326,792.1999998092651,665.6999998092651,678.7999999523163,666.8999998569489 +remote-fast-device,False,6,433.5,9.799999952316284,451.69999980926514,82.5,1.8999998569488525,520.7999999523163,434.2999999523163,493,444.09999990463257,151,468.39999985694885,650.5,549.2999999523163,417.7999999523163,445.89999985694885,418.59999990463257 +remote-fast-device,False,7,425,10.299999952316284,451.7999999523163,82.09999990463257,2,518.5999999046326,430.2999999523163,483.7999999523163,446.5,152,468.5,643.5999999046326,541.3999998569489,416.19999980926514,448.39999985694885,416.89999985694885 +remote-fast-device,True,7,441.7000000476837,10.5,456,82.60000014305115,80.20000004768372,566.2000000476837,441.10000014305115,498.7000000476837,431.80000019073486,152,456,674.3000001907349,554.8000001907349,419.90000009536743,434.2000000476837,421 +remote-fast-device,True,8,436.5,10,445.2000000476837,82.60000014305115,77.10000014305115,519.7000000476837,435.7000000476837,492,419.40000009536743,152,445.2000000476837,644.4000000953674,548.2000000476837,410.30000019073486,422,411.30000019073486 +remote-fast-device,False,8,431.2000000476837,9.799999952316284,445.7000000476837,81.5,75.89999985694885,539.3999998569489,445.2999999523163,489.5,443.09999990463257,153,460,647.7000000476837,545.0999999046326,410.09999990463257,445.09999990463257,411.09999990463257 +remote-fast-device,False,9,428.90000009536743,9.799999952316284,451.2999999523163,80.90000009536743,77.79999995231628,522,430.90000009536743,488.90000009536743,443.2000000476837,152,467.7000000476837,640.7999999523163,544.7000000476837,416.2000000476837,445.2000000476837,417.10000014305115 +remote-fast-device,True,9,440.5,10.099999904632568,452.59999990463257,82.59999990463257,76.5,530.3999998569489,439.5,497.5,431.39999985694885,153,452.59999990463257,648.2999999523163,552.5999999046326,420,433.59999990463257,421 +remote-slow-device,True,0,445.7000000476837,817.7999999523163,458.2999999523163,83,25.299999952316284,539.5999999046326,442.7000000476837,945.3999998569489,887.5999999046326,153,891.3999998569489,1146.2999999523163,1003.0999999046326,426.39999985694885,889.2999999523163,427.2000000476837 +remote-slow-device,False,0,1055.5,825.3000001907349,447.40000009536743,82.10000014305115,30.700000047683716,1188.4000000953674,1067,1113.9000000953674,1087.2000000476837,152,1114,1291.8000001907349,1171.4000000953674,422.80000019073486,1089.9000000953674,423.7000000476837 +remote-slow-device,False,1,1054.7999999523163,825.0999999046326,445.7000000476837,83.5,29.09999990463257,1160.7999999523163,1066.2000000476837,1114,1089,151,1112.2999999523163,1258.0999999046326,1172.2999999523163,411.7999999523163,1091.5,412.90000009536743 +remote-slow-device,True,1,429.7000000476837,817,435.2000000476837,82.79999995231628,17.799999952316284,511.09999990463257,435.7000000476837,951.9000000953674,892.2999999523163,152,901.2000000476837,1113.2000000476837,1008.5,406.59999990463257,894.4000000953674,407.7000000476837 +remote-slow-device,True,2,431.09999990463257,818.0999999046326,440.09999990463257,82.29999995231628,23.5,509.2000000476837,430.2999999523163,948.8999998569489,889.2000000476837,152,905.7999999523163,1137.7000000476837,1004.8999998569489,414.59999990463257,890.8999998569489,415.5 +remote-slow-device,False,2,1056.7999999523163,824.2999999523163,666.4000000953674,82.40000009536743,33,1154.7000000476837,1067.0999999046326,1115.2000000476837,1089,150,1116.4000000953674,1268.5999999046326,1166.0999999046326,638.2999999523163,1091.2000000476837,639.5 +remote-slow-device,False,3,1051.8999998569489,823,446.89999985694885,82.5,30.59999990463257,1187.7999999523163,1063.3999998569489,1110.0999999046326,1085.8999998569489,153,1114.2000000476837,1289.5999999046326,1164.7000000476837,413.09999990463257,1088.5999999046326,414.59999990463257 +remote-slow-device,True,3,425.2000000476837,818.2999999523163,442.39999985694885,81.89999985694885,25.799999952316284,561.8999998569489,432.5,945.5999999046326,888.7999999523163,153,908.7000000476837,1130.5999999046326,999.5,412.5,890.5,413.09999990463257 +remote-slow-device,True,4,435.2000000476837,824.7999999523163,443.7999999523163,82.90000009536743,27.59999990463257,521.4000000953674,447.2000000476837,954.7000000476837,895.5,152,910.2999999523163,1120.4000000953674,1011.5,411.09999990463257,897.7000000476837,412.09999990463257 +remote-slow-device,False,4,1052.2000000476837,822.0999999046326,615,83.20000004768372,31.700000047683716,1193.2999999523163,1063.7000000476837,1111.2999999523163,1086.7999999523163,153,1115.0999999046326,1307.8999998569489,1168.0999999046326,589.8999998569489,1088.7999999523163,591.5 +remote-slow-device,False,5,1060.8999998569489,824.1999998092651,442.69999980926514,82.79999995231628,25.699999809265137,1158.0999999046326,1073.5999999046326,1120,1096,153,1125.5999999046326,1259.8999998569489,1174.7999999523163,411.89999985694885,1098.5999999046326,413.09999990463257 +remote-slow-device,True,5,443.2000000476837,821.2000000476837,445.5,82.59999990463257,28.90000009536743,527.2999999523163,441.09999990463257,952.7999999523163,894.4000000953674,152,912.4000000953674,1145.2000000476837,1008.7999999523163,418.90000009536743,896.5999999046326,420 +remote-slow-device,True,6,649.7999999523163,822.0999999046326,668.2999999523163,83,2.0999999046325684,731.0999999046326,650.7999999523163,954.7000000476837,895.5,152,900.7000000476837,1108.5,1011.7000000476837,642.9000000953674,897.4000000953674,643.7000000476837 +remote-slow-device,False,6,1049.2999999523163,821.8999998569489,437.2999999523163,83.29999995231628,20.399999856948853,1155.6999998092651,1060.8999998569489,1106.2999999523163,1082.6999998092651,152,1104.6999998092651,1257.5999999046326,1161.3999998569489,412.2999999523163,1084.8999998569489,413.5 +remote-slow-device,False,7,1059.1000001430511,823.6000001430511,439.40000009536743,83.30000019073486,22.300000190734863,1162.6000001430511,1069.5,1118,1090.8000001907349,153,1122.7000000476837,1264.7000000476837,1174,411.7000000476837,1093,412.7000000476837 +remote-slow-device,True,7,454.10000014305115,821,458.10000014305115,82.5,24.90000009536743,556.1000001430511,459.5,957.8000001907349,898.4000000953674,151,908.1000001430511,1147.1000001430511,1009.5,438.90000009536743,900.5,439.90000009536743 +remote-slow-device,True,8,433,816.7999999523163,442.39999985694885,82.70000004768372,25.700000047683716,537.5,439.59999990463257,948.8999998569489,890,152,909.0999999046326,1140.3999998569489,1005.2999999523163,418.7999999523163,891.8999998569489,419.59999990463257 +remote-slow-device,False,8,1042.5999999046326,814.4000000953674,441.7000000476837,82.29999995231628,25,1160.5,1054,1100.5999999046326,1076.0999999046326,152,1107.9000000953674,1251.4000000953674,1154.7999999523163,421.7000000476837,1078.2999999523163,422.5 +remote-slow-device,False,9,1048.7000000476837,824.0999999046326,441.09999990463257,82.89999985694885,24,1162.2000000476837,1060.0999999046326,1165.7000000476837,1084.5,151,1107.8999998569489,1332.2000000476837,1224.0999999046326,421,1087.2000000476837,421.59999990463257 +remote-slow-device,True,9,434.10000014305115,837.7000000476837,441.7999999523163,82,25.90000009536743,572,440.7000000476837,972.5,915.9000000953674,152,924.9000000953674,1160.4000000953674,1029.2999999523163,419.7000000476837,917.7000000476837,420.40000009536743 +slow-credentials,True,0,1369,11.600000143051147,1368.1000001430511,1003.9000000953674,76.5,1470.6000001430511,1368.1000001430511,1426.4000000953674,1358.6000001430511,154,1385.7000000476837,1591.1000001430511,1485.1000001430511,1347,1360.8000001907349,1348 +slow-credentials,False,0,1366,9.699999809265137,1378,1003.2999999523163,77.39999985694885,1468.5,1363.7999999523163,1423.5,1372.6999998092651,153,1394.6999998092651,1603.5,1479.8999998569489,1347.6999998092651,1374.6999998092651,1348.8999998569489 +slow-credentials,False,1,1487.7999999523163,9.799999952316284,1512,1002.7999999523163,82.10000014305115,1605.7999999523163,1499.7999999523163,1544.2000000476837,1506.1000001430511,151,1527.6000001430511,1703,1600.1000001430511,1482.7000000476837,1509.2999999523163,1483.2999999523163 +slow-credentials,True,1,1367,14.099999904632568,1369.5999999046326,1003,84.59999990463257,1455.6999998092651,1363.8999998569489,1422.5,1355.0999999046326,153,1386.1999998092651,1585.5,1478.1999998092651,1344.6999998092651,1357.6999998092651,1345.5999999046326 +slow-credentials,True,2,1376,10.800000190734863,1391,1004.4000000953674,75.90000009536743,1470.4000000953674,1375.3000001907349,1431.1000001430511,1366.7000000476837,154,1391,1589.7000000476837,1487.8000001907349,1354.8000001907349,1368.7000000476837,1355.7000000476837 +slow-credentials,False,2,1366,12.900000095367432,1375,1003.2000000476837,83.80000019073486,1456.5,1365.1000001430511,1421.5,1372.4000000953674,152,1405,1575.4000000953674,1478.8000001907349,1349.7000000476837,1374.3000001907349,1350.6000001430511 +slow-credentials,False,3,1370.5999999046326,9.599999904632568,1380.7000000476837,1004.4000000953674,80.29999995231628,1473.5999999046326,1368.2999999523163,1428,1377.5999999046326,153,1397.2999999523163,1581.7000000476837,1484.9000000953674,1351.5999999046326,1379.7999999523163,1352.5999999046326 +slow-credentials,True,3,1591.5999999046326,9.799999952316284,1596.7999999523163,1003.5,81.20000004768372,1687,1592.5,1646.8999998569489,1574.5999999046326,152,1596.7999999523163,1801.7000000476837,1704,1567.5999999046326,1576.5999999046326,1568.3999998569489 +slow-credentials,True,4,1353.3999998569489,10.5,1387.2000000476837,1004.2999999523163,79.20000004768372,1470.7000000476837,1364.0999999046326,1414.5,1360,154,1387.2000000476837,1592.7000000476837,1470.7000000476837,1358.0999999046326,1362.2000000476837,1358.2999999523163 +slow-credentials,False,4,1710.8999998569489,10.700000047683716,1734.3999998569489,1004.2000000476837,78.70000004768372,1803.0999999046326,1711.8999998569489,1765.8999998569489,1716.5999999046326,159,1734.3999998569489,1925.7000000476837,1821.3999998569489,1697.7000000476837,1718.8999998569489,1698.7000000476837 +slow-credentials,False,5,1374.2999999523163,10.100000143051147,1382.7999999523163,1004.7999999523163,77.5,1471.6000001430511,1372.2999999523163,1429.5,1379.7999999523163,153,1397.2999999523163,1605.7000000476837,1488,1356.4000000953674,1381.9000000953674,1357.2999999523163 +slow-credentials,True,5,1371.0999999046326,10.799999952316284,1380.5999999046326,1004.5999999046326,80.79999995231628,1451.4000000953674,1368.7999999523163,1428.0999999046326,1360.2000000476837,154,1380.5999999046326,1573.0999999046326,1482.2999999523163,1350.0999999046326,1362.5,1351.0999999046326 +slow-credentials,True,6,1425,11.399999856948853,1436.0999999046326,1004.5999999046326,78.70000004768372,1522.7999999523163,1423.0999999046326,1480.2999999523163,1417.2000000476837,154,1436.0999999046326,1641.7000000476837,1539.0999999046326,1406.7000000476837,1419.2999999523163,1407.7000000476837 +slow-credentials,False,6,1361.7000000476837,11.200000047683716,1376.2000000476837,1005.1000001430511,81.20000004768372,1453.5,1362.7000000476837,1417.3000001907349,1379.2000000476837,155,1409.4000000953674,1572.8000001907349,1471.5,1347.6000001430511,1381.2000000476837,1348.5 +slow-credentials,False,7,1364.7000000476837,9.700000047683716,1368.2999999523163,1004.0999999046326,77.09999990463257,1446.9000000953674,1362.7000000476837,1423.9000000953674,1376.9000000953674,153,1401.5999999046326,1571.4000000953674,1478.5,1345.7999999523163,1378.7000000476837,1346.7999999523163 +slow-credentials,True,7,1377.2000000476837,10.200000047683716,1382.0999999046326,1005.2000000476837,79.40000009536743,1479.2000000476837,1376.5999999046326,1436.0999999046326,1367.2000000476837,156,1400.7000000476837,1593.7000000476837,1490.7999999523163,1357.2999999523163,1369.4000000953674,1358.5 +slow-credentials,True,8,1378.1000001430511,11.100000143051147,1389.5,1004.6000001430511,79,1447,1373.2000000476837,1435,1364.3000001907349,152,1389.6000001430511,1574.4000000953674,1490.5,1352.8000001907349,1366.6000001430511,1353.8000001907349 +slow-credentials,False,8,1363.2000000476837,9.5,1373.5,1003.5,78.79999995231628,1447.5,1360.8999998569489,1419,1370.3999998569489,152,1389.2000000476837,1583.0999999046326,1476.3999998569489,1344.2000000476837,1372.5999999046326,1345.2999999523163 +slow-credentials,False,9,1372,10.200000047683716,1386.8999998569489,1004.5,82.20000004768372,1486.2000000476837,1370,1423.8999998569489,1383.8999998569489,151,1400.7000000476837,1585.2999999523163,1481.0999999046326,1353.2000000476837,1386,1354.2999999523163 +slow-credentials,True,9,1610.7000000476837,10.599999904632568,1609.5,1003.5999999046326,77.29999995231628,1699.7000000476837,1609.5999999046326,1666.7999999523163,1591.8999998569489,154,1609.5,1821.7999999523163,1723.5,1582.7000000476837,1594.5,1583.8999998569489 +permission-prompt,True,0,438.2999999523163,2023.1000001430511,451.2999999523163,82.79999995231628,2.1000001430511475,528.4000000953674,436.2000000476837,2156.600000143051,2098.7000000476837,154,2115.9000000953674,2310.9000000953674,2209.7000000476837,421.10000014305115,2100.2999999523163,422.10000014305115 +permission-prompt,False,0,2265.4000000953674,2025.4000000953674,453.5,83.40000009536743,20.200000047683716,2372.7999999523163,2273.2000000476837,2322.7000000476837,2296.5,155,2320.2000000476837,2473.5,2377.7999999523163,418.2999999523163,2299.2000000476837,420.2000000476837 +permission-prompt,False,1,2248,2018.0999999046326,446.69999980926514,83.19999980926514,30,2350.199999809265,2260.399999856949,2306.2999999523163,2281.5,153,2311.2999999523163,2449.5999999046326,2361.5999999046326,421.09999990463257,2283.2999999523163,422.19999980926514 +permission-prompt,True,1,446.39999985694885,2023.5,444.09999990463257,83,28.699999809265137,515.5999999046326,449.7999999523163,2158.5999999046326,2100.199999809265,155,2110.2999999523163,2320.5,2214.5999999046326,421,2101.7999999523163,422 +permission-prompt,True,2,451.69999980926514,2025.0999999046326,454.7999999523163,83.59999990463257,21,539.0999999046326,452.39999985694885,2159.0999999046326,2099.199999809265,152,2121.5,2297.2999999523163,2211.199999809265,433.69999980926514,2101.199999809265,434.7999999523163 +permission-prompt,False,2,2265.7000000476837,2029.0999999046326,446.7999999523163,84.5,30,2395.5,2274.5,2322.2999999523163,2295.899999856949,155,2330,2508,2377.2000000476837,421.2000000476837,2298.5,422.59999990463257 +permission-prompt,False,3,2260.600000143051,2023.8000001907349,453.2000000476837,83,20.5,2373.100000143051,2269.600000143051,2320.9000000953674,2293.7000000476837,153,2320.4000000953674,2474.600000143051,2378.2000000476837,422.60000014305115,2296.4000000953674,423.60000014305115 +permission-prompt,True,3,438.59999990463257,2025.5999999046326,462,83.59999990463257,27.299999952316284,527.0999999046326,436.59999990463257,2160.2999999523163,2101.899999856949,153,2111.7999999523163,2310.2000000476837,2214.399999856949,427.2999999523163,2103.899999856949,428.2000000476837 +permission-prompt,True,4,436.5,2022.4000000953674,451.40000009536743,83.59999990463257,2.0999999046325684,519.7000000476837,437.2000000476837,2156.4000000953674,2096.7999999523163,155,2118.0999999046326,2288.0999999046326,2211.7000000476837,420.2999999523163,2098.7000000476837,421.09999990463257 +permission-prompt,False,4,2261.7000000476837,2026.9000000953674,455.60000014305115,83.40000009536743,20.90000009536743,2383.2000000476837,2270.5,2319.2999999523163,2292.4000000953674,155,2320.9000000953674,2481.100000143051,2371.9000000953674,430.40000009536743,2294.7000000476837,431.60000014305115 +permission-prompt,False,5,2267.100000143051,2026.3000001907349,447.7000000476837,83.5,31,2351,2275.800000190735,2325.100000143051,2297.9000000953674,154,2329.100000143051,2454.2000000476837,2380.2000000476837,416.30000019073486,2300.4000000953674,417.60000014305115 +permission-prompt,True,5,444.5,2022.8999998569489,461.7999999523163,83.39999985694885,28.5,540.8999998569489,445.7999999523163,2156.5999999046326,2096.2999999523163,154,2111.7999999523163,2320.5,2213.899999856949,428,2098.899999856949,428.89999985694885 +permission-prompt,True,6,700.2000000476837,2024.2999999523163,706,83.20000004768372,24,790.4000000953674,698.0999999046326,2166.0999999046326,2103.9000000953674,155,2123.7999999523163,2302.7999999523163,2222.2000000476837,680.5,2107.4000000953674,681.5999999046326 +permission-prompt,False,6,2255.4000000953674,2022.9000000953674,459.40000009536743,84.29999995231628,26.100000143051147,2369.7000000476837,2265.4000000953674,2369.9000000953674,2287.600000143051,154,2307.9000000953674,2531.4000000953674,2425.5,430.90000009536743,2289.7000000476837,432.40000009536743 +permission-prompt,False,7,2262.2999999523163,2024,461.89999985694885,83.59999990463257,28.5,2361.7000000476837,2272.2999999523163,2319.2000000476837,2293.2000000476837,153,2328.5,2461.7999999523163,2374.2999999523163,428.2000000476837,2295.7999999523163,429.2999999523163 +permission-prompt,True,7,440.10000014305115,2014.6000001430511,454.2000000476837,83.5,22.5,547.6000001430511,442.7999999523163,2154.100000143051,2089.2999999523163,154,2104,2322.4000000953674,2207.7999999523163,433.2000000476837,2090.9000000953674,434 +permission-prompt,True,8,439.69999980926514,2027.8999998569489,442.89999985694885,82.5,26.199999809265137,513.6999998092651,451.2999999523163,2158.5,2099.5999999046326,155,2108.5,2344.7999999523163,2212.0999999046326,421,2101.899999856949,421.7999999523163 +permission-prompt,False,8,2257.100000143051,2024,446.2000000476837,83.79999995231628,31,2368.2000000476837,2269.2999999523163,2317.2000000476837,2291,154,2314.2999999523163,2472,2373.2000000476837,425.5,2292.5,426.7999999523163 +permission-prompt,False,9,2249.7000000476837,2018.9000000953674,451.10000014305115,82.40000009536743,32.799999952316284,2353.5,2260.9000000953674,2308.4000000953674,2283.2999999523163,154,2317.600000143051,2455.2999999523163,2365.9000000953674,426.60000014305115,2285.600000143051,428 +permission-prompt,True,9,448.2000000476837,2025.4000000953674,447.60000014305115,83.80000019073486,30.600000143051147,534.5,446.5,2158.7000000476837,2097.800000190735,155,2114.2000000476837,2307.4000000953674,2214.9000000953674,426,2099.600000143051,427.2000000476837 diff --git a/scripts/README.md b/scripts/README.md index d55bd8c99..8fe92d843 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -13,3 +13,9 @@ The runner uses Playwright and `livekit-server-sdk` from the companion SDK's dev It checks both delayed capture (JOIN finishes first) and early capture (microphone samples are buffered while JOIN proceeds). An agent peer must receive microphone audio and decoded camera video, and its preconnect stream must finish, carrying samples in the early-capture case. Attachment may not emit AddTrack or an offer after connection and capture are ready. Initial transport setup may independently emit an offer before that point. Both browser contexts and the temporary Vite server are closed by the runner. Ordinary `useSession().start()` options remain unchanged. Components detects the SDK preparation capability at runtime; SDKs without it retain the existing enable-track methods. The SDK decides whether a prepared handle can use JOIN or needs ordinary publication. Screen sharing keeps its existing user-gesture path. + +# Click-to-ready experience benchmark + +`benchmark-session-experience.mjs` exercises the actual React `useSession` hook and a small status UI, with a warm agent peer and a real SFU. Its TCP/UDP relays delay signaling, ICE, DTLS and media for the user, while the agent connects directly to the SFU. The nominated ICE pair and observed RTT are verified on every run. + +See [the report](../benchmarks/session-experience/README.md) for results, raw data, network/device/credential scenarios, caveats and the exact reproduction command. The runner reads the baseline hook from Git without changing tracked source files. Its default output is `/tmp/session-experience-results.json`. diff --git a/scripts/benchmark-session-experience.mjs b/scripts/benchmark-session-experience.mjs new file mode 100644 index 000000000..bc20739bb --- /dev/null +++ b/scripts/benchmark-session-experience.mjs @@ -0,0 +1,341 @@ +// Measures click-to-ready milestones in the actual React useSession hook. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import net from 'node:net'; +import dgram from 'node:dgram'; +import { resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +const root = fileURLToPath(new URL('..', import.meta.url)); +const sdk = resolve(root, process.env.LIVEKIT_SDK_DIR ?? '../client-sdk-js'); +const sdkRequire = createRequire(resolve(sdk, 'package.json')); +const reactRequire = createRequire(resolve(root, 'packages/react/package.json')); +const { chromium } = sdkRequire('playwright'); +const { AccessToken } = sdkRequire('livekit-server-sdk'); +const { createServer } = await import(pathToFileURL(reactRequire.resolve('vite')).href); +const upstream = new URL(process.env.LIVEKIT_URL ?? 'ws://127.0.0.1:7880'); +if (upstream.protocol !== 'ws:' || upstream.hostname !== '127.0.0.1') + throw new Error('Use a local ws://127.0.0.1 SFU for the TCP/UDP latency relays'); +const baselineRef = + process.env.BASELINE_REF ?? + execFileSync('git', ['merge-base', 'HEAD', 'origin/main'], { + cwd: root, + encoding: 'utf8', + }).trim(); +const baseline = execFileSync( + 'git', + ['show', `${baselineRef}:packages/react/src/hooks/useSession.ts`], + { cwd: root, encoding: 'utf8' }, +); +if (!process.env.BASELINE_REF && baseline.includes('connectSessionWithTracks')) { + throw new Error( + 'Baseline already uses prepared publications; set BASELINE_REF to a pre-feature Session revision', + ); +} +const reactRoot = resolve(root, 'packages/react'); +const baselineId = resolve(reactRoot, 'src/hooks/__benchmark_baseline_session.ts'); +const runs = new Map(); +const scenarios = [ + { name: 'nearby-fast-device', networkRttMs: 20, tokenDelayMs: 80, captureDelayMs: 0 }, + { name: 'remote-fast-device', networkRttMs: 150, tokenDelayMs: 80, captureDelayMs: 0 }, + { name: 'remote-slow-device', networkRttMs: 150, tokenDelayMs: 80, captureDelayMs: 800 }, + { name: 'slow-credentials', networkRttMs: 150, tokenDelayMs: 1000, captureDelayMs: 0 }, + { name: 'permission-prompt', networkRttMs: 150, tokenDelayMs: 80, captureDelayMs: 2000 }, +]; +const repetitions = Number(process.env.REPETITIONS ?? 10); +assert.ok(Number.isInteger(repetitions) && repetitions > 0); +if (process.env.SCENARIO && !scenarios.some((scenario) => scenario.name === process.env.SCENARIO)) { + throw new Error(`Unknown SCENARIO: ${process.env.SCENARIO}`); +} + +async function createLatencyProxy(rttMs) { + const sockets = new Set(); + const timers = new Set(); + const server = net.createServer((client) => { + const target = net.connect(Number(upstream.port || 80), upstream.hostname); + sockets.add(client); + sockets.add(target); + const forward = (source, destination) => { + let queued = 0; + source.on('data', (chunk) => { + queued += chunk.byteLength; + if (queued + destination.writableLength > 1024 * 1024) source.pause(); + const timer = setTimeout(() => { + timers.delete(timer); + queued -= chunk.byteLength; + if (!destination.destroyed) destination.write(chunk); + if (queued + destination.writableLength < 512 * 1024 && !source.destroyed) + source.resume(); + }, rttMs / 2); + timers.add(timer); + }); + destination.on('drain', () => { + if (queued < 512 * 1024 && !source.destroyed) source.resume(); + }); + }; + forward(client, target); + forward(target, client); + for (const [socket, peer] of [ + [client, target], + [target, client], + ]) { + socket.on('error', () => peer.destroy()); + socket.on('close', () => { + sockets.delete(socket); + peer.destroy(); + }); + } + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { + url: `ws://127.0.0.1:${server.address().port}`, + async close() { + timers.forEach(clearTimeout); + timers.clear(); + sockets.forEach((socket) => socket.destroy()); + await new Promise((resolve) => server.close(resolve)); + }, + }; +} +// One relay per client PC. Rewrite only the client's SFU candidates to this port. +// STUN transaction IDs route checks back to the correct local candidate; DTLS then +// identifies the selected socket used for SRTP, RTCP and data channels. +async function createMediaRelay(rttMs) { + const socket = dgram.createSocket('udp4'); + const timers = new Set(); + const checks = new Map(); + let selected; + const serverPort = Number(process.env.LIVEKIT_UDP_PORT ?? Number(upstream.port || 7880) + 2); + socket.on('message', (packet, remote) => { + const stun = packet.length >= 20 && packet.readUInt32BE(4) === 0x2112a442; + const transaction = stun ? packet.subarray(8, 20).toString('hex') : undefined; + const fromServer = remote.port === serverPort && remote.address === upstream.hostname; + let destination; + if (fromServer) destination = (transaction && checks.get(transaction)) || selected; + else { + if (transaction) { + if (checks.size >= 128) checks.delete(checks.keys().next().value); + checks.set(transaction, { address: remote.address, port: remote.port }); + let nominated = false; + for (let offset = 20; offset + 4 <= packet.length;) { + const type = packet.readUInt16BE(offset), + length = packet.readUInt16BE(offset + 2); + if (type === 0x0025) nominated = true; // STUN USE-CANDIDATE + offset += 4 + ((length + 3) & ~3); + } + if (nominated || !selected) selected = { address: remote.address, port: remote.port }; + } else selected = { address: remote.address, port: remote.port }; + destination = { address: upstream.hostname, port: serverPort }; + } + if (!destination) return; + const timer = setTimeout(() => { + timers.delete(timer); + socket.send(packet, destination.port, destination.address); + }, rttMs / 2); + timers.add(timer); + }); + await new Promise((resolve) => socket.bind(0, '127.0.0.1', resolve)); + return { + port: socket.address().port, + async close() { + timers.forEach(clearTimeout); + timers.clear(); + checks.clear(); + await new Promise((resolve) => socket.close(resolve)); + }, + }; +} +const vite = await createServer({ + configFile: false, + root: reactRoot, + resolve: { + alias: [ + { find: /^livekit-client$/, replacement: resolve(sdk, 'dist/livekit-client.esm.mjs') }, + { find: /^react$/, replacement: reactRequire.resolve('react') }, + { find: /^react-dom\/client$/, replacement: reactRequire.resolve('react-dom/client') }, + { find: /^react-dom$/, replacement: reactRequire.resolve('react-dom') }, + ], + dedupe: ['react', 'react-dom', 'livekit-client'], + }, + server: { host: '127.0.0.1', port: 0, hmr: false, fs: { allow: [root, sdk] } }, + plugins: [ + { + name: 'session-ux-fixture', + resolveId(id) { + if (id === '/src/hooks/__benchmark_baseline_session.ts' || id === baselineId) + return baselineId; + }, + load(id) { + if (id === baselineId) return baseline; + }, + configureServer(server) { + server.middlewares.use((request, response, next) => { + const url = new URL(request.url, 'http://localhost'); + if (url.pathname === '/') { + response.setHeader('Content-Type', 'text/html'); + response.end(''); + } else if (url.pathname === '/favicon.ico') { + response.statusCode = 204; + response.end(); + } else if (url.pathname === '/benchmark-token') { + const run = runs.get(url.searchParams.get('id')); + if (!run) { + response.statusCode = 404; + response.end(); + return; + } + const timer = setTimeout(() => { + response.setHeader('Content-Type', 'application/json'); + response.end(JSON.stringify(run.credentials)); + }, run.tokenDelayMs); + response.once('close', () => clearTimeout(timer)); + } else next(); + }); + }, + }, + ], +}); +await vite.listen(); +const origin = `http://127.0.0.1:${vite.httpServer.address().port}`; +let browser; +const samples = []; +try { + browser = await chromium.launch({ + executablePath: process.env.CHROMIUM_EXECUTABLE_PATH, + headless: true, + args: [ + '--autoplay-policy=no-user-gesture-required', + '--use-fake-ui-for-media-stream', + '--use-fake-device-for-media-stream', + ], + }); + for (const scenario of scenarios.filter( + (s) => !process.env.SCENARIO || s.name === process.env.SCENARIO, + )) { + const proxy = await createLatencyProxy(scenario.networkRttMs); + try { + for (let repetition = -1; repetition < repetitions; repetition++) { + // Alternate A/B order to reduce drift; the first pair is excluded as warmup. + for (const prepared of repetition % 2 === 0 ? [true, false] : [false, true]) { + const id = `ux-${Date.now()}-${Math.random().toString(36).slice(2)}`; + const tokens = await Promise.all( + ['user', 'agent'].map(async (identity) => { + const token = new AccessToken( + process.env.LIVEKIT_API_KEY ?? 'devkey', + process.env.LIVEKIT_API_SECRET ?? 'secret', + { + identity, + attributes: identity === 'agent' ? { 'lk.agent.state': 'listening' } : undefined, + }, + ); + if (identity === 'agent') token.kind = 'agent'; + token.addGrant({ roomJoin: true, room: id, agent: identity === 'agent' }); + return token.toJwt(); + }), + ); + runs.set(id, { + tokenDelayMs: scenario.tokenDelayMs, + credentials: { serverUrl: proxy.url, participantToken: tokens[0] }, + }); + const media = await createMediaRelay(scenario.networkRttMs); + const page = await browser.newPage(); + try { + page.on('pageerror', (error) => console.error(error.message)); + await page.goto(origin); + const metrics = await page.evaluate( + async (config) => { + const { runSessionExperience } = await import(config.fixture); + return runSessionExperience(config); + }, + { + prepared, + fixture: `/@fs/${resolve(root, 'scripts/session-experience-browser.ts')}`, + serverUrl: proxy.url, + agentToken: tokens[1], + agentServerUrl: upstream.toString(), + mediaRelayPort: media.port, + tokenEndpoint: `${origin}/benchmark-token?id=${id}`, + captureDelayMs: scenario.captureDelayMs, + agentReactionMs: 50, + }, + ); + assert.ok( + metrics.observedMediaRttMs >= scenario.networkRttMs * 0.8, + 'ICE RTT must include the configured UDP delay', + ); + const sample = { scenario: scenario.name, prepared, repetition, ...metrics }; + if (repetition >= 0) samples.push(sample); + console.log(JSON.stringify(sample)); + } finally { + runs.delete(id); + await page.close(); + await media.close(); + } + } + } + } finally { + await proxy.close(); + } + } + const percentile = (values, p) => { + values.sort((a, b) => a - b); + return values[Math.max(0, Math.ceil(values.length * p) - 1)]; + }; + const summary = scenarios.flatMap((scenario) => + [false, true] + .map((prepared) => { + const selected = samples.filter( + (s) => s.scenario === scenario.name && s.prepared === prepared, + ); + if (!selected.length) return undefined; + const metrics = Object.fromEntries( + Object.keys(selected[0]) + .filter((key) => key.endsWith('Ms')) + .map((key) => [ + key, + { + p50: Math.round( + percentile( + selected.map((s) => s[key]), + 0.5, + ), + ), + p95: Math.round( + percentile( + selected.map((s) => s[key]), + 0.95, + ), + ), + }, + ]), + ); + return { ...scenario, prepared, count: selected.length, metrics }; + }) + .filter(Boolean), + ); + const result = { + baselineRef, + sdkCommit: execFileSync('git', ['rev-parse', 'HEAD'], { cwd: sdk, encoding: 'utf8' }).trim(), + componentsCommit: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: root, + encoding: 'utf8', + }).trim(), + runtime: process.version, + platform: `${process.platform}/${process.arch}`, + agentReactionMs: 50, + agentPlacement: 'direct local SFU connection', + browserVersion: browser.version(), + repetitions, + methodology: + 'Actual React useSession, same companion SDK build; delayed credential HTTP response and getUserMedia, client TCP signaling + UDP ICE/DTLS/SRTP/RTCP/SCTP RTT relays; agent directly connected to local SFU. Warm agent returns deterministic tones, no STT/LLM/TTS; loaded app, no production authentication or bundle-load measurement.', + samples, + summary, + }; + const output = process.env.OUTPUT ?? '/tmp/session-experience-results.json'; + await writeFile(output, JSON.stringify(result, null, 2)); + console.log(`Saved ${output}`); +} finally { + await browser?.close(); + await vite.close(); +} diff --git a/scripts/session-experience-browser.ts b/scripts/session-experience-browser.ts new file mode 100644 index 000000000..2bd8af930 --- /dev/null +++ b/scripts/session-experience-browser.ts @@ -0,0 +1,347 @@ +import * as React from 'react'; +import { createRoot } from 'react-dom/client'; +import { LocalAudioTrack, Room, RoomEvent, TokenSource, Track } from 'livekit-client'; + +/** Measures an already-loaded React Session UI against a deterministic browser agent. */ +export async function runSessionExperience(config: { + prepared: boolean; + tokenEndpoint: string; + serverUrl: string; + agentToken: string; + agentServerUrl: string; + mediaRelayPort: number; + captureDelayMs: number; + agentReactionMs: number; +}) { + const sessionModule = config.prepared + ? '/src/hooks/useSession.ts' + : '/src/hooks/__benchmark_baseline_session.ts'; + const { useSession } = (await import(/* @vite-ignore */ sessionModule)) as { + useSession: typeof import('../packages/react/src/hooks/useSession').useSession; + }; + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const metrics: Record = {}; + let started = 0; + const mark = (name: string) => { + if (started && metrics[name] === undefined) metrics[name] = performance.now() - started; + }; + const frame = (name: string) => { + requestAnimationFrame(() => requestAnimationFrame(() => mark(name))); + }; + const room = new Room({ audioCaptureDefaults: { channelCount: 1 } }); + const agent = new Room(); + type SignalResponse = { + message: { case: string; value: { candidateInit?: string; sdp?: string } }; + }; + const engine = ( + room as unknown as { + engine: { + client: { handleSignalResponse(response: SignalResponse): void }; + pcManager: { publisher: { getStats(): Promise } }; + }; + } + ).engine; + const signal = engine.client; + const handleSignalResponse = signal.handleSignalResponse.bind(signal); + const candidate = (line: string) => { + const fields = line.trim().split(/\s+/); + if (fields[2]?.toLowerCase() !== 'udp') return undefined; + fields[4] = '127.0.0.1'; + fields[5] = String(config.mediaRelayPort); + return fields.join(' '); + }; + signal.handleSignalResponse = (response) => { + const { message } = response; + if (message.case === 'trickle' && message.value.candidateInit) { + const init = JSON.parse(message.value.candidateInit); + if (init?.candidate) { + const rewritten = candidate(init.candidate); + if (!rewritten) return; + init.candidate = rewritten; + message.value.candidateInit = JSON.stringify(init); + } + } else if ((message.case === 'answer' || message.case === 'offer') && message.value.sdp) { + message.value.sdp = message.value.sdp + .split('\r\n') + .flatMap((line) => { + if (!line.startsWith('a=candidate:')) return [line]; + const rewritten = candidate(line.slice(2)); + return rewritten ? [`a=${rewritten}`] : []; + }) + .join('\r\n'); + } + handleSignalResponse(response); + }; + + const userAudio = new AudioContext(); + const agentAudio = new AudioContext(); + const destination = agentAudio.createMediaStreamDestination(); + const gains: GainNode[] = []; + const oscillators: OscillatorNode[] = []; + for (const frequency of [440, 1170]) { + const oscillator = agentAudio.createOscillator(); + const gain = agentAudio.createGain(); + oscillator.frequency.value = frequency; + gain.gain.value = 0; + oscillator.connect(gain).connect(destination); + oscillator.start(); + oscillators.push(oscillator); + gains.push(gain); + } + const tasks: Promise[] = []; + const sources: MediaStreamAudioSourceNode[] = []; + let running = true; + let failure: unknown; + let session: ReturnType | undefined; + let sessionStarted: Promise | undefined; + let greetingScheduled = false; + let responseScheduled = false; + const onGreeting = async () => { + if (greetingScheduled) return; + greetingScheduled = true; + await delay(config.agentReactionMs); + if (!running) return; + mark('greetingSentMs'); + gains[0].gain.value = 0.15; + await delay(1000); + gains[0].gain.value = 0; + }; + agent.on(RoomEvent.ParticipantConnected, (participant) => { + if (participant.identity === 'user') tasks.push(onGreeting()); + }); + agent.registerByteStreamHandler('lk.agent.pre-connect-audio-buffer', async (reader) => { + try { + for await (const _chunk of reader) { + /* drain the ordinary preconnect stream */ + } + } catch (error) { + if (running) failure = error; + } + }); + agent.on(RoomEvent.TrackSubscribed, (track) => { + if (track.kind !== Track.Kind.Audio || responseScheduled) return; + responseScheduled = true; + mark('agentTrackSubscribedMs'); + const audio = track.attach(); + audio.muted = true; + document.body.appendChild(audio); + void audio.play(); + tasks.push( + (async () => { + while (running && track.receiver) { + const stats = await track.receiver.getStats(); + let decoded = false; + stats.forEach((stat) => { + if (stat.type === 'inbound-rtp' && stat.totalSamplesReceived > 0) decoded = true; + }); + if (decoded) { + mark('micDecodedAtAgentMs'); + await delay(config.agentReactionMs); + if (running) { + mark('responseSentMs'); + gains[1].gain.value = 0.15; + } + break; + } + await delay(10); + } + })().catch((error) => { + if (running) failure = error; + }), + ); + }); + room.on(RoomEvent.Connected, () => mark('roomConnectedMs')); + room.on(RoomEvent.LocalTrackPublished, (publication) => { + if (publication.source === Track.Source.Microphone) mark('micPublishedMs'); + }); + room.on(RoomEvent.TrackSubscribed, (track) => { + if (track.kind !== Track.Kind.Audio) return; + mark('userTrackSubscribedMs'); + const audio = track.attach(); + audio.muted = true; + document.body.appendChild(audio); + void audio.play(); + const source = userAudio.createMediaStreamSource(new MediaStream([track.mediaStreamTrack])); + const analyser = userAudio.createAnalyser(); + analyser.fftSize = 2048; + analyser.smoothingTimeConstant = 0; + const sink = userAudio.createGain(); + sink.gain.value = 0; + source.connect(analyser).connect(sink).connect(userAudio.destination); + sources.push(source); + const bins = new Float32Array(analyser.frequencyBinCount); + tasks.push( + (async () => { + while (running && metrics.responseAudioMs === undefined) { + analyser.getFloatFrequencyData(bins); + for (const [frequency, name] of [ + [440, 'greetingAudioMs'], + [1170, 'responseAudioMs'], + ] as const) { + if (name === 'responseAudioMs' && metrics.responseSentMs === undefined) continue; + const bin = Math.round((frequency * analyser.fftSize) / userAudio.sampleRate); + const peak = Math.max(bins[bin - 1], bins[bin], bins[bin + 1]); + if (peak > -40) mark(name); + } + await delay(5); + } + source.disconnect(); + analyser.disconnect(); + sink.disconnect(); + })(), + ); + }); + const originalCapture = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); + navigator.mediaDevices.getUserMedia = async (constraints) => { + await delay(config.captureDelayMs); + const stream = await originalCapture(constraints); + mark('captureReadyMs'); + return stream; + }; + let resolveCredentials!: (credentials: { serverUrl: string; participantToken: string }) => void; + let rejectCredentials!: (error: unknown) => void; + const credentials = new Promise<{ serverUrl: string; participantToken: string }>( + (resolve, reject) => { + resolveCredentials = resolve; + rejectCredentials = reject; + }, + ); + const tokenSource = TokenSource.literal({ serverUrl: config.serverUrl, participantToken: '' }); + tokenSource.fetch = () => credentials; + const element = document.createElement('main'); + document.body.appendChild(element); + const root = createRoot(element); + function App() { + const current = useSession(tokenSource, { room }); + session = current; + const [clicked, setClicked] = React.useState(false); + const ready = current.isConnected && !!current.local.microphoneTrack?.publication; + React.useLayoutEffect(() => { + if (clicked) frame('feedbackUiMs'); + }, [clicked]); + React.useLayoutEffect(() => { + if (current.isConnected) frame('connectedUiMs'); + }, [current.isConnected]); + React.useLayoutEffect(() => { + if (ready) frame('readyToTalkUiMs'); + }, [ready]); + return React.createElement( + 'div', + {}, + React.createElement( + 'p', + { id: 'status' }, + ready + ? 'Ready to talk' + : current.isConnected + ? 'Connected · microphone starting' + : clicked + ? 'Connecting…' + : 'Ready to connect', + ), + React.createElement( + 'button', + { + id: 'connect', + disabled: clicked, + onClick() { + started = performance.now(); + setClicked(true); + void userAudio.resume(); + void agentAudio.resume(); + void fetch(config.tokenEndpoint) + .then((response) => response.json()) + .then((value) => { + mark('credentialsMs'); + resolveCredentials(value); + }, rejectCredentials); + sessionStarted = current.start().then( + () => { + mark('sessionStartResolvedMs'); + }, + (error) => { + failure = error; + }, + ); + }, + }, + 'Connect', + ), + ); + } + try { + await userAudio.resume(); + await agentAudio.resume(); + await agent.connect(config.agentServerUrl, config.agentToken); + await agent.localParticipant.publishTrack( + new LocalAudioTrack(destination.stream.getAudioTracks()[0]), + { + source: Track.Source.Microphone, + red: false, + dtx: false, + }, + ); + root.render(React.createElement(App)); + while (!document.getElementById('connect') || !session) await delay(5); + // Let mount effects install listeners and begin the same credential prefetch in both variants. + await delay(30); + (document.getElementById('connect') as HTMLButtonElement).click(); + const deadline = performance.now() + 15000; + const required = [ + 'feedbackUiMs', + 'credentialsMs', + 'roomConnectedMs', + 'connectedUiMs', + 'captureReadyMs', + 'micPublishedMs', + 'readyToTalkUiMs', + 'sessionStartResolvedMs', + 'micDecodedAtAgentMs', + 'greetingAudioMs', + 'responseSentMs', + 'responseAudioMs', + ]; + while (required.some((name) => metrics[name] === undefined)) { + if (failure) throw failure; + if (performance.now() > deadline) + throw new Error( + `Missing UX milestones: ${required.filter((name) => metrics[name] === undefined).join(', ')}; got ${JSON.stringify(metrics)}`, + ); + await delay(10); + } + if (userAudio.state !== 'running') throw new Error('Audio playout context is suspended'); + const stats = await engine.pcManager.publisher.getStats(); + let relayed = false; + const remoteCandidates = new Map(); + stats.forEach((stat) => { + if (stat.type === 'remote-candidate') remoteCandidates.set(stat.id, stat); + }); + stats.forEach((stat) => { + if (stat.type === 'candidate-pair' && stat.state === 'succeeded' && stat.nominated) { + const remote = remoteCandidates.get(stat.remoteCandidateId); + if (remote?.port === config.mediaRelayPort && remote.protocol === 'udp') { + relayed = true; + metrics.observedMediaRttMs = stat.currentRoundTripTime * 1000; + } + } + }); + if (!relayed) throw new Error('Selected ICE pair bypassed the media-latency relay'); + return metrics; + } finally { + running = false; + navigator.mediaDevices.getUserMedia = originalCapture; + await room.disconnect(); + await agent.disconnect(); + await sessionStarted; + root.unmount(); + element.remove(); + await Promise.allSettled(tasks); + sources.forEach((source) => source.disconnect()); + oscillators.forEach((oscillator) => oscillator.stop()); + destination.stream.getTracks().forEach((track) => track.stop()); + await userAudio.close(); + await agentAudio.close(); + await room.dispose(); + await agent.dispose(); + } +}