diff --git a/.changeset/prepared-publications.md b/.changeset/prepared-publications.md new file mode 100644 index 0000000000..a924fff9f8 --- /dev/null +++ b/.changeset/prepared-publications.md @@ -0,0 +1,4 @@ +--- +'livekit-client': minor +--- +Add experimental device-free publication preparation so initial microphone and camera senders can join with the room and attach captured media without another publication negotiation. diff --git a/AGENTS.md b/AGENTS.md index 3438d5a136..3401e65459 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -238,3 +238,13 @@ Managers (e.g. `RpcClientManager`, `RpcServerManager`, `OutgoingDataTrackManager `resolve`/`reject` callbacks and a `.promise` property. - Error classes use a `Reasoned` pattern: `DataTrackSubscribeError` with a `reason` enum and static factory methods (`.timeout()`, `.cancelled()`, `.disconnected()`). + +### Initial prepared publications + +`Room.preparePublication()` snapshots device-free intent; the V1 offer builder derives each +trackless sender's CID from its unique stream MSID before munging SDP. JOIN response waiters +must exist before signaling opens. `PreparedPublicationState` owns the sender until normal +publication commits; room teardown or cancellation reconciles it. Initial offers share codec +and start-bitrate processing with later publisher offers. Session integration lives in the +components repo and must start JOIN independently of capture. Legacy V0, Firefox, encryption, +and unsupported configurations defer to ordinary publication. diff --git a/examples/prepared-publications/README.md b/examples/prepared-publications/README.md new file mode 100644 index 0000000000..c848dfd811 --- /dev/null +++ b/examples/prepared-publications/README.md @@ -0,0 +1,70 @@ +# Prepared publications (experimental) + +Declare publication intent before requesting device access. The SDK snapshots publication defaults and options, chooses a codec and encoding envelope, and can register trackless senders in the initial JOIN offer. Opening a device never blocks JOIN. + +```ts +const microphone = room.preparePublication({ + source: Track.Source.Microphone, + format: { channelCount: 1 }, + publishOptions: { audioPreset: { maxBitrate: 32000 }, preConnectBuffer: true }, +}); +const camera = room.preparePublication({ + source: Track.Source.Camera, + format: { width: 1280, height: 720, frameRate: 30 }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { maxBitrate: 1_500_000 }, + }, +}); + +const connecting = room.connect(url, token, { + initialPublications: [microphone, camera], +}); +const capturing = room.localParticipant.createTracks({ + audio: { channelCount: 1 }, + video: { resolution: { width: 1280, height: 720, frameRate: 30 } }, +}); + +const publishing = capturing.then(async (tracks) => { + try { + // attachTrack waits for connection independently and starts the requested + // microphone preconnect buffer as soon as the track is supplied. + return await Promise.all( + tracks.map((track) => + (track.kind === Track.Kind.Audio ? microphone : camera).attachTrack(track), + ), + ); + } catch (error) { + // This also handles capture resolving after connection failed or was cancelled. + tracks.forEach((track) => track.stop()); + throw error; + } +}); +try { + await Promise.all([connecting, publishing]); +} catch (error) { + await room.disconnect(); + throw error; +} finally { + await Promise.all([microphone.cancel(), camera.cancel()]); +} +``` + +The handle belongs to its room and must appear once in that room's next `connect()` call. Preparation is synchronous and only allowed while disconnected. A handle accepts one track; it never opens devices or stops caller-owned tracks. Cancel an unused handle when capture fails or startup is abandoned. Disconnect and disposal cancel outstanding handles. Cancelling after successful attachment does nothing; ordinary publication and unpublication APIs own the track from then on. + +For compatible capture, attachment reuses the negotiated sender with `replaceTrack` and returns a normal `LocalTrackPublication`. No new AddTrack request or offer is required. The JOIN carries codec/layer/encoding intent; mute status and capture-dependent audio processing features are reconciled when media attaches. No placeholder audio/video frames are generated. + +The optimized path currently requires the SDK's V1 offer-with-JOIN path (single PC on supported non-Firefox browsers). Legacy dual PC, Firefox, React Native, unavailable codec capabilities, SVC/additional-codec configurations, E2EE, and frame-metadata configurations use normal publication after connection. A different actual capture format or server-selected codec also reconciles the reservation and falls back. This preserves join-first behavior, but those fallback cases do not save the publication exchanges. Screen sharing remains on its existing user-gesture capture path. + +The exchange reduction and elapsed-time improvement are separate: bundling avoids the standalone registration/negotiation exchanges, while current ordinary fast publication already overlaps those exchanges. Do not assume every session improves by exactly two RTTs. Capture, ICE/DTLS, server support, media pacing, and the first decoded frame still affect perceived latency. + +Run `pnpm build`, start `livekit-server --dev`, then run: + +```sh +LIVEKIT_URL=ws://127.0.0.1:7880 node scripts/benchmark-prepared-publication.mjs +``` + +The real-browser harness injects 50 ms each direction at SDK signal dispatch, delays capture until after JOIN, verifies subscriber audio samples/video decoding, and asserts no AddTrack/offer on compatible attachment. It covers prepared and ordinary publication with both PC modes, including concurrent audio/video publication. Set `CHROMIUM_EXECUTABLE_PATH` to use an installed Chrome. Keys default to the local dev server's `devkey`/`secret`; other servers require `LIVEKIT_API_KEY` and `LIVEKIT_API_SECRET`. + +For codec substitution coverage, point the same harness at a server whose `room.enabled_codecs` contains only `audio/opus` and `video/h264`, and set `PREPARED_EXPECT_CODEC_FALLBACK=1`. Its VP8 reservations must join successfully, reconcile, and publish H264 to the subscriber through the ordinary path. diff --git a/scripts/benchmark-prepared-publication.mjs b/scripts/benchmark-prepared-publication.mjs new file mode 100644 index 0000000000..b9b98ec703 --- /dev/null +++ b/scripts/benchmark-prepared-publication.mjs @@ -0,0 +1,234 @@ +// Run after `pnpm build` against a real SFU: LIVEKIT_URL=ws://localhost:7880 node scripts/benchmark-prepared-publication.mjs +// CHROMIUM_EXECUTABLE_PATH optionally selects an installed Chrome. Devices are browser test devices. +import { AccessToken } from 'livekit-server-sdk'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { chromium } from 'playwright'; + +const url = process.env.LIVEKIT_URL ?? 'ws://127.0.0.1:7880'; +// For a server restricted to H264, verify VP8 reservations fall back instead of blocking JOIN. +const expectCodecFallback = process.env.PREPARED_EXPECT_CODEC_FALLBACK === '1'; +const apiKey = process.env.LIVEKIT_API_KEY ?? 'devkey'; +const apiSecret = process.env.LIVEKIT_API_SECRET ?? 'secret'; +const bundle = await readFile(new URL('../dist/livekit-client.esm.mjs', import.meta.url)); +const server = createServer((request, response) => { + response.setHeader('Content-Type', request.url === '/sdk.js' ? 'text/javascript' : 'text/html'); + response.end(request.url === '/sdk.js' ? bundle : ''); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); +let browser; +try { + browser = await chromium.launch({ + executablePath: process.env.CHROMIUM_EXECUTABLE_PATH, + headless: true, + args: [ + '--autoplay-policy=no-user-gesture-required', + '--use-fake-ui-for-media-stream', + '--use-fake-device-for-media-stream', + ], + }); + const rows = []; + for (const singlePeerConnection of [true, false]) { + for (const prepared of [false, true]) { + for (const kind of ['audio', 'video', 'both']) { + const roomName = `prepared-${Date.now()}-${Math.random().toString(36).slice(2)}`; + const tokens = await Promise.all( + ['publisher', 'subscriber'].map(async (identity) => { + const token = new AccessToken(apiKey, apiSecret, { identity }); + token.addGrant({ room: roomName, roomJoin: true }); + return token.toJwt(); + }), + ); + const page = await browser.newPage(); + try { + await page.goto(`http://127.0.0.1:${server.address().port}`); + const row = await page.evaluate( + async ({ url, tokens, singlePeerConnection, prepared, kind }) => { + const { Room, RoomEvent, Track, createLocalTracks } = await import('/sdk.js'); + const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const waitFor = async (predicate) => { + const deadline = performance.now() + 10000; + while (!(await predicate())) { + if (performance.now() > deadline) throw new Error('real media did not arrive'); + await delay(20); + } + }; + const subscriber = new Room({ singlePeerConnection }); + const publisher = new Room({ singlePeerConnection }); + const remotes = new Map(); + subscriber.on(RoomEvent.TrackSubscribed, (track) => { + remotes.set(track.kind, track); + const element = track.attach(); + element.muted = true; + document.body.appendChild(element); + void element.play(); + }); + const trace = []; + // Inject 50 ms each direction in the SDK signal dispatch only (100 ms RTT). + // RTP/ICE are undelayed; this isolates the publication signaling critical path. + const client = publisher.engine.client; + const send = client.sendRequest.bind(client); + const receive = client.handleSignalResponse.bind(client); + const pending = new Set(); + client.sendRequest = async (message, ...args) => { + trace.push(message.case); + await delay(50); + return send(message, ...args); + }; + client.handleSignalResponse = (message) => { + const task = delay(50).then(() => receive(message)); + pending.add(task); + void task.finally(() => pending.delete(task)); + }; + let tracks = []; + try { + await subscriber.connect(url, tokens[1]); + const sources = + kind === 'both' + ? [Track.Source.Microphone, Track.Source.Camera] + : [kind === 'video' ? Track.Source.Camera : Track.Source.Microphone]; + const publicationOptions = (source) => + source === Track.Source.Camera + ? { videoCodec: 'vp8', simulcast: true } + : { red: false }; + const reservations = new Map( + prepared + ? sources.map((source) => [ + source, + publisher.preparePublication({ + source, + ...(source === Track.Source.Camera + ? { format: { width: 640, height: 480, frameRate: 30 } } + : {}), + publishOptions: publicationOptions(source), + }), + ]) + : [], + ); + const started = performance.now(); + // Permission/device initialization completes late, while JOIN runs independently. + const capture = (async () => { + await delay(650); + return await createLocalTracks({ + audio: kind !== 'video' && { channelCount: 1 }, + video: kind !== 'audio' && { + resolution: { width: 640, height: 480, frameRate: 30 }, + }, + }); + })(); + const connecting = publisher.connect(url, tokens[0], { + initialPublications: [...reservations.values()], + }); + await connecting; + const connectedMs = performance.now() - started; + const initialVideoStartBitrate = publisher.engine.pcManager.publisher + .getLocalDescription() + ?.sdp.includes('x-google-start-bitrate='); + tracks = await capture; + const captureReadyMs = performance.now() - started; + const traceStart = trace.length; + await Promise.all( + tracks.map((track) => { + const source = + track.kind === Track.Kind.Video + ? Track.Source.Camera + : Track.Source.Microphone; + const reservation = reservations.get(source); + return reservation + ? reservation.attachTrack(track) + : publisher.localParticipant.publishTrack(track, publicationOptions(source)); + }), + ); + const publishedMs = performance.now() - started; + await waitFor(async () => { + if (remotes.size !== sources.length) return false; + const received = await Promise.all( + [...remotes.values()].map(async (remote) => { + if (!remote.receiver) return false; + const stats = await remote.receiver.getStats(); + return [...stats.values()].some( + (stat) => + stat.type === 'inbound-rtp' && + (remote.kind === Track.Kind.Video + ? stat.framesDecoded > 0 + : stat.totalSamplesReceived > 0), + ); + }), + ); + return received.every(Boolean); + }); + const mediaMs = performance.now() - started; + const publicationSignals = trace + .slice(traceStart) + .filter((type) => type === 'addTrack' || type === 'offer'); + return { + singlePeerConnection, + prepared, + kind, + connectedMs: Math.round(connectedMs), + captureReadyMs: Math.round(captureReadyMs), + publishAfterCaptureMs: Math.round(publishedMs - captureReadyMs), + mediaAfterCaptureMs: Math.round(mediaMs - captureReadyMs), + publicationSignals, + videoStartBitrate: + kind === 'audio' || + initialVideoStartBitrate || + publisher.engine.pcManager.publisher + .getLocalDescription() + ?.sdp.includes('x-google-start-bitrate='), + }; + } finally { + await publisher.disconnect(); + await subscriber.disconnect(); + tracks.forEach((track) => track.stop()); + await Promise.all(pending); + await publisher.dispose(); + await subscriber.dispose(); + } + }, + { url, tokens, singlePeerConnection, prepared, kind }, + ); + console.log(JSON.stringify(row)); + assert.ok( + row.connectedMs < row.captureReadyMs, + 'JOIN must finish without device acquisition', + ); + if (prepared && singlePeerConnection && !(expectCodecFallback && kind !== 'audio')) + assert.deepEqual( + row.publicationSignals, + [], + 'late attachment must not register or renegotiate', + ); + else + assert.ok( + row.publicationSignals.includes('addTrack') && + row.publicationSignals.includes('offer'), + ); + assert.ok(row.videoStartBitrate, 'video must preserve its start-bitrate hint'); + rows.push(row); + } finally { + await page.close(); + } + } + } + } + for (const row of rows.filter( + (row) => + row.prepared && row.singlePeerConnection && !(expectCodecFallback && row.kind !== 'audio'), + )) { + const baseline = rows.find( + (other) => + !other.prepared && + other.kind === row.kind && + other.singlePeerConnection === row.singlePeerConnection, + ); + assert.ok( + row.publishAfterCaptureMs < baseline.publishAfterCaptureMs, + 'reservation must reduce publication latency', + ); + } +} finally { + await browser?.close(); + await new Promise((resolve) => server.close(resolve)); +} diff --git a/src/api/SignalClient.test.ts b/src/api/SignalClient.test.ts index e68a56eda4..d7fcd907a7 100644 --- a/src/api/SignalClient.test.ts +++ b/src/api/SignalClient.test.ts @@ -1,4 +1,5 @@ import { + AddTrackRequest, ClientInfo_Capability, ClientInfo_SDK, DisconnectReason, @@ -6,6 +7,7 @@ import { JoinResponse, LeaveRequest, ReconnectResponse, + SessionDescription, SignalRequest, SignalResponse, WrappedJoinRequest, @@ -338,6 +340,37 @@ describe('SignalClient.connect', () => { expect(result).toEqual(joinResponse); }); + it('bundles publication metadata with the publisher offer in JOIN', async () => { + let capturedUrl = ''; + mockWebSocketStream({ + connection: createMockConnection( + createMockReadableStream([createSignalResponse('join', createJoinResponse())]), + ), + onUrl: (url) => { + capturedUrl = url; + }, + }); + const request = new AddTrackRequest({ + cid: 'sdp-track-id', + name: 'microphone', + stereo: true, + disableDtx: true, + }); + const offer = new SessionDescription({ type: 'offer', sdp: 'test-offer', id: 1 }); + await signalClient.join( + 'wss://test.livekit.io', + 'test-token', + defaultOptions, + undefined, + false, + offer, + [request], + ); + const joined = await decodeJoinRequestFromUrl(capturedUrl); + expect(joined.addTrackRequests).toEqual([request]); + expect(joined.publisherOffer).toEqual(offer); + }); + it('does not advertise packet trailer capability by default', async () => { const joinResponse = createJoinResponse(); const signalResponse = createSignalResponse('join', joinResponse); diff --git a/src/api/SignalClient.ts b/src/api/SignalClient.ts index ffb5c62ee5..ebeba41f51 100644 --- a/src/api/SignalClient.ts +++ b/src/api/SignalClient.ts @@ -356,6 +356,7 @@ export class SignalClient { abortSignal?: AbortSignal, useV0Path: boolean = false, publisherOffer?: SessionDescription, + addTrackRequests: AddTrackRequest[] = [], ): Promise { if (this.lifecycleState === 'disconnecting') { await this.settleInFlightClose(); @@ -370,7 +371,15 @@ export class SignalClient { } this.options = opts; try { - const res = await this.connect(url, token, opts, abortSignal, useV0Path, publisherOffer); + const res = await this.connect( + url, + token, + opts, + abortSignal, + useV0Path, + publisherOffer, + addTrackRequests, + ); return res as JoinResponse; } catch (e) { // reported here rather than at each rejection site inside connect(), so that no failure @@ -433,6 +442,7 @@ export class SignalClient { /** setting this to true results in dual peer connection mode being used */ useV0Path: boolean = false, publisherOffer?: SessionDescription, + addTrackRequests: AddTrackRequest[] = [], ): Promise { const unlock = await this.connectionLock.lock(); @@ -445,7 +455,13 @@ export class SignalClient { const clientInfo = getClientInfo(opts.clientInfoCapabilities); const params = useV0Path ? createConnectionParams(token, clientInfo, opts) - : await createJoinRequestConnectionParams(token, clientInfo, opts, publisherOffer); + : await createJoinRequestConnectionParams( + token, + clientInfo, + opts, + publisherOffer, + addTrackRequests, + ); rtcUrl = createRtcUrl(url, params, useV0Path).toString(); validateUrl = createValidateUrl(rtcUrl).toString(); } catch (e) { @@ -1483,6 +1499,7 @@ async function createJoinRequestConnectionParams( info: ClientInfo, opts: ConnectOpts, publisherOffer?: SessionDescription, + addTrackRequests: AddTrackRequest[] = [], ): Promise { const params = new URLSearchParams(); params.set('access_token', token); @@ -1498,6 +1515,7 @@ async function createJoinRequestConnectionParams( reconnect: !!opts.reconnect, participantSid: opts.sid ? opts.sid : undefined, publisherOffer: publisherOffer, + addTrackRequests, }); if (opts.reconnectReason) { joinRequest.reconnectReason = opts.reconnectReason; diff --git a/src/exports.ts b/src/exports.ts index b3842a5435..8b5a5dd181 100644 --- a/src/exports.ts +++ b/src/exports.ts @@ -227,3 +227,9 @@ export { type SerializerOutput, serializers, } from './utils/serializer'; + +export type { + PreparedPublication, + PreparePublicationOptions, + PreparedVideoFormat, +} from './room/PreparedPublication'; diff --git a/src/options.ts b/src/options.ts index 0be6dc7b59..604519acf6 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,5 +1,6 @@ import type { E2EEOptions } from './e2ee/types'; import type { FrameMetadataOptions } from './frameMetadata/FrameMetadataManager'; +import type { PreparedPublication } from './room/PreparedPublication'; import type { ReconnectPolicy } from './room/ReconnectPolicy'; import type { AudioCaptureOptions, @@ -164,6 +165,9 @@ export interface RoomOptions extends Partial {} /** The resolved connect options. `RoomConnectOptions` is the partial form apps pass in. */ export interface InternalRoomConnectOptions { + /** Device-free publications to include in the initial JOIN. Connection never waits for capture. @experimental */ + initialPublications?: ReadonlyArray; + /** autosubscribe to room tracks after joining, defaults to true */ autoSubscribe: boolean; diff --git a/src/room/CoreRoom.ts b/src/room/CoreRoom.ts index 14b0e7be6c..0924d4866f 100644 --- a/src/room/CoreRoom.ts +++ b/src/room/CoreRoom.ts @@ -43,6 +43,11 @@ import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; import { BackOffStrategy } from './BackOffStrategy'; import DeviceManager from './DeviceManager'; +import { + type PreparePublicationOptions, + type PreparedPublication, + PreparedPublicationState, +} from './PreparedPublication'; import RTCEngine, { type RegionStrategy } from './RTCEngine'; import { DEFAULT_MAX_AGE_MS, RegionUrlProvider } from './RegionUrlProvider'; import type { DataChannelKind } from './data-channel/types'; @@ -451,6 +456,7 @@ export class CoreRoom extends (EventEmitter as new () => TypedEmitter TypedEmitter(); + + private readonly pendingInitialTrackSubscriptions = new Set(); + + /** Prepares publication metadata without opening a device. Supply the handle to connect. @experimental */ + preparePublication(options: PreparePublicationOptions): PreparedPublication { + if (this.disposed || this.state !== ConnectionState.Disconnected) { + throw new UnexpectedConnectionState('prepare publications before connecting'); + } + const prepared = this.localParticipant.preparePublication(options, (p) => { + this.preparedPublications.delete(p); + if (this.preparedPublications.size === 0) this.pendingInitialTrackSubscriptions.clear(); + if (this.engine) + this.engine.initialPublications = this.engine.initialPublications.filter( + (plan) => plan !== p, + ); + }); + this.preparedPublications.add(prepared); + return prepared; + } + connect = async (url: string, token: string, opts?: RoomConnectOptions): Promise => { if (this.disposed) { throw new UnexpectedConnectionState('room has been disposed'); @@ -761,6 +788,25 @@ export class CoreRoom extends (EventEmitter as new () => TypedEmitter TypedEmitter TypedEmitter TypedEmitter TypedEmitter plan.claimed)) { + // Device permission can take arbitrarily long. The preparation/room lifecycle, + // rather than the ordinary publish timeout, owns these deferred notifications. + this.pendingInitialTrackSubscriptions.add(subscribedSid); + return; + } + // the track publication may not be registered yet if the server signals // the subscription before publishTrack has finished adding the publication. // defer with a timeout until LocalTrackPublished fires for the matching trackSid @@ -1816,6 +1874,9 @@ export class CoreRoom extends (EventEmitter as new () => TypedEmitter TypedEmitter { return { transport, initial: (await transport.createInitialOffer())! }; }; + it('applies video start bitrate and conforms placeholders on an initial reservation', async () => { + const sdp = TWO_VIDEO_SECTIONS.replace('a=sendonly', 'a=recvonly').replace( + 'a=msid:PA_remote|camera other-track', + '', + ); + stubPeerConnectionReturning(sdp); + const transport = new PCTransport(undefined, {}, { singlePeerConnection: true }); + const initial = await transport.createInitialOffer(() => { + transport.setTrackCodecBitrate({ cid: 'camera-cid', codec: 'vp8', maxbr: 3000 }); + }); + expect(initial?.offer.sdp).toContain('x-google-start-bitrate='); + expect(transport.getLocalDescription()).toBeUndefined(); + const media = parse(initial!.offer.sdp!).media; + expect(fmtpOf(media, '0', 96)).toBe(fmtpOf(media, '1', 96)); + }); + it('offers Opus NACK on every audio section', async () => { const { initial } = await initialOfferAndTransport(); diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index c06f2b74d4..2d32271fa9 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -313,7 +313,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter private offerLock: Mutex; - private pendingInitialOffer?: { original: RTCSessionDescriptionInit; munged: string }; + private pendingInitialOffer?: { + original: RTCSessionDescriptionInit; + munged: string; + appliedVideoStartBitrate: boolean; + }; private readonly offeredAudioNackMids: string[]; @@ -499,10 +503,12 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter this.remoteNackMids = nackMids; } else if (sd.type === 'answer') { if (this.pendingInitialOffer && this._pc) { - const { original, munged } = this.pendingInitialOffer; + const { original, munged, appliedVideoStartBitrate } = this.pendingInitialOffer; this.pendingInitialOffer = undefined; this.log.debug('setting pending initial offer before processing answer'); await this.setMungedSDP(original, munged); + if (appliedVideoStartBitrate && original.sdp === munged) + this.hasAppliedVideoStartBitrate = true; } const sdpParsed = parse(sd.sdp ?? ''); sdpParsed.media.forEach((media) => { @@ -622,7 +628,75 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter } }, debounceInterval); - async createInitialOffer() { + /** Applies the same codec and start-bitrate policy to initial and later publisher offers. */ + private preparePublisherOffer( + sdpParsed: ReturnType, + stereo = true, + initial = false, + ) { + // One value for every video m-section, written only on the first offer that carries + // local video: the hint is connection-level in libwebrtc, so differing per-section + // values would be last-writer-wins on m-section order. Offers before any video is + // published (data channel or audio only) find no target and leave the latch unset. + const connectionStartBitrate = this.hasAppliedVideoStartBitrate + ? undefined + : computeConnectionStartBitrate( + sdpParsed.media, + this.trackBitrates, + this.connectionSetupTimeEstimateMs(), + ); + let appliedVideoStartBitrate = false; + sdpParsed.media.forEach((media) => { + ensureIPAddrMatchVersion(media); + if (media.type === 'audio') { + ensureAudioNackAndStereo(media, stereo ? ['all'] : [], this.offeredAudioNackMids); + } else if (media.type === 'video') { + this.trackBitrates.some((trackbr): boolean => { + if (!trackbr.cid) { + return false; + } + + const codecPayload = findTrackCodecPayload(media, trackbr.cid, trackbr.codec); + if (codecPayload === undefined) { + return false; + } + + if (codecPayload > 0) { + if (connectionStartBitrate !== undefined) { + appliedVideoStartBitrate = + applyVideoStartBitrate(media, codecPayload, connectionStartBitrate) || + appliedVideoStartBitrate; + } + + if (isSVCCodec(trackbr.codec) && !isSafari()) { + this.ddExtID = ensureVideoDDExtension(media, sdpParsed, this.ddExtID); + } + } + + return true; + }); + } + }); + // Conform the placeholder sections (pre-populated, or reverted after an + // unpublish on a server below protocol 19, which keeps the transceiver; from + // protocol 19 unpublish stops it) so every shared payload type carries + // identical fmtp across the bundle, otherwise libwebrtc flags a "bundled + // payload type collision". + // Detection is by transceiver (mids are stable across renegotiations) since + // an unpublished section keeps its `a=msid`. + const placeholderMids = this.getPlaceholderMids(); + if (initial || placeholderMids.size > 0) { + conformBundledCodecFmtp( + sdpParsed.media, + (media) => + placeholderMids.has(getMidString(media.mid!)) || + (initial && media.direction === 'recvonly'), + ); + } + return { connectionStartBitrate, appliedVideoStartBitrate }; + } + + async createInitialOffer(prepare?: (sdp: string) => void) { const unlock = await this.offerLock.lock(); try { if (this.pc.signalingState !== 'stable') { @@ -633,15 +707,11 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter this.latestOfferId = offerId; const offer = await this.pc.createOffer(); const original = { sdp: offer.sdp, type: offer.type }; + prepare?.(offer.sdp ?? ''); const sdpParsed = parse(offer.sdp ?? ''); - sdpParsed.media.forEach((media) => { - ensureIPAddrMatchVersion(media); - if (media.type === 'audio') { - ensureAudioNackAndStereo(media, [], this.offeredAudioNackMids); - } - }); + const { appliedVideoStartBitrate } = this.preparePublisherOffer(sdpParsed, false, true); offer.sdp = write(sdpParsed); - this.pendingInitialOffer = { original, munged: offer.sdp }; + this.pendingInitialOffer = { original, munged: offer.sdp, appliedVideoStartBitrate }; return { offer, offerId }; } finally { unlock(); @@ -697,62 +767,8 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter this.log.debug('original offer', { sdp: offer.sdp }); const sdpParsed = parse(offer.sdp ?? ''); - // One value for every video m-section, written only on the first offer that carries - // local video: the hint is connection-level in libwebrtc, so differing per-section - // values would be last-writer-wins on m-section order. Offers before any video is - // published (data channel or audio only) find no target and leave the latch unset. - const connectionStartBitrate = this.hasAppliedVideoStartBitrate - ? undefined - : computeConnectionStartBitrate( - sdpParsed.media, - this.trackBitrates, - this.connectionSetupTimeEstimateMs(), - ); - let appliedVideoStartBitrate = false; - sdpParsed.media.forEach((media) => { - ensureIPAddrMatchVersion(media); - if (media.type === 'audio') { - ensureAudioNackAndStereo(media, ['all'], this.offeredAudioNackMids); - } else if (media.type === 'video') { - this.trackBitrates.some((trackbr): boolean => { - if (!trackbr.cid) { - return false; - } - - const codecPayload = findTrackCodecPayload(media, trackbr.cid, trackbr.codec); - if (codecPayload === undefined) { - return false; - } - - if (codecPayload > 0) { - if (connectionStartBitrate !== undefined) { - appliedVideoStartBitrate = - applyVideoStartBitrate(media, codecPayload, connectionStartBitrate) || - appliedVideoStartBitrate; - } - - if (isSVCCodec(trackbr.codec) && !isSafari()) { - this.ddExtID = ensureVideoDDExtension(media, sdpParsed, this.ddExtID); - } - } - - return true; - }); - } - }); - // Conform the placeholder sections (pre-populated, or reverted after an - // unpublish on a server below protocol 19, which keeps the transceiver; from - // protocol 19 unpublish stops it) so every shared payload type carries - // identical fmtp across the bundle, otherwise libwebrtc flags a "bundled - // payload type collision". - // Detection is by transceiver (mids are stable across renegotiations) since - // an unpublished section keeps its `a=msid`. - const placeholderMids = this.getPlaceholderMids(); - if (placeholderMids.size > 0) { - conformBundledCodecFmtp(sdpParsed.media, (media) => - placeholderMids.has(getMidString(media.mid!)), - ); - } + const { connectionStartBitrate, appliedVideoStartBitrate } = + this.preparePublisherOffer(sdpParsed); if (this.latestOfferId > offerId) { this.log.warn('latestOfferId mismatch', { latestOfferId: this.latestOfferId, diff --git a/src/room/PreparedPublication.test.ts b/src/room/PreparedPublication.test.ts new file mode 100644 index 0000000000..555aae3134 --- /dev/null +++ b/src/room/PreparedPublication.test.ts @@ -0,0 +1,216 @@ +import { TrackInfo } from '@livekit/protocol'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { type InitialPublicationSender, PreparedPublicationState } from './PreparedPublication'; +import { Room } from './Room'; +import { EngineEvent, ParticipantEvent } from './events'; +import type LocalTrack from './track/LocalTrack'; +import LocalTrackPublication from './track/LocalTrackPublication'; +import { Track } from './track/Track'; + +function fixture() { + const publication = {} as LocalTrackPublication; + const publish = vi.fn(async () => publication); + const release = vi.fn(); + const plan = new PreparedPublicationState( + { source: Track.Source.Microphone }, + {}, + [], + 'opus', + true, + publish, + release, + ); + const track = { + kind: Track.Kind.Audio, + getSourceTrackSettings: () => ({ channelCount: 1 }), + stop: vi.fn(), + } as unknown as LocalTrack; + const transport = {}; + const engine = { + isClosed: false, + pcManager: { publisher: transport }, + cancelPendingTrack: vi.fn(), + negotiate: vi.fn(async () => {}), + }; + const transceiver = { stop: vi.fn(), sender: {} }; + const initial = { + engine, + transport, + transceiver, + accepted: Promise.resolve(new TrackInfo()), + } as unknown as InitialPublicationSender; + return { plan, publish, release, track, initial, engine, transceiver, publication }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); +describe('prepared publication lifecycle', () => { + it('does not request capture or publish while preparing', async () => { + const room = new Room(); + const createTracks = vi.spyOn(room.localParticipant, 'createTracks'); + const publish = vi.spyOn(room.localParticipant, 'publishTrack'); + const plan = room.preparePublication({ source: Track.Source.Microphone }); + expect(createTracks).not.toHaveBeenCalled(); + expect(publish).not.toHaveBeenCalled(); + await plan.cancel(); + await room.dispose(); + }); + it('uses the declared audio format over default stereo settings', async () => { + const room = new Room(); + const plan = room.preparePublication({ + source: Track.Source.Microphone, + format: { channelCount: 2 }, + }) as PreparedPublicationState; + expect(plan.request.stereo).toBe(true); + expect(() => + room.preparePublication({ + source: Track.Source.Microphone, + format: { channelCount: 2 }, + publishOptions: { forceStereo: false }, + }), + ).toThrow('disagree'); + await plan.cancel(); + await room.dispose(); + }); + it('attaches only after the explicitly claimed connect is ready', async () => { + const { plan, track, publish, publication, release } = fixture(); + const attached = plan.attachTrack(track); + plan.connectionReady(); + await Promise.resolve(); + expect(publish).not.toHaveBeenCalled(); + plan.claimed = true; + plan.connectionReady(); + expect(await attached).toBe(publication); + expect(track.source).toBe(Track.Source.Microphone); + expect(release).toHaveBeenCalledOnce(); + await plan.cancel(); + expect(publish).toHaveBeenCalledOnce(); + }); + it('rejects cancellation promptly before JOIN settles and never stops caller media', async () => { + const { plan, track, publish, release } = fixture(); + const attached = plan.attachTrack(track); + await plan.cancel(); + await expect(attached).rejects.toThrow('cancelled'); + expect(publish).not.toHaveBeenCalled(); + expect(track.stop).not.toHaveBeenCalled(); + expect(release).toHaveBeenCalledOnce(); + }); + it('keeps a single attachment future for duplicate calls', async () => { + const { plan, track } = fixture(); + const attached = plan.attachTrack(track); + track.sender = {} as RTCRtpSender; + expect(plan.attachTrack(track)).toBe(attached); + await expect(plan.attachTrack({ kind: Track.Kind.Audio } as LocalTrack)).rejects.toThrow( + 'already has a track', + ); + await plan.cancel(); + await expect(attached).rejects.toThrow('cancelled'); + }); + it('tears down a negotiated reservation once and releases it', async () => { + const { plan, initial, engine, transceiver, release } = fixture(); + plan.claimed = true; + plan.reserve(initial); + const cancelled = plan.cancel(); + expect(transceiver.stop).not.toHaveBeenCalled(); + plan.connectionReady(); + await cancelled; + await plan.cancel(); + expect(transceiver.stop).toHaveBeenCalledOnce(); + expect(engine.negotiate).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); + }); + it('lets PC teardown own a discarded connection attempt', async () => { + const { plan, initial, engine, transceiver, track } = fixture(); + plan.reserve(initial); + plan.fallback(); + expect(engine.cancelPendingTrack).toHaveBeenCalledOnce(); + expect(await plan.senderFor(track, initial.engine)).toBeUndefined(); + await plan.cancel(); + expect(transceiver.stop).not.toHaveBeenCalled(); + }); + it('accepts CID-only codec metadata before media reaches the SFU', async () => { + const { plan, initial, track } = fixture(); + initial.accepted = Promise.resolve(new TrackInfo({ codecs: [{ cid: 'prepared-cid' }] })); + plan.reserve(initial); + expect(await plan.senderFor(track, initial.engine)).toBe(initial); + plan.fallback(); + await plan.cancel(); + }); + it.each(['codec', 'format', 'connection'])( + 'falls back when %s no longer matches', + async (reason) => { + const { plan, initial, track, engine, transceiver } = fixture(); + plan.claimed = true; + if (reason === 'codec') + initial.accepted = Promise.resolve(new TrackInfo({ codecs: [{ mimeType: 'audio/pcmu' }] })); + if (reason === 'format') track.getSourceTrackSettings = () => ({ channelCount: 2 }); + plan.reserve(initial); + plan.connectionReady(); + if (reason === 'connection') engine.pcManager.publisher = {}; + expect(await plan.senderFor(track, initial.engine)).toBeUndefined(); + expect(transceiver.stop).toHaveBeenCalledTimes(reason === 'connection' ? 0 : 1); + await plan.cancel(); + }, + ); + it('rejects installing a sender after cancellation has completed', async () => { + const { plan, initial } = fixture(); + await plan.cancel(); + expect(() => plan.reserve(initial)).toThrow('cancelled'); + expect(plan.hasInitialSender).toBe(false); + }); + it('owns only the preconnect recorder it starts', async () => { + const { plan, track } = fixture(); + plan.options.preConnectBuffer = true; + track.startPreConnectBuffer = vi.fn(); + track.stopPreConnectBuffer = vi.fn(); + const attaching = plan.attachTrack(track); + expect(track.startPreConnectBuffer).toHaveBeenCalledOnce(); + await plan.cancel(); + await expect(attaching).rejects.toThrow('cancelled'); + expect(track.stopPreConnectBuffer).toHaveBeenCalledOnce(); + expect(track.stop).not.toHaveBeenCalled(); + }); + it.each([false, true])( + 'keeps early subscriptions past capture delay, cancelled: %s', + async (cancelled) => { + vi.useFakeTimers(); + const room = new Room(); + const internals = room as unknown as { + setupLocalParticipantEvents(): void; + onLocalTrackPublished(publication: LocalTrackPublication): Promise; + }; + const plan = room.preparePublication({ + source: Track.Source.Microphone, + }) as PreparedPublicationState; + plan.claimed = true; + internals.setupLocalParticipantEvents(); + const subscribed = vi.fn(); + room.localParticipant.on(ParticipantEvent.LocalTrackSubscribed, subscribed); + room.engine.emit(EngineEvent.LocalTrackSubscribed, 'early-sid'); + await vi.advanceTimersByTimeAsync(11000); + if (cancelled) await plan.cancel(); + const publication = new LocalTrackPublication( + Track.Kind.Audio, + new TrackInfo({ sid: 'early-sid' }), + ); + await internals.onLocalTrackPublished(publication); + expect(subscribed).toHaveBeenCalledTimes(cancelled ? 0 : 1); + await plan.cancel(); + await room.dispose(); + vi.useRealTimers(); + }, + ); + it('disconnect settles attachment and releases a reserved sender', async () => { + const { plan, initial, track, engine, release } = fixture(); + plan.claimed = true; + plan.reserve(initial); + const attached = plan.attachTrack(track); + engine.isClosed = true; + plan.disconnected(); + await expect(attached).rejects.toThrow(); + await plan.cancel(); + expect(release).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/room/PreparedPublication.ts b/src/room/PreparedPublication.ts new file mode 100644 index 0000000000..f24750b1ec --- /dev/null +++ b/src/room/PreparedPublication.ts @@ -0,0 +1,291 @@ +import { AddTrackRequest, type TrackInfo } from '@livekit/protocol'; +import type PCTransport from './PCTransport'; +import type RTCEngine from './RTCEngine'; +import { TrackInvalidError, UnexpectedConnectionState } from './errors'; +import type LocalTrack from './track/LocalTrack'; +import type LocalTrackPublication from './track/LocalTrackPublication'; +import { Track } from './track/Track'; +import type { TrackPublishOptions } from './track/options'; + +/** The video format to reserve before opening a camera. */ +export interface PreparedVideoFormat { + width: number; + height: number; + frameRate?: number; +} + +/** Publication intent that can be resolved without requesting device access. @experimental */ +export type PreparePublicationOptions = + | { + source: Track.Source.Microphone; + /** Otherwise inferred from forceStereo, defaulting to mono. */ + format?: { channelCount: 1 | 2 }; + publishOptions?: TrackPublishOptions; + } + | { + source: Track.Source.Camera; + format: PreparedVideoFormat; + publishOptions?: TrackPublishOptions; + }; + +/** A room-owned reservation; creating it never requests camera or microphone access. @experimental */ +export interface PreparedPublication { + /** + * Binds captured media after the room connects. Uses ordinary publication when the reservation + * cannot be negotiated or the captured format differs. Does not take ownership on failure. + */ + attachTrack(track: LocalTrack): Promise; + /** Cancels an unbound reservation without disconnecting the room or stopping caller-owned media. */ + cancel(): Promise; +} + +/** The negotiated sender owned by a preparation until normal publication takes over. */ +export interface InitialPublicationSender { + engine: RTCEngine; + transport: PCTransport; + transceiver: RTCRtpTransceiver; + accepted: Promise; +} + +/** Internal implementation shared by the room, engine, and ordinary publish pipeline. */ +export class PreparedPublicationState implements PreparedPublication { + readonly request: AddTrackRequest; + + readonly kind: Track.Kind; + + claimed = false; + + startBitrate?: number; + + private cancelled?: Error; + + private initial?: InitialPublicationSender; + + private boundTrack?: LocalTrack; + + private attaching?: Promise; + + private cancelling?: Promise; + + private committed = false; + + private ownsBuffer = false; + + private resolveConnected!: () => void; + + private rejectConnected!: (error: Error) => void; + + private rejectCancelled!: (error: Error) => void; + + private readonly connected: Promise; + + private readonly cancellation: Promise; + + constructor( + readonly specification: PreparePublicationOptions, + readonly options: TrackPublishOptions, + readonly encodings: RTCRtpEncodingParameters[], + readonly codec: string, + readonly canSendWithJoin: boolean, + private publish?: ( + track: LocalTrack, + prepared: PreparedPublicationState, + ) => Promise, + private release?: (prepared: PreparedPublicationState) => void, + ) { + this.kind = + specification.source === Track.Source.Microphone ? Track.Kind.Audio : Track.Kind.Video; + this.request = new AddTrackRequest({ + name: options.name, + type: Track.kindToProto(this.kind), + source: Track.sourceToProto(specification.source), + stream: options.stream, + stereo: !!options.forceStereo, + disableDtx: !(options.dtx ?? true), + disableRed: !(options.red ?? true), + }); + this.connected = new Promise((resolve, reject) => { + this.resolveConnected = resolve; + this.rejectConnected = reject; + }); + this.cancellation = new Promise((_, reject) => { + this.rejectCancelled = reject; + }); + // A caller may prepare now and attach much later, or never attach at all. + this.connected.catch(() => {}); + this.cancellation.catch(() => {}); + } + + attachTrack(track: LocalTrack): Promise { + if (this.cancelled) return Promise.reject(this.cancelled); + if (this.attaching) { + return this.boundTrack === track + ? this.attaching + : Promise.reject(new TrackInvalidError('publication already has a track')); + } + if (track.kind !== this.kind || track.sender) { + return Promise.reject( + new TrackInvalidError('track kind differs or track already has a sender'), + ); + } + this.boundTrack = track; + if ( + this.options.preConnectBuffer && + track.kind === Track.Kind.Audio && + !track.hasPreConnectBuffer + ) { + track.startPreConnectBuffer(); + this.ownsBuffer = true; + } + this.attaching = this.attach(track); + return this.attaching; + } + + private async attach(track: LocalTrack): Promise { + try { + await Promise.race([this.connected, this.cancellation]); + this.assertActive(); + track.source = this.specification.source; + const publication = await this.publish!(track, this); + this.committed = true; + this.initial = undefined; + this.finish(); + return publication; + } catch (error) { + // Reconciliation is room-owned and may wait for an in-flight JOIN to settle. + this.cancel().catch(() => {}); + throw error; + } + } + + /** Whether this preparation was cancelled before it entered an offer. */ + get isCancelled() { + return !!this.cancelled; + } + + /** Whether a connection attempt already owns the reservation. */ + get hasInitialSender() { + return !!this.initial; + } + + /** Marks only preparations explicitly supplied to the successful connect. */ + connectionReady() { + if (this.claimed) this.resolveConnected(); + } + + /** Fails pending attachment even if capture has not completed yet. */ + disconnected() { + this.rejectConnected(new UnexpectedConnectionState('room disconnected before publication')); + this.cancel().catch(() => {}); + } + + /** Installs a response waiter before the JOIN request can reach the server. */ + reserve(initial: InitialPublicationSender) { + this.assertActive(); + this.initial = initial; + initial.accepted.catch((error) => { + // An abandoned attempt must not cancel a newer fallback or connection attempt. + if (this.initial === initial) { + this.cancelled = error instanceof Error ? error : new Error(String(error)); + this.rejectCancelled(this.cancelled); + this.cancel().catch(() => {}); + } + }); + } + + /** Detaches a discarded connection attempt; attachment will use ordinary publication. */ + fallback() { + const initial = this.initial; + this.initial = undefined; + if (initial) initial.engine.cancelPendingTrack(this.request.cid); + } + + /** Returns a compatible reserved sender, otherwise removes it before ordinary publication. */ + async senderFor( + track: LocalTrack, + engine: RTCEngine, + ): Promise { + this.assertActive(); + const initial = this.initial; + if (!initial) return; + const info = await initial.accepted; + this.assertActive(); + const settings = track.getSourceTrackSettings(); + const mime = `${this.kind}/${this.codec}`.toLowerCase(); + // Before RTP arrives, the SFU may return codec entries with only CID/MID. + // The negotiated SDP already constrains those senders to our selected codec. + const advertisedCodecs = info.codecs.filter((codec) => codec.mimeType.length > 0); + const codecChanged = + advertisedCodecs.length > 0 && + !advertisedCodecs.some( + (c) => + c.mimeType.toLowerCase() === mime || + (this.kind === Track.Kind.Audio && c.mimeType.toLowerCase() === 'audio/red'), + ); + const formatChanged = + this.specification.source === Track.Source.Camera + ? (settings.width !== undefined && settings.width !== this.specification.format.width) || + (settings.height !== undefined && settings.height !== this.specification.format.height) + : settings.channelCount !== undefined && + settings.channelCount !== (this.options.forceStereo ? 2 : 1); + if ( + engine.e2eeManager || + engine.frameMetadataOptions || + initial.engine !== engine || + engine.pcManager?.publisher !== initial.transport || + codecChanged || + formatChanged + ) { + await this.removeInitial(); + return; + } + return initial; + } + + /** Rejects continuations after cancellation or teardown. */ + assertActive() { + if (this.cancelled) throw this.cancelled; + } + + cancel(): Promise { + if (this.committed) return Promise.resolve(); + if (!this.cancelled) this.cancelled = new TrackInvalidError('prepared publication cancelled'); + this.rejectCancelled(this.cancelled); + if (this.ownsBuffer) { + this.boundTrack?.stopPreConnectBuffer(); + this.ownsBuffer = false; + } + this.cancelling ??= this.removeInitial().finally(() => this.finish()); + return this.cancelling; + } + + private async removeInitial() { + const initial = this.initial; + if (!initial) return; + // Do not mutate a transceiver while its initial local description is still deferred. + try { + await this.connected; + } catch { + /* the connection's teardown owns the PC */ + } + if (this.initial !== initial) return; + this.initial = undefined; + initial.engine.cancelPendingTrack(this.request.cid); + if (initial.engine.isClosed || initial.engine.pcManager?.publisher !== initial.transport) + return; + if (this.boundTrack?.sender === initial.transceiver.sender) this.boundTrack.sender = undefined; + initial.transceiver.stop(); + try { + await initial.engine.negotiate(); + } catch { + /* disconnect also cancels reconciliation */ + } + } + + private finish() { + this.release?.(this); + this.release = undefined; + this.publish = undefined; + if (!this.committed) this.boundTrack = undefined; + } +} diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 04766c1ca2..7332d45928 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -45,6 +45,7 @@ import { type UserPacket, } from '@livekit/protocol'; import { EventEmitter } from 'events'; +import { parse as parseInitialSDP } from 'sdp-transform'; import type { MediaAttributes } from 'sdp-transform'; import type TypedEventEmitter from 'typed-emitter'; import type { SignalOptions } from '../api/SignalClient'; @@ -70,6 +71,7 @@ import { TTLMap } from '../utils/ttlmap'; import { PROTOCOL_NEGOTIATED_DATA_CHANNELS } from '../version'; import PCTransport, { PCEvents } from './PCTransport'; import { PCTransportManager, PCTransportState } from './PCTransportManager'; +import type { PreparedPublicationState } from './PreparedPublication'; import type { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy'; import { DataChannelManager } from './data-channel/DataChannelManager'; import type { FlowControlledDataChannel } from './data-channel/FlowControlledDataChannel'; @@ -93,7 +95,7 @@ import type LocalTrackPublication from './track/LocalTrackPublication'; import type LocalVideoTrack from './track/LocalVideoTrack'; import type { SimulcastTrackInfo } from './track/LocalVideoTrack'; import type RemoteTrackPublication from './track/RemoteTrackPublication'; -import type { Track } from './track/Track'; +import { Track } from './track/Track'; import type { TrackPublishOptions, VideoCodec } from './track/options'; import { getTrackPublicationInfo } from './track/utils'; import type { LoggerOptions } from './types'; @@ -218,6 +220,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit private _isNewlyCreated: boolean = true; + /** Startup reservations supplied by the room; never wait for device capture. @internal */ + initialPublications: PreparedPublicationState[] = []; + + private initialTrackRequests: AddTrackRequest[] = []; + private pendingTrackResolvers: { [key: string]: { resolve: (info: TrackInfo) => void; reject: () => void }; } = {}; @@ -399,9 +406,83 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit // channels and fall back to in-band ones below if the server is too old. this.applyInitialPublisherLayout(true); } - const offer = await this.pcManager?.publisher.createInitialOffer(); - if (offer) { - offerProto = toProtoSessionDescription(offer.offer, offer.offerId); + const canPreparePublications = + !this.e2eeManager && !this.frameMetadataOptions && !opts.e2eeEnabled; + const preparingPublications = + canPreparePublications && + this.initialPublications.some((plan) => plan.canSendWithJoin && !plan.isCancelled); + try { + const reservations: { + plan: PreparedPublicationState; + transceiver: RTCRtpTransceiver; + stream: string; + }[] = []; + for (const plan of canPreparePublications ? this.initialPublications : []) { + if (!plan.canSendWithJoin || plan.isCancelled || plan.hasInitialSender) continue; + const stream = new MediaStream(); + const transceiver = await this.pcManager!.addPublisherTransceiverOfKind( + plan.kind === Track.Kind.Audio ? 'audio' : 'video', + { + direction: 'sendonly', + streams: [stream], + sendEncodings: plan.encodings, + }, + ); + const mime = `${plan.kind}/${plan.codec}`.toLowerCase(); + const codecs = RTCRtpSender.getCapabilities( + plan.kind === Track.Kind.Audio ? 'audio' : 'video', + )!.codecs.filter( + (c) => + c.mimeType.toLowerCase() === mime || + (plan.kind === Track.Kind.Audio + ? !plan.request.disableRed && c.mimeType.toLowerCase() === 'audio/red' + : c.mimeType.toLowerCase() === 'video/rtx'), + ); + transceiver.setCodecPreferences(codecs); + reservations.push({ plan, transceiver, stream: stream.id }); + } + const requests: AddTrackRequest[] = []; + const offer = await this.pcManager?.publisher.createInitialOffer((sdp) => { + const media = parseInitialSDP(sdp).media; + for (const { plan, transceiver, stream } of reservations) { + const section = media.find( + (m) => typeof m.msid === 'string' && m.msid.startsWith(`${stream} `), + ); + const cid = section?.msid?.split(' ')[1]; + if (!cid) + throw new UnexpectedConnectionState('prepared sender has no SDP track identity'); + plan.assertActive(); + plan.request.cid = cid; + if (plan.request.simulcastCodecs[0]) plan.request.simulcastCodecs[0].cid = cid; + const accepted = this.waitForTrackPublished(plan.request); + plan.reserve({ + engine: this, + transport: this.pcManager!.publisher, + transceiver, + accepted, + }); + requests.push(plan.request); + if (plan.startBitrate) { + this.pcManager!.publisher.setTrackCodecBitrate({ + cid, + codec: plan.codec, + maxbr: plan.startBitrate / 1000, + }); + } + } + }); + if (offer) offerProto = toProtoSessionDescription(offer.offer, offer.offerId); + this.initialTrackRequests = requests; + } catch (error) { + if (!preparingPublications) throw error; + // A browser unable to reserve a trackless sender must still be able to join. + this.log.warn('initial publication preparation failed; deferring publication', { error }); + this.pcManager!.onStateChange = undefined; + await this.cleanupPeerConnections(); + await this.configure(); + this.applyInitialPublisherLayout(true); + const offer = await this.pcManager!.publisher.createInitialOffer(); + if (offer) offerProto = toProtoSessionDescription(offer.offer, offer.offerId); } } @@ -420,6 +501,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit abortSignal, useV0Path, offerProto, + this.initialTrackRequests, ); if (abortSignal?.aborted) { this.log.info('join cancelled after the signal join response'); @@ -485,6 +567,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit return { joinResponse, serverInfo }; } catch (e) { + if (this.initialTrackRequests.length) { + for (const plan of this.initialPublications) plan.fallback(); + this.initialTrackRequests = []; + if (this.pcManager) this.pcManager.onStateChange = undefined; + await this.cleanupPeerConnections(); + } if (e instanceof ConnectionError) { if (e.reason === ConnectionErrorReason.ServerUnreachable) { this.log.warn( @@ -539,6 +627,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } async cleanupPeerConnections() { + for (const plan of this.initialPublications) plan.fallback(); + this.initialTrackRequests = []; this.dataChannels.teardown(); await this.pcManager?.close(); @@ -567,6 +657,19 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } addTrack(req: AddTrackRequest): Promise { + const accepted = this.waitForTrackPublished(req); + this.client.sendAddTrack(req); + return accepted; + } + + /** Cancels only the waiter belonging to this publication. @internal */ + cancelPendingTrack(cid: string) { + const pending = this.pendingTrackResolvers[cid]; + delete this.pendingTrackResolvers[cid]; + pending?.reject(); + } + + private waitForTrackPublished(req: AddTrackRequest): Promise { if (this.pendingTrackResolvers[req.cid]) { throw new TrackInvalidError('a track with the same ID has already been published'); } @@ -587,7 +690,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit reject(new Error('Cancelled publication by calling unpublish')); }, }; - this.client.sendAddTrack(req); }); } diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index f723fe5d9b..5c95ef5f42 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -31,6 +31,7 @@ import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed- import TypedPromise from '../../utils/TypedPromise'; import DeviceManager from '../DeviceManager'; import { PCTransportState } from '../PCTransportManager'; +import { type PreparePublicationOptions, PreparedPublicationState } from '../PreparedPublication'; import type RTCEngine from '../RTCEngine'; import { DataChannelKind } from '../RTCEngine'; import { @@ -55,6 +56,7 @@ import type { ScreenShareCaptureOptions, TrackPublishOptions, VideoCaptureOptions, + VideoCodec, } from '../track/options'; import { getLogContextFromTrack, @@ -73,6 +75,7 @@ import { isLocalTrack, isLocalVideoTrack, isPermissionDeniedError, + isReactNative, isVideoTrack, isWeb, sleep, @@ -88,6 +91,13 @@ import type RemoteParticipant from './RemoteParticipant'; * @experimental */ export interface VideoPublisher { + /** Computes a conservative, device-free initial video envelope. Unsupported configurations defer. */ + prepareInitial?( + format: { width: number; height: number; frameRate?: number }, + opts: TrackPublishOptions, + req: AddTrackRequest, + ): { encodings: RTCRtpEncodingParameters[]; startBitrate: number } | undefined; + /** * Chooses the codec, settles simulcast and SVC and fills `req` (dimensions, layers, codecs). * Returns the encodings for the sender. @@ -633,6 +643,101 @@ export class LocalParticipant extends Participant { return this.slots.videoPublisher; } + /** Builds a room-owned, device-free publication plan. @internal */ + preparePublication( + specification: PreparePublicationOptions, + release: (prepared: PreparedPublicationState) => void, + ): PreparedPublicationState { + const opts: TrackPublishOptions = { + ...this.roomOptions.publishDefaults, + ...specification.publishOptions, + source: specification.source, + }; + if ( + specification.source !== Track.Source.Microphone && + specification.source !== Track.Source.Camera + ) { + throw new TrackInvalidError('only microphone and camera publications can be prepared'); + } + if (specification.source === Track.Source.Microphone) { + if ( + specification.format && + specification.publishOptions?.forceStereo !== undefined && + specification.publishOptions.forceStereo !== (specification.format.channelCount === 2) + ) { + throw new TrackInvalidError('audio format and forceStereo disagree'); + } + opts.forceStereo = specification.format + ? specification.format.channelCount === 2 + : (opts.forceStereo ?? false); + } + const req = new AddTrackRequest(); + const video = specification.source === Track.Source.Camera; + if ( + video && + (![specification.format.width, specification.format.height].every( + (n) => Number.isFinite(n) && n > 0, + ) || + (specification.format.frameRate !== undefined && + (!Number.isFinite(specification.format.frameRate) || + specification.format.frameRate <= 0))) + ) { + throw new TrackInvalidError( + 'video format must have positive finite dimensions and frame rate', + ); + } + const initialVideo = + specification.source === Track.Source.Camera + ? this.slots.videoPublisher?.prepareInitial?.(specification.format, opts, req) + : undefined; + const encodings = video + ? initialVideo?.encodings + : [ + { + maxBitrate: opts.audioPreset?.maxBitrate, + priority: opts.audioPreset?.priority ?? 'high', + networkPriority: opts.audioPreset?.priority ?? 'high', + } as RTCRtpEncodingParameters, + ]; + const codec = video ? (opts.videoCodec ?? 'vp8') : 'opus'; + const available = + typeof RTCRtpSender !== 'undefined' && + RTCRtpSender.getCapabilities?.(video ? 'video' : 'audio')?.codecs.some( + (c) => c.mimeType.toLowerCase() === `${video ? 'video' : 'audio'}/${codec}`, + ); + const eligible = + !!available && + !!encodings && + !isReactNative() && + !this.slots.e2eeManager && + !this.slots.frameMetadataOptions && + !opts.frameMetadata; + const prepared = new PreparedPublicationState( + specification.source === Track.Source.Camera + ? { ...specification, format: { ...specification.format } } + : { ...specification }, + opts, + encodings ?? [], + codec, + eligible, + (track, plan) => this.publishOrRepublishTrack(track, opts, false, false, plan), + release, + ); + if (specification.source === Track.Source.Camera) { + prepared.startBitrate = initialVideo?.startBitrate; + prepared.request.width = req.width; + prepared.request.height = req.height; + prepared.request.layers = req.layers; + prepared.request.simulcastCodecs = req.simulcastCodecs; + } else { + if (opts.forceStereo) prepared.request.audioFeatures.push(AudioTrackFeature.TF_STEREO); + if (opts.dtx === false) prepared.request.audioFeatures.push(AudioTrackFeature.TF_NO_DTX); + if (opts.preConnectBuffer) + prepared.request.audioFeatures.push(AudioTrackFeature.TF_PRECONNECT_BUFFER); + } + return prepared; + } + /** * Publish a new track to the room * @param track @@ -678,6 +783,7 @@ export class LocalParticipant extends Participant { options?: TrackPublishOptions, isRepublish = false, hasRetriedAfterNegotiationError = false, + prepared?: PreparedPublicationState, ): Promise { if (isLocalAudioTrack(track)) { await track.setAudioContext(this.audioContext); @@ -815,11 +921,11 @@ export class LocalParticipant extends Participant { if (publicationTimedOut) { return; } - const publication = await this.publish(track, opts, isStereo); + const publication = await this.publish(track, opts, isStereo, prepared); resolve(publication); } else { try { - const publication = await this.publish(track, opts, isStereo); + const publication = await this.publish(track, opts, isStereo, prepared); resolve(publication); } catch (e) { reject(e); @@ -872,7 +978,13 @@ export class LocalParticipant extends Participant { return false; } - private async publish(track: LocalTrack, opts: TrackPublishOptions, isStereo: boolean) { + private async publish( + track: LocalTrack, + opts: TrackPublishOptions, + isStereo: boolean, + prepared?: PreparedPublicationState, + ) { + const reserved = prepared ? await prepared.senderFor(track, this.engine) : undefined; if (!this.hasPermissionsToPublish(track)) { throw new PublishTrackError('failed to publish track, insufficient permissions', 403); } @@ -924,26 +1036,30 @@ export class LocalParticipant extends Participant { this.normalizeRequestedFrameMetadataOptions(track, opts); // create track publication from track - const req = new AddTrackRequest({ - // get local track id for use during publishing - cid: track.mediaStreamTrack.id, - name: opts.name, - type: Track.kindToProto(track.kind), - muted: track.isMuted, - source: Track.sourceToProto(track.source), - disableDtx, - encryption: this.encryptionType, - stereo: isStereo, - disableRed: this.isE2EEEnabled || !(opts.red ?? true), - stream: opts?.stream, - backupCodecPolicy: opts?.backupCodecPolicy as BackupCodecPolicy, - audioFeatures, - packetTrailerFeatures, - }); + const req = reserved + ? prepared!.request + : new AddTrackRequest({ + // get local track id for use during publishing + cid: track.mediaStreamTrack.id, + name: opts.name, + type: Track.kindToProto(track.kind), + muted: track.isMuted, + source: Track.sourceToProto(track.source), + disableDtx, + encryption: this.encryptionType, + stereo: isStereo, + disableRed: this.isE2EEEnabled || !(opts.red ?? true), + stream: opts?.stream, + backupCodecPolicy: opts?.backupCodecPolicy as BackupCodecPolicy, + audioFeatures, + packetTrailerFeatures, + }); // compute encodings and layers for video let encodings: RTCRtpEncodingParameters[] | undefined; - if (isLocalVideoTrack(track)) { + if (reserved) { + encodings = prepared!.encodings; + } else if (isLocalVideoTrack(track)) { encodings = await this.requireVideoPublisher().prepare(track, opts, req); } else if (track.kind === Track.Kind.Audio) { encodings = [ @@ -964,7 +1080,22 @@ export class LocalParticipant extends Participant { throw new UnexpectedConnectionState('pcManager is not ready'); } - track.sender = await this.engine.createSender(track, opts, encodings); + if (reserved) { + prepared!.assertActive(); + await reserved.transceiver.sender.replaceTrack(track.mediaStreamTrack); + prepared!.assertActive(); + if ( + this.engine !== reserved.engine || + this.engine.isClosed || + this.engine.pcManager?.publisher !== reserved.transport + ) { + throw new UnexpectedConnectionState('connection changed while attaching prepared media'); + } + track.sender = reserved.transceiver.sender; + if (isLocalVideoTrack(track)) track.codec = prepared!.codec as VideoCodec; + } else { + track.sender = await this.engine.createSender(track, opts, encodings); + } if (isLocalVideoTrack(track)) { track.publishOptions = opts; } @@ -1000,13 +1131,13 @@ export class LocalParticipant extends Participant { } } - await this.engine.negotiate(); + if (!reserved) await this.engine.negotiate(); }; let ti: TrackInfo; const addTrackPromise = new Promise(async (resolve, reject) => { try { - ti = await this.engine.addTrack(req); + ti = await (reserved ? reserved.accepted : this.engine.addTrack(req)); resolve(ti); } catch (err) { if (track.sender && this.engine.pcManager?.publisher) { @@ -1033,7 +1164,10 @@ export class LocalParticipant extends Participant { reject(err); } }); - if (this.enabledPublishVideoCodecs.length > 0 && packetTrailerFeatures.length === 0) { + if ( + reserved || + (this.enabledPublishVideoCodecs.length > 0 && packetTrailerFeatures.length === 0) + ) { const rets = await Promise.all([addTrackPromise, negotiate()]); ti = rets[0]; } else { @@ -1046,6 +1180,7 @@ export class LocalParticipant extends Participant { await negotiate(); } + prepared?.assertActive(); const publication = new LocalTrackPublication(track.kind, ti, track, { loggerName: this.roomOptions.loggerName, loggerContextCb: () => this.logContext, @@ -1056,6 +1191,12 @@ export class LocalParticipant extends Participant { // save options for when it needs to be republished again publication.options = opts; track.sid = ti.sid; + if (reserved) { + // Capture-dependent features and mute state are unavailable when constructing JOIN. + if (track.kind === Track.Kind.Audio) + this.engine.client.sendUpdateLocalAudioTrack(ti.sid, audioFeatures); + if (track.isMuted) this.engine.updateMuteStatus(ti.sid, true); + } // keep publish options on the video track so that it can recompute encoding // parameters when the MediaStreamTrack is restarted (e.g. after switching cameras). @@ -1077,26 +1218,25 @@ export class LocalParticipant extends Participant { } this.addTrackPublication(publication); - // send event for publication - this.emit(ParticipantEvent.LocalTrackPublished, publication); - if ( isLocalAudioTrack(track) && ti.audioFeatures.includes(AudioTrackFeature.TF_PRECONNECT_BUFFER) ) { const stream = track.getPreConnectBuffer(); const mimeType = track.getPreConnectBufferMimeType(); - // TODO: we're registering the listener after negotiation, so there might be a race - this.on(ParticipantEvent.LocalTrackSubscribed, (pub) => { - if (pub.trackSid === ti.sid) { - if (!track.hasPreConnectBuffer) { - this.log.warn('subscribe event came to late, buffer already closed'); - return; - } - this.log.debug('finished recording preconnect buffer', getLogContextFromTrack(track)); - track.stopPreConnectBuffer(); - } - }); + const finishBuffer = () => { + this.off(ParticipantEvent.LocalTrackSubscribed, onSubscribed); + this.off(ParticipantEvent.LocalTrackUnpublished, onUnpublished); + track.stopPreConnectBuffer(); + }; + const onSubscribed = (pub: LocalTrackPublication) => { + if (pub.trackSid === ti.sid) finishBuffer(); + }; + const onUnpublished = (pub: LocalTrackPublication) => { + if (pub.trackSid === ti.sid) finishBuffer(); + }; + this.on(ParticipantEvent.LocalTrackSubscribed, onSubscribed); + this.on(ParticipantEvent.LocalTrackUnpublished, onUnpublished); if (stream) { const bufferStreamPromise = new Promise(async (resolve, reject) => { @@ -1145,6 +1285,9 @@ export class LocalParticipant extends Participant { }); } } + // A subscription deferred during JOIN is replayed by the room synchronously here. + // Install buffer listeners first so that replay cannot be missed. + this.emit(ParticipantEvent.LocalTrackPublished, publication); return publication; } diff --git a/src/room/video/publisher.ts b/src/room/video/publisher.ts index 5e9c3914ee..9615efd8e9 100644 --- a/src/room/video/publisher.ts +++ b/src/room/video/publisher.ts @@ -53,6 +53,38 @@ export function createVideoPublisher(room: CoreRoom, log: StructuredLogger): Vid const roomOptions = room.options; return { + prepareInitial(format, opts, req) { + // SVC and additional-codec policy depend on server capabilities learned after JOIN. + opts.videoCodec ??= defaultVideoCodec; + if ( + !['vp8', 'h264'].includes(opts.videoCodec) || + opts.scalabilityMode || + (typeof opts.backupCodec === 'object' && opts.backupCodec.codec !== opts.videoCodec) + ) + return; + req.width = format.width; + req.height = format.height; + if (format.frameRate && opts.videoEncoding) { + opts.videoEncoding = { + ...opts.videoEncoding, + maxFramerate: opts.videoEncoding.maxFramerate ?? format.frameRate, + }; + } + const encodings = computeVideoEncodings(false, format.width, format.height, opts); + if (format.frameRate) { + for (const encoding of encodings) + encoding.maxFramerate = Math.min( + encoding.maxFramerate ?? format.frameRate, + format.frameRate, + ); + } + req.layers = videoLayersFromEncodings(format.width, format.height, encodings); + req.simulcastCodecs = [new SimulcastCodec({ codec: opts.videoCodec, layers: req.layers })]; + return { + encodings, + startBitrate: computeStartTargetBitrate(opts.videoCodec, opts, encodings), + }; + }, async prepare(track, opts, req) { // read at publish time: the e2ee extension can install after video if (!isE2EESimulcastSupported() && room.hasE2EESetup) { diff --git a/tests/public/types.ts b/tests/public/types.ts index 65337bbad2..3ba96fe421 100644 --- a/tests/public/types.ts +++ b/tests/public/types.ts @@ -64,3 +64,33 @@ export async function join(url: string, token: string): Promise { // The core entry: a room with only the listed extensions. export const coreRoom: CoreRoom = createRoom({ adaptiveStream: true }, [rpc]); + +// Reservation configuration is available without device access, including core-only audio. +export async function joinWithPreparedPublications(url: string, token: string): Promise { + const audio = coreRoom.preparePublication({ + source: Track.Source.Microphone, + format: { channelCount: 2 }, + publishOptions: { forceStereo: true, audioPreset: { maxBitrate: 96000 } }, + }); + await coreRoom.connect(url, token, { initialPublications: [audio] }); + await audio.cancel(); + const camera = room.preparePublication({ + source: Track.Source.Camera, + format: { width: 1280, height: 720, frameRate: 30 }, + publishOptions: { + videoCodec: 'h264', + simulcast: false, + videoEncoding: { maxBitrate: 1000000 }, + }, + }); + const connecting = room.connect(url, token, { initialPublications: [camera] }); + const tracks = await createLocalTracks({ + audio: false, + video: { resolution: { width: 1280, height: 720 } }, + }); + await connecting; + if (tracks[0]) { + const publication: LocalTrackPublication = await camera.attachTrack(tracks[0]); + publication.trackSid.toUpperCase(); + } +}