diff --git a/packages/tsunami/src/FishjamClient.ts b/packages/tsunami/src/FishjamClient.ts index 825b1e2f..5ad7e4e8 100644 --- a/packages/tsunami/src/FishjamClient.ts +++ b/packages/tsunami/src/FishjamClient.ts @@ -21,6 +21,7 @@ import type TypedEmitter from "typed-emitter"; import { ClientResourceScope } from "./ClientResourceScope"; import { DeviceOrchestrator } from "./controllers/DeviceOrchestrator"; +import type { ScreenShareConstraints } from "./controllers/ScreenShareController"; import type { TrackPublisher } from "./controllers/TrackPublisher"; import { VIDEO_TRACK_CONSTRAINTS } from "./devices/constraints"; import type { IDeviceManager, PlatformMediaStream, PlatformMediaStreamTrack } from "./devices/deviceManager"; @@ -31,6 +32,7 @@ import type { InitializeDevicesSettings, StreamConfig, TrackMiddleware, + TracksMiddleware, } from "./mediaTypes"; import { type ClientState, createInitialClientState } from "./state/clientState"; import { StateStore, type StoreListener } from "./state/StateStore"; @@ -224,6 +226,22 @@ export class FishjamClient { + return this.requireDevices().screenShare.start(constraints); + } + + public stopScreenShare(): Promise { + return this.requireDevices().screenShare.stop(); + } + + public setScreenShareTracksMiddleware(middleware: TracksMiddleware | null): Promise { + return this.requireDevices().screenShare.setMiddleware(middleware); + } + + public setCustomSource(sourceId: string, stream: PlatformMediaStream | null): Promise { + return this.requireDevices().customSources.setSource(sourceId, stream); + } + private requireDevices(): DeviceOrchestrator { this.resources.assertActive(); if (!this.deviceOrchestrator) throw new DeviceManagerMissingError(); diff --git a/packages/tsunami/src/controllers/CustomSourceController.ts b/packages/tsunami/src/controllers/CustomSourceController.ts new file mode 100644 index 00000000..eddd8d84 --- /dev/null +++ b/packages/tsunami/src/controllers/CustomSourceController.ts @@ -0,0 +1,144 @@ +import { type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client"; + +import type { PlatformMediaStream, PlatformMediaStreamTrack } from "../devices/deviceManager"; +import type { CustomSourceState, PeerStatus } from "../state/clientState"; +import type { TrackPublisher } from "./TrackPublisher"; + +export type CustomSourceControllerDeps = { + publisher: TrackPublisher; + logger: Logger; + getPeerStatus: () => PeerStatus; + onStateChanged: () => void; +}; + +/** + * Owns user-provided custom media sources. All mutations run through a promise + * queue: replacing a source's stream issues an unpublish and a publish in the + * same tick, and run concurrently they would race to remove the same track ids. + */ +export class CustomSourceController { + private sources: Record = {}; + private queue: Promise = Promise.resolve(); + private readonly sessionCleanups: (() => void)[]; + + public constructor(private readonly deps: CustomSourceControllerDeps) { + this.sessionCleanups = [ + deps.publisher.onJoined(() => { + void this.enqueue(() => this.publishPendingSources()).catch((error) => + this.deps.logger.error("Failed to publish custom sources", error), + ); + }), + deps.publisher.onDisconnected(() => { + const hasPublishedSources = Object.values(this.sources).some((source) => source.trackIds !== undefined); + if (!hasPublishedSources) return; + this.sources = Object.fromEntries( + Object.entries(this.sources).map(([id, source]) => [id, { stream: source.stream }]), + ); + this.notify(); + }), + ]; + } + + public snapshot(): Record { + return this.sources; + } + + public setSource(sourceId: string, stream: PlatformMediaStream | null): Promise { + return this.enqueue(() => this.applySetSource(sourceId, stream)); + } + + public dispose(): void { + for (const cleanup of this.sessionCleanups) cleanup(); + this.sources = {}; + } + + private enqueue(operation: () => Promise): Promise { + const run = this.queue.then(operation); + // Chain a never-rejecting link so one failed call cannot poison the queue; + // the caller still observes failures through the returned promise. + this.queue = run.catch(() => undefined); + return run; + } + + private async applySetSource(sourceId: string, stream: PlatformMediaStream | null): Promise { + const oldSource = this.sources[sourceId]; + if (stream === oldSource?.stream) return; + + if (oldSource?.trackIds) await this.removeTracks(oldSource.trackIds); + + if (stream !== null) { + this.sources = { ...this.sources, [sourceId]: { stream } }; + this.notify(); + if (this.deps.getPeerStatus() === "connected") await this.publishPendingSources(); + } else if (oldSource) { + this.sources = Object.fromEntries(Object.entries(this.sources).filter(([id]) => id !== sourceId)); + this.notify(); + } + } + + private async publishPendingSources(): Promise { + const pending = Object.entries(this.sources).filter(([, source]) => source.trackIds === undefined); + if (pending.length === 0) return; + + const published = await Promise.all( + pending.map(async ([id, source]) => [id, await this.publishSource(source)] as const), + ); + + // The queue serializes mutations, but publishing awaits addTrack — verify + // each entry is still the one we published before recording its track ids. + const isStillCurrent = ([id, started]: (typeof published)[number]) => { + const current = this.sources[id]; + return current !== undefined && current.stream === started.stream && current.trackIds === undefined; + }; + const toPatch = published.filter(isStillCurrent); + const orphans = published.filter((entry) => !isStillCurrent(entry)); + + if (toPatch.length > 0) { + this.sources = { ...this.sources, ...Object.fromEntries(toPatch) }; + this.notify(); + } + for (const [, started] of orphans) { + if (started.trackIds) await this.removeTracks(started.trackIds); + } + } + + private async publishSource(source: CustomSourceState): Promise { + const video = source.stream.getVideoTracks().at(0); + const audio = source.stream.getAudioTracks().at(0); + + const displayName = this.deps.publisher.getDisplayName(); + const promises = []; + if (video) promises.push(this.addTrack(video, { type: "customVideo", displayName, paused: false })); + if (audio) promises.push(this.addTrack(audio, { type: "customAudio", displayName, paused: false })); + + if (promises.length === 0) { + this.deps.logger.warn("Attempted to add empty PlatformMediaStream as custom source."); + return source; + } + const [videoId, audioId] = await Promise.all(promises); + return { ...source, trackIds: { videoId, audioId } }; + } + + private async removeTracks({ videoId, audioId }: { videoId?: string; audioId?: string }): Promise { + const promises = []; + if (videoId) promises.push(this.deps.publisher.removeTrack(videoId)); + if (audioId) promises.push(this.deps.publisher.removeTrack(audioId)); + await Promise.all(promises); + } + + private async addTrack(track: PlatformMediaStreamTrack, metadata: TrackMetadata): Promise { + try { + return await this.deps.publisher.addTrack(track, metadata); + } catch (err) { + if (err instanceof TrackTypeError) { + this.deps.logger.warn(err.message); + return undefined; + } + throw err; + } + } + + private notify(): void { + this.deps.onStateChanged(); + } +} diff --git a/packages/tsunami/src/controllers/DeviceOrchestrator.ts b/packages/tsunami/src/controllers/DeviceOrchestrator.ts index 0ed1e0e0..bb668eff 100644 --- a/packages/tsunami/src/controllers/DeviceOrchestrator.ts +++ b/packages/tsunami/src/controllers/DeviceOrchestrator.ts @@ -6,6 +6,8 @@ import { getAvailableMedia,recoverPersistedDevices } from "../devices/mediaIniti import type { BandwidthLimits, InitializeDevicesResult, InitializeDevicesSettings, StreamConfig } from "../mediaTypes"; import type { ClientState } from "../state/clientState"; import type { StateStore } from "../state/StateStore"; +import { CustomSourceController } from "./CustomSourceController"; +import { ScreenShareController } from "./ScreenShareController"; import { TrackDeviceController } from "./TrackDeviceController"; import type { TrackPublisher } from "./TrackPublisher"; @@ -32,6 +34,8 @@ export type DeviceOrchestratorDeps = { export class DeviceOrchestrator { public readonly camera: TrackDeviceController; public readonly microphone: TrackDeviceController; + public readonly screenShare: ScreenShareController; + public readonly customSources: CustomSourceController; private deviceList: DeviceItem[] = []; private availableCameras: DeviceItem[] = []; @@ -74,6 +78,9 @@ export class DeviceOrchestrator { onSelectedDeviceChanged: (device) => this.persistLastDevice("audio", device), }); + this.screenShare = new ScreenShareController(commonControllerDeps); + this.customSources = new CustomSourceController(commonControllerDeps); + this.deviceChangeCleanup = deps.deviceManager.onDeviceChange(() => { void this.refreshDeviceList().catch((error) => deps.logger.error("Failed to refresh device list", error)); }); @@ -164,6 +171,8 @@ export class DeviceOrchestrator { this.deviceChangeCleanup(); this.camera.dispose(); this.microphone.dispose(); + this.screenShare.dispose(); + this.customSources.dispose(); } private async getInitialStream(): Promise { @@ -188,6 +197,8 @@ export class DeviceOrchestrator { this.deps.store.update({ camera: this.camera.snapshot(), microphone: this.microphone.snapshot(), + screenShare: this.screenShare.snapshot(), + customSources: this.customSources.snapshot(), availableCameras: this.availableCameras, availableMicrophones: this.availableMicrophones, cameraError: this.camera.error, diff --git a/packages/tsunami/src/controllers/ScreenShareController.ts b/packages/tsunami/src/controllers/ScreenShareController.ts new file mode 100644 index 00000000..0a473043 --- /dev/null +++ b/packages/tsunami/src/controllers/ScreenShareController.ts @@ -0,0 +1,205 @@ +import { type Logger, type TrackMetadata, TrackTypeError } from "@fishjam-cloud/ts-client"; + +import type { IDeviceManager, PlatformMediaStream, PlatformMediaStreamTrack } from "../devices/deviceManager"; +import type { TracksMiddleware } from "../mediaTypes"; +import type { PeerStatus, ScreenShareState } from "../state/clientState"; +import type { TrackPublisher } from "./TrackPublisher"; + +export type ScreenShareConstraints = { + audioConstraints?: boolean | MediaTrackConstraints; + videoConstraints?: boolean | MediaTrackConstraints; +}; + +export type ScreenShareControllerDeps = { + publisher: TrackPublisher; + deviceManager: IDeviceManager; + logger: Logger; + getPeerStatus: () => PeerStatus; + onStateChanged: () => void; +}; + +/** Owns the screen-share lifecycle: prompt, publish, middleware, teardown. */ +export class ScreenShareController { + private stream: PlatformMediaStream | null = null; + private trackIds: { videoId?: string; audioId?: string } | null = null; + private middleware: TracksMiddleware | null = null; + private middlewareCleanup: (() => void) | null = null; + + private trackEndCleanups: (() => void)[] = []; + private readonly sessionCleanups: (() => void)[]; + private lastSnapshot: ScreenShareState | null = null; + + public constructor(private readonly deps: ScreenShareControllerDeps) { + this.sessionCleanups = [ + deps.publisher.onDisconnected(() => { + if (!this.stream) return; + void this.stop().catch((err) => this.deps.logger.error(err)); + }), + ]; + } + + public snapshot(): ScreenShareState { + const [videoTrack, audioTrack] = this.stream ? getTracksFromStream(this.stream) : [null, null]; + const next: ScreenShareState = { stream: this.stream, videoTrack, audioTrack, middleware: this.middleware }; + + const previous = this.lastSnapshot; + const isUnchanged = + previous && + previous.stream === next.stream && + previous.videoTrack === next.videoTrack && + previous.audioTrack === next.audioTrack && + previous.middleware === next.middleware; + if (isUnchanged) return previous; + + this.lastSnapshot = next; + return next; + } + + public async start(constraints?: ScreenShareConstraints): Promise { + const displayStream = await this.deps.deviceManager.getDisplayMedia({ + video: constraints?.videoConstraints ?? true, + audio: constraints?.audioConstraints ?? true, + }); + + const displayName = this.deps.publisher.getDisplayName(); + + let [video, audio] = getTracksFromStream(displayStream); + + if (this.middleware && video) { + const { videoTrack, audioTrack, onClear } = await this.middleware(video, audio); + video = videoTrack; + audio = audioTrack; + this.middlewareCleanup = onClear; + } + + if (this.deps.publisher.isSignallingActive() && video) { + const addTrackPromises = [this.addTrack(video, { displayName, type: "screenShareVideo", paused: false })]; + if (audio) addTrackPromises.push(this.addTrack(audio, { displayName, type: "screenShareAudio", paused: false })); + + const [videoId, audioId] = await Promise.all(addTrackPromises); + this.stream = displayStream; + this.trackIds = { videoId, audioId }; + } else { + this.stream = displayStream; + this.trackIds = {}; + } + + this.attachTrackEndListeners(displayStream); + this.notify(); + } + + public async stop(): Promise { + if (!this.stream) { + this.deps.logger.warn("No stream to stop"); + return; + } + const [video, audio] = getTracksFromStream(this.stream); + + video?.stop(); + audio?.stop(); + + if (this.deps.getPeerStatus() === "connected") { + const removeTrackPromises: Promise[] = []; + if (this.trackIds?.videoId) removeTrackPromises.push(this.deps.publisher.removeTrack(this.trackIds.videoId)); + if (this.trackIds?.audioId) removeTrackPromises.push(this.deps.publisher.removeTrack(this.trackIds.audioId)); + + await Promise.all(removeTrackPromises); + } + + this.detachTrackEndListeners(); + this.middlewareCleanup?.(); + this.middlewareCleanup = null; + this.stream = null; + this.trackIds = null; + this.notify(); + } + + public async setMiddleware(middleware: TracksMiddleware | null): Promise { + this.middleware = middleware; + if (!this.stream) { + this.notify(); + return; + } + + const [video, audio] = getTracksFromStream(this.stream); + if (!video) return; + + this.middlewareCleanup?.(); + this.middlewareCleanup = null; + + const { videoTrack, audioTrack, onClear } = (await middleware?.(video, audio)) ?? { + videoTrack: video, + audioTrack: audio, + onClear: null, + }; + this.middlewareCleanup = onClear; + + const replacePromises: Promise[] = []; + if (videoTrack && this.trackIds?.videoId) + replacePromises.push(this.deps.publisher.replaceTrack(this.trackIds.videoId, videoTrack)); + if (audioTrack && this.trackIds?.audioId) + replacePromises.push(this.deps.publisher.replaceTrack(this.trackIds.audioId, audioTrack)); + await Promise.all(replacePromises); + + this.notify(); + } + + public dispose(): void { + for (const cleanup of this.sessionCleanups) cleanup(); + this.detachTrackEndListeners(); + this.middlewareCleanup?.(); + this.middlewareCleanup = null; + if (this.stream) { + const [video, audio] = getTracksFromStream(this.stream); + video?.stop(); + audio?.stop(); + } + this.stream = null; + this.trackIds = null; + } + + private async addTrack(track: PlatformMediaStreamTrack, metadata: TrackMetadata): Promise { + try { + return await this.deps.publisher.addTrack(track, metadata); + } catch (err) { + if (err instanceof TrackTypeError) { + this.deps.logger.warn(err.message); + return undefined; + } + throw err; + } + } + + private attachTrackEndListeners(stream: PlatformMediaStream): void { + this.detachTrackEndListeners(); + const [video, audio] = getTracksFromStream(stream); + + const handleTrackEnded = () => { + void this.stop().catch((err) => this.deps.logger.error(err)); + }; + + for (const track of [video, audio]) { + if (!track) continue; + track.addEventListener?.("ended", handleTrackEnded); + this.trackEndCleanups.push(() => track.removeEventListener?.("ended", handleTrackEnded)); + } + } + + private detachTrackEndListeners(): void { + for (const cleanup of this.trackEndCleanups) cleanup(); + this.trackEndCleanups = []; + } + + private notify(): void { + this.deps.onStateChanged(); + } +} + +const getTracksFromStream = ( + stream: PlatformMediaStream, +): [PlatformMediaStreamTrack | null, PlatformMediaStreamTrack | null] => { + const video = stream.getVideoTracks()[0] ?? null; + const audio = stream.getAudioTracks()[0] ?? null; + + return [video, audio]; +}; diff --git a/packages/tsunami/src/index.ts b/packages/tsunami/src/index.ts index 32f6218d..08a7c527 100644 --- a/packages/tsunami/src/index.ts +++ b/packages/tsunami/src/index.ts @@ -4,6 +4,7 @@ * @packageDocumentation */ export type { DeviceOrchestrator } from "./controllers/DeviceOrchestrator"; +export type { ScreenShareConstraints } from "./controllers/ScreenShareController"; export type { TrackDeviceController } from "./controllers/TrackDeviceController"; export type { DeviceItem, @@ -36,12 +37,16 @@ export type { SimulcastBandwidthLimits, StreamConfig, TrackMiddleware, + TracksMiddleware, + TracksMiddlewareResult, } from "./mediaTypes"; export { type ClientState, createInitialClientState, + type CustomSourceState, type LocalDeviceState, type PeerStatus, + type ScreenShareState, } from "./state/clientState"; export { StateStore, type StateStoreOptions, type StoreListener } from "./state/StateStore"; export * from "@fishjam-cloud/ts-client"; diff --git a/packages/tsunami/src/state/clientState.ts b/packages/tsunami/src/state/clientState.ts index 7cd4bffa..a40bbb3f 100644 --- a/packages/tsunami/src/state/clientState.ts +++ b/packages/tsunami/src/state/clientState.ts @@ -2,7 +2,7 @@ import type { Component, GenericMetadata, Peer, ReconnectionStatus } from "@fish import type { DeviceItem, PlatformMediaStream, PlatformMediaStreamTrack } from "../devices/deviceManager"; import type { DeviceError } from "../devices/errors"; -import type { TrackMiddleware } from "../mediaTypes"; +import type { TrackMiddleware, TracksMiddleware } from "../mediaTypes"; /** * Represents the possible statuses of a peer connection. @@ -27,6 +27,18 @@ export interface LocalDeviceState { middleware: TrackMiddleware; } +export interface ScreenShareState { + stream: PlatformMediaStream | null; + videoTrack: PlatformMediaStreamTrack | null; + audioTrack: PlatformMediaStreamTrack | null; + middleware: TracksMiddleware | null; +} + +export interface CustomSourceState { + stream: PlatformMediaStream; + trackIds?: { videoId?: string; audioId?: string }; +} + /** * Flat, synchronously readable snapshot of the client's observable state. * @@ -46,6 +58,8 @@ export interface ClientState; // available hardware availableCameras: DeviceItem[]; @@ -75,6 +89,8 @@ export const createInitialClientState = (): Client components: {}, camera: createInitialDeviceState(), microphone: createInitialDeviceState(), + screenShare: { stream: null, videoTrack: null, audioTrack: null, middleware: null }, + customSources: {}, availableCameras: [], availableMicrophones: [], cameraError: null,