From 825315de7da643a1c0d95d42e964689331b752c7 Mon Sep 17 00:00:00 2001 From: David Liu Date: Fri, 20 Nov 2026 03:23:14 +0900 Subject: [PATCH] Add frame metadata for video tracks Publishers can send a capture timestamp, a frame ID and their own user data with each video frame, in the packet trailer the JS and Rust SDKs use. Subscribers look the values up by RTP timestamp. This needs a webrtc-sdk build with the frame metadata APIs; the webrtc version isn't bumped here. API: - `frameMetadata` (FrameMetadataPublishOptions) on VideoTrackPublishOptions and VideoTrackPublishDefaults turns frame metadata on, picks the fields the SDK fills in, and declares them in the AddTrackRequest's packet_trailer_features. - Beta: FrameMetadata, FrameMetadataProvider, LocalVideoTrack.frameMetadataProvider, RemoteVideoTrack.lookupFrameMetadata, TrackPublication.frameMetadataFeatures and FrameMetadataFeature. - E2EEManager's add/remove track methods are now internal. They no longer install the cryptor, so calling them directly would send unencrypted media. Publishing: - A stamping processor at the end of every SDK-created camera and screencast track's processor chain fills in the defaults (capture time as wall-clock microseconds, a per-track frame ID) and calls the provider, while the track is published with an option on. - FrameMetadataManager creates the send transformer and links it to the video source. Backup codec senders share it, and restartTrack relinks it to the new source. - restartTrack now also moves backup codec senders to the new track. Before, backup codec video stopped after a restart. Subscribing: - A track whose publisher declared features gets a receive transformer, which strips the trailer and keeps the values of the last 300 frames. A resubscribe disposes the previous subscription's transformer. - The client always advertises CAP_PACKET_TRAILER, so the server forwards trailers instead of stripping them. E2EE: FrameTransformerInstaller installs a track's cryptor and metadata transformer as one chain, in the right order for the direction. E2EEManager now creates cryptors without installing them. Threading: code that uses a transformer holds the lock its disposal waits on (the track's lock for send, FrameMetadataManager's for receive). An unpublish or participant leave on another thread therefore can't dispose it mid-use. E2EE cryptors aren't covered yet. ScaleCropVideoProcessor keeps a frame's metadata. Tests: FrameMetadataMockE2ETest (27) and FrameMetadataStampingTest (16). Verified on a device with instrumented tests covering VP8, H.264, VP9, AV1, E2EE, simulcast, a backup codec, restartTrack, republishing and reconnects. Interop with the JS and Rust SDKs tested by hand. Co-Authored-By: Claude Opus 5.5 --- .changeset/frame-metadata.md | 11 + .../detekt-baseline-release.xml | 4 +- .../io/livekit/android/dagger/RTCModule.kt | 7 + .../io/livekit/android/e2ee/E2EEManager.kt | 39 +- .../livekit/android/room/ClientCapability.kt | 5 + .../main/java/io/livekit/android/room/Room.kt | 61 +- .../framemetadata/FrameMetadataManager.kt | 182 +++++ .../FrameMetadataStampingProcessor.kt | 189 ++++++ .../framemetadata/FrameTransformerFactory.kt | 92 +++ .../FrameTransformerInstaller.kt | 67 ++ .../room/participant/LocalParticipant.kt | 26 + .../android/room/track/FrameMetadata.kt | 144 ++++ .../room/track/LocalScreencastVideoTrack.kt | 6 +- .../android/room/track/LocalVideoTrack.kt | 86 ++- .../android/room/track/RemoteVideoTrack.kt | 40 +- .../android/room/track/TrackPublication.kt | 14 +- .../track/video/ScaleCropVideoProcessor.kt | 4 +- .../test/mock/MockFrameTransformerFactory.kt | 80 +++ .../android/test/mock/MockVideoSource.kt | 22 + .../test/mock/dagger/TestLiveKitComponent.kt | 3 + .../android/test/mock/dagger/TestRTCModule.kt | 13 + .../org/webrtc/MockPeerConnectionFactory.kt | 9 +- .../android/room/ConnectionParamsTest.kt | 5 +- .../java/io/livekit/android/room/RoomTest.kt | 7 + .../framemetadata/FrameMetadataMockE2ETest.kt | 623 ++++++++++++++++++ .../FrameMetadataStampingTest.kt | 255 +++++++ 26 files changed, 1963 insertions(+), 31 deletions(-) create mode 100644 .changeset/frame-metadata.md create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt create mode 100644 livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt create mode 100644 livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt create mode 100644 livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt diff --git a/.changeset/frame-metadata.md b/.changeset/frame-metadata.md new file mode 100644 index 000000000..ff7586156 --- /dev/null +++ b/.changeset/frame-metadata.md @@ -0,0 +1,11 @@ +--- +"client-sdk-android": minor +--- + +Add frame metadata for video tracks. A publisher can send a capture timestamp, a frame ID and its own user data with each video frame, carried in a trailer after the encoded frame, and subscribers look them up by RTP timestamp. The format is the same as the JavaScript and Rust SDKs, so metadata works between them, with or without end-to-end encryption. + +To publish, set `frameMetadata` on `VideoTrackPublishOptions` or `VideoTrackPublishDefaults`. By default the SDK sends the frame's capture time as wall-clock time and a per-track frame ID; supply your own values with `LocalVideoTrack.frameMetadataProvider`, or on the frames your video processor emits. To read them, call `RemoteVideoTrack.lookupFrameMetadata(frame.rtpTimestamp)` from a video sink. `TrackPublication.frameMetadataFeatures` lists the fields a track's publisher declared, as `FrameMetadataFeature` values. `FrameMetadata`, `FrameMetadataFeature`, `FrameMetadataProvider`, `frameMetadataProvider`, `frameMetadataFeatures` and `lookupFrameMetadata` are in beta and need an opt-in to `@Beta`; the publish options don't, so `copy()` on the publish option classes works as before. User data isn't encrypted under end-to-end encryption, so the server can read it. + +`E2EEManager.addPublishedTrack`, `addSubscribedTrack`, `removePublishedTrack` and `removeSubscribedTrack` are now internal. The room calls them for you, and they no longer install the cryptor themselves, so an app calling them directly would have sent unencrypted media. + +`LocalVideoTrack.restartTrack` now moves backup codec senders to the new track as well. Before, backup codec video stopped after a restart. diff --git a/livekit-android-sdk/detekt-baseline-release.xml b/livekit-android-sdk/detekt-baseline-release.xml index c8efe8b1f..9bdd1aaf7 100644 --- a/livekit-android-sdk/detekt-baseline-release.xml +++ b/livekit-android-sdk/detekt-baseline-release.xml @@ -43,7 +43,7 @@ LongParameterList:KeyProvider.kt$BaseKeyProvider$( ratchetSalt: String = defaultRatchetSalt, uncryptedMagicBytes: String = defaultMagicBytes, ratchetWindowSize: Int = defaultRatchetWindowSize, override var enableSharedKey: Boolean = true, failureTolerance: Int = defaultFailureTolerance, keyRingSize: Int = defaultKeyRingSize, discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady, keyDerivationAlgorithm: FrameCryptorKeyDerivationAlgorithm = defaultKeyDerivationAlgorithm, ) LongParameterList:LiveKitOverrides.kt$AudioOptions$( /** * Override the default output [AudioType]. * * This affects the audio routing and how the audio is handled. Default is [AudioType.CallAudioType]. * * Note: if [audioHandler] is also passed, the values from [audioOutputType] will not be reflected in it, * and must be set yourself. */ val audioOutputType: AudioType? = null, /** * Override the default [AudioHandler]. * * Default is [AudioSwitchHandler]. * * Use [NoAudioHandler] to turn off automatic audio handling or * [AudioFocusHandler] to get simple audio focus handling. */ val audioHandler: AudioHandler? = null, /** * Override the default [AudioDeviceModule]. * * If a non-null value is passed, the library does not * take ownership of the object and will not release it upon [Room.release]. * It is the responsibility of the owner to call [AudioDeviceModule.release] when finished * with it to prevent memory leaks. */ val audioDeviceModule: AudioDeviceModule? = null, /** * Called after default setup to allow for customizations on the [JavaAudioDeviceModule]. * * Not used if [audioDeviceModule] is provided. * * Note: We require setting the [JavaAudioDeviceModule.Builder.setSamplesReadyCallback] to provide * support for [LocalAudioTrack.addSink]. If you wish to grab the audio samples * from the local microphone track, use [LocalAudioTrack.addSink] instead of setting your own * callback. */ val javaAudioDeviceModuleCustomizer: ((builder: JavaAudioDeviceModule.Builder) -> Unit)? = null, /** * On Android 11+, the audio mode will reset itself from [AudioManager.MODE_IN_COMMUNICATION] if * there is no audio playback or capture for 6 seconds (for example when joining a room with * no speakers and the local mic is muted.) This mode reset will cause unexpected * behavior when trying to change the volume, causing it to not properly change the volume. * * We use a workaround by playing a silent audio track to keep the communication mode from * resetting. * * Setting this flag to true will disable the workaround. * * This flag is a no-op when the audio mode is set to anything other than * [AudioManager.MODE_IN_COMMUNICATION]. */ val disableCommunicationModeWorkaround: Boolean = false, /** * Options for processing the mic and incoming audio. */ val audioProcessorOptions: AudioProcessorOptions? = null, /** * Devices may take some time initializing the audio stack for recording. * Prewarming allows starting up the underlying audio recording prior to publish, letting * the audio device be ready immediately when the track is fully published. * * If set to true, disables audio recording prewarming (and the related * [LocalAudioTrack.prewarm] function), and audio resources are only used while the * track is connected and published. Defaults to false. */ val disableAudioPrewarming: Boolean = false, ) LongParameterList:LocalAudioTrack.kt$LocalAudioTrack$( @Assisted name: String, @Assisted mediaTrack: livekit.org.webrtc.AudioTrack, @Assisted options: LocalAudioTrackOptions, private val audioProcessingController: AudioProcessingController, @Named(InjectionNames.DISPATCHER_DEFAULT) private val dispatcher: CoroutineDispatcher, @Named(InjectionNames.LOCAL_AUDIO_RECORD_SAMPLES_DISPATCHER) private val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher, @Named(InjectionNames.LOCAL_AUDIO_BUFFER_CALLBACK_DISPATCHER) private val audioBufferCallbackDispatcher: AudioBufferCallbackDispatcher, private val audioRecordPrewarmer: AudioRecordPrewarmer, rtcThreadToken: RTCThreadToken, ) - LongParameterList:LocalParticipant.kt$LocalParticipant$( @Assisted internal var dynacast: Boolean, internal val engine: RTCEngine, private val peerConnectionFactory: PeerConnectionFactory, private val context: Context, private val eglBase: EglBase, private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory, private val videoTrackFactory: LocalVideoTrack.Factory, private val audioTrackFactory: LocalAudioTrack.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) coroutineDispatcher: CoroutineDispatcher, @Named(InjectionNames.SENDER) private val capabilitiesGetter: CapabilitiesGetter, private val outgoingDataStreamManager: OutgoingDataStreamManager, private val outgoingDataTrackManager: OutgoingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, ) + LongParameterList:LocalParticipant.kt$LocalParticipant$( @Assisted internal var dynacast: Boolean, internal val engine: RTCEngine, private val peerConnectionFactory: PeerConnectionFactory, private val context: Context, private val eglBase: EglBase, private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory, private val videoTrackFactory: LocalVideoTrack.Factory, private val audioTrackFactory: LocalAudioTrack.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) coroutineDispatcher: CoroutineDispatcher, @Named(InjectionNames.SENDER) private val capabilitiesGetter: CapabilitiesGetter, private val outgoingDataStreamManager: OutgoingDataStreamManager, private val outgoingDataTrackManager: OutgoingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val frameMetadataManager: FrameMetadataManager, ) LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack$( @Assisted capturer: VideoCapturer, @Assisted source: VideoSource, @Assisted name: String, @Assisted options: LocalVideoTrackOptions, @Assisted rtcTrack: livekit.org.webrtc.VideoTrack, @Assisted mediaProjectionCallback: MediaProjectionCallback, peerConnectionFactory: PeerConnectionFactory, context: Context, eglBase: EglBase, defaultsManager: DefaultsManager, videoTrackFactory: LocalVideoTrack.Factory, rtcThreadToken: RTCThreadToken, ) LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack.Companion$( mediaProjectionPermissionResultData: Intent, peerConnectionFactory: PeerConnectionFactory, context: Context, name: String, options: LocalVideoTrackOptions, rootEglBase: EglBase, screencastVideoTrackFactory: Factory, videoProcessor: VideoProcessor?, onStop: (Track) -> Unit, ) LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack.Factory$( capturer: VideoCapturer, source: VideoSource, name: String, options: LocalVideoTrackOptions, rtcTrack: livekit.org.webrtc.VideoTrack, mediaProjectionCallback: MediaProjectionCallback, ) @@ -60,7 +60,7 @@ LongParameterList:RTCStatsExt.kt$( trackIdentifier: String, ssrcs: Set<Long?>, codecIds: Set<String?>, localCandidateId: String?, remoteCandidateId: String?, statsMap: Map<String, RTCStats>, ) LongParameterList:RemoteParticipant.kt$RemoteParticipant$( mediaTrack: MediaStreamTrack, sid: String, statsGetter: RTCStatsGetter, receiver: RtpReceiver, autoManageVideo: Boolean = false, triesLeft: Int = 20, ) LongParameterList:RemoteParticipant.kt$RemoteParticipant$( sid: Sid, identity: Identity? = null, internal val signalClient: SignalClient, private val ioDispatcher: CoroutineDispatcher, defaultDispatcher: CoroutineDispatcher, private val audioTrackFactory: RemoteAudioTrack.Factory, private val videoTrackFactory: RemoteVideoTrack.Factory, ) - LongParameterList:Room.kt$Room$( @Assisted private val context: Context, internal val engine: RTCEngine, private val eglBase: EglBase, localParticipantFactory: LocalParticipant.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) private val defaultDispatcher: CoroutineDispatcher, @Named(InjectionNames.DISPATCHER_IO) private val ioDispatcher: CoroutineDispatcher, /** * The [AudioHandler] for setting up the audio as need. * * By default, this is an instance of [AudioSwitchHandler]. * * This can be substituted for your own custom implementation through * [LiveKitOverrides.audioOptions] when creating the room with [LiveKit.create]. * * @see [audioSwitchHandler] * @see [AudioSwitchHandler] */ val audioHandler: AudioHandler, private val closeableManager: CloseableManager, private val e2EEManagerFactory: E2EEManager.Factory, private val communicationWorkaround: CommunicationWorkaround, val audioProcessingController: AudioProcessingController, /** * A holder for objects that are used internally within LiveKit. */ val lkObjects: LKObjects, networkCallbackManagerFactory: NetworkCallbackManagerFactory, private val audioDeviceModule: AudioDeviceModule, private val regionUrlProviderFactory: RegionUrlProvider.Factory, private val connectionWarmer: ConnectionWarmer, private val audioRecordPrewarmer: AudioRecordPrewarmer, private val incomingDataStreamManager: IncomingDataStreamManager, private val dataStreams: DataStreams, private val incomingDataTrackManager: IncomingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val remoteParticipantFactory: RemoteParticipant.Factory, ) + LongParameterList:Room.kt$Room$( @Assisted private val context: Context, internal val engine: RTCEngine, private val eglBase: EglBase, localParticipantFactory: LocalParticipant.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) private val defaultDispatcher: CoroutineDispatcher, @Named(InjectionNames.DISPATCHER_IO) private val ioDispatcher: CoroutineDispatcher, /** * The [AudioHandler] for setting up the audio as need. * * By default, this is an instance of [AudioSwitchHandler]. * * This can be substituted for your own custom implementation through * [LiveKitOverrides.audioOptions] when creating the room with [LiveKit.create]. * * @see [audioSwitchHandler] * @see [AudioSwitchHandler] */ val audioHandler: AudioHandler, private val closeableManager: CloseableManager, private val e2EEManagerFactory: E2EEManager.Factory, private val communicationWorkaround: CommunicationWorkaround, val audioProcessingController: AudioProcessingController, /** * A holder for objects that are used internally within LiveKit. */ val lkObjects: LKObjects, networkCallbackManagerFactory: NetworkCallbackManagerFactory, private val audioDeviceModule: AudioDeviceModule, private val regionUrlProviderFactory: RegionUrlProvider.Factory, private val connectionWarmer: ConnectionWarmer, private val audioRecordPrewarmer: AudioRecordPrewarmer, private val incomingDataStreamManager: IncomingDataStreamManager, private val dataStreams: DataStreams, private val incomingDataTrackManager: IncomingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val remoteParticipantFactory: RemoteParticipant.Factory, private val frameMetadataManager: FrameMetadataManager, private val frameTransformerInstaller: FrameTransformerInstaller, ) MapGetWithNotNullAssertionOperator:LocalParticipant.kt$LocalParticipant$sourcePubLocks[source]!! NestedBlockDepth:ByteStreamSender.kt$@CheckResult suspend fun ByteStreamSender.write(source: Source): Result<Unit> NestedBlockDepth:LocalParticipant.kt$LocalParticipant$@Throws(TrackException.PublishException::class) private suspend fun publishTrackImpl( track: Track, options: TrackPublishOptions, requestConfig: AddTrackRequest.Builder.() -> Unit, encodings: List<RtpParameters.Encoding> = emptyList(), publishListener: PublishListener? = null, ): LocalTrackPublication? diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt index 91e3ad0ff..86b574361 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt @@ -38,6 +38,8 @@ import io.livekit.android.e2ee.DataPacketCryptorManagerImpl import io.livekit.android.memory.CloseableManager import io.livekit.android.room.datatrack.LocalDataTrackManagerFactory import io.livekit.android.room.datatrack.RemoteDataTrackManagerFactory +import io.livekit.android.room.framemetadata.DefaultFrameTransformerFactory +import io.livekit.android.room.framemetadata.FrameTransformerFactory import io.livekit.android.util.LKLog import io.livekit.android.util.LoggingLevel import io.livekit.android.webrtc.CustomAudioProcessingFactory @@ -405,6 +407,11 @@ internal object RTCModule { return DataPacketCryptorManagerImpl.Factory } + @Provides + internal fun frameTransformerFactory(peerConnectionFactory: PeerConnectionFactory): FrameTransformerFactory { + return DefaultFrameTransformerFactory(peerConnectionFactory) + } + @Provides fun localDataTrackManagerFactory(): LocalDataTrackManagerFactory { return LocalDataTrackManagerFactory { delegate, encryptionProvider -> diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt b/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt index 40913db63..0fd14f811 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt @@ -22,6 +22,8 @@ import dagger.assisted.AssistedInject import io.livekit.android.annotations.Beta import io.livekit.android.events.RoomEvent import io.livekit.android.room.Room +import io.livekit.android.room.framemetadata.DefaultFrameTransformerFactory +import io.livekit.android.room.framemetadata.FrameTransformerFactory import io.livekit.android.room.participant.LocalParticipant import io.livekit.android.room.participant.Participant import io.livekit.android.room.participant.RemoteParticipant @@ -36,7 +38,6 @@ import livekit.LivekitModels.Encryption import livekit.org.webrtc.FrameCryptor import livekit.org.webrtc.FrameCryptor.FrameCryptionState import livekit.org.webrtc.FrameCryptorAlgorithm -import livekit.org.webrtc.FrameCryptorFactory import livekit.org.webrtc.PeerConnectionFactory import livekit.org.webrtc.RtpReceiver import livekit.org.webrtc.RtpSender @@ -55,6 +56,8 @@ constructor( internal var dataPacketCryptorManager: DataPacketCryptorManager = dataPacketCryptorManagerFactory.create(keyProvider) + internal var frameTransformerFactory: FrameTransformerFactory = DefaultFrameTransformerFactory(peerConnectionFactory) + var enabled: Boolean = false set(value) { field = value @@ -99,22 +102,29 @@ constructor( this.room?.localParticipant?.trackPublications?.forEach { item -> val participant = this.room!!.localParticipant val publication = item.value - if (publication.track != null) { - addPublishedTrack(publication.track!!, publication, participant, room) + val track = publication.track + if (track != null) { + val frameCryptor = addPublishedTrack(track, publication, participant, room) + room.installSendTransformers(track, frameCryptor) } } this.room?.remoteParticipants?.forEach { item -> val participant = item.value participant.trackPublications.forEach { item -> val publication = item.value - if (publication.track != null) { - addSubscribedTrack(publication.track!!, publication, participant, room) + val track = publication.track + if (track != null) { + val frameCryptor = addSubscribedTrack(track, publication, participant, room) + room.installReceiveTransformers(track, frameCryptor) } } } } - fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) { + /** + * Creates the cryptor for a subscribed track, without installing it. + */ + internal fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room): FrameCryptor { val rtpReceiver: RtpReceiver? = when (publication.track!!) { is RemoteAudioTrack -> (publication.track!! as RemoteAudioTrack).receiver is RemoteVideoTrack -> (publication.track!! as RemoteVideoTrack).receiver @@ -135,9 +145,10 @@ constructor( ), ) } + return frameCryptor } - fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) { + internal fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) { val trackId = publication.sid val participantId = participant.identity val frameCryptor = frameCryptors.get(trackId to participantId) @@ -148,7 +159,10 @@ constructor( } } - fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) { + /** + * Creates the cryptor for a published track, without installing it. + */ + internal fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room): FrameCryptor { val rtpSender: RtpSender = when (publication.track!!) { is LocalAudioTrack -> (publication.track!! as LocalAudioTrack).sender is LocalVideoTrack -> (publication.track!! as LocalVideoTrack).sender @@ -170,9 +184,10 @@ constructor( ), ) } + return frameCryptor } - fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) { + internal fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) { val trackId = publication.sid val participantId = participant.identity val frameCryptor = frameCryptors.get(trackId to participantId) @@ -197,8 +212,7 @@ constructor( } private fun addRtpSender(sender: RtpSender, participantId: Participant.Identity, trackId: String, kind: String): FrameCryptor { - val frameCryptor = FrameCryptorFactory.createFrameCryptorForRtpSender( - peerConnectionFactory, + val frameCryptor = frameTransformerFactory.createFrameCryptor( sender, participantId.value, algorithm, @@ -212,8 +226,7 @@ constructor( } private fun addRtpReceiver(receiver: RtpReceiver, participantId: Participant.Identity, trackId: String, kind: String): FrameCryptor { - val frameCryptor = FrameCryptorFactory.createFrameCryptorForRtpReceiver( - peerConnectionFactory, + val frameCryptor = frameTransformerFactory.createFrameCryptor( receiver, participantId.value, algorithm, diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt index e5f86daa2..cf2129746 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt @@ -76,8 +76,13 @@ enum class ClientCapability(val value: Int) { * compression is advertised unconditionally: deflate-raw is compressed and decompressed by the * Rust data stream core, so support does not vary by device, API level, or room options. * + * Packet trailers are also advertised unconditionally: the SDK handles the trailer itself on every + * subscribed video track whose publisher sends one. Without this, the server strips it first and + * frame metadata never arrives. + * * @suppress */ internal val ADVERTISED_CLIENT_CAPABILITIES = listOf( + ClientCapability.PACKET_TRAILER, ClientCapability.COMPRESSION_DEFLATE_RAW, ) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt index b4642f829..5a74be946 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt @@ -54,6 +54,8 @@ import io.livekit.android.room.datatrack.DataTrackSid import io.livekit.android.room.datatrack.IncomingDataTrackEvent import io.livekit.android.room.datatrack.IncomingDataTrackManager import io.livekit.android.room.datatrack.RemoteDataTrack +import io.livekit.android.room.framemetadata.FrameMetadataManager +import io.livekit.android.room.framemetadata.FrameTransformerInstaller import io.livekit.android.room.metrics.collectMetrics import io.livekit.android.room.network.NetworkCallbackManagerFactory import io.livekit.android.room.network.ReconnectPolicy @@ -76,10 +78,14 @@ import io.livekit.android.room.rpc.RPC_RESPONSE_DATA_STREAM_TOPIC import io.livekit.android.room.rpc.RpcClientManager import io.livekit.android.room.rpc.RpcManager import io.livekit.android.room.rpc.RpcServerManager +import io.livekit.android.room.track.LocalAudioTrack import io.livekit.android.room.track.LocalAudioTrackOptions import io.livekit.android.room.track.LocalTrackPublication +import io.livekit.android.room.track.LocalVideoTrack import io.livekit.android.room.track.LocalVideoTrackOptions +import io.livekit.android.room.track.RemoteAudioTrack import io.livekit.android.room.track.RemoteTrackPublication +import io.livekit.android.room.track.RemoteVideoTrack import io.livekit.android.room.track.Track import io.livekit.android.room.track.TrackPublication import io.livekit.android.room.types.toSDKType @@ -110,6 +116,7 @@ import kotlinx.serialization.Serializable import livekit.LivekitModels import livekit.LivekitRtc import livekit.org.webrtc.EglBase +import livekit.org.webrtc.FrameCryptor import livekit.org.webrtc.MediaStream import livekit.org.webrtc.MediaStreamTrack import livekit.org.webrtc.RTCStatsCollectorCallback @@ -165,6 +172,8 @@ constructor( private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val remoteParticipantFactory: RemoteParticipant.Factory, + private val frameMetadataManager: FrameMetadataManager, + private val frameTransformerInstaller: FrameTransformerInstaller, ) : RTCEngine.Listener, ParticipantListener, RpcManager, IncomingDataStreamManager by incomingDataStreamManager { private lateinit var coroutineScope: CoroutineScope @@ -515,6 +524,7 @@ constructor( if (roomOptions.e2eeOptions != null) { e2eeManager = e2EEManagerFactory.create(roomOptions.e2eeOptions.keyProvider).apply { + frameTransformerFactory = frameTransformerInstaller.factory setup(this@Room) { event -> coroutineScope.launch { emitWhenConnected(event) @@ -1126,6 +1136,7 @@ constructor( isRecording = false sidToIdentity.clear() incomingDataStreamManager.clearOpenStreams() + frameMetadataManager.cleanup() } private suspend fun sendSyncState() { @@ -1629,8 +1640,10 @@ constructor( * @suppress */ override fun onTrackPublished(publication: LocalTrackPublication, participant: LocalParticipant) { - if (e2eeManager != null) { - e2eeManager!!.addPublishedTrack(publication.track!!, publication, participant, this) + val track = publication.track + if (track != null) { + val frameCryptor = e2eeManager?.addPublishedTrack(track, publication, participant, this) + installSendTransformers(track, frameCryptor) } eventBus.postEvent(RoomEvent.TrackPublished(this, publication, participant), coroutineScope) } @@ -1640,6 +1653,7 @@ constructor( */ override fun onTrackUnpublished(publication: LocalTrackPublication, participant: LocalParticipant) { e2eeManager?.removePublishedTrack(publication.track!!, publication, participant, this) + publication.track?.let { frameMetadataManager.removePublishedTrack(it) } eventBus.postEvent(RoomEvent.TrackUnpublished(this, publication, participant), coroutineScope) } @@ -1647,12 +1661,48 @@ constructor( * @suppress */ override fun onTrackSubscribed(track: Track, publication: RemoteTrackPublication, participant: RemoteParticipant) { - if (e2eeManager != null) { - e2eeManager!!.addSubscribedTrack(track, publication, participant, this) - } + val frameCryptor = e2eeManager?.addSubscribedTrack(track, publication, participant, this) + frameMetadataManager.addSubscribedTrack(track, publication) + installReceiveTransformers(track, frameCryptor) eventBus.postEvent(RoomEvent.TrackSubscribed(this, track, publication, participant), coroutineScope) } + /** + * Installs a published track's cryptor and frame metadata transformer, whichever it has, on its + * sender in a single install. + */ + internal fun installSendTransformers(track: Track, frameCryptor: FrameCryptor?) { + val sender = when (track) { + is LocalVideoTrack -> track.sender + is LocalAudioTrack -> track.sender + else -> null + } ?: return + // Installs under the track's lock so an unpublish can't dispose the transformer in the + // meantime. The chain is kept after that lock is released: the manager's lock is taken + // before a track's, never after. setChain disposes the chain if the track was removed. + val chain = if (track is LocalVideoTrack) { + track.withFrameMetadataTransformer { frameTransformerInstaller.installSend(sender, frameCryptor, it) } + } else { + frameTransformerInstaller.installSend(sender, frameCryptor, null) + } + frameMetadataManager.setChain(track, chain) + } + + /** + * Installs a subscribed track's frame metadata transformer and cryptor, whichever it has, on its + * receiver in a single install. + */ + internal fun installReceiveTransformers(track: Track, frameCryptor: FrameCryptor?) { + val receiver = when (track) { + is RemoteVideoTrack -> track.receiver + is RemoteAudioTrack -> track.receiver + else -> null + } ?: return + frameMetadataManager.installReceive(track) { metadataTransformer -> + frameTransformerInstaller.installReceive(receiver, metadataTransformer, frameCryptor) + } + } + /** * @suppress */ @@ -1673,6 +1723,7 @@ constructor( participant: RemoteParticipant, ) { e2eeManager?.removeSubscribedTrack(track, publication, participant, this) + frameMetadataManager.removeSubscribedTrack(track) eventBus.postEvent(RoomEvent.TrackUnsubscribed(this, track, publication, participant), coroutineScope) } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt new file mode 100644 index 000000000..b23b23b54 --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt @@ -0,0 +1,182 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import io.livekit.android.room.track.FrameMetadataPublishOptions +import io.livekit.android.room.track.LocalVideoTrack +import io.livekit.android.room.track.RemoteVideoTrack +import io.livekit.android.room.track.Track +import io.livekit.android.room.track.TrackPublication +import io.livekit.android.room.track.hasFeatures +import io.livekit.android.util.LKLog +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.FrameTransformerChain +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Owns the frame metadata transformers of a room's video tracks, and any chain built around one. + * + * It creates and disposes them; [FrameTransformerInstaller] installs them. Only video tracks get + * one: a published track when its options turn on a field, a subscribed track when its publisher + * sends metadata. + * + * Anything that uses a transformer holds the lock its disposal waits on: this manager's lock for + * receive transformers, the track's for send transformers (see + * [LocalVideoTrack.withFrameMetadataTransformer]). This manager's lock is taken before a track's, + * never after. + * + * @suppress + */ +@Singleton +class FrameMetadataManager +@Inject +internal constructor( + private val factory: FrameTransformerFactory, +) { + private class Entry(val transformer: FrameMetadataTransformer) { + var chain: FrameTransformerChain? = null + } + + private val lock = Any() + private val published = mutableMapOf() + private val subscribed = mutableMapOf() + + /** + * Sets up a track that's about to be published: creates its send transformer, links it to the + * track's source and starts stamping. Call before the publication becomes visible, so a backup + * codec sender created right after can find the transformer. + */ + internal fun addPublishedTrack(track: LocalVideoTrack, options: FrameMetadataPublishOptions?) { + if (options == null || !options.hasFeatures()) { + return + } + synchronized(lock) { + if (published.containsKey(track)) { + return + } + val transformer = factory.createFrameMetadataTransformer(FrameMetadataTransformer.Direction.SEND) + track.attachFrameMetadataTransformer(transformer) + track.frameMetadataStampState.arm(options) + if (track.frameMetadataStamper == null) { + LKLog.w { "Track ${track.name} has no frame metadata stamping processor; only values set on its frames will be sent." } + } + published[track] = Entry(transformer) + } + } + + internal fun removePublishedTrack(track: Track) { + val entry = synchronized(lock) { published.remove(track) } ?: return + val localTrack = track as LocalVideoTrack + localTrack.frameMetadataStampState.disarm() + localTrack.detachFrameMetadataTransformer() + entry.chain?.dispose() + entry.transformer.dispose() + } + + /** + * Creates the receive transformer for a subscribed track if its publisher sends metadata, and + * hands it to the track for lookups. + * + * @return the transformer to install, or null if the track doesn't get one. + */ + internal fun addSubscribedTrack(track: Track, publication: TrackPublication): FrameMetadataTransformer? { + if (track !is RemoteVideoTrack) { + return null + } + if (publication.trackInfo?.packetTrailerFeaturesList.isNullOrEmpty()) { + return null + } + // The client isn't told when the server stops sending a track, so a resubscribe arrives + // with the earlier subscription's track still here. Every subscribe builds a new track + // object, so that track is found by sid. This also gives a track added twice a fresh + // transformer. + val previousTracks = synchronized(lock) { subscribed.keys.filter { it.sid == publication.sid } } + previousTracks.forEach { removeSubscribedTrack(it) } + val transformer = factory.createFrameMetadataTransformer(FrameMetadataTransformer.Direction.RECEIVE) + synchronized(lock) { + subscribed[track] = Entry(transformer) + } + track.setFrameMetadataTransformer(transformer) + return transformer + } + + /** + * Runs [install] with [track]'s receive transformer, or null if it has none, and keeps the + * chain it returns as [setChain] does. + * + * It all runs under the lock, so [removeSubscribedTrack] can't dispose the transformer while + * it's being installed: either the track was removed first and [install] gets null, or the + * removal waits until the chain is kept and then disposes it too. So [install] must not wait + * on the RTC thread or call app code. + */ + internal fun installReceive(track: Track, install: (FrameMetadataTransformer?) -> FrameTransformerChain?) { + val unused = synchronized(lock) { + val entry = (track as? RemoteVideoTrack)?.let { subscribed[it] } + keepChain(entry, install(entry?.transformer)) + } + unused?.dispose() + } + + internal fun removeSubscribedTrack(track: Track) { + val entry = synchronized(lock) { subscribed.remove(track) } ?: return + // Stops lookups before disposing. A disabled receive transformer stops stripping trailers, + // so it's disposed without being disabled first. + (track as RemoteVideoTrack).setFrameMetadataTransformer(null) + entry.chain?.dispose() + entry.transformer.dispose() + } + + /** + * Keeps the chain installed on [track]'s sender or receiver so it's disposed with the track's + * transformer. Replaces and disposes a previous chain. A track without a metadata transformer + * never has a chain, and one that was removed in the meantime has its chain disposed here. + */ + internal fun setChain(track: Track, chain: FrameTransformerChain?) { + val unused = synchronized(lock) { + val entry = when (track) { + is LocalVideoTrack -> published[track] + is RemoteVideoTrack -> subscribed[track] + else -> null + } + keepChain(entry, chain) + } + unused?.dispose() + } + + /** + * Keeps [chain] on [entry], and returns the chain to dispose: [chain] itself if the track has + * no entry, or the chain it replaces. Call it under the lock, in the same block that found + * [entry], so a removal can't take the entry and read its chain in between. + */ + private fun keepChain(entry: Entry?, chain: FrameTransformerChain?): FrameTransformerChain? { + if (entry == null) { + return chain + } + val previous = entry.chain + entry.chain = chain + return previous?.takeIf { it !== chain } + } + + /** Disposes everything left, for a room that's disconnecting. */ + internal fun cleanup() { + val publishedTracks = synchronized(lock) { published.keys.toList() } + val subscribedTracks = synchronized(lock) { subscribed.keys.toList() } + publishedTracks.forEach { removePublishedTrack(it) } + subscribedTracks.forEach { removeSubscribedTrack(it) } + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt new file mode 100644 index 000000000..05e7ae9cd --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt @@ -0,0 +1,189 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import io.livekit.android.room.track.FrameMetadata +import io.livekit.android.room.track.FrameMetadataProvider +import io.livekit.android.room.track.FrameMetadataPublishOptions +import io.livekit.android.room.track.toRtc +import io.livekit.android.room.track.toSdk +import io.livekit.android.util.LKLog +import livekit.org.webrtc.VideoFrame +import livekit.org.webrtc.VideoProcessor +import livekit.org.webrtc.VideoSink + +/** + * The last step of a local video track's processing chain. It wraps whatever processor the track + * was created with, so it sees each frame after every other processor, and sets the frame metadata + * from [state] on it. + * + * It's installed when the track is created and does nothing until [state] is armed at publish. + */ +internal class FrameMetadataStampingProcessor( + private val inner: VideoProcessor?, +) : VideoProcessor { + + @Volatile + var state = FrameMetadataStampState() + + @Volatile + private var sink: VideoSink? = null + + private val stampingSink = VideoSink { frame -> deliver(frame) } + + override fun setSink(sink: VideoSink?) { + this.sink = sink + inner?.setSink(if (sink != null) stampingSink else null) + } + + override fun onCapturerStarted(success: Boolean) { + inner?.onCapturerStarted(success) + } + + override fun onCapturerStopped() { + inner?.onCapturerStopped() + } + + // Forwards the two-argument version, which some processors override (see NoDropVideoProcessor). + override fun onFrameCaptured(frame: VideoFrame, parameters: VideoProcessor.FrameAdaptationParameters) { + val inner = inner + if (inner != null) { + inner.onFrameCaptured(frame, parameters) + return + } + val adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters) ?: return + deliver(adaptedFrame) + adaptedFrame.release() + } + + override fun onFrameCaptured(frame: VideoFrame) { + val inner = inner + if (inner != null) { + inner.onFrameCaptured(frame) + } else { + deliver(frame) + } + } + + private fun deliver(frame: VideoFrame) { + val sink = this.sink ?: return + // A new frame shares the buffer without retaining it, so only the original is released. + sink.onFrame(state.stamp(frame)) + } +} + +/** + * The frame metadata settings of one local video track: whether it's armed, the frame ID counter + * and the app's [FrameMetadataProvider]. It belongs to the track, so it carries over when the + * track's source and processor are replaced. + */ +internal class FrameMetadataStampState( + // WebRTC's clock, which is CLOCK_MONOTONIC on Android, the same as System.nanoTime(). Not + // TimestampAligner.getRtcTimeNanos(): it creates a native Environment on every call and never + // frees it. + private val rtcTimeNanos: () -> Long = { System.nanoTime() }, + private val wallTimeMillis: () -> Long = { System.currentTimeMillis() }, + initialFrameId: Long = 1L, +) { + @Volatile + var provider: FrameMetadataProvider? = null + + @Volatile + private var options: FrameMetadataPublishOptions? = null + + private val frameIdLock = Any() + private var nextFrameId = initialFrameId + + @Volatile + private var loggedProviderError = false + + val isArmed: Boolean + get() = options != null + + fun arm(options: FrameMetadataPublishOptions) { + this.options = options + } + + fun disarm() { + options = null + } + + /** + * Returns [frame] if there's nothing to change, or a new frame with the metadata to send. + * + * Starts from any metadata already on the frame, fills fields that are 0 with the defaults + * turned on in the options, then lets the provider replace the result. + */ + fun stamp(frame: VideoFrame): VideoFrame { + val options = this.options ?: return frame + val existing = frame.frameMetadata?.toSdk() + val withDefaults = fillDefaults(frame, options, existing ?: FrameMetadata()) + return frameWith(frame, existing, applyProvider(frame, withDefaults)) + } + + private fun fillDefaults(frame: VideoFrame, options: FrameMetadataPublishOptions, metadata: FrameMetadata): FrameMetadata { + val timestampUs = if (options.timestamp && metadata.userTimestampUs == 0L) { + defaultTimestampUs(frame) + } else { + metadata.userTimestampUs + } + val frameId = if (options.frameId && metadata.frameId == 0L) takeFrameId() else metadata.frameId + return metadata.copy(userTimestampUs = timestampUs, frameId = frameId) + } + + private fun applyProvider(frame: VideoFrame, current: FrameMetadata): FrameMetadata { + val provider = this.provider ?: return current + return try { + provider.provide(frame, current) ?: current + } catch (e: Exception) { + if (!loggedProviderError) { + loggedProviderError = true + LKLog.w(e) { "FrameMetadataProvider threw; sending the values it was given. Logged once." } + } + current + } + } + + private fun frameWith(frame: VideoFrame, existing: FrameMetadata?, result: FrameMetadata): VideoFrame { + val isEmpty = result.userTimestampUs == 0L && result.frameId == 0L && result.userData?.isEmpty() != false + return when { + isEmpty && existing == null -> frame + // Drop the metadata a processor set, since nothing is left to send. + isEmpty -> VideoFrame(frame.buffer, frame.rotation, frame.timestampNs) + result == existing -> frame + else -> VideoFrame(frame.buffer, frame.rotation, frame.timestampNs, result.toRtc()) + } + } + + // The frame's capture time, moved from WebRTC's monotonic clock to wall-clock time. + private fun defaultTimestampUs(frame: VideoFrame): Long { + val offsetNanos = wallTimeMillis() * 1_000_000L - rtcTimeNanos() + val timestampUs = (frame.timestampNs + offsetNanos) / 1_000L + return if (timestampUs > 0) timestampUs else 0L + } + + // A uint32 that starts at 1 and wraps back to 1, since 0 means "not sent". + private fun takeFrameId(): Long = synchronized(frameIdLock) { + val id = nextFrameId + nextFrameId = if (id >= MAX_FRAME_ID) 1L else id + 1 + id + } + + companion object { + const val MAX_FRAME_ID = 0xFFFFFFFFL + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt new file mode 100644 index 000000000..0be9a784f --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import livekit.org.webrtc.FrameCryptor +import livekit.org.webrtc.FrameCryptorAlgorithm +import livekit.org.webrtc.FrameCryptorFactory +import livekit.org.webrtc.FrameCryptorKeyProvider +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.FrameMetadataTransformerFactory +import livekit.org.webrtc.FrameTransformer +import livekit.org.webrtc.FrameTransformerChain +import livekit.org.webrtc.PeerConnectionFactory +import livekit.org.webrtc.RtpReceiver +import livekit.org.webrtc.RtpSender + +/** + * Creates the frame transformers that go on senders and receivers. None of them are installed on + * creation; [FrameTransformerInstaller] does that. + * + * Behind an interface so tests can replace the native objects. + */ +internal interface FrameTransformerFactory { + fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer + + /** Runs each frame through [first], then [second]. */ + fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain + + fun createFrameCryptor( + sender: RtpSender, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor + + fun createFrameCryptor( + receiver: RtpReceiver, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor +} + +internal class DefaultFrameTransformerFactory( + private val peerConnectionFactory: PeerConnectionFactory, +) : FrameTransformerFactory { + override fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer = + FrameMetadataTransformerFactory.createFrameMetadataTransformer(direction) + + override fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain = + FrameTransformerChain(first, second) + + override fun createFrameCryptor( + sender: RtpSender, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor = FrameCryptorFactory.createUninstalledFrameCryptor( + peerConnectionFactory, + sender, + participantId, + algorithm, + keyProvider, + ) + + override fun createFrameCryptor( + receiver: RtpReceiver, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor = FrameCryptorFactory.createUninstalledFrameCryptor( + peerConnectionFactory, + receiver, + participantId, + algorithm, + keyProvider, + ) +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt new file mode 100644 index 000000000..b52e20e1b --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt @@ -0,0 +1,67 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import livekit.org.webrtc.FrameCryptor +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.FrameTransformer +import livekit.org.webrtc.FrameTransformerChain +import livekit.org.webrtc.RtpReceiver +import livekit.org.webrtc.RtpSender +import javax.inject.Inject + +/** + * Installs a track's transformers on its sender or receiver in a single install. + * + * With both a cryptor and a metadata transformer, it chains them in the order for the direction: + * a sender encrypts and then adds the trailer, so the trailer stays readable; a receiver strips the + * trailer and then decrypts. With only one, it installs that one directly. + * + * It holds no state. A chain it creates is returned so the caller can dispose it. + * + * @suppress + */ +class FrameTransformerInstaller +@Inject +internal constructor( + internal val factory: FrameTransformerFactory, +) { + internal fun installSend( + sender: RtpSender, + cryptor: FrameCryptor?, + metadataTransformer: FrameMetadataTransformer?, + ): FrameTransformerChain? = install(first = cryptor, second = metadataTransformer) { sender.setFrameTransformer(it) } + + internal fun installReceive( + receiver: RtpReceiver, + metadataTransformer: FrameMetadataTransformer?, + cryptor: FrameCryptor?, + ): FrameTransformerChain? = install(first = metadataTransformer, second = cryptor) { receiver.setFrameTransformer(it) } + + private inline fun install( + first: FrameTransformer?, + second: FrameTransformer?, + setTransformer: (FrameTransformer) -> Unit, + ): FrameTransformerChain? { + if (first != null && second != null) { + return factory.createChain(first, second).also { setTransformer(it) } + } + val single = first ?: second ?: return null + setTransformer(single) + return null + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 714f28d23..38408e678 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -44,11 +44,13 @@ import io.livekit.android.room.datatrack.DataTrackSchemaException import io.livekit.android.room.datatrack.DataTrackSchemaId import io.livekit.android.room.datatrack.LocalDataTrack import io.livekit.android.room.datatrack.OutgoingDataTrackManager +import io.livekit.android.room.framemetadata.FrameMetadataManager import io.livekit.android.room.isSVCCodec import io.livekit.android.room.rpc.RpcClientManager import io.livekit.android.room.rpc.RpcManager import io.livekit.android.room.rpc.RpcServerManager import io.livekit.android.room.track.DataPublishReliability +import io.livekit.android.room.track.FrameMetadataPublishOptions import io.livekit.android.room.track.LocalAudioTrack import io.livekit.android.room.track.LocalAudioTrackOptions import io.livekit.android.room.track.LocalScreencastVideoTrack @@ -63,6 +65,7 @@ import io.livekit.android.room.track.VideoCodec import io.livekit.android.room.track.VideoEncoding import io.livekit.android.room.track.VideoPreset import io.livekit.android.room.track.screencapture.ScreenCaptureParams +import io.livekit.android.room.track.toPacketTrailerFeatures import io.livekit.android.room.util.EncodingUtils import io.livekit.android.rpc.RpcError import io.livekit.android.util.LKLog @@ -126,6 +129,7 @@ internal constructor( private val outgoingDataTrackManager: OutgoingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, + private val frameMetadataManager: FrameMetadataManager, ) : Participant(Sid(""), null, coroutineDispatcher), OutgoingDataStreamManager by outgoingDataStreamManager, RpcManager { @@ -621,6 +625,7 @@ internal constructor( }, ) } + options.frameMetadata?.let { addAllPacketTrailerFeatures(it.toPacketTrailerFeatures()) } }, encodings = encodings, publishListener = publishListener, @@ -856,6 +861,11 @@ internal constructor( participant = this, options = options, ) + if (track is LocalVideoTrack) { + // Before the publication is visible, so a backup codec sender created after it + // finds the transformer. + frameMetadataManager.addPublishedTrack(track, (options as? VideoTrackPublishOptions)?.frameMetadata) + } addTrackPublication(publication) LKLog.v { "add track publication $publication" } @@ -1439,6 +1449,12 @@ internal constructor( return@launch } simulcastTrack.transceiver = transceiver + // The backup codec sender shares the primary's metadata transformer: the source and the + // frames are the same. No send stream exists yet, so this doesn't recreate one. It's + // installed under the track's lock so an unpublish can't dispose it in the meantime. + track.withFrameMetadataTransformer { transformer -> + transformer?.let { transceiver.sender.setFrameTransformer(it) } + } val trackRequest = AddTrackRequest.newBuilder().apply { sid = existingPublication.sid muted = !track.enabled @@ -1733,6 +1749,13 @@ abstract class BaseVideoTrackPublishOptions { * When set to null, it defaults to H180 and H360. */ abstract val simulcastLayers: List? + + /** + * Frame metadata to send with each frame, such as a capture timestamp and a frame ID. + * Off when null. See [FrameMetadataPublishOptions]. + */ + open val frameMetadata: FrameMetadataPublishOptions? + get() = null } // Remember when adding any defaults to add it in the copy constructor of VideoTrackPublishOptions. @@ -1744,6 +1767,7 @@ data class VideoTrackPublishDefaults( override val backupCodec: BackupVideoCodec? = null, override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, + override val frameMetadata: FrameMetadataPublishOptions? = null, ) : BaseVideoTrackPublishOptions() data class VideoTrackPublishOptions( @@ -1757,6 +1781,7 @@ data class VideoTrackPublishOptions( override val stream: String? = null, override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, + override val frameMetadata: FrameMetadataPublishOptions? = null, ) : BaseVideoTrackPublishOptions(), TrackPublishOptions { constructor( name: String? = null, @@ -1774,6 +1799,7 @@ data class VideoTrackPublishOptions( stream = stream, degradationPreference = base.degradationPreference, simulcastLayers = base.simulcastLayers, + frameMetadata = base.frameMetadata, ) fun createBackupOptions(): VideoTrackPublishOptions? { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt new file mode 100644 index 000000000..db2d3c9d7 --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import io.livekit.android.annotations.Beta +import livekit.LivekitModels.PacketTrailerFeature +import livekit.org.webrtc.VideoFrame +import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata + +/** + * Metadata sent along with a single video frame. + * + * A field set to 0, or a null or empty [userData], isn't sent. + * + * @property userTimestampUs A timestamp in microseconds. By default, the publisher's capture time + * as wall-clock time. See [FrameMetadataPublishOptions.timestamp]. + * @property frameId A frame ID from 0 to 4294967295. By default, a per-track counter. + * See [FrameMetadataPublishOptions.frameId]. + * @property userData Bytes supplied by the publishing app. See [FrameMetadataPublishOptions.userData]. + */ +@Beta +data class FrameMetadata( + val userTimestampUs: Long = 0, + val frameId: Long = 0, + val userData: ByteArray? = null, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is FrameMetadata) return false + return userTimestampUs == other.userTimestampUs && + frameId == other.frameId && + userData.contentEquals(other.userData) + } + + override fun hashCode(): Int { + var result = userTimestampUs.hashCode() + result = 31 * result + frameId.hashCode() + result = 31 * result + (userData?.contentHashCode() ?: 0) + return result + } +} + +/** + * Declares which frame metadata to send with a published video track. Subscribers read it with + * [RemoteVideoTrack.lookupFrameMetadata]. + * + * Each option turns on the associated [FrameMetadata] field, which the SDK fills in by default. + * To send your own values, use [LocalVideoTrack.frameMetadataProvider]. They're sent even for a field whose option is off, as + * long as at least one option is on. + * + * @property timestamp Send a timestamp with each frame. By default it's the frame's capture time, + * in wall-clock microseconds. If your video processor changes frame timestamps, supply your own. + * @property frameId Send a frame ID with each frame. By default it's a per-track counter starting + * at 1. IDs skip frames that are dropped, and can repeat when a subscriber switches simulcast + * layers. + * @property userData Send user data, which you supply. **It isn't encrypted, even with end-to-end + * encryption.** Each frame can carry up to 232 bytes, or 248 with the other options off. User data + * that doesn't fit, or that would form an H.264 or H.265 start code, is dropped from that frame. + */ +data class FrameMetadataPublishOptions( + val timestamp: Boolean = false, + val frameId: Boolean = false, + val userData: Boolean = false, +) + +/** + * Supplies frame metadata for a local video track, through [LocalVideoTrack.frameMetadataProvider]. + * + * It's called for every frame on the video capture path, so should be kept fast and non-blocking. + */ +@Beta +fun interface FrameMetadataProvider { + /** + * @param frame The frame about to be encoded. + * @param current The values so far: any metadata a video processor set on the frame, with the + * defaults turned on in [FrameMetadataPublishOptions] filled into fields that are 0. + * @return New values to send instead of [current], or null to send [current]. A field set to + * 0 isn't sent. If this throws, [current] is sent. + */ + fun provide(frame: VideoFrame, current: FrameMetadata): FrameMetadata? +} + +/** + * A frame metadata field that a publisher declared for a video track, so that subscribers know to + * expect it. See [TrackPublication.frameMetadataFeatures]. + */ +@Beta +enum class FrameMetadataFeature { + /** Frames carry a timestamp. See [FrameMetadata.userTimestampUs]. */ + TIMESTAMP, + + /** Frames carry a frame ID. See [FrameMetadata.frameId]. */ + FRAME_ID, + + /** Frames carry user data. See [FrameMetadata.userData]. */ + USER_DATA, + ; + + internal fun toProto(): PacketTrailerFeature = when (this) { + TIMESTAMP -> PacketTrailerFeature.PTF_USER_TIMESTAMP + FRAME_ID -> PacketTrailerFeature.PTF_FRAME_ID + USER_DATA -> PacketTrailerFeature.PTF_USER_DATA + } + + internal companion object { + /** + * Returns null for a value this SDK build doesn't know, so a publisher declaring a newer + * feature isn't an error. + */ + fun fromProto(feature: PacketTrailerFeature): FrameMetadataFeature? = when (feature) { + PacketTrailerFeature.PTF_USER_TIMESTAMP -> TIMESTAMP + PacketTrailerFeature.PTF_FRAME_ID -> FRAME_ID + PacketTrailerFeature.PTF_USER_DATA -> USER_DATA + PacketTrailerFeature.UNRECOGNIZED -> null + } + } +} + +internal fun FrameMetadataPublishOptions?.hasFeatures(): Boolean = + this != null && (timestamp || frameId || userData) + +internal fun FrameMetadataPublishOptions.toPacketTrailerFeatures(): List = buildList { + if (timestamp) add(FrameMetadataFeature.TIMESTAMP) + if (frameId) add(FrameMetadataFeature.FRAME_ID) + if (userData) add(FrameMetadataFeature.USER_DATA) +}.map { it.toProto() } + +internal fun FrameMetadata.toRtc(): RtcFrameMetadata = RtcFrameMetadata(userTimestampUs, frameId, userData) + +internal fun RtcFrameMetadata.toSdk(): FrameMetadata = FrameMetadata(userTimestampUs, frameId, userData) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt index 563c7dc5e..d9e9d7d2b 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt @@ -30,6 +30,7 @@ import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject import io.livekit.android.memory.SurfaceTextureHelperCloser import io.livekit.android.room.DefaultsManager +import io.livekit.android.room.framemetadata.FrameMetadataStampingProcessor import io.livekit.android.room.participant.LocalParticipant import io.livekit.android.room.track.screencapture.ScreenCaptureConnection import io.livekit.android.room.track.screencapture.ScreenCaptureService @@ -247,7 +248,9 @@ constructor( onStop: (Track) -> Unit, ): LocalScreencastVideoTrack { val source = peerConnectionFactory.createVideoSource(options.isScreencast) - source.setVideoProcessor(videoProcessor) + // Installed now and armed at publish, so publishing doesn't have to swap processors. + val stampingProcessor = FrameMetadataStampingProcessor(videoProcessor) + source.setVideoProcessor(stampingProcessor) val callback = MediaProjectionCallback().apply { addOnStopCallback(onStop) } @@ -268,6 +271,7 @@ constructor( rtcTrack = rtcTrack, mediaProjectionCallback = callback, ) + track.frameMetadataStamper = stampingProcessor // The track owns the SurfaceTextureHelper; dispose() releases its capture thread. track.closeableManager.registerResource( rtcTrack, diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt index 75f93ea84..05bc78eb0 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt @@ -23,9 +23,12 @@ import androidx.core.content.ContextCompat import dagger.assisted.Assisted import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject +import io.livekit.android.annotations.Beta import io.livekit.android.memory.CloseableManager import io.livekit.android.memory.SurfaceTextureHelperCloser import io.livekit.android.room.DefaultsManager +import io.livekit.android.room.framemetadata.FrameMetadataStampState +import io.livekit.android.room.framemetadata.FrameMetadataStampingProcessor import io.livekit.android.room.track.video.CameraCapturerUtils import io.livekit.android.room.track.video.CameraCapturerUtils.CameraDeviceInfo import io.livekit.android.room.track.video.CameraCapturerUtils.createCameraEnumerator @@ -45,6 +48,7 @@ import livekit.LivekitRtc.SubscribedCodec import livekit.org.webrtc.CameraVideoCapturer import livekit.org.webrtc.CameraVideoCapturer.CameraEventsHandler import livekit.org.webrtc.EglBase +import livekit.org.webrtc.FrameMetadataTransformer import livekit.org.webrtc.MediaStreamTrack import livekit.org.webrtc.PeerConnectionFactory import livekit.org.webrtc.RtpParameters @@ -122,6 +126,68 @@ constructor( internal val closeableManager = CloseableManager() + /** + * This track's frame metadata settings. They belong to the track rather than its stamping + * processor, so they carry over when [restartTrack] replaces the source and processor. + */ + internal val frameMetadataStampState = FrameMetadataStampState() + + /** + * The processor at the end of this track's chain that sets frame metadata on its frames. + * Null if the track wasn't created by the SDK's track factories. + */ + internal var frameMetadataStamper: FrameMetadataStampingProcessor? = null + set(value) { + value?.state = frameMetadataStampState + field = value + } + + private val frameMetadataLock = Any() + + /** + * The send transformer linked to this track's source while it's published with frame metadata. + * Backup codec senders reuse it. Use it only through [withFrameMetadataTransformer]. + */ + private var frameMetadataTransformer: FrameMetadataTransformer? = null + + /** + * Supplies this track's frame metadata in place of, or on top of, the defaults turned on in + * [VideoTrackPublishOptions.frameMetadata]. See [FrameMetadataProvider]. + * + * It's only called while the track is published with frame metadata on. + */ + @Beta + var frameMetadataProvider: FrameMetadataProvider? + get() = frameMetadataStampState.provider + set(value) { + frameMetadataStampState.provider = value + } + + /** + * Runs [action] with this track's send transformer, or null if it has none. + * + * It holds the lock that [detachFrameMetadataTransformer] takes, and the transformer is only + * disposed after it's detached, so it can't be disposed while [action] uses it. So [action] + * must not wait on the RTC thread, take the frame metadata manager's lock, or call app code. + */ + internal fun withFrameMetadataTransformer(action: (FrameMetadataTransformer?) -> T): T = + synchronized(frameMetadataLock) { action(frameMetadataTransformer) } + + internal fun attachFrameMetadataTransformer(transformer: FrameMetadataTransformer) { + synchronized(frameMetadataLock) { + frameMetadataTransformer = transformer + source.addFrameMetadataTransformer(transformer) + } + } + + internal fun detachFrameMetadataTransformer() { + synchronized(frameMetadataLock) { + val transformer = frameMetadataTransformer ?: return + frameMetadataTransformer = null + source.removeFrameMetadataTransformer(transformer) + } + } + /** * Starts the [capturer] with the capture params contained in [options]. */ @@ -280,6 +346,7 @@ constructor( oldCapturer.stopCapture() oldCapturer.dispose() + withFrameMetadataTransformer { transformer -> transformer?.let { oldSource.removeFrameMetadataTransformer(it) } } oldSource.dispose() oldRtcTrack.setEnabled(false) @@ -312,11 +379,23 @@ constructor( } capturer = newTrack.capturer - source = newTrack.source rtcTrack = newTrack.rtcTrack this.options = options + // Keeps this track's frame metadata settings and frame ID counter on the new processor. + frameMetadataStamper = newTrack.frameMetadataStamper + // The source changes under the same lock as the link, so a detach running now unlinks the + // transformer from whichever source has it, and can't dispose it while it's being linked. + synchronized(frameMetadataLock) { + source = newTrack.source + frameMetadataTransformer?.let { source.addFrameMetadataTransformer(it) } + } startCapture() sender?.setTrack(newTrack.rtcTrack, false) + // Backup codec senders send the same track, so they move to the new one too. + for (simulcastTrackInfo in simulcastCodecs.values) { + simulcastTrackInfo.rtcTrack = newTrack.rtcTrack + (simulcastTrackInfo.transceiver?.sender ?: simulcastTrackInfo.sender)?.setTrack(newTrack.rtcTrack, false) + } } internal fun setPublishingLayers( @@ -536,7 +615,9 @@ constructor( } else { videoProcessor } - source.setVideoProcessor(finalVideoProcessor) + // Installed now and armed at publish, so publishing doesn't have to swap processors. + val stampingProcessor = FrameMetadataStampingProcessor(finalVideoProcessor) + source.setVideoProcessor(stampingProcessor) val surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureThread", rootEglBase.eglBaseContext) @@ -564,6 +645,7 @@ constructor( rtcTrack = rtcTrack, dispatchObserver = dispatchObserver, ) + track.frameMetadataStamper = stampingProcessor track.closeableManager.registerResource( rtcTrack, diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt index 4c1179842..80bdbcbd4 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import androidx.annotation.VisibleForTesting import dagger.assisted.Assisted import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject +import io.livekit.android.annotations.Beta import io.livekit.android.dagger.InjectionNames import io.livekit.android.events.TrackEvent import io.livekit.android.room.track.video.VideoSinkVisibility @@ -32,6 +33,7 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch +import livekit.org.webrtc.FrameMetadataTransformer import livekit.org.webrtc.RtpReceiver import livekit.org.webrtc.VideoSink import javax.inject.Named @@ -74,6 +76,42 @@ constructor( this.receiver = receiver } + // Guards the transformer, so a lookup can't use it while it's being disposed. + private val frameMetadataLock = Any() + private var frameMetadataTransformer: FrameMetadataTransformer? = null + + internal fun setFrameMetadataTransformer(transformer: FrameMetadataTransformer?) { + synchronized(frameMetadataLock) { + frameMetadataTransformer = transformer + } + } + + /** + * Looks up the frame metadata the publisher sent with a frame of this track. + * + * Get the RTP timestamp from the frame in your video sink, with + * [VideoFrame.getRtpTimestamp][livekit.org.webrtc.VideoFrame.getRtpTimestamp]: + * + * ``` + * remoteVideoTrack.addRenderer { frame -> + * val metadata = remoteVideoTrack.lookupFrameMetadata(frame.rtpTimestamp) + * } + * ``` + * + * Values for the last ~300 frames are kept, so look up a frame soon after it arrives. + * + * @return the values, or null if the publisher didn't send any for this frame, the frame is no + * longer available, or the track has been unsubscribed. + */ + @Beta + fun lookupFrameMetadata(rtpTimestamp: Long): FrameMetadata? { + // The transformer is cleared under this lock before it's disposed, so a lookup never sees + // a disposed one. + synchronized(frameMetadataLock) { + return frameMetadataTransformer?.lookup(rtpTimestamp)?.toSdk() + } + } + /** * If `autoManageVideo` is enabled, a VideoSinkVisibility should be passed, using * [ViewVisibility] if using a traditional View layout, or ComposeVisibility diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt index 181e3f8bc..6b93e1ff6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package io.livekit.android.room.track +import io.livekit.android.annotations.Beta import io.livekit.android.events.BroadcastEventBus import io.livekit.android.events.RoomEvent import io.livekit.android.events.TrackPublicationEvent @@ -67,6 +68,16 @@ open class TrackPublication( var mimeType: String? = null internal set + /** + * The frame metadata fields the publisher declared for this track, so that subscribers know to + * expect them. Empty if the track sends no frame metadata. Metadata is unsupported for audio. + * + * Subscribers read the values with [RemoteVideoTrack.lookupFrameMetadata]. + */ + @Beta + var frameMetadataFeatures: List = emptyList() // Declared before init, which sets it. + internal set + internal var trackInfo: LivekitModels.TrackInfo? = null var participant: WeakReference @@ -90,6 +101,7 @@ open class TrackPublication( dimensions = Track.Dimensions(info.width, info.height) } mimeType = info.mimeType + frameMetadataFeatures = info.packetTrailerFeaturesList.mapNotNull { FrameMetadataFeature.fromProto(it) } trackInfo = info } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt index 7b55366ee..3b1dbad2c 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt @@ -1,5 +1,5 @@ /* - * Copyright 2024 LiveKit, Inc. + * Copyright 2024-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,7 +89,7 @@ class ScaleCropVideoProcessor( scaleHeight, ) - val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs) + val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs, frame.frameMetadata) continueChain(croppedFrame) croppedFrame.release() } diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt new file mode 100644 index 000000000..d8a230f87 --- /dev/null +++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt @@ -0,0 +1,80 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.test.mock + +import io.livekit.android.room.framemetadata.FrameTransformerFactory +import livekit.org.webrtc.FrameCryptor +import livekit.org.webrtc.FrameCryptorAlgorithm +import livekit.org.webrtc.FrameCryptorKeyProvider +import livekit.org.webrtc.FrameMetadata +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.FrameTransformer +import livekit.org.webrtc.FrameTransformerChain +import livekit.org.webrtc.RtpReceiver +import livekit.org.webrtc.RtpSender +import org.mockito.Mockito +import org.mockito.kotlin.whenever + +/** + * Records the frame transformers the SDK creates, as Mockito mocks in place of the native objects. + */ +class MockFrameTransformerFactory { + + data class CreatedChain( + val chain: FrameTransformerChain, + val first: FrameTransformer, + val second: FrameTransformer, + ) + + val metadataTransformers = mutableListOf() + val chains = mutableListOf() + val senderCryptors = mutableListOf() + val receiverCryptors = mutableListOf() + + /** What a receive transformer's `lookup` returns. */ + var lookupResult: (rtpTimestamp: Long) -> FrameMetadata? = { null } + + internal val factory: FrameTransformerFactory = object : FrameTransformerFactory { + override fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer { + val transformer = Mockito.mock(FrameMetadataTransformer::class.java) + whenever(transformer.direction).thenReturn(direction) + whenever(transformer.lookup(Mockito.anyLong())).thenAnswer { lookupResult(it.getArgument(0)) } + metadataTransformers.add(transformer) + return transformer + } + + override fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain { + val chain = Mockito.mock(FrameTransformerChain::class.java) + chains.add(CreatedChain(chain, first, second)) + return chain + } + + override fun createFrameCryptor( + sender: RtpSender, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor = Mockito.mock(FrameCryptor::class.java).also { senderCryptors.add(it) } + + override fun createFrameCryptor( + receiver: RtpReceiver, + participantId: String, + algorithm: FrameCryptorAlgorithm, + keyProvider: FrameCryptorKeyProvider, + ): FrameCryptor = Mockito.mock(FrameCryptor::class.java).also { receiverCryptors.add(it) } + } +} diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt index 9280dc6f4..8bfea3b37 100644 --- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt +++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt @@ -16,9 +16,31 @@ package io.livekit.android.test.mock +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.VideoProcessor import livekit.org.webrtc.VideoSource class MockVideoSource(nativeSource: Long = 100) : VideoSource(nativeSource) { + + /** The send transformers currently linked to this source. */ + val frameMetadataTransformers = mutableListOf() + + var videoProcessor: VideoProcessor? = null + private set + override fun dispose() { } + + override fun addFrameMetadataTransformer(transformer: FrameMetadataTransformer) { + frameMetadataTransformers.add(transformer) + } + + override fun removeFrameMetadataTransformer(transformer: FrameMetadataTransformer) { + frameMetadataTransformers.remove(transformer) + } + + override fun setVideoProcessor(newVideoProcessor: VideoProcessor?) { + videoProcessor = newVideoProcessor + super.setVideoProcessor(newVideoProcessor) + } } diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt index f0ba8fba6..c6fd42c5a 100644 --- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt +++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt @@ -25,6 +25,7 @@ import io.livekit.android.dagger.LiveKitComponent import io.livekit.android.dagger.MemoryModule import io.livekit.android.dagger.UniffiModule import io.livekit.android.room.RTCEngine +import io.livekit.android.test.mock.MockFrameTransformerFactory import io.livekit.android.test.mock.MockNetworkCallbackRegistry import io.livekit.android.test.mock.MockWebSocketFactory import io.livekit.android.test.mock.room.datatrack.MockLocalDataTrackManagerFactory @@ -56,6 +57,8 @@ interface TestLiveKitComponent : LiveKitComponent { fun remoteDataTrackManagerFactory(): MockRemoteDataTrackManagerFactory + fun frameTransformerFactory(): MockFrameTransformerFactory + @Component.Factory interface Factory { fun create( diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt index 67d448344..803f78d23 100644 --- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt +++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt @@ -32,9 +32,11 @@ import io.livekit.android.e2ee.DataPacketCryptorManager import io.livekit.android.e2ee.KeyProvider import io.livekit.android.room.datatrack.LocalDataTrackManagerFactory import io.livekit.android.room.datatrack.RemoteDataTrackManagerFactory +import io.livekit.android.room.framemetadata.FrameTransformerFactory import io.livekit.android.test.mock.MockAudioDeviceModule import io.livekit.android.test.mock.MockAudioProcessingController import io.livekit.android.test.mock.MockEglBase +import io.livekit.android.test.mock.MockFrameTransformerFactory import io.livekit.android.test.mock.e2ee.ReversingDataPacketCryptorManager import io.livekit.android.test.mock.room.datatrack.MockLocalDataTrackManagerFactory import io.livekit.android.test.mock.room.datatrack.MockRemoteDataTrackManagerFactory @@ -143,6 +145,17 @@ object TestRTCModule { } } + @Provides + @Singleton + fun mockFrameTransformerFactory(): MockFrameTransformerFactory { + return MockFrameTransformerFactory() + } + + @Provides + internal fun frameTransformerFactory( + factory: MockFrameTransformerFactory, + ): FrameTransformerFactory = factory.factory + @Provides @Singleton fun mockLocalDataTrackManagerFactory(): MockLocalDataTrackManagerFactory { diff --git a/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt b/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt index 529b13dd2..2bfc935c6 100644 --- a/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt +++ b/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,12 +40,15 @@ class MockPeerConnectionFactory : PeerConnectionFactory(1L) { return MockAudioStreamTrack(id = id) } + /** Every video source created, in order. */ + val createdVideoSources = mutableListOf() + override fun createVideoSource(isScreencast: Boolean, alignTimestamps: Boolean): VideoSource { - return MockVideoSource() + return MockVideoSource().also { createdVideoSources.add(it) } } override fun createVideoSource(isScreencast: Boolean): VideoSource { - return MockVideoSource() + return MockVideoSource().also { createdVideoSources.add(it) } } override fun createVideoTrack(id: String, source: VideoSource?): VideoTrack { diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt index 78c6b4330..441c1cacd 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt @@ -77,7 +77,10 @@ class ConnectionParamsTest : MockE2ETest() { assertEquals(ClientProtocolVersion.DATA_STREAM_V2.value, clientInfo.clientProtocol) assertEquals( - listOf(LivekitModels.ClientInfo.Capability.CAP_COMPRESSION_DEFLATE_RAW), + listOf( + LivekitModels.ClientInfo.Capability.CAP_PACKET_TRAILER, + LivekitModels.ClientInfo.Capability.CAP_COMPRESSION_DEFLATE_RAW, + ), clientInfo.capabilitiesList, ) } diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt index 41a7133e0..bdb86b429 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt @@ -33,6 +33,8 @@ import io.livekit.android.room.datastream.DataStreams import io.livekit.android.room.datastream.incoming.IncomingDataStreamManagerImpl import io.livekit.android.room.datatrack.IncomingDataTrackEvent import io.livekit.android.room.datatrack.IncomingDataTrackManager +import io.livekit.android.room.framemetadata.FrameMetadataManager +import io.livekit.android.room.framemetadata.FrameTransformerInstaller import io.livekit.android.room.network.NetworkCallbackManagerImpl import io.livekit.android.room.participant.LocalParticipant import io.livekit.android.test.assert.assertIsClassList @@ -41,6 +43,7 @@ import io.livekit.android.test.events.EventCollector import io.livekit.android.test.mock.MockAudioDeviceModule import io.livekit.android.test.mock.MockAudioProcessingController import io.livekit.android.test.mock.MockEglBase +import io.livekit.android.test.mock.MockFrameTransformerFactory import io.livekit.android.test.mock.MockLKObjects import io.livekit.android.test.mock.MockNetworkCallbackRegistry import io.livekit.android.test.mock.TestData @@ -170,9 +173,13 @@ class RoomTest { ioDispatcher = coroutineRule.dispatcher, defaultDispatcher = coroutineRule.dispatcher, ), + frameMetadataManager = FrameMetadataManager(frameTransformerFactory.factory), + frameTransformerInstaller = FrameTransformerInstaller(frameTransformerFactory.factory), ) } + private val frameTransformerFactory = MockFrameTransformerFactory() + suspend fun connect(joinResponse: JoinResponse = TestData.JOIN.join) { rtcEngine.stub { onBlocking { rtcEngine.join(any(), any(), anyOrNull(), anyOrNull()) } diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt new file mode 100644 index 000000000..43c15fd48 --- /dev/null +++ b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt @@ -0,0 +1,623 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import android.Manifest +import android.app.Application +import io.livekit.android.e2ee.E2EEOptions +import io.livekit.android.room.DefaultsManager +import io.livekit.android.room.participant.BackupVideoCodec +import io.livekit.android.room.participant.VideoTrackPublishOptions +import io.livekit.android.room.track.FrameMetadata +import io.livekit.android.room.track.FrameMetadataFeature +import io.livekit.android.room.track.FrameMetadataPublishOptions +import io.livekit.android.room.track.LocalVideoTrack +import io.livekit.android.room.track.LocalVideoTrackOptions +import io.livekit.android.room.track.RemoteTrackPublication +import io.livekit.android.room.track.RemoteVideoTrack +import io.livekit.android.room.track.Track +import io.livekit.android.room.track.VideoCaptureParameter +import io.livekit.android.room.track.VideoCodec +import io.livekit.android.test.MockE2ETest +import io.livekit.android.test.mock.MockEglBase +import io.livekit.android.test.mock.MockMediaStream +import io.livekit.android.test.mock.MockRTCThreadToken +import io.livekit.android.test.mock.MockRtpReceiver +import io.livekit.android.test.mock.MockVideoCapturer +import io.livekit.android.test.mock.MockVideoSource +import io.livekit.android.test.mock.MockVideoStreamTrack +import io.livekit.android.test.mock.TestData +import io.livekit.android.test.mock.camera.MockCameraProvider +import io.livekit.android.test.mock.createMediaStreamId +import io.livekit.android.test.mock.e2ee.NoopKeyProvider +import io.livekit.android.test.mock.room.track.createMockLocalAudioTrack +import io.livekit.android.test.util.toPBByteString +import io.livekit.android.util.toOkioByteString +import kotlinx.coroutines.ExperimentalCoroutinesApi +import livekit.LivekitModels +import livekit.LivekitModels.PacketTrailerFeature +import livekit.LivekitRtc +import livekit.org.webrtc.FrameCryptor +import livekit.org.webrtc.FrameMetadataTransformer +import livekit.org.webrtc.FrameTransformer +import livekit.org.webrtc.FrameTransformerChain +import livekit.org.webrtc.MockPeerConnectionFactory +import livekit.org.webrtc.RtpReceiver +import livekit.org.webrtc.RtpSender +import livekit.org.webrtc.VideoSource +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNotSame +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito +import org.mockito.Mockito.mock +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows +import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +class FrameMetadataMockE2ETest : MockE2ETest() { + + private val factory + get() = component.frameTransformerFactory() + + private val metadataOptions = FrameMetadataPublishOptions(timestamp = true, frameId = true) + + private fun createLocalTrack(source: VideoSource = MockVideoSource()) = LocalVideoTrack( + capturer = MockVideoCapturer(), + source = source, + name = "", + options = LocalVideoTrackOptions( + isScreencast = false, + deviceId = null, + position = null, + captureParams = VideoCaptureParameter(width = 1280, height = 720, maxFps = 30), + ), + rtcTrack = MockVideoStreamTrack(), + peerConnectionFactory = component.peerConnectionFactory(), + context = context, + eglBase = MockEglBase(), + defaultsManager = DefaultsManager(), + trackFactory = mock(LocalVideoTrack.Factory::class.java), + rtcThreadToken = MockRTCThreadToken(), + ) + + private fun sentAddTrackRequests(): List = + wsFactory.ws.sentRequests + .map { LivekitRtc.SignalRequest.newBuilder().mergeFrom(it.toPBByteString()).build() } + .filter { it.hasAddTrack() } + .map { it.addTrack } + + private fun sendTransformers() = factory.metadataTransformers.filter { it.direction == FrameMetadataTransformer.Direction.SEND } + + private fun receiveTransformers() = factory.metadataTransformers.filter { it.direction == FrameMetadataTransformer.Direction.RECEIVE } + + private fun enableE2EE() { + room.e2eeOptions = E2EEOptions(keyProvider = NoopKeyProvider()) + } + + private fun verifySingleInstall(sender: RtpSender, transformer: FrameTransformer) { + Mockito.verify(sender).setFrameTransformer(transformer) + Mockito.verify(sender, Mockito.times(1)).setFrameTransformer(Mockito.any()) + } + + private fun verifySingleInstall(receiver: RtpReceiver, transformer: FrameTransformer) { + Mockito.verify(receiver).setFrameTransformer(transformer) + Mockito.verify(receiver, Mockito.times(1)).setFrameTransformer(Mockito.any()) + } + + // Publishing + + @Test + fun publishSendsThePacketTrailerFeatures() = runTest { + connect() + wsFactory.ws.clearRequests() + + room.localParticipant.publishVideoTrack( + createLocalTrack(), + VideoTrackPublishOptions(frameMetadata = FrameMetadataPublishOptions(timestamp = true, frameId = true, userData = true)), + ) + + assertEquals( + listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID, PacketTrailerFeature.PTF_USER_DATA), + sentAddTrackRequests().single().packetTrailerFeaturesList, + ) + } + + @Test + fun publishDefaultsCarryFrameMetadata() = runTest { + room.videoTrackPublishDefaults = room.videoTrackPublishDefaults.copy(frameMetadata = metadataOptions) + connect() + wsFactory.ws.clearRequests() + + room.localParticipant.publishVideoTrack(createLocalTrack()) + + assertEquals( + listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID), + sentAddTrackRequests().single().packetTrailerFeaturesList, + ) + assertEquals(1, sendTransformers().size) + } + + @Test + fun publishWithoutFrameMetadataSendsNoFeaturesAndCreatesNoTransformer() = runTest { + connect() + wsFactory.ws.clearRequests() + val track = createLocalTrack() + + room.localParticipant.publishVideoTrack(track) + + assertTrue(sentAddTrackRequests().single().packetTrailerFeaturesList.isEmpty()) + assertTrue(factory.metadataTransformers.isEmpty()) + Mockito.verify(track.sender!!, Mockito.never()).setFrameTransformer(Mockito.any()) + assertFalse(track.frameMetadataStampState.isArmed) + } + + @Test + fun publishLinksArmsAndInstallsTheSendTransformer() = runTest { + connect() + val source = MockVideoSource() + val track = createLocalTrack(source) + + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + + val transformer = sendTransformers().single() + assertSame(transformer, track.withFrameMetadataTransformer { it }) + assertEquals(listOf(transformer), source.frameMetadataTransformers) + assertTrue(track.frameMetadataStampState.isArmed) + verifySingleInstall(track.sender!!, transformer) + assertTrue(factory.chains.isEmpty()) + } + + @Test + fun unpublishUnlinksDisarmsAndDisposes() = runTest { + connect() + val source = MockVideoSource() + val track = createLocalTrack(source) + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + val transformer = sendTransformers().single() + + room.localParticipant.unpublishTrack(track) + + assertTrue(source.frameMetadataTransformers.isEmpty()) + assertNull(track.withFrameMetadataTransformer { it }) + assertFalse(track.frameMetadataStampState.isArmed) + Mockito.verify(transformer).dispose() + } + + @Test + fun audioPublishCreatesNoTransformer() = runTest { + connect() + wsFactory.ws.clearRequests() + + room.localParticipant.publishAudioTrack(createMockLocalAudioTrack()) + + assertTrue(sentAddTrackRequests().single().packetTrailerFeaturesList.isEmpty()) + assertTrue(factory.metadataTransformers.isEmpty()) + } + + // Publishing with E2EE + + @Test + fun e2eeChainsTheCryptorBeforeTheMetadataTransformerOnSend() = runTest { + enableE2EE() + connect() + val track = createLocalTrack() + + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + + val created = factory.chains.single() + assertSame(factory.senderCryptors.single(), created.first) + assertSame(sendTransformers().single(), created.second) + verifySingleInstall(track.sender!!, created.chain) + } + + @Test + fun e2eeOnlyVideoInstallsTheCryptorWithoutAChain() = runTest { + enableE2EE() + connect() + val track = createLocalTrack() + + room.localParticipant.publishVideoTrack(track) + + assertTrue(factory.chains.isEmpty()) + verifySingleInstall(track.sender!!, factory.senderCryptors.single()) + } + + @Test + fun e2eeAudioInstallsTheCryptorOnce() = runTest { + enableE2EE() + connect() + val track = createMockLocalAudioTrack() + + room.localParticipant.publishAudioTrack(track) + + assertTrue(factory.chains.isEmpty()) + assertTrue(factory.metadataTransformers.isEmpty()) + verifySingleInstall(track.sender!!, factory.senderCryptors.single()) + } + + @Test + fun e2eeUnpublishDisposesTheChain() = runTest { + enableE2EE() + connect() + val track = createLocalTrack() + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + val created = factory.chains.single() + + room.localParticipant.unpublishTrack(track) + + Mockito.verify(created.chain).dispose() + Mockito.verify(sendTransformers().single()).dispose() + } + + @Test + fun chainForATrackRemovedBeforeItIsKeptIsDisposed() = runTest { + // A send install keeps its chain after releasing the track's lock, so an unpublish can + // remove the track in between. + val manager = FrameMetadataManager(factory.factory) + val chain = mock(FrameTransformerChain::class.java) + + manager.setChain(createLocalTrack(), chain) + + Mockito.verify(chain).dispose() + } + + // Backup codecs and restarts + + private suspend fun publishVp9WithBackup(track: LocalVideoTrack) { + room.localParticipant.publishVideoTrack( + track, + VideoTrackPublishOptions( + videoCodec = VideoCodec.VP9.codecName, + backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName), + frameMetadata = metadataOptions, + ), + ) + } + + private fun requestBackupCodec() { + val quality = LivekitRtc.SubscribedQuality.newBuilder() + .setQuality(LivekitModels.VideoQuality.HIGH) + .setEnabled(true) + .build() + wsFactory.receiveMessage( + with(LivekitRtc.SignalResponse.newBuilder()) { + subscribedQualityUpdate = with(LivekitRtc.SubscribedQualityUpdate.newBuilder()) { + trackSid = room.localParticipant.videoTrackPublications.first().first.sid + addSubscribedCodecs(LivekitRtc.SubscribedCodec.newBuilder().setCodec("vp9").addQualities(quality)) + addSubscribedCodecs(LivekitRtc.SubscribedCodec.newBuilder().setCodec("vp8").addQualities(quality)) + build() + } + build().toOkioByteString() + }, + ) + } + + @Test + fun backupCodecSenderGetsThePrimarysTransformer() = runTest { + connect() + val track = createLocalTrack() + publishVp9WithBackup(track) + + requestBackupCodec() + + val transceivers = getPublisherPeerConnection().transceivers + assertEquals(2, transceivers.size) + val transformer = sendTransformers().single() + verifySingleInstall(transceivers.last().sender, transformer) + } + + private fun grantCamera() { + MockCameraProvider.register() + Shadows.shadowOf(context as Application).grantPermissions(Manifest.permission.CAMERA) + } + + private val createdVideoSources + get() = (component.peerConnectionFactory() as MockPeerConnectionFactory).createdVideoSources + + @Test + fun restartTrackRelinksTheTransformerAndKeepsTheStampState() = runTest { + connect() + grantCamera() + val track = room.localParticipant.createVideoTrack() + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + val transformer = sendTransformers().single() + val oldSource = createdVideoSources.last() + val oldStamper = track.frameMetadataStamper!! + assertSame(oldStamper, oldSource.videoProcessor) + + track.restartTrack() + + val newSource = createdVideoSources.last() + assertTrue(oldSource.frameMetadataTransformers.isEmpty()) + assertEquals(listOf(transformer), newSource.frameMetadataTransformers) + val newStamper = track.frameMetadataStamper!! + assertSame(newStamper, newSource.videoProcessor) + assertTrue(newStamper !== oldStamper) + assertSame(track.frameMetadataStampState, newStamper.state) + assertTrue(track.frameMetadataStampState.isArmed) + } + + @Test + fun restartTrackMovesBackupSendersToTheNewTrack() = runTest { + connect() + grantCamera() + val track = room.localParticipant.createVideoTrack() + publishVp9WithBackup(track) + requestBackupCodec() + val backupSender = getPublisherPeerConnection().transceivers.last().sender + + track.restartTrack() + + Mockito.verify(backupSender).setTrack(track.rtcTrack, false) + } + + @Test + fun restartTrackAfterUnpublishLinksNothing() = runTest { + connect() + grantCamera() + val track = room.localParticipant.createVideoTrack() + room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions)) + room.localParticipant.unpublishTrack(track, stopOnUnpublish = false) + + track.restartTrack() + + assertTrue(createdVideoSources.last().frameMetadataTransformers.isEmpty()) + } + + // Subscribing + + private fun joinRemoteParticipant(features: List) { + val videoTrack = TestData.REMOTE_VIDEO_TRACK.toBuilder().addAllPacketTrailerFeatures(features).build() + val participant = TestData.REMOTE_PARTICIPANT.toBuilder() + .clearTracks() + .addTracks(TestData.REMOTE_AUDIO_TRACK) + .addTracks(videoTrack) + .build() + simulateMessageFromServer( + LivekitRtc.SignalResponse.newBuilder() + .setUpdate(LivekitRtc.ParticipantUpdate.newBuilder().addParticipants(participant)) + .build(), + ) + } + + private fun subscribeRemoteVideo(): RtpReceiver { + val receiver = MockRtpReceiver.create() + room.onAddTrack( + receiver, + MockVideoStreamTrack(), + arrayOf( + MockMediaStream( + id = createMediaStreamId(TestData.REMOTE_PARTICIPANT.sid, TestData.REMOTE_VIDEO_TRACK.sid), + ), + ), + ) + return receiver + } + + private fun remoteVideoPublication(): RemoteTrackPublication { + val participant = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!! + return participant.getTrackPublication(Track.Source.CAMERA) as RemoteTrackPublication + } + + private fun remoteVideoTrack(): RemoteVideoTrack = remoteVideoPublication().track as RemoteVideoTrack + + /** Sends a participant update without the video track, which unpublishes it. */ + private fun unpublishRemoteVideo() { + val participant = TestData.REMOTE_PARTICIPANT.toBuilder() + .clearTracks() + .addTracks(TestData.REMOTE_AUDIO_TRACK) + .build() + simulateMessageFromServer( + LivekitRtc.SignalResponse.newBuilder() + .setUpdate(LivekitRtc.ParticipantUpdate.newBuilder().addParticipants(participant)) + .build(), + ) + } + + @Test + fun subscribeWithFeaturesInstallsTheReceiveTransformerAndLooksUp() = runTest { + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + factory.lookupResult = { rtpTimestamp -> if (rtpTimestamp == 1234L) RtcFrameMetadata(55L, 6L, null) else null } + + val receiver = subscribeRemoteVideo() + + val transformer = receiveTransformers().single() + verifySingleInstall(receiver, transformer) + val track = remoteVideoTrack() + assertEquals(FrameMetadata(userTimestampUs = 55L, frameId = 6L), track.lookupFrameMetadata(1234L)) + assertNull(track.lookupFrameMetadata(1L)) + } + + @Test + fun remotePublicationExposesTheDeclaredFeatures() = runTest { + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_USER_DATA)) + + val publication = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!!.getTrackPublication(Track.Source.CAMERA)!! + assertEquals(listOf(FrameMetadataFeature.TIMESTAMP, FrameMetadataFeature.USER_DATA), publication.frameMetadataFeatures) + } + + @Test + fun remotePublicationSkipsFeaturesThisBuildDoesNotKnow() = runTest { + connect() + val videoTrack = TestData.REMOTE_VIDEO_TRACK.toBuilder() + .addPacketTrailerFeatures(PacketTrailerFeature.PTF_FRAME_ID) + .addPacketTrailerFeaturesValue(99) + .build() + simulateMessageFromServer( + LivekitRtc.SignalResponse.newBuilder() + .setUpdate( + LivekitRtc.ParticipantUpdate.newBuilder().addParticipants( + TestData.REMOTE_PARTICIPANT.toBuilder().clearTracks().addTracks(videoTrack), + ), + ) + .build(), + ) + + val publication = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!!.getTrackPublication(Track.Source.CAMERA)!! + assertEquals(listOf(FrameMetadataFeature.FRAME_ID), publication.frameMetadataFeatures) + } + + @Test + fun subscribeWithoutFeaturesInstallsNothing() = runTest { + connect() + joinRemoteParticipant(emptyList()) + + val receiver = subscribeRemoteVideo() + + assertTrue(factory.metadataTransformers.isEmpty()) + Mockito.verify(receiver, Mockito.never()).setFrameTransformer(Mockito.any()) + assertNull(remoteVideoTrack().lookupFrameMetadata(1234L)) + } + + @Test + fun e2eeChainsTheMetadataTransformerBeforeTheCryptorOnReceive() = runTest { + enableE2EE() + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_FRAME_ID)) + + val receiver = subscribeRemoteVideo() + + val created = factory.chains.single() + assertSame(receiveTransformers().single(), created.first) + assertSame(factory.receiverCryptors.single(), created.second) + verifySingleInstall(receiver, created.chain) + } + + @Test + fun unsubscribeDisposesWithoutDisablingAndLookupReturnsNull() = runTest { + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + factory.lookupResult = { RtcFrameMetadata(55L, 6L, null) } + subscribeRemoteVideo() + val track = remoteVideoTrack() + val transformer = receiveTransformers().single() + assertNotNull(track.lookupFrameMetadata(1234L)) + + simulateMessageFromServer(TestData.PARTICIPANT_DISCONNECT) + + Mockito.verify(transformer).dispose() + Mockito.verify(transformer, Mockito.never()).setEnabled(false) + assertNull(track.lookupFrameMetadata(1234L)) + } + + @Test + fun receiveInstallAfterRemovalInstallsOnlyTheCryptor() = runTest { + // What an install sees when a removal gets the manager's lock first. + enableE2EE() + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + val receiver = subscribeRemoteVideo() + val track = remoteVideoTrack() + unpublishRemoteVideo() + val cryptor = mock(FrameCryptor::class.java) + + room.installReceiveTransformers(track, cryptor) + + assertEquals(1, factory.chains.size) + Mockito.verify(receiver).setFrameTransformer(cryptor) + } + + @Test + fun receiveReinstallReplacesAndDisposesThePreviousChain() = runTest { + enableE2EE() + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + subscribeRemoteVideo() + val first = factory.chains.single() + + room.installReceiveTransformers(remoteVideoTrack(), mock(FrameCryptor::class.java)) + + val second = factory.chains.single { it !== first } + assertSame(first.first, second.first) + Mockito.verify(first.chain).dispose() + Mockito.verify(second.chain, Mockito.never()).dispose() + } + + @Test + fun resubscribeDisposesThePreviousTransformerAndChain() = runTest { + enableE2EE() + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + factory.lookupResult = { RtcFrameMetadata(55L, 6L, null) } + subscribeRemoteVideo() + val firstTrack = remoteVideoTrack() + val firstTransformer = receiveTransformers().single() + val firstChain = factory.chains.single().chain + + // The client isn't told when the server stops sending the track, so the next subscribe + // arrives with the first track still in place. + remoteVideoPublication().setSubscribed(false) + remoteVideoPublication().setSubscribed(true) + subscribeRemoteVideo() + + val secondTrack = remoteVideoTrack() + assertNotSame(firstTrack, secondTrack) + Mockito.verify(firstTransformer).dispose() + Mockito.verify(firstChain).dispose() + assertNull(firstTrack.lookupFrameMetadata(1234L)) + val secondTransformer = receiveTransformers().single { it !== firstTransformer } + Mockito.verify(secondTransformer, Mockito.never()).dispose() + Mockito.verify(factory.chains.single { it.chain !== firstChain }.chain, Mockito.never()).dispose() + assertNotNull(secondTrack.lookupFrameMetadata(1234L)) + } + + @Test + fun unpublishAfterResubscribeDisposesEachTransformerOnce() = runTest { + connect() + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + subscribeRemoteVideo() + subscribeRemoteVideo() + + unpublishRemoteVideo() + + assertEquals(2, receiveTransformers().size) + receiveTransformers().forEach { Mockito.verify(it).dispose() } + } + + @Test + fun disconnectDisposesEverything() = runTest { + connect() + room.localParticipant.publishVideoTrack(createLocalTrack(), VideoTrackPublishOptions(frameMetadata = metadataOptions)) + joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP)) + subscribeRemoteVideo() + + room.disconnect() + + assertEquals(2, factory.metadataTransformers.size) + factory.metadataTransformers.forEach { Mockito.verify(it).dispose() } + } + + // Capabilities + + @Test + fun connectAdvertisesPacketTrailers() = runTest { + connect() + + val capabilities = wsFactory.request.url.queryParameter("capabilities")!!.split(",") + assertTrue(capabilities.contains("CAP_PACKET_TRAILER")) + } +} diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt new file mode 100644 index 000000000..98644a212 --- /dev/null +++ b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt @@ -0,0 +1,255 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.framemetadata + +import io.livekit.android.room.track.FrameMetadata +import io.livekit.android.room.track.FrameMetadataProvider +import io.livekit.android.room.track.FrameMetadataPublishOptions +import livekit.org.webrtc.VideoFrame +import livekit.org.webrtc.VideoProcessor +import livekit.org.webrtc.VideoSink +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito +import org.robolectric.RobolectricTestRunner +import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata + +@RunWith(RobolectricTestRunner::class) +class FrameMetadataStampingTest { + + private val buffer = Mockito.mock(VideoFrame.Buffer::class.java) + + private fun frame(timestampNs: Long = 4_000_000_000L, metadata: RtcFrameMetadata? = null) = + VideoFrame(buffer, 0, timestampNs, metadata) + + // WebRTC's clock at 5 s and the wall clock at 1,700,000,000,000 ms, so a frame captured at + // 4 s on WebRTC's clock is 1 s before the wall-clock time. + private fun state(initialFrameId: Long = 1L) = FrameMetadataStampState( + rtcTimeNanos = { 5_000_000_000L }, + wallTimeMillis = { 1_700_000_000_000L }, + initialFrameId = initialFrameId, + ) + + private val allOn = FrameMetadataPublishOptions(timestamp = true, frameId = true) + + @Test + fun nothingIsStampedBeforeArming() { + val input = frame() + val output = state().stamp(input) + + assertSame(input, output) + assertNull(output.frameMetadata) + } + + @Test + fun nothingIsStampedAfterDisarming() { + val state = state() + state.arm(allOn) + state.disarm() + + val input = frame() + assertSame(input, state.stamp(input)) + } + + @Test + fun defaultTimestampIsTheCaptureTimeOnTheWallClock() { + val state = state() + state.arm(FrameMetadataPublishOptions(timestamp = true, frameId = false)) + + val metadata = state.stamp(frame()).frameMetadata!! + + assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs) + assertEquals(0L, metadata.frameId) + } + + @Test + fun onlyEnabledFieldsAreFilled() { + val state = state() + state.arm(FrameMetadataPublishOptions(timestamp = false, frameId = true)) + + val metadata = state.stamp(frame()).frameMetadata!! + + assertEquals(0L, metadata.userTimestampUs) + assertEquals(1L, metadata.frameId) + assertNull(metadata.userData) + } + + @Test + fun frameIdsCountUpFromOne() { + val state = state() + state.arm(allOn) + + val ids = (1..3).map { state.stamp(frame()).frameMetadata!!.frameId } + + assertEquals(listOf(1L, 2L, 3L), ids) + } + + @Test + fun frameIdWrapsToOne() { + val state = state(initialFrameId = FrameMetadataStampState.MAX_FRAME_ID) + state.arm(allOn) + + val ids = (1..2).map { state.stamp(frame()).frameMetadata!!.frameId } + + assertEquals(listOf(FrameMetadataStampState.MAX_FRAME_ID, 1L), ids) + } + + @Test + fun metadataOnTheFrameWinsOverDefaults() { + val state = state() + state.arm(allOn) + val userData = byteArrayOf(1, 2, 3) + + val metadata = state.stamp(frame(metadata = RtcFrameMetadata(123L, 0L, userData))).frameMetadata!! + + assertEquals(123L, metadata.userTimestampUs) + // The frame ID was 0, so the default fills it. + assertEquals(1L, metadata.frameId) + assertArrayEquals(userData, metadata.userData) + } + + @Test + fun appValuesAreSentWithEveryOptionOff() { + val state = state() + state.arm(FrameMetadataPublishOptions(timestamp = false, frameId = false, userData = false)) + state.provider = FrameMetadataProvider { _, current -> current.copy(userData = byteArrayOf(9)) } + + val metadata = state.stamp(frame()).frameMetadata!! + + assertArrayEquals(byteArrayOf(9), metadata.userData) + } + + @Test + fun providerSeesTheDefaultsAndReplacesThem() { + val state = state() + state.arm(allOn) + var seen: FrameMetadata? = null + state.provider = FrameMetadataProvider { _, current -> + seen = current + FrameMetadata(userTimestampUs = 42L, frameId = 7L, userData = byteArrayOf(5)) + } + + val metadata = state.stamp(frame()).frameMetadata!! + + assertEquals(FrameMetadata(1_699_999_999_000_000L, 1L, null), seen) + assertEquals(42L, metadata.userTimestampUs) + assertEquals(7L, metadata.frameId) + assertArrayEquals(byteArrayOf(5), metadata.userData) + } + + @Test + fun providerReturningNullKeepsTheValues() { + val state = state() + state.arm(allOn) + state.provider = FrameMetadataProvider { _, _ -> null } + + val metadata = state.stamp(frame()).frameMetadata!! + + assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs) + assertEquals(1L, metadata.frameId) + } + + @Test + fun throwingProviderFallsBackToTheValuesItWasGiven() { + val state = state() + state.arm(allOn) + state.provider = FrameMetadataProvider { _, _ -> throw IllegalStateException("app bug") } + + val metadata = state.stamp(frame()).frameMetadata!! + + assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs) + assertEquals(1L, metadata.frameId) + } + + @Test + fun providerZeroingEverythingRemovesMetadataFromTheFrame() { + val state = state() + state.arm(allOn) + state.provider = FrameMetadataProvider { _, _ -> FrameMetadata() } + + val output = state.stamp(frame(metadata = RtcFrameMetadata(123L, 4L, null))) + + assertNull(output.frameMetadata) + } + + @Test + fun processorStampsAfterTheInnerProcessor() { + // An inner processor that replaces each frame, as virtual background does. + val inner = object : VideoProcessor { + var output: VideoSink? = null + override fun onCapturerStarted(success: Boolean) {} + override fun onCapturerStopped() {} + override fun setSink(sink: VideoSink?) { + output = sink + } + + override fun onFrameCaptured(frame: VideoFrame) { + output?.onFrame(VideoFrame(frame.buffer, frame.rotation, frame.timestampNs)) + } + } + val processor = FrameMetadataStampingProcessor(inner) + processor.state = state().apply { arm(allOn) } + val received = mutableListOf() + processor.setSink { received.add(it) } + + processor.onFrameCaptured(frame()) + + assertEquals(1, received.size) + assertEquals(1L, received.single().frameMetadata!!.frameId) + } + + @Test + fun processorForwardsTheTwoArgumentCallToTheInnerProcessor() { + val inner = Mockito.mock(VideoProcessor::class.java) + val processor = FrameMetadataStampingProcessor(inner) + val input = frame() + val parameters = VideoProcessor.FrameAdaptationParameters(0, 0, 1, 1, 1, 1, 0L, false) + + processor.onFrameCaptured(input, parameters) + + Mockito.verify(inner).onFrameCaptured(input, parameters) + } + + @Test + fun processorPassesFramesThroughWithoutAnInnerProcessor() { + val processor = FrameMetadataStampingProcessor(null) + val received = mutableListOf() + processor.setSink { received.add(it) } + val input = frame() + + processor.onFrameCaptured(input) + + assertSame(input, received.single()) + } + + @Test + fun clearingTheSinkClearsTheInnerProcessorsSink() { + val inner = Mockito.mock(VideoProcessor::class.java) + val processor = FrameMetadataStampingProcessor(inner) + + processor.setSink { } + processor.setSink(null) + + val inOrder = Mockito.inOrder(inner) + inOrder.verify(inner).setSink(Mockito.notNull()) + inOrder.verify(inner).setSink(null) + } +}