diff --git a/.changeset/frame-metadata.md b/.changeset/frame-metadata.md
new file mode 100644
index 000000000..ff7586156
--- /dev/null
+++ b/.changeset/frame-metadata.md
@@ -0,0 +1,11 @@
+---
+"client-sdk-android": minor
+---
+
+Add frame metadata for video tracks. A publisher can send a capture timestamp, a frame ID and its own user data with each video frame, carried in a trailer after the encoded frame, and subscribers look them up by RTP timestamp. The format is the same as the JavaScript and Rust SDKs, so metadata works between them, with or without end-to-end encryption.
+
+To publish, set `frameMetadata` on `VideoTrackPublishOptions` or `VideoTrackPublishDefaults`. By default the SDK sends the frame's capture time as wall-clock time and a per-track frame ID; supply your own values with `LocalVideoTrack.frameMetadataProvider`, or on the frames your video processor emits. To read them, call `RemoteVideoTrack.lookupFrameMetadata(frame.rtpTimestamp)` from a video sink. `TrackPublication.frameMetadataFeatures` lists the fields a track's publisher declared, as `FrameMetadataFeature` values. `FrameMetadata`, `FrameMetadataFeature`, `FrameMetadataProvider`, `frameMetadataProvider`, `frameMetadataFeatures` and `lookupFrameMetadata` are in beta and need an opt-in to `@Beta`; the publish options don't, so `copy()` on the publish option classes works as before. User data isn't encrypted under end-to-end encryption, so the server can read it.
+
+`E2EEManager.addPublishedTrack`, `addSubscribedTrack`, `removePublishedTrack` and `removeSubscribedTrack` are now internal. The room calls them for you, and they no longer install the cryptor themselves, so an app calling them directly would have sent unencrypted media.
+
+`LocalVideoTrack.restartTrack` now moves backup codec senders to the new track as well. Before, backup codec video stopped after a restart.
diff --git a/livekit-android-sdk/detekt-baseline-release.xml b/livekit-android-sdk/detekt-baseline-release.xml
index c8efe8b1f..9bdd1aaf7 100644
--- a/livekit-android-sdk/detekt-baseline-release.xml
+++ b/livekit-android-sdk/detekt-baseline-release.xml
@@ -43,7 +43,7 @@
LongParameterList:KeyProvider.kt$BaseKeyProvider$( ratchetSalt: String = defaultRatchetSalt, uncryptedMagicBytes: String = defaultMagicBytes, ratchetWindowSize: Int = defaultRatchetWindowSize, override var enableSharedKey: Boolean = true, failureTolerance: Int = defaultFailureTolerance, keyRingSize: Int = defaultKeyRingSize, discardFrameWhenCryptorNotReady: Boolean = defaultDiscardFrameWhenCryptorNotReady, keyDerivationAlgorithm: FrameCryptorKeyDerivationAlgorithm = defaultKeyDerivationAlgorithm, )
LongParameterList:LiveKitOverrides.kt$AudioOptions$( /** * Override the default output [AudioType]. * * This affects the audio routing and how the audio is handled. Default is [AudioType.CallAudioType]. * * Note: if [audioHandler] is also passed, the values from [audioOutputType] will not be reflected in it, * and must be set yourself. */ val audioOutputType: AudioType? = null, /** * Override the default [AudioHandler]. * * Default is [AudioSwitchHandler]. * * Use [NoAudioHandler] to turn off automatic audio handling or * [AudioFocusHandler] to get simple audio focus handling. */ val audioHandler: AudioHandler? = null, /** * Override the default [AudioDeviceModule]. * * If a non-null value is passed, the library does not * take ownership of the object and will not release it upon [Room.release]. * It is the responsibility of the owner to call [AudioDeviceModule.release] when finished * with it to prevent memory leaks. */ val audioDeviceModule: AudioDeviceModule? = null, /** * Called after default setup to allow for customizations on the [JavaAudioDeviceModule]. * * Not used if [audioDeviceModule] is provided. * * Note: We require setting the [JavaAudioDeviceModule.Builder.setSamplesReadyCallback] to provide * support for [LocalAudioTrack.addSink]. If you wish to grab the audio samples * from the local microphone track, use [LocalAudioTrack.addSink] instead of setting your own * callback. */ val javaAudioDeviceModuleCustomizer: ((builder: JavaAudioDeviceModule.Builder) -> Unit)? = null, /** * On Android 11+, the audio mode will reset itself from [AudioManager.MODE_IN_COMMUNICATION] if * there is no audio playback or capture for 6 seconds (for example when joining a room with * no speakers and the local mic is muted.) This mode reset will cause unexpected * behavior when trying to change the volume, causing it to not properly change the volume. * * We use a workaround by playing a silent audio track to keep the communication mode from * resetting. * * Setting this flag to true will disable the workaround. * * This flag is a no-op when the audio mode is set to anything other than * [AudioManager.MODE_IN_COMMUNICATION]. */ val disableCommunicationModeWorkaround: Boolean = false, /** * Options for processing the mic and incoming audio. */ val audioProcessorOptions: AudioProcessorOptions? = null, /** * Devices may take some time initializing the audio stack for recording. * Prewarming allows starting up the underlying audio recording prior to publish, letting * the audio device be ready immediately when the track is fully published. * * If set to true, disables audio recording prewarming (and the related * [LocalAudioTrack.prewarm] function), and audio resources are only used while the * track is connected and published. Defaults to false. */ val disableAudioPrewarming: Boolean = false, )
LongParameterList:LocalAudioTrack.kt$LocalAudioTrack$( @Assisted name: String, @Assisted mediaTrack: livekit.org.webrtc.AudioTrack, @Assisted options: LocalAudioTrackOptions, private val audioProcessingController: AudioProcessingController, @Named(InjectionNames.DISPATCHER_DEFAULT) private val dispatcher: CoroutineDispatcher, @Named(InjectionNames.LOCAL_AUDIO_RECORD_SAMPLES_DISPATCHER) private val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher, @Named(InjectionNames.LOCAL_AUDIO_BUFFER_CALLBACK_DISPATCHER) private val audioBufferCallbackDispatcher: AudioBufferCallbackDispatcher, private val audioRecordPrewarmer: AudioRecordPrewarmer, rtcThreadToken: RTCThreadToken, )
- LongParameterList:LocalParticipant.kt$LocalParticipant$( @Assisted internal var dynacast: Boolean, internal val engine: RTCEngine, private val peerConnectionFactory: PeerConnectionFactory, private val context: Context, private val eglBase: EglBase, private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory, private val videoTrackFactory: LocalVideoTrack.Factory, private val audioTrackFactory: LocalAudioTrack.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) coroutineDispatcher: CoroutineDispatcher, @Named(InjectionNames.SENDER) private val capabilitiesGetter: CapabilitiesGetter, private val outgoingDataStreamManager: OutgoingDataStreamManager, private val outgoingDataTrackManager: OutgoingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, )
+ LongParameterList:LocalParticipant.kt$LocalParticipant$( @Assisted internal var dynacast: Boolean, internal val engine: RTCEngine, private val peerConnectionFactory: PeerConnectionFactory, private val context: Context, private val eglBase: EglBase, private val screencastVideoTrackFactory: LocalScreencastVideoTrack.Factory, private val videoTrackFactory: LocalVideoTrack.Factory, private val audioTrackFactory: LocalAudioTrack.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) coroutineDispatcher: CoroutineDispatcher, @Named(InjectionNames.SENDER) private val capabilitiesGetter: CapabilitiesGetter, private val outgoingDataStreamManager: OutgoingDataStreamManager, private val outgoingDataTrackManager: OutgoingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val frameMetadataManager: FrameMetadataManager, )
LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack$( @Assisted capturer: VideoCapturer, @Assisted source: VideoSource, @Assisted name: String, @Assisted options: LocalVideoTrackOptions, @Assisted rtcTrack: livekit.org.webrtc.VideoTrack, @Assisted mediaProjectionCallback: MediaProjectionCallback, peerConnectionFactory: PeerConnectionFactory, context: Context, eglBase: EglBase, defaultsManager: DefaultsManager, videoTrackFactory: LocalVideoTrack.Factory, rtcThreadToken: RTCThreadToken, )
LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack.Companion$( mediaProjectionPermissionResultData: Intent, peerConnectionFactory: PeerConnectionFactory, context: Context, name: String, options: LocalVideoTrackOptions, rootEglBase: EglBase, screencastVideoTrackFactory: Factory, videoProcessor: VideoProcessor?, onStop: (Track) -> Unit, )
LongParameterList:LocalScreencastVideoTrack.kt$LocalScreencastVideoTrack.Factory$( capturer: VideoCapturer, source: VideoSource, name: String, options: LocalVideoTrackOptions, rtcTrack: livekit.org.webrtc.VideoTrack, mediaProjectionCallback: MediaProjectionCallback, )
@@ -60,7 +60,7 @@
LongParameterList:RTCStatsExt.kt$( trackIdentifier: String, ssrcs: Set<Long?>, codecIds: Set<String?>, localCandidateId: String?, remoteCandidateId: String?, statsMap: Map<String, RTCStats>, )
LongParameterList:RemoteParticipant.kt$RemoteParticipant$( mediaTrack: MediaStreamTrack, sid: String, statsGetter: RTCStatsGetter, receiver: RtpReceiver, autoManageVideo: Boolean = false, triesLeft: Int = 20, )
LongParameterList:RemoteParticipant.kt$RemoteParticipant$( sid: Sid, identity: Identity? = null, internal val signalClient: SignalClient, private val ioDispatcher: CoroutineDispatcher, defaultDispatcher: CoroutineDispatcher, private val audioTrackFactory: RemoteAudioTrack.Factory, private val videoTrackFactory: RemoteVideoTrack.Factory, )
- LongParameterList:Room.kt$Room$( @Assisted private val context: Context, internal val engine: RTCEngine, private val eglBase: EglBase, localParticipantFactory: LocalParticipant.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) private val defaultDispatcher: CoroutineDispatcher, @Named(InjectionNames.DISPATCHER_IO) private val ioDispatcher: CoroutineDispatcher, /** * The [AudioHandler] for setting up the audio as need. * * By default, this is an instance of [AudioSwitchHandler]. * * This can be substituted for your own custom implementation through * [LiveKitOverrides.audioOptions] when creating the room with [LiveKit.create]. * * @see [audioSwitchHandler] * @see [AudioSwitchHandler] */ val audioHandler: AudioHandler, private val closeableManager: CloseableManager, private val e2EEManagerFactory: E2EEManager.Factory, private val communicationWorkaround: CommunicationWorkaround, val audioProcessingController: AudioProcessingController, /** * A holder for objects that are used internally within LiveKit. */ val lkObjects: LKObjects, networkCallbackManagerFactory: NetworkCallbackManagerFactory, private val audioDeviceModule: AudioDeviceModule, private val regionUrlProviderFactory: RegionUrlProvider.Factory, private val connectionWarmer: ConnectionWarmer, private val audioRecordPrewarmer: AudioRecordPrewarmer, private val incomingDataStreamManager: IncomingDataStreamManager, private val dataStreams: DataStreams, private val incomingDataTrackManager: IncomingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val remoteParticipantFactory: RemoteParticipant.Factory, )
+ LongParameterList:Room.kt$Room$( @Assisted private val context: Context, internal val engine: RTCEngine, private val eglBase: EglBase, localParticipantFactory: LocalParticipant.Factory, private val defaultsManager: DefaultsManager, @Named(InjectionNames.DISPATCHER_DEFAULT) private val defaultDispatcher: CoroutineDispatcher, @Named(InjectionNames.DISPATCHER_IO) private val ioDispatcher: CoroutineDispatcher, /** * The [AudioHandler] for setting up the audio as need. * * By default, this is an instance of [AudioSwitchHandler]. * * This can be substituted for your own custom implementation through * [LiveKitOverrides.audioOptions] when creating the room with [LiveKit.create]. * * @see [audioSwitchHandler] * @see [AudioSwitchHandler] */ val audioHandler: AudioHandler, private val closeableManager: CloseableManager, private val e2EEManagerFactory: E2EEManager.Factory, private val communicationWorkaround: CommunicationWorkaround, val audioProcessingController: AudioProcessingController, /** * A holder for objects that are used internally within LiveKit. */ val lkObjects: LKObjects, networkCallbackManagerFactory: NetworkCallbackManagerFactory, private val audioDeviceModule: AudioDeviceModule, private val regionUrlProviderFactory: RegionUrlProvider.Factory, private val connectionWarmer: ConnectionWarmer, private val audioRecordPrewarmer: AudioRecordPrewarmer, private val incomingDataStreamManager: IncomingDataStreamManager, private val dataStreams: DataStreams, private val incomingDataTrackManager: IncomingDataTrackManager, private val rpcClientManager: RpcClientManager, private val rpcServerManager: RpcServerManager, private val remoteParticipantFactory: RemoteParticipant.Factory, private val frameMetadataManager: FrameMetadataManager, private val frameTransformerInstaller: FrameTransformerInstaller, )
MapGetWithNotNullAssertionOperator:LocalParticipant.kt$LocalParticipant$sourcePubLocks[source]!!
NestedBlockDepth:ByteStreamSender.kt$@CheckResult suspend fun ByteStreamSender.write(source: Source): Result<Unit>
NestedBlockDepth:LocalParticipant.kt$LocalParticipant$@Throws(TrackException.PublishException::class) private suspend fun publishTrackImpl( track: Track, options: TrackPublishOptions, requestConfig: AddTrackRequest.Builder.() -> Unit, encodings: List<RtpParameters.Encoding> = emptyList(), publishListener: PublishListener? = null, ): LocalTrackPublication?
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt
index 91e3ad0ff..86b574361 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt
@@ -38,6 +38,8 @@ import io.livekit.android.e2ee.DataPacketCryptorManagerImpl
import io.livekit.android.memory.CloseableManager
import io.livekit.android.room.datatrack.LocalDataTrackManagerFactory
import io.livekit.android.room.datatrack.RemoteDataTrackManagerFactory
+import io.livekit.android.room.framemetadata.DefaultFrameTransformerFactory
+import io.livekit.android.room.framemetadata.FrameTransformerFactory
import io.livekit.android.util.LKLog
import io.livekit.android.util.LoggingLevel
import io.livekit.android.webrtc.CustomAudioProcessingFactory
@@ -405,6 +407,11 @@ internal object RTCModule {
return DataPacketCryptorManagerImpl.Factory
}
+ @Provides
+ internal fun frameTransformerFactory(peerConnectionFactory: PeerConnectionFactory): FrameTransformerFactory {
+ return DefaultFrameTransformerFactory(peerConnectionFactory)
+ }
+
@Provides
fun localDataTrackManagerFactory(): LocalDataTrackManagerFactory {
return LocalDataTrackManagerFactory { delegate, encryptionProvider ->
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt b/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt
index 40913db63..0fd14f811 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/e2ee/E2EEManager.kt
@@ -22,6 +22,8 @@ import dagger.assisted.AssistedInject
import io.livekit.android.annotations.Beta
import io.livekit.android.events.RoomEvent
import io.livekit.android.room.Room
+import io.livekit.android.room.framemetadata.DefaultFrameTransformerFactory
+import io.livekit.android.room.framemetadata.FrameTransformerFactory
import io.livekit.android.room.participant.LocalParticipant
import io.livekit.android.room.participant.Participant
import io.livekit.android.room.participant.RemoteParticipant
@@ -36,7 +38,6 @@ import livekit.LivekitModels.Encryption
import livekit.org.webrtc.FrameCryptor
import livekit.org.webrtc.FrameCryptor.FrameCryptionState
import livekit.org.webrtc.FrameCryptorAlgorithm
-import livekit.org.webrtc.FrameCryptorFactory
import livekit.org.webrtc.PeerConnectionFactory
import livekit.org.webrtc.RtpReceiver
import livekit.org.webrtc.RtpSender
@@ -55,6 +56,8 @@ constructor(
internal var dataPacketCryptorManager: DataPacketCryptorManager = dataPacketCryptorManagerFactory.create(keyProvider)
+ internal var frameTransformerFactory: FrameTransformerFactory = DefaultFrameTransformerFactory(peerConnectionFactory)
+
var enabled: Boolean = false
set(value) {
field = value
@@ -99,22 +102,29 @@ constructor(
this.room?.localParticipant?.trackPublications?.forEach { item ->
val participant = this.room!!.localParticipant
val publication = item.value
- if (publication.track != null) {
- addPublishedTrack(publication.track!!, publication, participant, room)
+ val track = publication.track
+ if (track != null) {
+ val frameCryptor = addPublishedTrack(track, publication, participant, room)
+ room.installSendTransformers(track, frameCryptor)
}
}
this.room?.remoteParticipants?.forEach { item ->
val participant = item.value
participant.trackPublications.forEach { item ->
val publication = item.value
- if (publication.track != null) {
- addSubscribedTrack(publication.track!!, publication, participant, room)
+ val track = publication.track
+ if (track != null) {
+ val frameCryptor = addSubscribedTrack(track, publication, participant, room)
+ room.installReceiveTransformers(track, frameCryptor)
}
}
}
}
- fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
+ /**
+ * Creates the cryptor for a subscribed track, without installing it.
+ */
+ internal fun addSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room): FrameCryptor {
val rtpReceiver: RtpReceiver? = when (publication.track!!) {
is RemoteAudioTrack -> (publication.track!! as RemoteAudioTrack).receiver
is RemoteVideoTrack -> (publication.track!! as RemoteVideoTrack).receiver
@@ -135,9 +145,10 @@ constructor(
),
)
}
+ return frameCryptor
}
- fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
+ internal fun removeSubscribedTrack(track: Track, publication: TrackPublication, participant: RemoteParticipant, room: Room) {
val trackId = publication.sid
val participantId = participant.identity
val frameCryptor = frameCryptors.get(trackId to participantId)
@@ -148,7 +159,10 @@ constructor(
}
}
- fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
+ /**
+ * Creates the cryptor for a published track, without installing it.
+ */
+ internal fun addPublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room): FrameCryptor {
val rtpSender: RtpSender = when (publication.track!!) {
is LocalAudioTrack -> (publication.track!! as LocalAudioTrack).sender
is LocalVideoTrack -> (publication.track!! as LocalVideoTrack).sender
@@ -170,9 +184,10 @@ constructor(
),
)
}
+ return frameCryptor
}
- fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
+ internal fun removePublishedTrack(track: Track, publication: TrackPublication, participant: LocalParticipant, room: Room) {
val trackId = publication.sid
val participantId = participant.identity
val frameCryptor = frameCryptors.get(trackId to participantId)
@@ -197,8 +212,7 @@ constructor(
}
private fun addRtpSender(sender: RtpSender, participantId: Participant.Identity, trackId: String, kind: String): FrameCryptor {
- val frameCryptor = FrameCryptorFactory.createFrameCryptorForRtpSender(
- peerConnectionFactory,
+ val frameCryptor = frameTransformerFactory.createFrameCryptor(
sender,
participantId.value,
algorithm,
@@ -212,8 +226,7 @@ constructor(
}
private fun addRtpReceiver(receiver: RtpReceiver, participantId: Participant.Identity, trackId: String, kind: String): FrameCryptor {
- val frameCryptor = FrameCryptorFactory.createFrameCryptorForRtpReceiver(
- peerConnectionFactory,
+ val frameCryptor = frameTransformerFactory.createFrameCryptor(
receiver,
participantId.value,
algorithm,
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt
index e5f86daa2..cf2129746 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/ClientCapability.kt
@@ -76,8 +76,13 @@ enum class ClientCapability(val value: Int) {
* compression is advertised unconditionally: deflate-raw is compressed and decompressed by the
* Rust data stream core, so support does not vary by device, API level, or room options.
*
+ * Packet trailers are also advertised unconditionally: the SDK handles the trailer itself on every
+ * subscribed video track whose publisher sends one. Without this, the server strips it first and
+ * frame metadata never arrives.
+ *
* @suppress
*/
internal val ADVERTISED_CLIENT_CAPABILITIES = listOf(
+ ClientCapability.PACKET_TRAILER,
ClientCapability.COMPRESSION_DEFLATE_RAW,
)
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt
index b4642f829..5a74be946 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/Room.kt
@@ -54,6 +54,8 @@ import io.livekit.android.room.datatrack.DataTrackSid
import io.livekit.android.room.datatrack.IncomingDataTrackEvent
import io.livekit.android.room.datatrack.IncomingDataTrackManager
import io.livekit.android.room.datatrack.RemoteDataTrack
+import io.livekit.android.room.framemetadata.FrameMetadataManager
+import io.livekit.android.room.framemetadata.FrameTransformerInstaller
import io.livekit.android.room.metrics.collectMetrics
import io.livekit.android.room.network.NetworkCallbackManagerFactory
import io.livekit.android.room.network.ReconnectPolicy
@@ -76,10 +78,14 @@ import io.livekit.android.room.rpc.RPC_RESPONSE_DATA_STREAM_TOPIC
import io.livekit.android.room.rpc.RpcClientManager
import io.livekit.android.room.rpc.RpcManager
import io.livekit.android.room.rpc.RpcServerManager
+import io.livekit.android.room.track.LocalAudioTrack
import io.livekit.android.room.track.LocalAudioTrackOptions
import io.livekit.android.room.track.LocalTrackPublication
+import io.livekit.android.room.track.LocalVideoTrack
import io.livekit.android.room.track.LocalVideoTrackOptions
+import io.livekit.android.room.track.RemoteAudioTrack
import io.livekit.android.room.track.RemoteTrackPublication
+import io.livekit.android.room.track.RemoteVideoTrack
import io.livekit.android.room.track.Track
import io.livekit.android.room.track.TrackPublication
import io.livekit.android.room.types.toSDKType
@@ -110,6 +116,7 @@ import kotlinx.serialization.Serializable
import livekit.LivekitModels
import livekit.LivekitRtc
import livekit.org.webrtc.EglBase
+import livekit.org.webrtc.FrameCryptor
import livekit.org.webrtc.MediaStream
import livekit.org.webrtc.MediaStreamTrack
import livekit.org.webrtc.RTCStatsCollectorCallback
@@ -165,6 +172,8 @@ constructor(
private val rpcClientManager: RpcClientManager,
private val rpcServerManager: RpcServerManager,
private val remoteParticipantFactory: RemoteParticipant.Factory,
+ private val frameMetadataManager: FrameMetadataManager,
+ private val frameTransformerInstaller: FrameTransformerInstaller,
) : RTCEngine.Listener, ParticipantListener, RpcManager, IncomingDataStreamManager by incomingDataStreamManager {
private lateinit var coroutineScope: CoroutineScope
@@ -515,6 +524,7 @@ constructor(
if (roomOptions.e2eeOptions != null) {
e2eeManager = e2EEManagerFactory.create(roomOptions.e2eeOptions.keyProvider).apply {
+ frameTransformerFactory = frameTransformerInstaller.factory
setup(this@Room) { event ->
coroutineScope.launch {
emitWhenConnected(event)
@@ -1126,6 +1136,7 @@ constructor(
isRecording = false
sidToIdentity.clear()
incomingDataStreamManager.clearOpenStreams()
+ frameMetadataManager.cleanup()
}
private suspend fun sendSyncState() {
@@ -1629,8 +1640,10 @@ constructor(
* @suppress
*/
override fun onTrackPublished(publication: LocalTrackPublication, participant: LocalParticipant) {
- if (e2eeManager != null) {
- e2eeManager!!.addPublishedTrack(publication.track!!, publication, participant, this)
+ val track = publication.track
+ if (track != null) {
+ val frameCryptor = e2eeManager?.addPublishedTrack(track, publication, participant, this)
+ installSendTransformers(track, frameCryptor)
}
eventBus.postEvent(RoomEvent.TrackPublished(this, publication, participant), coroutineScope)
}
@@ -1640,6 +1653,7 @@ constructor(
*/
override fun onTrackUnpublished(publication: LocalTrackPublication, participant: LocalParticipant) {
e2eeManager?.removePublishedTrack(publication.track!!, publication, participant, this)
+ publication.track?.let { frameMetadataManager.removePublishedTrack(it) }
eventBus.postEvent(RoomEvent.TrackUnpublished(this, publication, participant), coroutineScope)
}
@@ -1647,12 +1661,48 @@ constructor(
* @suppress
*/
override fun onTrackSubscribed(track: Track, publication: RemoteTrackPublication, participant: RemoteParticipant) {
- if (e2eeManager != null) {
- e2eeManager!!.addSubscribedTrack(track, publication, participant, this)
- }
+ val frameCryptor = e2eeManager?.addSubscribedTrack(track, publication, participant, this)
+ frameMetadataManager.addSubscribedTrack(track, publication)
+ installReceiveTransformers(track, frameCryptor)
eventBus.postEvent(RoomEvent.TrackSubscribed(this, track, publication, participant), coroutineScope)
}
+ /**
+ * Installs a published track's cryptor and frame metadata transformer, whichever it has, on its
+ * sender in a single install.
+ */
+ internal fun installSendTransformers(track: Track, frameCryptor: FrameCryptor?) {
+ val sender = when (track) {
+ is LocalVideoTrack -> track.sender
+ is LocalAudioTrack -> track.sender
+ else -> null
+ } ?: return
+ // Installs under the track's lock so an unpublish can't dispose the transformer in the
+ // meantime. The chain is kept after that lock is released: the manager's lock is taken
+ // before a track's, never after. setChain disposes the chain if the track was removed.
+ val chain = if (track is LocalVideoTrack) {
+ track.withFrameMetadataTransformer { frameTransformerInstaller.installSend(sender, frameCryptor, it) }
+ } else {
+ frameTransformerInstaller.installSend(sender, frameCryptor, null)
+ }
+ frameMetadataManager.setChain(track, chain)
+ }
+
+ /**
+ * Installs a subscribed track's frame metadata transformer and cryptor, whichever it has, on its
+ * receiver in a single install.
+ */
+ internal fun installReceiveTransformers(track: Track, frameCryptor: FrameCryptor?) {
+ val receiver = when (track) {
+ is RemoteVideoTrack -> track.receiver
+ is RemoteAudioTrack -> track.receiver
+ else -> null
+ } ?: return
+ frameMetadataManager.installReceive(track) { metadataTransformer ->
+ frameTransformerInstaller.installReceive(receiver, metadataTransformer, frameCryptor)
+ }
+ }
+
/**
* @suppress
*/
@@ -1673,6 +1723,7 @@ constructor(
participant: RemoteParticipant,
) {
e2eeManager?.removeSubscribedTrack(track, publication, participant, this)
+ frameMetadataManager.removeSubscribedTrack(track)
eventBus.postEvent(RoomEvent.TrackUnsubscribed(this, track, publication, participant), coroutineScope)
}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt
new file mode 100644
index 000000000..b23b23b54
--- /dev/null
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataManager.kt
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import io.livekit.android.room.track.FrameMetadataPublishOptions
+import io.livekit.android.room.track.LocalVideoTrack
+import io.livekit.android.room.track.RemoteVideoTrack
+import io.livekit.android.room.track.Track
+import io.livekit.android.room.track.TrackPublication
+import io.livekit.android.room.track.hasFeatures
+import io.livekit.android.util.LKLog
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.FrameTransformerChain
+import javax.inject.Inject
+import javax.inject.Singleton
+
+/**
+ * Owns the frame metadata transformers of a room's video tracks, and any chain built around one.
+ *
+ * It creates and disposes them; [FrameTransformerInstaller] installs them. Only video tracks get
+ * one: a published track when its options turn on a field, a subscribed track when its publisher
+ * sends metadata.
+ *
+ * Anything that uses a transformer holds the lock its disposal waits on: this manager's lock for
+ * receive transformers, the track's for send transformers (see
+ * [LocalVideoTrack.withFrameMetadataTransformer]). This manager's lock is taken before a track's,
+ * never after.
+ *
+ * @suppress
+ */
+@Singleton
+class FrameMetadataManager
+@Inject
+internal constructor(
+ private val factory: FrameTransformerFactory,
+) {
+ private class Entry(val transformer: FrameMetadataTransformer) {
+ var chain: FrameTransformerChain? = null
+ }
+
+ private val lock = Any()
+ private val published = mutableMapOf()
+ private val subscribed = mutableMapOf()
+
+ /**
+ * Sets up a track that's about to be published: creates its send transformer, links it to the
+ * track's source and starts stamping. Call before the publication becomes visible, so a backup
+ * codec sender created right after can find the transformer.
+ */
+ internal fun addPublishedTrack(track: LocalVideoTrack, options: FrameMetadataPublishOptions?) {
+ if (options == null || !options.hasFeatures()) {
+ return
+ }
+ synchronized(lock) {
+ if (published.containsKey(track)) {
+ return
+ }
+ val transformer = factory.createFrameMetadataTransformer(FrameMetadataTransformer.Direction.SEND)
+ track.attachFrameMetadataTransformer(transformer)
+ track.frameMetadataStampState.arm(options)
+ if (track.frameMetadataStamper == null) {
+ LKLog.w { "Track ${track.name} has no frame metadata stamping processor; only values set on its frames will be sent." }
+ }
+ published[track] = Entry(transformer)
+ }
+ }
+
+ internal fun removePublishedTrack(track: Track) {
+ val entry = synchronized(lock) { published.remove(track) } ?: return
+ val localTrack = track as LocalVideoTrack
+ localTrack.frameMetadataStampState.disarm()
+ localTrack.detachFrameMetadataTransformer()
+ entry.chain?.dispose()
+ entry.transformer.dispose()
+ }
+
+ /**
+ * Creates the receive transformer for a subscribed track if its publisher sends metadata, and
+ * hands it to the track for lookups.
+ *
+ * @return the transformer to install, or null if the track doesn't get one.
+ */
+ internal fun addSubscribedTrack(track: Track, publication: TrackPublication): FrameMetadataTransformer? {
+ if (track !is RemoteVideoTrack) {
+ return null
+ }
+ if (publication.trackInfo?.packetTrailerFeaturesList.isNullOrEmpty()) {
+ return null
+ }
+ // The client isn't told when the server stops sending a track, so a resubscribe arrives
+ // with the earlier subscription's track still here. Every subscribe builds a new track
+ // object, so that track is found by sid. This also gives a track added twice a fresh
+ // transformer.
+ val previousTracks = synchronized(lock) { subscribed.keys.filter { it.sid == publication.sid } }
+ previousTracks.forEach { removeSubscribedTrack(it) }
+ val transformer = factory.createFrameMetadataTransformer(FrameMetadataTransformer.Direction.RECEIVE)
+ synchronized(lock) {
+ subscribed[track] = Entry(transformer)
+ }
+ track.setFrameMetadataTransformer(transformer)
+ return transformer
+ }
+
+ /**
+ * Runs [install] with [track]'s receive transformer, or null if it has none, and keeps the
+ * chain it returns as [setChain] does.
+ *
+ * It all runs under the lock, so [removeSubscribedTrack] can't dispose the transformer while
+ * it's being installed: either the track was removed first and [install] gets null, or the
+ * removal waits until the chain is kept and then disposes it too. So [install] must not wait
+ * on the RTC thread or call app code.
+ */
+ internal fun installReceive(track: Track, install: (FrameMetadataTransformer?) -> FrameTransformerChain?) {
+ val unused = synchronized(lock) {
+ val entry = (track as? RemoteVideoTrack)?.let { subscribed[it] }
+ keepChain(entry, install(entry?.transformer))
+ }
+ unused?.dispose()
+ }
+
+ internal fun removeSubscribedTrack(track: Track) {
+ val entry = synchronized(lock) { subscribed.remove(track) } ?: return
+ // Stops lookups before disposing. A disabled receive transformer stops stripping trailers,
+ // so it's disposed without being disabled first.
+ (track as RemoteVideoTrack).setFrameMetadataTransformer(null)
+ entry.chain?.dispose()
+ entry.transformer.dispose()
+ }
+
+ /**
+ * Keeps the chain installed on [track]'s sender or receiver so it's disposed with the track's
+ * transformer. Replaces and disposes a previous chain. A track without a metadata transformer
+ * never has a chain, and one that was removed in the meantime has its chain disposed here.
+ */
+ internal fun setChain(track: Track, chain: FrameTransformerChain?) {
+ val unused = synchronized(lock) {
+ val entry = when (track) {
+ is LocalVideoTrack -> published[track]
+ is RemoteVideoTrack -> subscribed[track]
+ else -> null
+ }
+ keepChain(entry, chain)
+ }
+ unused?.dispose()
+ }
+
+ /**
+ * Keeps [chain] on [entry], and returns the chain to dispose: [chain] itself if the track has
+ * no entry, or the chain it replaces. Call it under the lock, in the same block that found
+ * [entry], so a removal can't take the entry and read its chain in between.
+ */
+ private fun keepChain(entry: Entry?, chain: FrameTransformerChain?): FrameTransformerChain? {
+ if (entry == null) {
+ return chain
+ }
+ val previous = entry.chain
+ entry.chain = chain
+ return previous?.takeIf { it !== chain }
+ }
+
+ /** Disposes everything left, for a room that's disconnecting. */
+ internal fun cleanup() {
+ val publishedTracks = synchronized(lock) { published.keys.toList() }
+ val subscribedTracks = synchronized(lock) { subscribed.keys.toList() }
+ publishedTracks.forEach { removePublishedTrack(it) }
+ subscribedTracks.forEach { removeSubscribedTrack(it) }
+ }
+}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt
new file mode 100644
index 000000000..05e7ae9cd
--- /dev/null
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameMetadataStampingProcessor.kt
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import io.livekit.android.room.track.FrameMetadata
+import io.livekit.android.room.track.FrameMetadataProvider
+import io.livekit.android.room.track.FrameMetadataPublishOptions
+import io.livekit.android.room.track.toRtc
+import io.livekit.android.room.track.toSdk
+import io.livekit.android.util.LKLog
+import livekit.org.webrtc.VideoFrame
+import livekit.org.webrtc.VideoProcessor
+import livekit.org.webrtc.VideoSink
+
+/**
+ * The last step of a local video track's processing chain. It wraps whatever processor the track
+ * was created with, so it sees each frame after every other processor, and sets the frame metadata
+ * from [state] on it.
+ *
+ * It's installed when the track is created and does nothing until [state] is armed at publish.
+ */
+internal class FrameMetadataStampingProcessor(
+ private val inner: VideoProcessor?,
+) : VideoProcessor {
+
+ @Volatile
+ var state = FrameMetadataStampState()
+
+ @Volatile
+ private var sink: VideoSink? = null
+
+ private val stampingSink = VideoSink { frame -> deliver(frame) }
+
+ override fun setSink(sink: VideoSink?) {
+ this.sink = sink
+ inner?.setSink(if (sink != null) stampingSink else null)
+ }
+
+ override fun onCapturerStarted(success: Boolean) {
+ inner?.onCapturerStarted(success)
+ }
+
+ override fun onCapturerStopped() {
+ inner?.onCapturerStopped()
+ }
+
+ // Forwards the two-argument version, which some processors override (see NoDropVideoProcessor).
+ override fun onFrameCaptured(frame: VideoFrame, parameters: VideoProcessor.FrameAdaptationParameters) {
+ val inner = inner
+ if (inner != null) {
+ inner.onFrameCaptured(frame, parameters)
+ return
+ }
+ val adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters) ?: return
+ deliver(adaptedFrame)
+ adaptedFrame.release()
+ }
+
+ override fun onFrameCaptured(frame: VideoFrame) {
+ val inner = inner
+ if (inner != null) {
+ inner.onFrameCaptured(frame)
+ } else {
+ deliver(frame)
+ }
+ }
+
+ private fun deliver(frame: VideoFrame) {
+ val sink = this.sink ?: return
+ // A new frame shares the buffer without retaining it, so only the original is released.
+ sink.onFrame(state.stamp(frame))
+ }
+}
+
+/**
+ * The frame metadata settings of one local video track: whether it's armed, the frame ID counter
+ * and the app's [FrameMetadataProvider]. It belongs to the track, so it carries over when the
+ * track's source and processor are replaced.
+ */
+internal class FrameMetadataStampState(
+ // WebRTC's clock, which is CLOCK_MONOTONIC on Android, the same as System.nanoTime(). Not
+ // TimestampAligner.getRtcTimeNanos(): it creates a native Environment on every call and never
+ // frees it.
+ private val rtcTimeNanos: () -> Long = { System.nanoTime() },
+ private val wallTimeMillis: () -> Long = { System.currentTimeMillis() },
+ initialFrameId: Long = 1L,
+) {
+ @Volatile
+ var provider: FrameMetadataProvider? = null
+
+ @Volatile
+ private var options: FrameMetadataPublishOptions? = null
+
+ private val frameIdLock = Any()
+ private var nextFrameId = initialFrameId
+
+ @Volatile
+ private var loggedProviderError = false
+
+ val isArmed: Boolean
+ get() = options != null
+
+ fun arm(options: FrameMetadataPublishOptions) {
+ this.options = options
+ }
+
+ fun disarm() {
+ options = null
+ }
+
+ /**
+ * Returns [frame] if there's nothing to change, or a new frame with the metadata to send.
+ *
+ * Starts from any metadata already on the frame, fills fields that are 0 with the defaults
+ * turned on in the options, then lets the provider replace the result.
+ */
+ fun stamp(frame: VideoFrame): VideoFrame {
+ val options = this.options ?: return frame
+ val existing = frame.frameMetadata?.toSdk()
+ val withDefaults = fillDefaults(frame, options, existing ?: FrameMetadata())
+ return frameWith(frame, existing, applyProvider(frame, withDefaults))
+ }
+
+ private fun fillDefaults(frame: VideoFrame, options: FrameMetadataPublishOptions, metadata: FrameMetadata): FrameMetadata {
+ val timestampUs = if (options.timestamp && metadata.userTimestampUs == 0L) {
+ defaultTimestampUs(frame)
+ } else {
+ metadata.userTimestampUs
+ }
+ val frameId = if (options.frameId && metadata.frameId == 0L) takeFrameId() else metadata.frameId
+ return metadata.copy(userTimestampUs = timestampUs, frameId = frameId)
+ }
+
+ private fun applyProvider(frame: VideoFrame, current: FrameMetadata): FrameMetadata {
+ val provider = this.provider ?: return current
+ return try {
+ provider.provide(frame, current) ?: current
+ } catch (e: Exception) {
+ if (!loggedProviderError) {
+ loggedProviderError = true
+ LKLog.w(e) { "FrameMetadataProvider threw; sending the values it was given. Logged once." }
+ }
+ current
+ }
+ }
+
+ private fun frameWith(frame: VideoFrame, existing: FrameMetadata?, result: FrameMetadata): VideoFrame {
+ val isEmpty = result.userTimestampUs == 0L && result.frameId == 0L && result.userData?.isEmpty() != false
+ return when {
+ isEmpty && existing == null -> frame
+ // Drop the metadata a processor set, since nothing is left to send.
+ isEmpty -> VideoFrame(frame.buffer, frame.rotation, frame.timestampNs)
+ result == existing -> frame
+ else -> VideoFrame(frame.buffer, frame.rotation, frame.timestampNs, result.toRtc())
+ }
+ }
+
+ // The frame's capture time, moved from WebRTC's monotonic clock to wall-clock time.
+ private fun defaultTimestampUs(frame: VideoFrame): Long {
+ val offsetNanos = wallTimeMillis() * 1_000_000L - rtcTimeNanos()
+ val timestampUs = (frame.timestampNs + offsetNanos) / 1_000L
+ return if (timestampUs > 0) timestampUs else 0L
+ }
+
+ // A uint32 that starts at 1 and wraps back to 1, since 0 means "not sent".
+ private fun takeFrameId(): Long = synchronized(frameIdLock) {
+ val id = nextFrameId
+ nextFrameId = if (id >= MAX_FRAME_ID) 1L else id + 1
+ id
+ }
+
+ companion object {
+ const val MAX_FRAME_ID = 0xFFFFFFFFL
+ }
+}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt
new file mode 100644
index 000000000..0be9a784f
--- /dev/null
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerFactory.kt
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import livekit.org.webrtc.FrameCryptor
+import livekit.org.webrtc.FrameCryptorAlgorithm
+import livekit.org.webrtc.FrameCryptorFactory
+import livekit.org.webrtc.FrameCryptorKeyProvider
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.FrameMetadataTransformerFactory
+import livekit.org.webrtc.FrameTransformer
+import livekit.org.webrtc.FrameTransformerChain
+import livekit.org.webrtc.PeerConnectionFactory
+import livekit.org.webrtc.RtpReceiver
+import livekit.org.webrtc.RtpSender
+
+/**
+ * Creates the frame transformers that go on senders and receivers. None of them are installed on
+ * creation; [FrameTransformerInstaller] does that.
+ *
+ * Behind an interface so tests can replace the native objects.
+ */
+internal interface FrameTransformerFactory {
+ fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer
+
+ /** Runs each frame through [first], then [second]. */
+ fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain
+
+ fun createFrameCryptor(
+ sender: RtpSender,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor
+
+ fun createFrameCryptor(
+ receiver: RtpReceiver,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor
+}
+
+internal class DefaultFrameTransformerFactory(
+ private val peerConnectionFactory: PeerConnectionFactory,
+) : FrameTransformerFactory {
+ override fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer =
+ FrameMetadataTransformerFactory.createFrameMetadataTransformer(direction)
+
+ override fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain =
+ FrameTransformerChain(first, second)
+
+ override fun createFrameCryptor(
+ sender: RtpSender,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor = FrameCryptorFactory.createUninstalledFrameCryptor(
+ peerConnectionFactory,
+ sender,
+ participantId,
+ algorithm,
+ keyProvider,
+ )
+
+ override fun createFrameCryptor(
+ receiver: RtpReceiver,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor = FrameCryptorFactory.createUninstalledFrameCryptor(
+ peerConnectionFactory,
+ receiver,
+ participantId,
+ algorithm,
+ keyProvider,
+ )
+}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt
new file mode 100644
index 000000000..b52e20e1b
--- /dev/null
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/framemetadata/FrameTransformerInstaller.kt
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import livekit.org.webrtc.FrameCryptor
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.FrameTransformer
+import livekit.org.webrtc.FrameTransformerChain
+import livekit.org.webrtc.RtpReceiver
+import livekit.org.webrtc.RtpSender
+import javax.inject.Inject
+
+/**
+ * Installs a track's transformers on its sender or receiver in a single install.
+ *
+ * With both a cryptor and a metadata transformer, it chains them in the order for the direction:
+ * a sender encrypts and then adds the trailer, so the trailer stays readable; a receiver strips the
+ * trailer and then decrypts. With only one, it installs that one directly.
+ *
+ * It holds no state. A chain it creates is returned so the caller can dispose it.
+ *
+ * @suppress
+ */
+class FrameTransformerInstaller
+@Inject
+internal constructor(
+ internal val factory: FrameTransformerFactory,
+) {
+ internal fun installSend(
+ sender: RtpSender,
+ cryptor: FrameCryptor?,
+ metadataTransformer: FrameMetadataTransformer?,
+ ): FrameTransformerChain? = install(first = cryptor, second = metadataTransformer) { sender.setFrameTransformer(it) }
+
+ internal fun installReceive(
+ receiver: RtpReceiver,
+ metadataTransformer: FrameMetadataTransformer?,
+ cryptor: FrameCryptor?,
+ ): FrameTransformerChain? = install(first = metadataTransformer, second = cryptor) { receiver.setFrameTransformer(it) }
+
+ private inline fun install(
+ first: FrameTransformer?,
+ second: FrameTransformer?,
+ setTransformer: (FrameTransformer) -> Unit,
+ ): FrameTransformerChain? {
+ if (first != null && second != null) {
+ return factory.createChain(first, second).also { setTransformer(it) }
+ }
+ val single = first ?: second ?: return null
+ setTransformer(single)
+ return null
+ }
+}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt
index 714f28d23..38408e678 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt
@@ -44,11 +44,13 @@ import io.livekit.android.room.datatrack.DataTrackSchemaException
import io.livekit.android.room.datatrack.DataTrackSchemaId
import io.livekit.android.room.datatrack.LocalDataTrack
import io.livekit.android.room.datatrack.OutgoingDataTrackManager
+import io.livekit.android.room.framemetadata.FrameMetadataManager
import io.livekit.android.room.isSVCCodec
import io.livekit.android.room.rpc.RpcClientManager
import io.livekit.android.room.rpc.RpcManager
import io.livekit.android.room.rpc.RpcServerManager
import io.livekit.android.room.track.DataPublishReliability
+import io.livekit.android.room.track.FrameMetadataPublishOptions
import io.livekit.android.room.track.LocalAudioTrack
import io.livekit.android.room.track.LocalAudioTrackOptions
import io.livekit.android.room.track.LocalScreencastVideoTrack
@@ -63,6 +65,7 @@ import io.livekit.android.room.track.VideoCodec
import io.livekit.android.room.track.VideoEncoding
import io.livekit.android.room.track.VideoPreset
import io.livekit.android.room.track.screencapture.ScreenCaptureParams
+import io.livekit.android.room.track.toPacketTrailerFeatures
import io.livekit.android.room.util.EncodingUtils
import io.livekit.android.rpc.RpcError
import io.livekit.android.util.LKLog
@@ -126,6 +129,7 @@ internal constructor(
private val outgoingDataTrackManager: OutgoingDataTrackManager,
private val rpcClientManager: RpcClientManager,
private val rpcServerManager: RpcServerManager,
+ private val frameMetadataManager: FrameMetadataManager,
) : Participant(Sid(""), null, coroutineDispatcher),
OutgoingDataStreamManager by outgoingDataStreamManager,
RpcManager {
@@ -621,6 +625,7 @@ internal constructor(
},
)
}
+ options.frameMetadata?.let { addAllPacketTrailerFeatures(it.toPacketTrailerFeatures()) }
},
encodings = encodings,
publishListener = publishListener,
@@ -856,6 +861,11 @@ internal constructor(
participant = this,
options = options,
)
+ if (track is LocalVideoTrack) {
+ // Before the publication is visible, so a backup codec sender created after it
+ // finds the transformer.
+ frameMetadataManager.addPublishedTrack(track, (options as? VideoTrackPublishOptions)?.frameMetadata)
+ }
addTrackPublication(publication)
LKLog.v { "add track publication $publication" }
@@ -1439,6 +1449,12 @@ internal constructor(
return@launch
}
simulcastTrack.transceiver = transceiver
+ // The backup codec sender shares the primary's metadata transformer: the source and the
+ // frames are the same. No send stream exists yet, so this doesn't recreate one. It's
+ // installed under the track's lock so an unpublish can't dispose it in the meantime.
+ track.withFrameMetadataTransformer { transformer ->
+ transformer?.let { transceiver.sender.setFrameTransformer(it) }
+ }
val trackRequest = AddTrackRequest.newBuilder().apply {
sid = existingPublication.sid
muted = !track.enabled
@@ -1733,6 +1749,13 @@ abstract class BaseVideoTrackPublishOptions {
* When set to null, it defaults to H180 and H360.
*/
abstract val simulcastLayers: List?
+
+ /**
+ * Frame metadata to send with each frame, such as a capture timestamp and a frame ID.
+ * Off when null. See [FrameMetadataPublishOptions].
+ */
+ open val frameMetadata: FrameMetadataPublishOptions?
+ get() = null
}
// Remember when adding any defaults to add it in the copy constructor of VideoTrackPublishOptions.
@@ -1744,6 +1767,7 @@ data class VideoTrackPublishDefaults(
override val backupCodec: BackupVideoCodec? = null,
override val degradationPreference: RtpParameters.DegradationPreference? = null,
override val simulcastLayers: List? = null,
+ override val frameMetadata: FrameMetadataPublishOptions? = null,
) : BaseVideoTrackPublishOptions()
data class VideoTrackPublishOptions(
@@ -1757,6 +1781,7 @@ data class VideoTrackPublishOptions(
override val stream: String? = null,
override val degradationPreference: RtpParameters.DegradationPreference? = null,
override val simulcastLayers: List? = null,
+ override val frameMetadata: FrameMetadataPublishOptions? = null,
) : BaseVideoTrackPublishOptions(), TrackPublishOptions {
constructor(
name: String? = null,
@@ -1774,6 +1799,7 @@ data class VideoTrackPublishOptions(
stream = stream,
degradationPreference = base.degradationPreference,
simulcastLayers = base.simulcastLayers,
+ frameMetadata = base.frameMetadata,
)
fun createBackupOptions(): VideoTrackPublishOptions? {
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt
new file mode 100644
index 000000000..db2d3c9d7
--- /dev/null
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FrameMetadata.kt
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.track
+
+import io.livekit.android.annotations.Beta
+import livekit.LivekitModels.PacketTrailerFeature
+import livekit.org.webrtc.VideoFrame
+import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata
+
+/**
+ * Metadata sent along with a single video frame.
+ *
+ * A field set to 0, or a null or empty [userData], isn't sent.
+ *
+ * @property userTimestampUs A timestamp in microseconds. By default, the publisher's capture time
+ * as wall-clock time. See [FrameMetadataPublishOptions.timestamp].
+ * @property frameId A frame ID from 0 to 4294967295. By default, a per-track counter.
+ * See [FrameMetadataPublishOptions.frameId].
+ * @property userData Bytes supplied by the publishing app. See [FrameMetadataPublishOptions.userData].
+ */
+@Beta
+data class FrameMetadata(
+ val userTimestampUs: Long = 0,
+ val frameId: Long = 0,
+ val userData: ByteArray? = null,
+) {
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (other !is FrameMetadata) return false
+ return userTimestampUs == other.userTimestampUs &&
+ frameId == other.frameId &&
+ userData.contentEquals(other.userData)
+ }
+
+ override fun hashCode(): Int {
+ var result = userTimestampUs.hashCode()
+ result = 31 * result + frameId.hashCode()
+ result = 31 * result + (userData?.contentHashCode() ?: 0)
+ return result
+ }
+}
+
+/**
+ * Declares which frame metadata to send with a published video track. Subscribers read it with
+ * [RemoteVideoTrack.lookupFrameMetadata].
+ *
+ * Each option turns on the associated [FrameMetadata] field, which the SDK fills in by default.
+ * To send your own values, use [LocalVideoTrack.frameMetadataProvider]. They're sent even for a field whose option is off, as
+ * long as at least one option is on.
+ *
+ * @property timestamp Send a timestamp with each frame. By default it's the frame's capture time,
+ * in wall-clock microseconds. If your video processor changes frame timestamps, supply your own.
+ * @property frameId Send a frame ID with each frame. By default it's a per-track counter starting
+ * at 1. IDs skip frames that are dropped, and can repeat when a subscriber switches simulcast
+ * layers.
+ * @property userData Send user data, which you supply. **It isn't encrypted, even with end-to-end
+ * encryption.** Each frame can carry up to 232 bytes, or 248 with the other options off. User data
+ * that doesn't fit, or that would form an H.264 or H.265 start code, is dropped from that frame.
+ */
+data class FrameMetadataPublishOptions(
+ val timestamp: Boolean = false,
+ val frameId: Boolean = false,
+ val userData: Boolean = false,
+)
+
+/**
+ * Supplies frame metadata for a local video track, through [LocalVideoTrack.frameMetadataProvider].
+ *
+ * It's called for every frame on the video capture path, so should be kept fast and non-blocking.
+ */
+@Beta
+fun interface FrameMetadataProvider {
+ /**
+ * @param frame The frame about to be encoded.
+ * @param current The values so far: any metadata a video processor set on the frame, with the
+ * defaults turned on in [FrameMetadataPublishOptions] filled into fields that are 0.
+ * @return New values to send instead of [current], or null to send [current]. A field set to
+ * 0 isn't sent. If this throws, [current] is sent.
+ */
+ fun provide(frame: VideoFrame, current: FrameMetadata): FrameMetadata?
+}
+
+/**
+ * A frame metadata field that a publisher declared for a video track, so that subscribers know to
+ * expect it. See [TrackPublication.frameMetadataFeatures].
+ */
+@Beta
+enum class FrameMetadataFeature {
+ /** Frames carry a timestamp. See [FrameMetadata.userTimestampUs]. */
+ TIMESTAMP,
+
+ /** Frames carry a frame ID. See [FrameMetadata.frameId]. */
+ FRAME_ID,
+
+ /** Frames carry user data. See [FrameMetadata.userData]. */
+ USER_DATA,
+ ;
+
+ internal fun toProto(): PacketTrailerFeature = when (this) {
+ TIMESTAMP -> PacketTrailerFeature.PTF_USER_TIMESTAMP
+ FRAME_ID -> PacketTrailerFeature.PTF_FRAME_ID
+ USER_DATA -> PacketTrailerFeature.PTF_USER_DATA
+ }
+
+ internal companion object {
+ /**
+ * Returns null for a value this SDK build doesn't know, so a publisher declaring a newer
+ * feature isn't an error.
+ */
+ fun fromProto(feature: PacketTrailerFeature): FrameMetadataFeature? = when (feature) {
+ PacketTrailerFeature.PTF_USER_TIMESTAMP -> TIMESTAMP
+ PacketTrailerFeature.PTF_FRAME_ID -> FRAME_ID
+ PacketTrailerFeature.PTF_USER_DATA -> USER_DATA
+ PacketTrailerFeature.UNRECOGNIZED -> null
+ }
+ }
+}
+
+internal fun FrameMetadataPublishOptions?.hasFeatures(): Boolean =
+ this != null && (timestamp || frameId || userData)
+
+internal fun FrameMetadataPublishOptions.toPacketTrailerFeatures(): List = buildList {
+ if (timestamp) add(FrameMetadataFeature.TIMESTAMP)
+ if (frameId) add(FrameMetadataFeature.FRAME_ID)
+ if (userData) add(FrameMetadataFeature.USER_DATA)
+}.map { it.toProto() }
+
+internal fun FrameMetadata.toRtc(): RtcFrameMetadata = RtcFrameMetadata(userTimestampUs, frameId, userData)
+
+internal fun RtcFrameMetadata.toSdk(): FrameMetadata = FrameMetadata(userTimestampUs, frameId, userData)
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt
index 563c7dc5e..d9e9d7d2b 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalScreencastVideoTrack.kt
@@ -30,6 +30,7 @@ import dagger.assisted.AssistedFactory
import dagger.assisted.AssistedInject
import io.livekit.android.memory.SurfaceTextureHelperCloser
import io.livekit.android.room.DefaultsManager
+import io.livekit.android.room.framemetadata.FrameMetadataStampingProcessor
import io.livekit.android.room.participant.LocalParticipant
import io.livekit.android.room.track.screencapture.ScreenCaptureConnection
import io.livekit.android.room.track.screencapture.ScreenCaptureService
@@ -247,7 +248,9 @@ constructor(
onStop: (Track) -> Unit,
): LocalScreencastVideoTrack {
val source = peerConnectionFactory.createVideoSource(options.isScreencast)
- source.setVideoProcessor(videoProcessor)
+ // Installed now and armed at publish, so publishing doesn't have to swap processors.
+ val stampingProcessor = FrameMetadataStampingProcessor(videoProcessor)
+ source.setVideoProcessor(stampingProcessor)
val callback = MediaProjectionCallback().apply {
addOnStopCallback(onStop)
}
@@ -268,6 +271,7 @@ constructor(
rtcTrack = rtcTrack,
mediaProjectionCallback = callback,
)
+ track.frameMetadataStamper = stampingProcessor
// The track owns the SurfaceTextureHelper; dispose() releases its capture thread.
track.closeableManager.registerResource(
rtcTrack,
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt
index 75f93ea84..05bc78eb0 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt
@@ -23,9 +23,12 @@ import androidx.core.content.ContextCompat
import dagger.assisted.Assisted
import dagger.assisted.AssistedFactory
import dagger.assisted.AssistedInject
+import io.livekit.android.annotations.Beta
import io.livekit.android.memory.CloseableManager
import io.livekit.android.memory.SurfaceTextureHelperCloser
import io.livekit.android.room.DefaultsManager
+import io.livekit.android.room.framemetadata.FrameMetadataStampState
+import io.livekit.android.room.framemetadata.FrameMetadataStampingProcessor
import io.livekit.android.room.track.video.CameraCapturerUtils
import io.livekit.android.room.track.video.CameraCapturerUtils.CameraDeviceInfo
import io.livekit.android.room.track.video.CameraCapturerUtils.createCameraEnumerator
@@ -45,6 +48,7 @@ import livekit.LivekitRtc.SubscribedCodec
import livekit.org.webrtc.CameraVideoCapturer
import livekit.org.webrtc.CameraVideoCapturer.CameraEventsHandler
import livekit.org.webrtc.EglBase
+import livekit.org.webrtc.FrameMetadataTransformer
import livekit.org.webrtc.MediaStreamTrack
import livekit.org.webrtc.PeerConnectionFactory
import livekit.org.webrtc.RtpParameters
@@ -122,6 +126,68 @@ constructor(
internal val closeableManager = CloseableManager()
+ /**
+ * This track's frame metadata settings. They belong to the track rather than its stamping
+ * processor, so they carry over when [restartTrack] replaces the source and processor.
+ */
+ internal val frameMetadataStampState = FrameMetadataStampState()
+
+ /**
+ * The processor at the end of this track's chain that sets frame metadata on its frames.
+ * Null if the track wasn't created by the SDK's track factories.
+ */
+ internal var frameMetadataStamper: FrameMetadataStampingProcessor? = null
+ set(value) {
+ value?.state = frameMetadataStampState
+ field = value
+ }
+
+ private val frameMetadataLock = Any()
+
+ /**
+ * The send transformer linked to this track's source while it's published with frame metadata.
+ * Backup codec senders reuse it. Use it only through [withFrameMetadataTransformer].
+ */
+ private var frameMetadataTransformer: FrameMetadataTransformer? = null
+
+ /**
+ * Supplies this track's frame metadata in place of, or on top of, the defaults turned on in
+ * [VideoTrackPublishOptions.frameMetadata]. See [FrameMetadataProvider].
+ *
+ * It's only called while the track is published with frame metadata on.
+ */
+ @Beta
+ var frameMetadataProvider: FrameMetadataProvider?
+ get() = frameMetadataStampState.provider
+ set(value) {
+ frameMetadataStampState.provider = value
+ }
+
+ /**
+ * Runs [action] with this track's send transformer, or null if it has none.
+ *
+ * It holds the lock that [detachFrameMetadataTransformer] takes, and the transformer is only
+ * disposed after it's detached, so it can't be disposed while [action] uses it. So [action]
+ * must not wait on the RTC thread, take the frame metadata manager's lock, or call app code.
+ */
+ internal fun withFrameMetadataTransformer(action: (FrameMetadataTransformer?) -> T): T =
+ synchronized(frameMetadataLock) { action(frameMetadataTransformer) }
+
+ internal fun attachFrameMetadataTransformer(transformer: FrameMetadataTransformer) {
+ synchronized(frameMetadataLock) {
+ frameMetadataTransformer = transformer
+ source.addFrameMetadataTransformer(transformer)
+ }
+ }
+
+ internal fun detachFrameMetadataTransformer() {
+ synchronized(frameMetadataLock) {
+ val transformer = frameMetadataTransformer ?: return
+ frameMetadataTransformer = null
+ source.removeFrameMetadataTransformer(transformer)
+ }
+ }
+
/**
* Starts the [capturer] with the capture params contained in [options].
*/
@@ -280,6 +346,7 @@ constructor(
oldCapturer.stopCapture()
oldCapturer.dispose()
+ withFrameMetadataTransformer { transformer -> transformer?.let { oldSource.removeFrameMetadataTransformer(it) } }
oldSource.dispose()
oldRtcTrack.setEnabled(false)
@@ -312,11 +379,23 @@ constructor(
}
capturer = newTrack.capturer
- source = newTrack.source
rtcTrack = newTrack.rtcTrack
this.options = options
+ // Keeps this track's frame metadata settings and frame ID counter on the new processor.
+ frameMetadataStamper = newTrack.frameMetadataStamper
+ // The source changes under the same lock as the link, so a detach running now unlinks the
+ // transformer from whichever source has it, and can't dispose it while it's being linked.
+ synchronized(frameMetadataLock) {
+ source = newTrack.source
+ frameMetadataTransformer?.let { source.addFrameMetadataTransformer(it) }
+ }
startCapture()
sender?.setTrack(newTrack.rtcTrack, false)
+ // Backup codec senders send the same track, so they move to the new one too.
+ for (simulcastTrackInfo in simulcastCodecs.values) {
+ simulcastTrackInfo.rtcTrack = newTrack.rtcTrack
+ (simulcastTrackInfo.transceiver?.sender ?: simulcastTrackInfo.sender)?.setTrack(newTrack.rtcTrack, false)
+ }
}
internal fun setPublishingLayers(
@@ -536,7 +615,9 @@ constructor(
} else {
videoProcessor
}
- source.setVideoProcessor(finalVideoProcessor)
+ // Installed now and armed at publish, so publishing doesn't have to swap processors.
+ val stampingProcessor = FrameMetadataStampingProcessor(finalVideoProcessor)
+ source.setVideoProcessor(stampingProcessor)
val surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureThread", rootEglBase.eglBaseContext)
@@ -564,6 +645,7 @@ constructor(
rtcTrack = rtcTrack,
dispatchObserver = dispatchObserver,
)
+ track.frameMetadataStamper = stampingProcessor
track.closeableManager.registerResource(
rtcTrack,
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt
index 4c1179842..80bdbcbd4 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteVideoTrack.kt
@@ -1,5 +1,5 @@
/*
- * Copyright 2023-2025 LiveKit, Inc.
+ * Copyright 2023-2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import androidx.annotation.VisibleForTesting
import dagger.assisted.Assisted
import dagger.assisted.AssistedFactory
import dagger.assisted.AssistedInject
+import io.livekit.android.annotations.Beta
import io.livekit.android.dagger.InjectionNames
import io.livekit.android.events.TrackEvent
import io.livekit.android.room.track.video.VideoSinkVisibility
@@ -32,6 +33,7 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
+import livekit.org.webrtc.FrameMetadataTransformer
import livekit.org.webrtc.RtpReceiver
import livekit.org.webrtc.VideoSink
import javax.inject.Named
@@ -74,6 +76,42 @@ constructor(
this.receiver = receiver
}
+ // Guards the transformer, so a lookup can't use it while it's being disposed.
+ private val frameMetadataLock = Any()
+ private var frameMetadataTransformer: FrameMetadataTransformer? = null
+
+ internal fun setFrameMetadataTransformer(transformer: FrameMetadataTransformer?) {
+ synchronized(frameMetadataLock) {
+ frameMetadataTransformer = transformer
+ }
+ }
+
+ /**
+ * Looks up the frame metadata the publisher sent with a frame of this track.
+ *
+ * Get the RTP timestamp from the frame in your video sink, with
+ * [VideoFrame.getRtpTimestamp][livekit.org.webrtc.VideoFrame.getRtpTimestamp]:
+ *
+ * ```
+ * remoteVideoTrack.addRenderer { frame ->
+ * val metadata = remoteVideoTrack.lookupFrameMetadata(frame.rtpTimestamp)
+ * }
+ * ```
+ *
+ * Values for the last ~300 frames are kept, so look up a frame soon after it arrives.
+ *
+ * @return the values, or null if the publisher didn't send any for this frame, the frame is no
+ * longer available, or the track has been unsubscribed.
+ */
+ @Beta
+ fun lookupFrameMetadata(rtpTimestamp: Long): FrameMetadata? {
+ // The transformer is cleared under this lock before it's disposed, so a lookup never sees
+ // a disposed one.
+ synchronized(frameMetadataLock) {
+ return frameMetadataTransformer?.lookup(rtpTimestamp)?.toSdk()
+ }
+ }
+
/**
* If `autoManageVideo` is enabled, a VideoSinkVisibility should be passed, using
* [ViewVisibility] if using a traditional View layout, or ComposeVisibility
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt
index 181e3f8bc..6b93e1ff6 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/TrackPublication.kt
@@ -1,5 +1,5 @@
/*
- * Copyright 2023-2024 LiveKit, Inc.
+ * Copyright 2023-2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
package io.livekit.android.room.track
+import io.livekit.android.annotations.Beta
import io.livekit.android.events.BroadcastEventBus
import io.livekit.android.events.RoomEvent
import io.livekit.android.events.TrackPublicationEvent
@@ -67,6 +68,16 @@ open class TrackPublication(
var mimeType: String? = null
internal set
+ /**
+ * The frame metadata fields the publisher declared for this track, so that subscribers know to
+ * expect them. Empty if the track sends no frame metadata. Metadata is unsupported for audio.
+ *
+ * Subscribers read the values with [RemoteVideoTrack.lookupFrameMetadata].
+ */
+ @Beta
+ var frameMetadataFeatures: List = emptyList() // Declared before init, which sets it.
+ internal set
+
internal var trackInfo: LivekitModels.TrackInfo? = null
var participant: WeakReference
@@ -90,6 +101,7 @@ open class TrackPublication(
dimensions = Track.Dimensions(info.width, info.height)
}
mimeType = info.mimeType
+ frameMetadataFeatures = info.packetTrailerFeaturesList.mapNotNull { FrameMetadataFeature.fromProto(it) }
trackInfo = info
}
diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt
index 7b55366ee..3b1dbad2c 100644
--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt
+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt
@@ -1,5 +1,5 @@
/*
- * Copyright 2024 LiveKit, Inc.
+ * Copyright 2024-2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -89,7 +89,7 @@ class ScaleCropVideoProcessor(
scaleHeight,
)
- val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs)
+ val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs, frame.frameMetadata)
continueChain(croppedFrame)
croppedFrame.release()
}
diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt
new file mode 100644
index 000000000..d8a230f87
--- /dev/null
+++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockFrameTransformerFactory.kt
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.test.mock
+
+import io.livekit.android.room.framemetadata.FrameTransformerFactory
+import livekit.org.webrtc.FrameCryptor
+import livekit.org.webrtc.FrameCryptorAlgorithm
+import livekit.org.webrtc.FrameCryptorKeyProvider
+import livekit.org.webrtc.FrameMetadata
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.FrameTransformer
+import livekit.org.webrtc.FrameTransformerChain
+import livekit.org.webrtc.RtpReceiver
+import livekit.org.webrtc.RtpSender
+import org.mockito.Mockito
+import org.mockito.kotlin.whenever
+
+/**
+ * Records the frame transformers the SDK creates, as Mockito mocks in place of the native objects.
+ */
+class MockFrameTransformerFactory {
+
+ data class CreatedChain(
+ val chain: FrameTransformerChain,
+ val first: FrameTransformer,
+ val second: FrameTransformer,
+ )
+
+ val metadataTransformers = mutableListOf()
+ val chains = mutableListOf()
+ val senderCryptors = mutableListOf()
+ val receiverCryptors = mutableListOf()
+
+ /** What a receive transformer's `lookup` returns. */
+ var lookupResult: (rtpTimestamp: Long) -> FrameMetadata? = { null }
+
+ internal val factory: FrameTransformerFactory = object : FrameTransformerFactory {
+ override fun createFrameMetadataTransformer(direction: FrameMetadataTransformer.Direction): FrameMetadataTransformer {
+ val transformer = Mockito.mock(FrameMetadataTransformer::class.java)
+ whenever(transformer.direction).thenReturn(direction)
+ whenever(transformer.lookup(Mockito.anyLong())).thenAnswer { lookupResult(it.getArgument(0)) }
+ metadataTransformers.add(transformer)
+ return transformer
+ }
+
+ override fun createChain(first: FrameTransformer, second: FrameTransformer): FrameTransformerChain {
+ val chain = Mockito.mock(FrameTransformerChain::class.java)
+ chains.add(CreatedChain(chain, first, second))
+ return chain
+ }
+
+ override fun createFrameCryptor(
+ sender: RtpSender,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor = Mockito.mock(FrameCryptor::class.java).also { senderCryptors.add(it) }
+
+ override fun createFrameCryptor(
+ receiver: RtpReceiver,
+ participantId: String,
+ algorithm: FrameCryptorAlgorithm,
+ keyProvider: FrameCryptorKeyProvider,
+ ): FrameCryptor = Mockito.mock(FrameCryptor::class.java).also { receiverCryptors.add(it) }
+ }
+}
diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt
index 9280dc6f4..8bfea3b37 100644
--- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt
+++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/MockVideoSource.kt
@@ -16,9 +16,31 @@
package io.livekit.android.test.mock
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.VideoProcessor
import livekit.org.webrtc.VideoSource
class MockVideoSource(nativeSource: Long = 100) : VideoSource(nativeSource) {
+
+ /** The send transformers currently linked to this source. */
+ val frameMetadataTransformers = mutableListOf()
+
+ var videoProcessor: VideoProcessor? = null
+ private set
+
override fun dispose() {
}
+
+ override fun addFrameMetadataTransformer(transformer: FrameMetadataTransformer) {
+ frameMetadataTransformers.add(transformer)
+ }
+
+ override fun removeFrameMetadataTransformer(transformer: FrameMetadataTransformer) {
+ frameMetadataTransformers.remove(transformer)
+ }
+
+ override fun setVideoProcessor(newVideoProcessor: VideoProcessor?) {
+ videoProcessor = newVideoProcessor
+ super.setVideoProcessor(newVideoProcessor)
+ }
}
diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt
index f0ba8fba6..c6fd42c5a 100644
--- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt
+++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestLiveKitComponent.kt
@@ -25,6 +25,7 @@ import io.livekit.android.dagger.LiveKitComponent
import io.livekit.android.dagger.MemoryModule
import io.livekit.android.dagger.UniffiModule
import io.livekit.android.room.RTCEngine
+import io.livekit.android.test.mock.MockFrameTransformerFactory
import io.livekit.android.test.mock.MockNetworkCallbackRegistry
import io.livekit.android.test.mock.MockWebSocketFactory
import io.livekit.android.test.mock.room.datatrack.MockLocalDataTrackManagerFactory
@@ -56,6 +57,8 @@ interface TestLiveKitComponent : LiveKitComponent {
fun remoteDataTrackManagerFactory(): MockRemoteDataTrackManagerFactory
+ fun frameTransformerFactory(): MockFrameTransformerFactory
+
@Component.Factory
interface Factory {
fun create(
diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt
index 67d448344..803f78d23 100644
--- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt
+++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt
@@ -32,9 +32,11 @@ import io.livekit.android.e2ee.DataPacketCryptorManager
import io.livekit.android.e2ee.KeyProvider
import io.livekit.android.room.datatrack.LocalDataTrackManagerFactory
import io.livekit.android.room.datatrack.RemoteDataTrackManagerFactory
+import io.livekit.android.room.framemetadata.FrameTransformerFactory
import io.livekit.android.test.mock.MockAudioDeviceModule
import io.livekit.android.test.mock.MockAudioProcessingController
import io.livekit.android.test.mock.MockEglBase
+import io.livekit.android.test.mock.MockFrameTransformerFactory
import io.livekit.android.test.mock.e2ee.ReversingDataPacketCryptorManager
import io.livekit.android.test.mock.room.datatrack.MockLocalDataTrackManagerFactory
import io.livekit.android.test.mock.room.datatrack.MockRemoteDataTrackManagerFactory
@@ -143,6 +145,17 @@ object TestRTCModule {
}
}
+ @Provides
+ @Singleton
+ fun mockFrameTransformerFactory(): MockFrameTransformerFactory {
+ return MockFrameTransformerFactory()
+ }
+
+ @Provides
+ internal fun frameTransformerFactory(
+ factory: MockFrameTransformerFactory,
+ ): FrameTransformerFactory = factory.factory
+
@Provides
@Singleton
fun mockLocalDataTrackManagerFactory(): MockLocalDataTrackManagerFactory {
diff --git a/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt b/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt
index 529b13dd2..2bfc935c6 100644
--- a/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt
+++ b/livekit-android-test/src/main/java/livekit/org/webrtc/MockPeerConnectionFactory.kt
@@ -1,5 +1,5 @@
/*
- * Copyright 2023-2025 LiveKit, Inc.
+ * Copyright 2023-2026 LiveKit, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,12 +40,15 @@ class MockPeerConnectionFactory : PeerConnectionFactory(1L) {
return MockAudioStreamTrack(id = id)
}
+ /** Every video source created, in order. */
+ val createdVideoSources = mutableListOf()
+
override fun createVideoSource(isScreencast: Boolean, alignTimestamps: Boolean): VideoSource {
- return MockVideoSource()
+ return MockVideoSource().also { createdVideoSources.add(it) }
}
override fun createVideoSource(isScreencast: Boolean): VideoSource {
- return MockVideoSource()
+ return MockVideoSource().also { createdVideoSources.add(it) }
}
override fun createVideoTrack(id: String, source: VideoSource?): VideoTrack {
diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt
index 78c6b4330..441c1cacd 100644
--- a/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt
+++ b/livekit-android-test/src/test/java/io/livekit/android/room/ConnectionParamsTest.kt
@@ -77,7 +77,10 @@ class ConnectionParamsTest : MockE2ETest() {
assertEquals(ClientProtocolVersion.DATA_STREAM_V2.value, clientInfo.clientProtocol)
assertEquals(
- listOf(LivekitModels.ClientInfo.Capability.CAP_COMPRESSION_DEFLATE_RAW),
+ listOf(
+ LivekitModels.ClientInfo.Capability.CAP_PACKET_TRAILER,
+ LivekitModels.ClientInfo.Capability.CAP_COMPRESSION_DEFLATE_RAW,
+ ),
clientInfo.capabilitiesList,
)
}
diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt
index 41a7133e0..bdb86b429 100644
--- a/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt
+++ b/livekit-android-test/src/test/java/io/livekit/android/room/RoomTest.kt
@@ -33,6 +33,8 @@ import io.livekit.android.room.datastream.DataStreams
import io.livekit.android.room.datastream.incoming.IncomingDataStreamManagerImpl
import io.livekit.android.room.datatrack.IncomingDataTrackEvent
import io.livekit.android.room.datatrack.IncomingDataTrackManager
+import io.livekit.android.room.framemetadata.FrameMetadataManager
+import io.livekit.android.room.framemetadata.FrameTransformerInstaller
import io.livekit.android.room.network.NetworkCallbackManagerImpl
import io.livekit.android.room.participant.LocalParticipant
import io.livekit.android.test.assert.assertIsClassList
@@ -41,6 +43,7 @@ import io.livekit.android.test.events.EventCollector
import io.livekit.android.test.mock.MockAudioDeviceModule
import io.livekit.android.test.mock.MockAudioProcessingController
import io.livekit.android.test.mock.MockEglBase
+import io.livekit.android.test.mock.MockFrameTransformerFactory
import io.livekit.android.test.mock.MockLKObjects
import io.livekit.android.test.mock.MockNetworkCallbackRegistry
import io.livekit.android.test.mock.TestData
@@ -170,9 +173,13 @@ class RoomTest {
ioDispatcher = coroutineRule.dispatcher,
defaultDispatcher = coroutineRule.dispatcher,
),
+ frameMetadataManager = FrameMetadataManager(frameTransformerFactory.factory),
+ frameTransformerInstaller = FrameTransformerInstaller(frameTransformerFactory.factory),
)
}
+ private val frameTransformerFactory = MockFrameTransformerFactory()
+
suspend fun connect(joinResponse: JoinResponse = TestData.JOIN.join) {
rtcEngine.stub {
onBlocking { rtcEngine.join(any(), any(), anyOrNull(), anyOrNull()) }
diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt
new file mode 100644
index 000000000..43c15fd48
--- /dev/null
+++ b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataMockE2ETest.kt
@@ -0,0 +1,623 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import android.Manifest
+import android.app.Application
+import io.livekit.android.e2ee.E2EEOptions
+import io.livekit.android.room.DefaultsManager
+import io.livekit.android.room.participant.BackupVideoCodec
+import io.livekit.android.room.participant.VideoTrackPublishOptions
+import io.livekit.android.room.track.FrameMetadata
+import io.livekit.android.room.track.FrameMetadataFeature
+import io.livekit.android.room.track.FrameMetadataPublishOptions
+import io.livekit.android.room.track.LocalVideoTrack
+import io.livekit.android.room.track.LocalVideoTrackOptions
+import io.livekit.android.room.track.RemoteTrackPublication
+import io.livekit.android.room.track.RemoteVideoTrack
+import io.livekit.android.room.track.Track
+import io.livekit.android.room.track.VideoCaptureParameter
+import io.livekit.android.room.track.VideoCodec
+import io.livekit.android.test.MockE2ETest
+import io.livekit.android.test.mock.MockEglBase
+import io.livekit.android.test.mock.MockMediaStream
+import io.livekit.android.test.mock.MockRTCThreadToken
+import io.livekit.android.test.mock.MockRtpReceiver
+import io.livekit.android.test.mock.MockVideoCapturer
+import io.livekit.android.test.mock.MockVideoSource
+import io.livekit.android.test.mock.MockVideoStreamTrack
+import io.livekit.android.test.mock.TestData
+import io.livekit.android.test.mock.camera.MockCameraProvider
+import io.livekit.android.test.mock.createMediaStreamId
+import io.livekit.android.test.mock.e2ee.NoopKeyProvider
+import io.livekit.android.test.mock.room.track.createMockLocalAudioTrack
+import io.livekit.android.test.util.toPBByteString
+import io.livekit.android.util.toOkioByteString
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import livekit.LivekitModels
+import livekit.LivekitModels.PacketTrailerFeature
+import livekit.LivekitRtc
+import livekit.org.webrtc.FrameCryptor
+import livekit.org.webrtc.FrameMetadataTransformer
+import livekit.org.webrtc.FrameTransformer
+import livekit.org.webrtc.FrameTransformerChain
+import livekit.org.webrtc.MockPeerConnectionFactory
+import livekit.org.webrtc.RtpReceiver
+import livekit.org.webrtc.RtpSender
+import livekit.org.webrtc.VideoSource
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertNotSame
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertSame
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.Mockito
+import org.mockito.Mockito.mock
+import org.robolectric.RobolectricTestRunner
+import org.robolectric.Shadows
+import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata
+
+@OptIn(ExperimentalCoroutinesApi::class)
+@RunWith(RobolectricTestRunner::class)
+class FrameMetadataMockE2ETest : MockE2ETest() {
+
+ private val factory
+ get() = component.frameTransformerFactory()
+
+ private val metadataOptions = FrameMetadataPublishOptions(timestamp = true, frameId = true)
+
+ private fun createLocalTrack(source: VideoSource = MockVideoSource()) = LocalVideoTrack(
+ capturer = MockVideoCapturer(),
+ source = source,
+ name = "",
+ options = LocalVideoTrackOptions(
+ isScreencast = false,
+ deviceId = null,
+ position = null,
+ captureParams = VideoCaptureParameter(width = 1280, height = 720, maxFps = 30),
+ ),
+ rtcTrack = MockVideoStreamTrack(),
+ peerConnectionFactory = component.peerConnectionFactory(),
+ context = context,
+ eglBase = MockEglBase(),
+ defaultsManager = DefaultsManager(),
+ trackFactory = mock(LocalVideoTrack.Factory::class.java),
+ rtcThreadToken = MockRTCThreadToken(),
+ )
+
+ private fun sentAddTrackRequests(): List =
+ wsFactory.ws.sentRequests
+ .map { LivekitRtc.SignalRequest.newBuilder().mergeFrom(it.toPBByteString()).build() }
+ .filter { it.hasAddTrack() }
+ .map { it.addTrack }
+
+ private fun sendTransformers() = factory.metadataTransformers.filter { it.direction == FrameMetadataTransformer.Direction.SEND }
+
+ private fun receiveTransformers() = factory.metadataTransformers.filter { it.direction == FrameMetadataTransformer.Direction.RECEIVE }
+
+ private fun enableE2EE() {
+ room.e2eeOptions = E2EEOptions(keyProvider = NoopKeyProvider())
+ }
+
+ private fun verifySingleInstall(sender: RtpSender, transformer: FrameTransformer) {
+ Mockito.verify(sender).setFrameTransformer(transformer)
+ Mockito.verify(sender, Mockito.times(1)).setFrameTransformer(Mockito.any())
+ }
+
+ private fun verifySingleInstall(receiver: RtpReceiver, transformer: FrameTransformer) {
+ Mockito.verify(receiver).setFrameTransformer(transformer)
+ Mockito.verify(receiver, Mockito.times(1)).setFrameTransformer(Mockito.any())
+ }
+
+ // Publishing
+
+ @Test
+ fun publishSendsThePacketTrailerFeatures() = runTest {
+ connect()
+ wsFactory.ws.clearRequests()
+
+ room.localParticipant.publishVideoTrack(
+ createLocalTrack(),
+ VideoTrackPublishOptions(frameMetadata = FrameMetadataPublishOptions(timestamp = true, frameId = true, userData = true)),
+ )
+
+ assertEquals(
+ listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID, PacketTrailerFeature.PTF_USER_DATA),
+ sentAddTrackRequests().single().packetTrailerFeaturesList,
+ )
+ }
+
+ @Test
+ fun publishDefaultsCarryFrameMetadata() = runTest {
+ room.videoTrackPublishDefaults = room.videoTrackPublishDefaults.copy(frameMetadata = metadataOptions)
+ connect()
+ wsFactory.ws.clearRequests()
+
+ room.localParticipant.publishVideoTrack(createLocalTrack())
+
+ assertEquals(
+ listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_FRAME_ID),
+ sentAddTrackRequests().single().packetTrailerFeaturesList,
+ )
+ assertEquals(1, sendTransformers().size)
+ }
+
+ @Test
+ fun publishWithoutFrameMetadataSendsNoFeaturesAndCreatesNoTransformer() = runTest {
+ connect()
+ wsFactory.ws.clearRequests()
+ val track = createLocalTrack()
+
+ room.localParticipant.publishVideoTrack(track)
+
+ assertTrue(sentAddTrackRequests().single().packetTrailerFeaturesList.isEmpty())
+ assertTrue(factory.metadataTransformers.isEmpty())
+ Mockito.verify(track.sender!!, Mockito.never()).setFrameTransformer(Mockito.any())
+ assertFalse(track.frameMetadataStampState.isArmed)
+ }
+
+ @Test
+ fun publishLinksArmsAndInstallsTheSendTransformer() = runTest {
+ connect()
+ val source = MockVideoSource()
+ val track = createLocalTrack(source)
+
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+
+ val transformer = sendTransformers().single()
+ assertSame(transformer, track.withFrameMetadataTransformer { it })
+ assertEquals(listOf(transformer), source.frameMetadataTransformers)
+ assertTrue(track.frameMetadataStampState.isArmed)
+ verifySingleInstall(track.sender!!, transformer)
+ assertTrue(factory.chains.isEmpty())
+ }
+
+ @Test
+ fun unpublishUnlinksDisarmsAndDisposes() = runTest {
+ connect()
+ val source = MockVideoSource()
+ val track = createLocalTrack(source)
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+ val transformer = sendTransformers().single()
+
+ room.localParticipant.unpublishTrack(track)
+
+ assertTrue(source.frameMetadataTransformers.isEmpty())
+ assertNull(track.withFrameMetadataTransformer { it })
+ assertFalse(track.frameMetadataStampState.isArmed)
+ Mockito.verify(transformer).dispose()
+ }
+
+ @Test
+ fun audioPublishCreatesNoTransformer() = runTest {
+ connect()
+ wsFactory.ws.clearRequests()
+
+ room.localParticipant.publishAudioTrack(createMockLocalAudioTrack())
+
+ assertTrue(sentAddTrackRequests().single().packetTrailerFeaturesList.isEmpty())
+ assertTrue(factory.metadataTransformers.isEmpty())
+ }
+
+ // Publishing with E2EE
+
+ @Test
+ fun e2eeChainsTheCryptorBeforeTheMetadataTransformerOnSend() = runTest {
+ enableE2EE()
+ connect()
+ val track = createLocalTrack()
+
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+
+ val created = factory.chains.single()
+ assertSame(factory.senderCryptors.single(), created.first)
+ assertSame(sendTransformers().single(), created.second)
+ verifySingleInstall(track.sender!!, created.chain)
+ }
+
+ @Test
+ fun e2eeOnlyVideoInstallsTheCryptorWithoutAChain() = runTest {
+ enableE2EE()
+ connect()
+ val track = createLocalTrack()
+
+ room.localParticipant.publishVideoTrack(track)
+
+ assertTrue(factory.chains.isEmpty())
+ verifySingleInstall(track.sender!!, factory.senderCryptors.single())
+ }
+
+ @Test
+ fun e2eeAudioInstallsTheCryptorOnce() = runTest {
+ enableE2EE()
+ connect()
+ val track = createMockLocalAudioTrack()
+
+ room.localParticipant.publishAudioTrack(track)
+
+ assertTrue(factory.chains.isEmpty())
+ assertTrue(factory.metadataTransformers.isEmpty())
+ verifySingleInstall(track.sender!!, factory.senderCryptors.single())
+ }
+
+ @Test
+ fun e2eeUnpublishDisposesTheChain() = runTest {
+ enableE2EE()
+ connect()
+ val track = createLocalTrack()
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+ val created = factory.chains.single()
+
+ room.localParticipant.unpublishTrack(track)
+
+ Mockito.verify(created.chain).dispose()
+ Mockito.verify(sendTransformers().single()).dispose()
+ }
+
+ @Test
+ fun chainForATrackRemovedBeforeItIsKeptIsDisposed() = runTest {
+ // A send install keeps its chain after releasing the track's lock, so an unpublish can
+ // remove the track in between.
+ val manager = FrameMetadataManager(factory.factory)
+ val chain = mock(FrameTransformerChain::class.java)
+
+ manager.setChain(createLocalTrack(), chain)
+
+ Mockito.verify(chain).dispose()
+ }
+
+ // Backup codecs and restarts
+
+ private suspend fun publishVp9WithBackup(track: LocalVideoTrack) {
+ room.localParticipant.publishVideoTrack(
+ track,
+ VideoTrackPublishOptions(
+ videoCodec = VideoCodec.VP9.codecName,
+ backupCodec = BackupVideoCodec(codec = VideoCodec.VP8.codecName),
+ frameMetadata = metadataOptions,
+ ),
+ )
+ }
+
+ private fun requestBackupCodec() {
+ val quality = LivekitRtc.SubscribedQuality.newBuilder()
+ .setQuality(LivekitModels.VideoQuality.HIGH)
+ .setEnabled(true)
+ .build()
+ wsFactory.receiveMessage(
+ with(LivekitRtc.SignalResponse.newBuilder()) {
+ subscribedQualityUpdate = with(LivekitRtc.SubscribedQualityUpdate.newBuilder()) {
+ trackSid = room.localParticipant.videoTrackPublications.first().first.sid
+ addSubscribedCodecs(LivekitRtc.SubscribedCodec.newBuilder().setCodec("vp9").addQualities(quality))
+ addSubscribedCodecs(LivekitRtc.SubscribedCodec.newBuilder().setCodec("vp8").addQualities(quality))
+ build()
+ }
+ build().toOkioByteString()
+ },
+ )
+ }
+
+ @Test
+ fun backupCodecSenderGetsThePrimarysTransformer() = runTest {
+ connect()
+ val track = createLocalTrack()
+ publishVp9WithBackup(track)
+
+ requestBackupCodec()
+
+ val transceivers = getPublisherPeerConnection().transceivers
+ assertEquals(2, transceivers.size)
+ val transformer = sendTransformers().single()
+ verifySingleInstall(transceivers.last().sender, transformer)
+ }
+
+ private fun grantCamera() {
+ MockCameraProvider.register()
+ Shadows.shadowOf(context as Application).grantPermissions(Manifest.permission.CAMERA)
+ }
+
+ private val createdVideoSources
+ get() = (component.peerConnectionFactory() as MockPeerConnectionFactory).createdVideoSources
+
+ @Test
+ fun restartTrackRelinksTheTransformerAndKeepsTheStampState() = runTest {
+ connect()
+ grantCamera()
+ val track = room.localParticipant.createVideoTrack()
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+ val transformer = sendTransformers().single()
+ val oldSource = createdVideoSources.last()
+ val oldStamper = track.frameMetadataStamper!!
+ assertSame(oldStamper, oldSource.videoProcessor)
+
+ track.restartTrack()
+
+ val newSource = createdVideoSources.last()
+ assertTrue(oldSource.frameMetadataTransformers.isEmpty())
+ assertEquals(listOf(transformer), newSource.frameMetadataTransformers)
+ val newStamper = track.frameMetadataStamper!!
+ assertSame(newStamper, newSource.videoProcessor)
+ assertTrue(newStamper !== oldStamper)
+ assertSame(track.frameMetadataStampState, newStamper.state)
+ assertTrue(track.frameMetadataStampState.isArmed)
+ }
+
+ @Test
+ fun restartTrackMovesBackupSendersToTheNewTrack() = runTest {
+ connect()
+ grantCamera()
+ val track = room.localParticipant.createVideoTrack()
+ publishVp9WithBackup(track)
+ requestBackupCodec()
+ val backupSender = getPublisherPeerConnection().transceivers.last().sender
+
+ track.restartTrack()
+
+ Mockito.verify(backupSender).setTrack(track.rtcTrack, false)
+ }
+
+ @Test
+ fun restartTrackAfterUnpublishLinksNothing() = runTest {
+ connect()
+ grantCamera()
+ val track = room.localParticipant.createVideoTrack()
+ room.localParticipant.publishVideoTrack(track, VideoTrackPublishOptions(frameMetadata = metadataOptions))
+ room.localParticipant.unpublishTrack(track, stopOnUnpublish = false)
+
+ track.restartTrack()
+
+ assertTrue(createdVideoSources.last().frameMetadataTransformers.isEmpty())
+ }
+
+ // Subscribing
+
+ private fun joinRemoteParticipant(features: List) {
+ val videoTrack = TestData.REMOTE_VIDEO_TRACK.toBuilder().addAllPacketTrailerFeatures(features).build()
+ val participant = TestData.REMOTE_PARTICIPANT.toBuilder()
+ .clearTracks()
+ .addTracks(TestData.REMOTE_AUDIO_TRACK)
+ .addTracks(videoTrack)
+ .build()
+ simulateMessageFromServer(
+ LivekitRtc.SignalResponse.newBuilder()
+ .setUpdate(LivekitRtc.ParticipantUpdate.newBuilder().addParticipants(participant))
+ .build(),
+ )
+ }
+
+ private fun subscribeRemoteVideo(): RtpReceiver {
+ val receiver = MockRtpReceiver.create()
+ room.onAddTrack(
+ receiver,
+ MockVideoStreamTrack(),
+ arrayOf(
+ MockMediaStream(
+ id = createMediaStreamId(TestData.REMOTE_PARTICIPANT.sid, TestData.REMOTE_VIDEO_TRACK.sid),
+ ),
+ ),
+ )
+ return receiver
+ }
+
+ private fun remoteVideoPublication(): RemoteTrackPublication {
+ val participant = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!!
+ return participant.getTrackPublication(Track.Source.CAMERA) as RemoteTrackPublication
+ }
+
+ private fun remoteVideoTrack(): RemoteVideoTrack = remoteVideoPublication().track as RemoteVideoTrack
+
+ /** Sends a participant update without the video track, which unpublishes it. */
+ private fun unpublishRemoteVideo() {
+ val participant = TestData.REMOTE_PARTICIPANT.toBuilder()
+ .clearTracks()
+ .addTracks(TestData.REMOTE_AUDIO_TRACK)
+ .build()
+ simulateMessageFromServer(
+ LivekitRtc.SignalResponse.newBuilder()
+ .setUpdate(LivekitRtc.ParticipantUpdate.newBuilder().addParticipants(participant))
+ .build(),
+ )
+ }
+
+ @Test
+ fun subscribeWithFeaturesInstallsTheReceiveTransformerAndLooksUp() = runTest {
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ factory.lookupResult = { rtpTimestamp -> if (rtpTimestamp == 1234L) RtcFrameMetadata(55L, 6L, null) else null }
+
+ val receiver = subscribeRemoteVideo()
+
+ val transformer = receiveTransformers().single()
+ verifySingleInstall(receiver, transformer)
+ val track = remoteVideoTrack()
+ assertEquals(FrameMetadata(userTimestampUs = 55L, frameId = 6L), track.lookupFrameMetadata(1234L))
+ assertNull(track.lookupFrameMetadata(1L))
+ }
+
+ @Test
+ fun remotePublicationExposesTheDeclaredFeatures() = runTest {
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP, PacketTrailerFeature.PTF_USER_DATA))
+
+ val publication = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!!.getTrackPublication(Track.Source.CAMERA)!!
+ assertEquals(listOf(FrameMetadataFeature.TIMESTAMP, FrameMetadataFeature.USER_DATA), publication.frameMetadataFeatures)
+ }
+
+ @Test
+ fun remotePublicationSkipsFeaturesThisBuildDoesNotKnow() = runTest {
+ connect()
+ val videoTrack = TestData.REMOTE_VIDEO_TRACK.toBuilder()
+ .addPacketTrailerFeatures(PacketTrailerFeature.PTF_FRAME_ID)
+ .addPacketTrailerFeaturesValue(99)
+ .build()
+ simulateMessageFromServer(
+ LivekitRtc.SignalResponse.newBuilder()
+ .setUpdate(
+ LivekitRtc.ParticipantUpdate.newBuilder().addParticipants(
+ TestData.REMOTE_PARTICIPANT.toBuilder().clearTracks().addTracks(videoTrack),
+ ),
+ )
+ .build(),
+ )
+
+ val publication = room.getParticipantBySid(TestData.REMOTE_PARTICIPANT.sid)!!.getTrackPublication(Track.Source.CAMERA)!!
+ assertEquals(listOf(FrameMetadataFeature.FRAME_ID), publication.frameMetadataFeatures)
+ }
+
+ @Test
+ fun subscribeWithoutFeaturesInstallsNothing() = runTest {
+ connect()
+ joinRemoteParticipant(emptyList())
+
+ val receiver = subscribeRemoteVideo()
+
+ assertTrue(factory.metadataTransformers.isEmpty())
+ Mockito.verify(receiver, Mockito.never()).setFrameTransformer(Mockito.any())
+ assertNull(remoteVideoTrack().lookupFrameMetadata(1234L))
+ }
+
+ @Test
+ fun e2eeChainsTheMetadataTransformerBeforeTheCryptorOnReceive() = runTest {
+ enableE2EE()
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_FRAME_ID))
+
+ val receiver = subscribeRemoteVideo()
+
+ val created = factory.chains.single()
+ assertSame(receiveTransformers().single(), created.first)
+ assertSame(factory.receiverCryptors.single(), created.second)
+ verifySingleInstall(receiver, created.chain)
+ }
+
+ @Test
+ fun unsubscribeDisposesWithoutDisablingAndLookupReturnsNull() = runTest {
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ factory.lookupResult = { RtcFrameMetadata(55L, 6L, null) }
+ subscribeRemoteVideo()
+ val track = remoteVideoTrack()
+ val transformer = receiveTransformers().single()
+ assertNotNull(track.lookupFrameMetadata(1234L))
+
+ simulateMessageFromServer(TestData.PARTICIPANT_DISCONNECT)
+
+ Mockito.verify(transformer).dispose()
+ Mockito.verify(transformer, Mockito.never()).setEnabled(false)
+ assertNull(track.lookupFrameMetadata(1234L))
+ }
+
+ @Test
+ fun receiveInstallAfterRemovalInstallsOnlyTheCryptor() = runTest {
+ // What an install sees when a removal gets the manager's lock first.
+ enableE2EE()
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ val receiver = subscribeRemoteVideo()
+ val track = remoteVideoTrack()
+ unpublishRemoteVideo()
+ val cryptor = mock(FrameCryptor::class.java)
+
+ room.installReceiveTransformers(track, cryptor)
+
+ assertEquals(1, factory.chains.size)
+ Mockito.verify(receiver).setFrameTransformer(cryptor)
+ }
+
+ @Test
+ fun receiveReinstallReplacesAndDisposesThePreviousChain() = runTest {
+ enableE2EE()
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ subscribeRemoteVideo()
+ val first = factory.chains.single()
+
+ room.installReceiveTransformers(remoteVideoTrack(), mock(FrameCryptor::class.java))
+
+ val second = factory.chains.single { it !== first }
+ assertSame(first.first, second.first)
+ Mockito.verify(first.chain).dispose()
+ Mockito.verify(second.chain, Mockito.never()).dispose()
+ }
+
+ @Test
+ fun resubscribeDisposesThePreviousTransformerAndChain() = runTest {
+ enableE2EE()
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ factory.lookupResult = { RtcFrameMetadata(55L, 6L, null) }
+ subscribeRemoteVideo()
+ val firstTrack = remoteVideoTrack()
+ val firstTransformer = receiveTransformers().single()
+ val firstChain = factory.chains.single().chain
+
+ // The client isn't told when the server stops sending the track, so the next subscribe
+ // arrives with the first track still in place.
+ remoteVideoPublication().setSubscribed(false)
+ remoteVideoPublication().setSubscribed(true)
+ subscribeRemoteVideo()
+
+ val secondTrack = remoteVideoTrack()
+ assertNotSame(firstTrack, secondTrack)
+ Mockito.verify(firstTransformer).dispose()
+ Mockito.verify(firstChain).dispose()
+ assertNull(firstTrack.lookupFrameMetadata(1234L))
+ val secondTransformer = receiveTransformers().single { it !== firstTransformer }
+ Mockito.verify(secondTransformer, Mockito.never()).dispose()
+ Mockito.verify(factory.chains.single { it.chain !== firstChain }.chain, Mockito.never()).dispose()
+ assertNotNull(secondTrack.lookupFrameMetadata(1234L))
+ }
+
+ @Test
+ fun unpublishAfterResubscribeDisposesEachTransformerOnce() = runTest {
+ connect()
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ subscribeRemoteVideo()
+ subscribeRemoteVideo()
+
+ unpublishRemoteVideo()
+
+ assertEquals(2, receiveTransformers().size)
+ receiveTransformers().forEach { Mockito.verify(it).dispose() }
+ }
+
+ @Test
+ fun disconnectDisposesEverything() = runTest {
+ connect()
+ room.localParticipant.publishVideoTrack(createLocalTrack(), VideoTrackPublishOptions(frameMetadata = metadataOptions))
+ joinRemoteParticipant(listOf(PacketTrailerFeature.PTF_USER_TIMESTAMP))
+ subscribeRemoteVideo()
+
+ room.disconnect()
+
+ assertEquals(2, factory.metadataTransformers.size)
+ factory.metadataTransformers.forEach { Mockito.verify(it).dispose() }
+ }
+
+ // Capabilities
+
+ @Test
+ fun connectAdvertisesPacketTrailers() = runTest {
+ connect()
+
+ val capabilities = wsFactory.request.url.queryParameter("capabilities")!!.split(",")
+ assertTrue(capabilities.contains("CAP_PACKET_TRAILER"))
+ }
+}
diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt
new file mode 100644
index 000000000..98644a212
--- /dev/null
+++ b/livekit-android-test/src/test/java/io/livekit/android/room/framemetadata/FrameMetadataStampingTest.kt
@@ -0,0 +1,255 @@
+/*
+ * Copyright 2026 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.framemetadata
+
+import io.livekit.android.room.track.FrameMetadata
+import io.livekit.android.room.track.FrameMetadataProvider
+import io.livekit.android.room.track.FrameMetadataPublishOptions
+import livekit.org.webrtc.VideoFrame
+import livekit.org.webrtc.VideoProcessor
+import livekit.org.webrtc.VideoSink
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertSame
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mockito.Mockito
+import org.robolectric.RobolectricTestRunner
+import livekit.org.webrtc.FrameMetadata as RtcFrameMetadata
+
+@RunWith(RobolectricTestRunner::class)
+class FrameMetadataStampingTest {
+
+ private val buffer = Mockito.mock(VideoFrame.Buffer::class.java)
+
+ private fun frame(timestampNs: Long = 4_000_000_000L, metadata: RtcFrameMetadata? = null) =
+ VideoFrame(buffer, 0, timestampNs, metadata)
+
+ // WebRTC's clock at 5 s and the wall clock at 1,700,000,000,000 ms, so a frame captured at
+ // 4 s on WebRTC's clock is 1 s before the wall-clock time.
+ private fun state(initialFrameId: Long = 1L) = FrameMetadataStampState(
+ rtcTimeNanos = { 5_000_000_000L },
+ wallTimeMillis = { 1_700_000_000_000L },
+ initialFrameId = initialFrameId,
+ )
+
+ private val allOn = FrameMetadataPublishOptions(timestamp = true, frameId = true)
+
+ @Test
+ fun nothingIsStampedBeforeArming() {
+ val input = frame()
+ val output = state().stamp(input)
+
+ assertSame(input, output)
+ assertNull(output.frameMetadata)
+ }
+
+ @Test
+ fun nothingIsStampedAfterDisarming() {
+ val state = state()
+ state.arm(allOn)
+ state.disarm()
+
+ val input = frame()
+ assertSame(input, state.stamp(input))
+ }
+
+ @Test
+ fun defaultTimestampIsTheCaptureTimeOnTheWallClock() {
+ val state = state()
+ state.arm(FrameMetadataPublishOptions(timestamp = true, frameId = false))
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs)
+ assertEquals(0L, metadata.frameId)
+ }
+
+ @Test
+ fun onlyEnabledFieldsAreFilled() {
+ val state = state()
+ state.arm(FrameMetadataPublishOptions(timestamp = false, frameId = true))
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertEquals(0L, metadata.userTimestampUs)
+ assertEquals(1L, metadata.frameId)
+ assertNull(metadata.userData)
+ }
+
+ @Test
+ fun frameIdsCountUpFromOne() {
+ val state = state()
+ state.arm(allOn)
+
+ val ids = (1..3).map { state.stamp(frame()).frameMetadata!!.frameId }
+
+ assertEquals(listOf(1L, 2L, 3L), ids)
+ }
+
+ @Test
+ fun frameIdWrapsToOne() {
+ val state = state(initialFrameId = FrameMetadataStampState.MAX_FRAME_ID)
+ state.arm(allOn)
+
+ val ids = (1..2).map { state.stamp(frame()).frameMetadata!!.frameId }
+
+ assertEquals(listOf(FrameMetadataStampState.MAX_FRAME_ID, 1L), ids)
+ }
+
+ @Test
+ fun metadataOnTheFrameWinsOverDefaults() {
+ val state = state()
+ state.arm(allOn)
+ val userData = byteArrayOf(1, 2, 3)
+
+ val metadata = state.stamp(frame(metadata = RtcFrameMetadata(123L, 0L, userData))).frameMetadata!!
+
+ assertEquals(123L, metadata.userTimestampUs)
+ // The frame ID was 0, so the default fills it.
+ assertEquals(1L, metadata.frameId)
+ assertArrayEquals(userData, metadata.userData)
+ }
+
+ @Test
+ fun appValuesAreSentWithEveryOptionOff() {
+ val state = state()
+ state.arm(FrameMetadataPublishOptions(timestamp = false, frameId = false, userData = false))
+ state.provider = FrameMetadataProvider { _, current -> current.copy(userData = byteArrayOf(9)) }
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertArrayEquals(byteArrayOf(9), metadata.userData)
+ }
+
+ @Test
+ fun providerSeesTheDefaultsAndReplacesThem() {
+ val state = state()
+ state.arm(allOn)
+ var seen: FrameMetadata? = null
+ state.provider = FrameMetadataProvider { _, current ->
+ seen = current
+ FrameMetadata(userTimestampUs = 42L, frameId = 7L, userData = byteArrayOf(5))
+ }
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertEquals(FrameMetadata(1_699_999_999_000_000L, 1L, null), seen)
+ assertEquals(42L, metadata.userTimestampUs)
+ assertEquals(7L, metadata.frameId)
+ assertArrayEquals(byteArrayOf(5), metadata.userData)
+ }
+
+ @Test
+ fun providerReturningNullKeepsTheValues() {
+ val state = state()
+ state.arm(allOn)
+ state.provider = FrameMetadataProvider { _, _ -> null }
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs)
+ assertEquals(1L, metadata.frameId)
+ }
+
+ @Test
+ fun throwingProviderFallsBackToTheValuesItWasGiven() {
+ val state = state()
+ state.arm(allOn)
+ state.provider = FrameMetadataProvider { _, _ -> throw IllegalStateException("app bug") }
+
+ val metadata = state.stamp(frame()).frameMetadata!!
+
+ assertEquals(1_699_999_999_000_000L, metadata.userTimestampUs)
+ assertEquals(1L, metadata.frameId)
+ }
+
+ @Test
+ fun providerZeroingEverythingRemovesMetadataFromTheFrame() {
+ val state = state()
+ state.arm(allOn)
+ state.provider = FrameMetadataProvider { _, _ -> FrameMetadata() }
+
+ val output = state.stamp(frame(metadata = RtcFrameMetadata(123L, 4L, null)))
+
+ assertNull(output.frameMetadata)
+ }
+
+ @Test
+ fun processorStampsAfterTheInnerProcessor() {
+ // An inner processor that replaces each frame, as virtual background does.
+ val inner = object : VideoProcessor {
+ var output: VideoSink? = null
+ override fun onCapturerStarted(success: Boolean) {}
+ override fun onCapturerStopped() {}
+ override fun setSink(sink: VideoSink?) {
+ output = sink
+ }
+
+ override fun onFrameCaptured(frame: VideoFrame) {
+ output?.onFrame(VideoFrame(frame.buffer, frame.rotation, frame.timestampNs))
+ }
+ }
+ val processor = FrameMetadataStampingProcessor(inner)
+ processor.state = state().apply { arm(allOn) }
+ val received = mutableListOf()
+ processor.setSink { received.add(it) }
+
+ processor.onFrameCaptured(frame())
+
+ assertEquals(1, received.size)
+ assertEquals(1L, received.single().frameMetadata!!.frameId)
+ }
+
+ @Test
+ fun processorForwardsTheTwoArgumentCallToTheInnerProcessor() {
+ val inner = Mockito.mock(VideoProcessor::class.java)
+ val processor = FrameMetadataStampingProcessor(inner)
+ val input = frame()
+ val parameters = VideoProcessor.FrameAdaptationParameters(0, 0, 1, 1, 1, 1, 0L, false)
+
+ processor.onFrameCaptured(input, parameters)
+
+ Mockito.verify(inner).onFrameCaptured(input, parameters)
+ }
+
+ @Test
+ fun processorPassesFramesThroughWithoutAnInnerProcessor() {
+ val processor = FrameMetadataStampingProcessor(null)
+ val received = mutableListOf()
+ processor.setSink { received.add(it) }
+ val input = frame()
+
+ processor.onFrameCaptured(input)
+
+ assertSame(input, received.single())
+ }
+
+ @Test
+ fun clearingTheSinkClearsTheInnerProcessorsSink() {
+ val inner = Mockito.mock(VideoProcessor::class.java)
+ val processor = FrameMetadataStampingProcessor(inner)
+
+ processor.setSink { }
+ processor.setSink(null)
+
+ val inOrder = Mockito.inOrder(inner)
+ inOrder.verify(inner).setSink(Mockito.notNull())
+ inOrder.verify(inner).setSink(null)
+ }
+}