From 1acac756e57e8303d98136ecab1c701797311056 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 12 Nov 2026 18:49:41 +0800 Subject: [PATCH 1/3] Add FlexFEC negotiation and subscriber FEC protection LiveKit.enableFlexFec turns on the FlexFEC field trials when libwebrtc is initialized, so flexfec-03 is negotiated. Codec preferences already keep unmatched codecs, so the repair codec is offered without further changes. RemoteTrackPublication.setFecProtection requests FlexFEC repair packets from the server for a subscribed video track through the new UpdateTrackSettings.fec field, which requires livekit/protocol#1987. FecProtection is a sealed class mirroring the protocol's modes: Disabled, Adaptive, and Fixed and Minimum, which carry a FecProtectionLevel (low, medium or high). The protection is only sent once requested, so other settings updates leave the server's setting unchanged. Unlike quality and dimensions it also applies while adaptive stream manages the track. FlexFecOnDeviceTest subscribes to a video track on a real server and checks the repairs received for each mode through the inbound-rtp FEC counters. It is skipped unless the lkUrl and lkToken instrumentation arguments are given. The protocol submodule points at livekit/protocol#1987 applied on top of the current protocol commit. --- .changeset/flexfec-subscription.md | 5 + .../main/java/io/livekit/android/LiveKit.kt | 20 ++ .../io/livekit/android/dagger/RTCModule.kt | 6 +- .../io/livekit/android/room/SignalClient.kt | 6 + .../android/room/track/FecProtection.kt | 92 ++++++++ .../room/track/RemoteTrackPublication.kt | 35 +++- .../src/androidTest/AndroidManifest.xml | 6 + .../android/room/track/FlexFecOnDeviceTest.kt | 198 ++++++++++++++++++ .../room/track/RemoteTrackPublicationTest.kt | 100 ++++++++- protocol | 2 +- 10 files changed, 466 insertions(+), 4 deletions(-) create mode 100644 .changeset/flexfec-subscription.md create mode 100644 livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt create mode 100644 livekit-android-test/src/androidTest/AndroidManifest.xml create mode 100644 livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt diff --git a/.changeset/flexfec-subscription.md b/.changeset/flexfec-subscription.md new file mode 100644 index 000000000..27599e3ab --- /dev/null +++ b/.changeset/flexfec-subscription.md @@ -0,0 +1,5 @@ +--- +"client-sdk-android": minor +--- + +Add `LiveKit.enableFlexFec` to negotiate FlexFEC, and `RemoteTrackPublication.setFecProtection` to request FlexFEC repair packets from the server for a subscribed video track, with disabled, adaptive, fixed and minimum modes and low, medium and high levels. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt index c281b5aa8..30db2d437 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt @@ -64,6 +64,26 @@ object LiveKit { @JvmStatic var enableWebRTCLogging: Boolean = false + /** + * Enables FlexFEC forward error correction for video. + * + * When enabled, connections negotiate `flexfec-03`, which lets subscribed video tracks + * receive repair packets from the server, see + * [io.livekit.android.room.track.RemoteTrackPublication.setFecProtection]. + * Published video tracks then also send FlexFEC as WebRTC adapts it to observed loss. + * Disabled by default. + * + * This is a process-wide WebRTC setting, so it must be set before the first [Room] is + * created or [init] is called; later changes have no effect. + * + * ``` + * LiveKit.enableFlexFec = true + * val room = LiveKit.create(appContext) + * ``` + */ + @JvmStatic + var enableFlexFec: Boolean = false + /** * Certain WebRTC classes need to be initialized prior to use. * diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt index 91e3ad0ff..6f1aafbf6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt @@ -91,6 +91,10 @@ internal object RTCModule { * it. */ private const val DEFAULT_FIELD_TRIALS = "WebRTC-IceHandshakeDtls/Enabled/" + private const val FLEXFEC_FIELD_TRIALS = "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/" + + private val fieldTrials: String + get() = if (LiveKit.enableFlexFec) DEFAULT_FIELD_TRIALS + FLEXFEC_FIELD_TRIALS else DEFAULT_FIELD_TRIALS /** * Certain classes require libwebrtc to be initialized prior to use. @@ -122,7 +126,7 @@ internal object RTCModule { PeerConnectionFactory.InitializationOptions .builder(appContext) .setNativeLibraryName("lkjingle_peerconnection_so") - .setFieldTrials(DEFAULT_FIELD_TRIALS) + .setFieldTrials(fieldTrials) .setInjectableLogger( { s, severity, s2 -> if (!LiveKit.enableWebRTCLogging) { diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt index 875b52051..2903c71df 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/SignalClient.kt @@ -529,12 +529,14 @@ constructor( sendRequest(request) } + @Suppress("LongParameterList") fun sendUpdateTrackSettings( sid: String, disabled: Boolean, videoDimensions: Track.Dimensions?, videoQuality: LivekitModels.VideoQuality?, fps: Int?, + fecProtection: LivekitRtc.FECProtection? = null, ) { val trackSettings = LivekitRtc.UpdateTrackSettings.newBuilder() .addTrackSids(sid) @@ -553,6 +555,10 @@ constructor( if (fps != null) { setFps(fps) } + + if (fecProtection != null) { + fec = fecProtection + } } val request = LivekitRtc.SignalRequest.newBuilder() diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt new file mode 100644 index 000000000..d9807d076 --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/FecProtection.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import livekit.LivekitRtc + +/** + * Proactive FlexFEC protection the server adds to a subscribed video track. + * + * The server sends repair packets alongside the track's media, so the receiver can recover lost + * packets without waiting for a retransmission. + * + * @see RemoteTrackPublication.setFecProtection + */ +sealed class FecProtection { + /** No repair packets. This is the default. */ + data object Disabled : FecProtection() + + /** + * Protection follows the observed packet loss, using the same algorithm as libwebrtc. + */ + data object Adaptive : FecProtection() + + /** + * Constant protection at [level], regardless of packet loss. + */ + data class Fixed(val level: FecProtectionLevel) : FecProtection() + + /** + * Protection never drops below [level]. It rises adaptively under higher packet loss, and + * returns to [level] once the loss subsides. + */ + data class Minimum(val level: FecProtectionLevel) : FecProtection() + + internal fun toProto(): LivekitRtc.FECProtection { + val builder = LivekitRtc.FECProtection.newBuilder() + when (this) { + Disabled -> builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_DISABLED + Adaptive -> builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_ADAPTIVE + is Fixed -> { + builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_FIXED + builder.level = level.toProto() + } + + is Minimum -> { + builder.mode = LivekitRtc.FECProtectionMode.FEC_MODE_MINIMUM + builder.level = level.toProto() + } + } + return builder.build() + } +} + +/** + * The share of packets spent on FlexFEC repairs, used by [FecProtection.Fixed] and + * [FecProtection.Minimum]. + * + * Frames that span only a few packets get proportionally more repairs. + */ +enum class FecProtectionLevel { + /** Approximately 15% repair packet overhead. */ + LOW, + + /** Approximately 25% repair packet overhead. */ + MEDIUM, + + /** Approximately 35% repair packet overhead. */ + HIGH, + ; + + internal fun toProto(): LivekitRtc.FECProtectionLevel { + return when (this) { + LOW -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_LOW + MEDIUM -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_MEDIUM + HIGH -> LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH + } + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt index 0f04a588d..6537e2481 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/RemoteTrackPublication.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -75,6 +75,15 @@ class RemoteTrackPublication( private var videoDimensions: Track.Dimensions? = null private var fps: Int? = null + /** + * The FlexFEC protection requested with [setFecProtection]. + */ + val fecProtection: FecProtection + get() = requestedFecProtection ?: FecProtection.Disabled + + // null until requested, so updates leave the server's setting unchanged + private var requestedFecProtection: FecProtection? = null + var subscriptionAllowed: Boolean = true internal set @@ -204,6 +213,29 @@ class RemoteTrackPublication( sendUpdateTrackSettings.invoke() } + /** + * Requests proactive FlexFEC repair packets from the server for this video track. + * + * Repairs let the receiver recover lost packets without waiting for a retransmission, at the + * cost of extra bandwidth. The protection changes without renegotiation, and is applied even + * when the track's quality is managed by adaptive stream. + * + * Repairs are only sent when the server has downstream FlexFEC enabled and FlexFEC was + * negotiated, which requires [io.livekit.android.LiveKit.enableFlexFec]. Otherwise the request + * has no effect. + * + * ``` + * publication.setFecProtection(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + * ``` + */ + fun setFecProtection(protection: FecProtection) { + if (!subscribed || track !is VideoTrack || protection == requestedFecProtection) { + return + } + requestedFecProtection = protection + sendUpdateTrackSettings.invoke() + } + private fun handleVisibilityChanged(isVisible: Boolean) { disabled = !isVisible sendUpdateTrackSettings.invoke() @@ -237,6 +269,7 @@ class RemoteTrackPublication( videoDimensions, videoQuality?.toProto(), fps, + requestedFecProtection?.toProto(), ) } diff --git a/livekit-android-test/src/androidTest/AndroidManifest.xml b/livekit-android-test/src/androidTest/AndroidManifest.xml new file mode 100644 index 000000000..5f9b23641 --- /dev/null +++ b/livekit-android-test/src/androidTest/AndroidManifest.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt new file mode 100644 index 000000000..2d7358f9f --- /dev/null +++ b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecOnDeviceTest.kt @@ -0,0 +1,198 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import android.util.Log +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.livekit.android.LiveKit +import io.livekit.android.RoomOptions +import io.livekit.android.events.RoomEvent +import io.livekit.android.room.Room +import io.livekit.android.util.LoggingLevel +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withTimeout +import livekit.org.webrtc.RTCStatsReport +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import kotlin.coroutines.resume + +/** + * Subscribes to a video track published by another participant on a real LiveKit server, and + * checks that the server sends FlexFEC repair packets as requested through + * [RemoteTrackPublication.setFecProtection]. + * + * Needs a server with `rtc.flexfec.downstream_enabled` and a participant already publishing video + * in the room, so it is skipped unless the instrumentation arguments `lkUrl` and `lkToken` are + * given: + * + * ``` + * ./gradlew :livekit-android-test:connectedDebugAndroidTest \ + * -Pandroid.testInstrumentationRunnerArguments.class=io.livekit.android.room.track.FlexFecOnDeviceTest \ + * -Pandroid.testInstrumentationRunnerArguments.lkUrl=ws://192.168.1.2:7880 \ + * -Pandroid.testInstrumentationRunnerArguments.lkToken= + * ``` + * + * Repairs are counted per frame and rounded like libwebrtc's `NumFecPackets`, so the measured + * ratios depend on the frame sizes and only approximate the nominal 15/25/35%. The highest layer + * is subscribed throughout to keep the frame sizes comparable between modes. No loss is + * simulated, so adaptive protection is only checked to stay below a fixed level. + */ +@RunWith(AndroidJUnit4::class) +class FlexFecOnDeviceTest { + + private companion object { + const val TAG = "FlexFecOnDeviceTest" + const val SUBSCRIBE_TIMEOUT_MS = 30_000L + const val SETTLE_MS = 2_000L + const val MEASURE_MS = 6_000L + const val STATS_POLL_MS = 500L + } + + private data class Counters(val mediaPackets: Long, val fecPackets: Long, val fecDiscarded: Long) + + private data class Window(val mediaPackets: Long, val fecPackets: Long, val fecDiscarded: Long) { + val fecRatio: Double + get() = if (mediaPackets == 0L) 0.0 else fecPackets.toDouble() / mediaPackets + } + + private lateinit var room: Room + private lateinit var publication: RemoteTrackPublication + private lateinit var trackSid: String + + @Before + fun setUp(): Unit = runBlocking { + val args = InstrumentationRegistry.getArguments() + val url = args.getString("lkUrl") + val token = args.getString("lkToken") + assumeTrue("lkUrl and lkToken instrumentation arguments are required", url != null && token != null) + + LiveKit.loggingLevel = LoggingLevel.INFO + LiveKit.enableFlexFec = true + + val context = InstrumentationRegistry.getInstrumentation().targetContext + room = LiveKit.create(context, RoomOptions(adaptiveStream = false)) + + withTimeout(SUBSCRIBE_TIMEOUT_MS) { + val subscribed = room.events.events + .filterIsInstance() + val connect = async { subscribed.first { it.track is VideoTrack } } + room.connect(url!!, token!!) + val event = connect.await() + publication = event.publication as RemoteTrackPublication + trackSid = event.publication.sid + } + Log.i(TAG, "subscribed to video track $trackSid") + + publication.setVideoQuality(VideoQuality.HIGH) + withTimeout(SUBSCRIBE_TIMEOUT_MS) { + while (readCounters()?.mediaPackets?.let { it > 0 } != true) { + delay(STATS_POLL_MS) + } + } + } + + @After + fun tearDown() { + if (::room.isInitialized) { + room.disconnect() + room.release() + } + } + + @Test + fun serverSendsRepairsAsRequested(): Unit = runBlocking { + val disabledAtStart = measure("initial (not requested)") + assertTrue("media is flowing", disabledAtStart.mediaPackets > 0) + assertEquals("no repairs before FEC is requested", 0L, disabledAtStart.fecPackets) + + val fixedLow = measure(FecProtection.Fixed(FecProtectionLevel.LOW)) + val fixedMedium = measure(FecProtection.Fixed(FecProtectionLevel.MEDIUM)) + val fixedHigh = measure(FecProtection.Fixed(FecProtectionLevel.HIGH)) + val minimumMedium = measure(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + val adaptive = measure(FecProtection.Adaptive) + val disabled = measure(FecProtection.Disabled) + + assertTrue("media keeps flowing", disabled.mediaPackets > 0) + assertTrue("fixed LOW sends repairs", fixedLow.fecRatio >= 0.10) + assertTrue("fixed MEDIUM protects at least as much as LOW", fixedMedium.fecRatio >= fixedLow.fecRatio * 0.9) + assertTrue("fixed HIGH protects more than LOW", fixedHigh.fecRatio > fixedLow.fecRatio) + assertTrue("fixed HIGH protects at least 20%", fixedHigh.fecRatio >= 0.20) + assertEquals( + "minimum MEDIUM matches fixed MEDIUM without loss", + fixedMedium.fecRatio, + minimumMedium.fecRatio, + 0.05, + ) + assertTrue("adaptive without loss stays below fixed LOW", adaptive.fecRatio < fixedLow.fecRatio) + assertEquals("disabling stops the repairs", 0L, disabled.fecPackets) + listOf(fixedLow, fixedMedium, fixedHigh, minimumMedium).forEach { + assertEquals("repairs are not discarded", 0L, it.fecDiscarded) + } + } + + private suspend fun measure(protection: FecProtection): Window { + publication.setFecProtection(protection) + assertEquals(protection, publication.fecProtection) + return measure(protection.toString()) + } + + private suspend fun measure(label: String): Window { + delay(SETTLE_MS) + val start = requireNotNull(readCounters()) { "no inbound video stats" } + delay(MEASURE_MS) + val end = requireNotNull(readCounters()) { "no inbound video stats" } + val window = Window( + mediaPackets = end.mediaPackets - start.mediaPackets, + fecPackets = end.fecPackets - start.fecPackets, + fecDiscarded = end.fecDiscarded - start.fecDiscarded, + ) + Log.i( + TAG, + "$label: media=${window.mediaPackets} fec=${window.fecPackets} " + + "discarded=${window.fecDiscarded} ratio=${"%.3f".format(window.fecRatio)}", + ) + return window + } + + private suspend fun readCounters(): Counters? { + val report = suspendCancellableCoroutine { cont -> + room.getSubscriberRTCStats { cont.resume(it) } + } + val inbound = report.statsMap.values.firstOrNull { + it.type == "inbound-rtp" && it.members["kind"] == "video" + } + val members = inbound?.members ?: return null + return Counters( + mediaPackets = members.long("packetsReceived"), + fecPackets = members.long("fecPacketsReceived"), + fecDiscarded = members.long("fecPacketsDiscarded"), + ) + } + + private fun Map.long(key: String): Long = (this[key] as? Number)?.toLong() ?: 0L +} diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt index 77a433452..f28996f73 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/track/RemoteTrackPublicationTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2025 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,6 +84,104 @@ class RemoteTrackPublicationTest : MockE2ETest() { assertEquals(VideoQuality.LOW, lastRequest.trackSetting.quality) } + private suspend fun subscribeRemoteVideo(): RemoteTrackPublication { + connect() + + wsFactory.listener.onMessage( + wsFactory.ws, + TestData.PARTICIPANT_JOIN.toOkioByteString(), + ) + + room.onAddTrack( + MockRtpReceiver.create(), + MockVideoStreamTrack(), + arrayOf( + MockMediaStream( + id = createMediaStreamId( + TestData.REMOTE_PARTICIPANT.sid, + TestData.REMOTE_VIDEO_TRACK.sid, + ), + ), + ), + ) + + return room.remoteParticipants.values.first() + .videoTrackPublications.first() + .first as RemoteTrackPublication + } + + private fun lastTrackSetting(): LivekitRtc.UpdateTrackSettings { + val requests = wsFactory.ws.sentRequests.map { + LivekitRtc.SignalRequest.newBuilder().mergeFrom(it.toPBByteString()).build() + } + val lastTrackSetting = requests.lastOrNull { it.hasTrackSetting() } + assertTrue("no track setting sent", lastTrackSetting != null) + return lastTrackSetting!!.trackSetting + } + + @Test + fun fecProtectionOmittedUntilRequested() = runTest { + room.adaptiveStream = false + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + remoteVideoPub.setVideoFps(30) + advanceUntilIdle() + + assertFalse(lastTrackSetting().hasFec()) + assertEquals(FecProtection.Disabled, remoteVideoPub.fecProtection) + } + + @Test + fun fecProtectionSentAndKept() = runTest { + room.adaptiveStream = false + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + remoteVideoPub.setFecProtection(FecProtection.Fixed(FecProtectionLevel.HIGH)) + advanceUntilIdle() + + assertEquals(FecProtection.Fixed(FecProtectionLevel.HIGH), remoteVideoPub.fecProtection) + assertTrue(lastTrackSetting().hasFec()) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_FIXED, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH, lastTrackSetting().fec.level) + + // later settings updates keep the requested level + remoteVideoPub.setVideoFps(30) + advanceUntilIdle() + assertEquals(30, lastTrackSetting().fps) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_FIXED, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_HIGH, lastTrackSetting().fec.level) + + remoteVideoPub.setFecProtection(FecProtection.Adaptive) + advanceUntilIdle() + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_ADAPTIVE, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_UNSPECIFIED, lastTrackSetting().fec.level) + + // disabling is sent explicitly rather than omitted + remoteVideoPub.setFecProtection(FecProtection.Disabled) + advanceUntilIdle() + assertTrue(lastTrackSetting().hasFec()) + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_DISABLED, lastTrackSetting().fec.mode) + } + + @Test + fun fecProtectionWithAdaptiveStream() = runTest { + room.adaptiveStream = true + val remoteVideoPub = subscribeRemoteVideo() + advanceUntilIdle() + wsFactory.ws.clearRequests() + + // applies even when adaptive stream manages the track's quality + remoteVideoPub.setFecProtection(FecProtection.Minimum(FecProtectionLevel.MEDIUM)) + advanceUntilIdle() + + assertEquals(LivekitRtc.FECProtectionMode.FEC_MODE_MINIMUM, lastTrackSetting().fec.mode) + assertEquals(LivekitRtc.FECProtectionLevel.FEC_LEVEL_MEDIUM, lastTrackSetting().fec.level) + } + @Test fun subscriptionStatus() = runTest { connect() diff --git a/protocol b/protocol index 30d320b79..3a115c89c 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 30d320b796396930fbfa1d82932bfbfdea314353 +Subproject commit 3a115c89c9285e738e6db8e681af0acfd10570db From c91fdee8448e723eb71619b81a0fe297da216118 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Tue, 17 Nov 2026 10:25:17 +0800 Subject: [PATCH 2/3] Note that FlexFEC is only sent for tracks published without simulcast libwebrtc only protects a single media stream with FlexFEC, so it signals no FlexFEC SSRC for a sender with simulcast layers. --- .../src/main/java/io/livekit/android/LiveKit.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt index 30db2d437..967532ea6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/LiveKit.kt @@ -70,7 +70,8 @@ object LiveKit { * When enabled, connections negotiate `flexfec-03`, which lets subscribed video tracks * receive repair packets from the server, see * [io.livekit.android.room.track.RemoteTrackPublication.setFecProtection]. - * Published video tracks then also send FlexFEC as WebRTC adapts it to observed loss. + * Video tracks published without simulcast also send FlexFEC, at a rate WebRTC adapts to the + * reported loss; WebRTC does not protect simulcast tracks with FlexFEC. * Disabled by default. * * This is a process-wide WebRTC setting, so it must be set before the first [Room] is From 8a1a9004f0ec0bd47aa5711638c4d084cb2ec580 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Tue, 17 Nov 2026 10:25:17 +0800 Subject: [PATCH 3/3] Add an on-device test for publishing with FlexFEC FlexFecPublishOnDeviceTest publishes a synthetic video track to a real server and checks the publisher negotiation: without simulcast the offer includes flexfec-03 and a FEC-FR SSRC and the server accepts flexfec-03, and with simulcast no FlexFEC SSRC is signaled. The track can be held for a while to watch the repairs arrive at the server under upstream loss. It is skipped unless the lkUrl and lkPublisherToken instrumentation arguments are given. --- .../room/track/FlexFecPublishOnDeviceTest.kt | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt diff --git a/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt new file mode 100644 index 000000000..18e36072e --- /dev/null +++ b/livekit-android-test/src/androidTest/java/io/livekit/android/room/track/FlexFecPublishOnDeviceTest.kt @@ -0,0 +1,222 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track + +import android.content.Context +import android.os.SystemClock +import android.util.Log +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import io.livekit.android.LiveKit +import io.livekit.android.RoomOptions +import io.livekit.android.room.Room +import io.livekit.android.room.participant.VideoTrackPublishOptions +import io.livekit.android.util.LoggingLevel +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import livekit.org.webrtc.CapturerObserver +import livekit.org.webrtc.JavaI420Buffer +import livekit.org.webrtc.SurfaceTextureHelper +import livekit.org.webrtc.VideoCapturer +import livekit.org.webrtc.VideoFrame +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import kotlin.random.Random + +/** + * Publishes a synthetic video track to a real LiveKit server with [LiveKit.enableFlexFec], and + * checks how FlexFEC is negotiated on the publisher peer connection. + * + * libwebrtc only protects a single media stream with FlexFEC, so a FlexFEC SSRC is only signaled + * when the track is published without simulcast. The repairs themselves follow libwebrtc's + * loss-based protection, so none are sent without loss, and the publisher's stats do not count + * them. To see them arrive, run against a server with `rtc.flexfec.upstream_enabled`, add upstream + * loss, and watch the server's `livekit_fec_total{direction="incoming"}` metric while the track is + * held for `lkPublishSeconds`. + * + * Skipped unless the instrumentation arguments `lkUrl` and `lkPublisherToken` are given: + * + * ``` + * ./gradlew :livekit-android-test:connectedDebugAndroidTest \ + * -Pandroid.testInstrumentationRunnerArguments.class=io.livekit.android.room.track.FlexFecPublishOnDeviceTest \ + * -Pandroid.testInstrumentationRunnerArguments.lkUrl=ws://192.168.1.2:7880 \ + * -Pandroid.testInstrumentationRunnerArguments.lkPublisherToken= \ + * -Pandroid.testInstrumentationRunnerArguments.lkPublishSeconds=30 + * ``` + */ +@RunWith(AndroidJUnit4::class) +class FlexFecPublishOnDeviceTest { + + private companion object { + const val TAG = "FlexFecPublishTest" + const val WIDTH = 640 + const val HEIGHT = 480 + const val FPS = 30 + const val NEGOTIATION_MS = 3_000L + const val STATS_INTERVAL_MS = 5_000L + const val DEFAULT_PUBLISH_SECONDS = 10L + } + + private data class PublisherSdp(val offer: String, val answer: String) { + val offersFlexFec: Boolean + get() = offer.lines().any { it.startsWith("a=rtpmap:") && "flexfec-03" in it } + val signalsFlexFecSsrc: Boolean + get() = offer.lines().any { it.startsWith("a=ssrc-group:FEC-FR") } + val answersFlexFec: Boolean + get() = answer.lines().any { it.startsWith("a=rtpmap:") && "flexfec-03" in it } + } + + private lateinit var room: Room + private lateinit var url: String + private lateinit var token: String + private var publishSeconds = DEFAULT_PUBLISH_SECONDS + + @Before + fun setUp() { + val args = InstrumentationRegistry.getArguments() + val url = args.getString("lkUrl") + val token = args.getString("lkPublisherToken") + assumeTrue("lkUrl and lkPublisherToken instrumentation arguments are required", url != null && token != null) + this.url = url!! + this.token = token!! + publishSeconds = args.getString("lkPublishSeconds")?.toLongOrNull() ?: DEFAULT_PUBLISH_SECONDS + + LiveKit.loggingLevel = LoggingLevel.INFO + LiveKit.enableFlexFec = true + } + + @After + fun tearDown() { + if (::room.isInitialized) { + room.disconnect() + room.release() + } + } + + @Test + fun withoutSimulcastSignalsFlexFec(): Unit = runBlocking { + val sdp = publish(simulcast = false) + + assertTrue("offer includes flexfec-03", sdp.offersFlexFec) + assertTrue("offer signals a FEC-FR SSRC", sdp.signalsFlexFecSsrc) + assertTrue("server accepts flexfec-03", sdp.answersFlexFec) + } + + @Test + fun withSimulcastDoesNotSignalFlexFec(): Unit = runBlocking { + val sdp = publish(simulcast = true) + + assertFalse("libwebrtc only protects a single stream with FlexFEC", sdp.signalsFlexFecSsrc) + } + + private suspend fun publish(simulcast: Boolean): PublisherSdp { + val context = InstrumentationRegistry.getInstrumentation().targetContext + room = LiveKit.create(context, RoomOptions(adaptiveStream = false)) + room.connect(url, token) + + val local = room.localParticipant + val track = local.createVideoTrack("synthetic", NoiseCapturer()) + track.startCapture() + local.publishVideoTrack(track, VideoTrackPublishOptions(simulcast = simulcast)) + delay(NEGOTIATION_MS) + + val sdp = requireNotNull( + room.engine.transport?.publisher?.withPeerConnection { + PublisherSdp(localDescription?.description.orEmpty(), remoteDescription?.description.orEmpty()) + }, + ) { "publisher peer connection is closed" } + Log.i( + TAG, + "simulcast=$simulcast offersFlexFec=${sdp.offersFlexFec} " + + "signalsFlexFecSsrc=${sdp.signalsFlexFecSsrc} answersFlexFec=${sdp.answersFlexFec}", + ) + + val end = SystemClock.elapsedRealtime() + publishSeconds * 1000 + var packetsSent = 0L + while (SystemClock.elapsedRealtime() < end) { + delay(STATS_INTERVAL_MS) + val stats = track.getRTCStats() ?: continue + val outbound = stats.statsMap.values.filter { it.type == "outbound-rtp" } + packetsSent = outbound.sumOf { (it.members["packetsSent"] as? Number)?.toLong() ?: 0L } + val retransmitted = outbound.sumOf { (it.members["retransmittedPacketsSent"] as? Number)?.toLong() ?: 0L } + val fractionLost = stats.statsMap.values + .filter { it.type == "remote-inbound-rtp" } + .map { it.members["fractionLost"] } + Log.i( + TAG, + "simulcast=$simulcast packetsSent=$packetsSent retransmitted=$retransmitted " + + "reportedFractionLost=$fractionLost", + ) + } + track.stopCapture() + + assertTrue("media is sent", packetsSent > 0) + return sdp + } + + /** + * Pushes noisy I420 frames, so the encoder produces multi-packet frames without a camera. + */ + private class NoiseCapturer : VideoCapturer { + private var observer: CapturerObserver? = null + private var executor: ScheduledExecutorService? = null + private val random = Random(0) + + override fun initialize(helper: SurfaceTextureHelper?, context: Context?, observer: CapturerObserver) { + this.observer = observer + } + + override fun startCapture(width: Int, height: Int, framerate: Int) { + observer?.onCapturerStarted(true) + executor = Executors.newSingleThreadScheduledExecutor().also { + it.scheduleAtFixedRate(::pushFrame, 0, 1000L / FPS, TimeUnit.MILLISECONDS) + } + } + + private fun pushFrame() { + val buffer = JavaI420Buffer.allocate(WIDTH, HEIGHT) + val luma = ByteArray(buffer.dataY.capacity()) + random.nextBytes(luma) + buffer.dataY.put(luma) + val frame = VideoFrame(buffer, 0, System.nanoTime()) + observer?.onFrameCaptured(frame) + frame.release() + } + + override fun stopCapture() { + executor?.shutdownNow() + executor = null + observer?.onCapturerStopped() + } + + override fun changeCaptureFormat(width: Int, height: Int, framerate: Int) = Unit + + override fun dispose() { + stopCapture() + } + + override fun isScreencast(): Boolean = false + } +}