diff --git a/.changes/single-pc-opus-nack b/.changes/single-pc-opus-nack new file mode 100644 index 000000000..88ed31025 --- /dev/null +++ b/.changes/single-pc-opus-nack @@ -0,0 +1 @@ +patch type="fixed" "Negotiate Opus NACK on every audio section in single peer connection mode, including send-only offers created before any subscriptions, enabling retransmission of lost audio in both publish and subscribe directions when RED is disabled, such as with end-to-end encryption. Accept server-offered Opus NACK in dual peer connection subscriber answers, with fallback if the modified answer is rejected" diff --git a/lib/src/core/engine.dart b/lib/src/core/engine.dart index a2f60975e..430390bd5 100644 --- a/lib/src/core/engine.dart +++ b/lib/src/core/engine.dart @@ -885,6 +885,7 @@ class Engine extends Disposable with EventsEmittable { publisher?.onNegotiationError = _onPublisherNegotiationError; // subscribed video arrives on the publisher's recvonly sections in single PC mode publisher?.negotiateReceivedVideoDependencyDescriptor = _singlePCMode; + publisher?.negotiateOpusNack = _singlePCMode; // In single PC mode, server opens data channels on publisher. // In dual PC subscriber-primary mode, server opens data channels on subscriber. @@ -1721,10 +1722,9 @@ class Engine extends Disposable with EventsEmittable { await subscriber!.setRemoteDescription(event.sd); try { - final answer = await subscriber!.pc.createAnswer(); + final answer = await subscriber!.createAndSetAnswer(event.sd); logger.fine('Created answer'); logger.finer('sdp: ${answer.sdp}'); - await subscriber!.pc.setLocalDescription(answer); signalClient.sendAnswer(answer, offerId: event.offerId); } catch (_) { logger.severe('[$objectId] Failed to createAnswer()'); diff --git a/lib/src/core/transport.dart b/lib/src/core/transport.dart index 132170268..590efed95 100644 --- a/lib/src/core/transport.dart +++ b/lib/src/core/transport.dart @@ -204,6 +204,69 @@ void applyVideoStartBitrate(Map media, int codecPayload, int st media['fmtp'] = fmtpList; } +/// Adds plain NACK to every Opus payload in an audio media section. +/// +/// The SFU negotiates audio feedback from the first audio section, so single-PC +/// send-only sections must advertise it before subscriptions exist. Existing plain +/// or wildcard NACK is preserved; `nack pli` does not imply plain NACK. +@internal +void ensureOpusNack(Map media) { + if (media['type'] != 'audio') return; + final opusPayloads = (media['rtp'] as List? ?? const []) + .where((rtp) => (rtp['codec'] as String?)?.toLowerCase() == 'opus') + .map((rtp) => rtp['payload'] as int) + .toSet(); + if (opusPayloads.isEmpty) return; + + final feedback = media['rtcpFb'] as List? ?? []; + for (final payload in opusPayloads) { + final hasNack = feedback.any( + (fb) => + (fb['payload'] == payload || fb['payload'] == '*') && + fb['type'] == 'nack' && + (fb['subtype'] == null || fb['subtype'] == ''), + ); + if (!hasNack) { + feedback.add({'payload': payload, 'type': 'nack'}); + } + } + media['rtcpFb'] = feedback; +} + +/// Accepts Opus NACK only on answer sections whose MID offered it. +@internal +String mungeOpusNackForAnswer(String answer, String offer) { + final offeredMedia = sdp_transform.parse(offer)['media'] as List? ?? const []; + final nackMids = {}; + for (final media in offeredMedia) { + if (media['type'] != 'audio' || media['mid'] == null) continue; + final opus = (media['rtp'] as List? ?? const []) + .where((rtp) => (rtp['codec'] as String?)?.toLowerCase() == 'opus') + .map((rtp) => rtp['payload']) + .toSet(); + final feedback = media['rtcpFb'] as List? ?? const []; + if (opus.isNotEmpty && + feedback.any( + (fb) => + (fb['payload'] == '*' || opus.contains(fb['payload'])) && + fb['type'] == 'nack' && + (fb['subtype'] == null || fb['subtype'] == ''), + )) { + nackMids.add(media['mid'].toString()); + } + } + if (nackMids.isEmpty) return answer; + final document = sdp_transform.parse(answer); + var changed = false; + for (final media in document['media'] as List? ?? const []) { + if (media['mid'] == null || !nackMids.contains(media['mid'].toString())) continue; + final before = (media['rtcpFb'] as List? ?? const []).length; + ensureOpusNack(media); + changed = changed || (media['rtcpFb'] as List? ?? const []).length != before; + } + return changed ? sdp_transform.write(document, null) : answer; +} + typedef TransportOnOffer = void Function(rtc.RTCSessionDescription offer); /// [offerId] numbers the offer, starting at 1, so its answer can be told apart @@ -245,6 +308,11 @@ class Transport extends Disposable { @internal bool negotiateReceivedVideoDependencyDescriptor = false; + /// Advertise Opus NACK on local audio offers when this peer connection also receives. + /// Set from the effective single-PC mode, which can differ from the requested mode after fallback. + @internal + bool negotiateOpusNack = false; + /// Id of the last offer created on this transport, 0 before the first one. @internal int latestOfferId = 0; @@ -467,6 +535,9 @@ class Transport extends Disposable { } var appliedVideoStartBitrate = false; sdpParsed['media']?.forEach((media) { + if (negotiateOpusNack) { + ensureOpusNack(media); + } if (media['type'] == 'video') { final codec = (media['rtp']?[0]?['codec'] as String?) ?? ''; final receiveOnly = media['direction'] == 'recvonly'; @@ -652,6 +723,14 @@ class Transport extends Disposable { return maxId + 1 == 15 ? 16 : maxId + 1; } + /// Creates and applies an answer accepting the server's offered Opus NACK. + Future createAndSetAnswer(rtc.RTCSessionDescription offer) async { + final answer = await pc.createAnswer(); + final munged = mungeOpusNackForAnswer(answer.sdp ?? '', offer.sdp ?? ''); + await setMungedSDP(sd: answer, munged: munged == answer.sdp ? null : munged); + return answer; + } + Future setMungedSDP({required rtc.RTCSessionDescription sd, String? munged, bool? remote}) async { if (munged != null) { final originalSdp = sd.sdp; diff --git a/test/core/transport_opus_nack_test.dart b/test/core/transport_opus_nack_test.dart new file mode 100644 index 000000000..d5cbc2780 --- /dev/null +++ b/test/core/transport_opus_nack_test.dart @@ -0,0 +1,325 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; +import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; + +import 'package:livekit_client/src/core/transport.dart' show Transport, ensureOpusNack, mungeOpusNackForAnswer; +import 'package:livekit_client/src/exceptions.dart' show NegotiationError; +import 'package:livekit_client/src/options.dart' show ConnectOptions; +import '../mock/peerconnection_mock.dart'; + +const _offer = + 'v=0\r\n' + 'o=- 1 1 IN IP4 127.0.0.1\r\n' + 's=-\r\n' + 't=0 0\r\n' + 'm=audio 9 UDP/TLS/RTP/SAVPF 109\r\n' + 'a=mid:0\r\n' + 'a=sendonly\r\n' + 'a=rtpmap:109 opus/48000/2\r\n' + 'm=audio 9 UDP/TLS/RTP/SAVPF 111\r\n' + 'a=mid:1\r\n' + 'a=recvonly\r\n' + 'a=rtpmap:111 opus/48000/2\r\n' + 'm=audio 9 UDP/TLS/RTP/SAVPF 112\r\n' + 'a=mid:2\r\n' + 'a=sendrecv\r\n' + 'a=rtpmap:112 opus/48000/2\r\n'; + +class _AudioPeerConnection extends MockPeerConnection { + String offer = _offer; + String answer = _offer; + bool rejectNack = false; + bool rejectAll = false; + int rejectedNackOffers = 0; + + @override + Future createOffer([Map? constraints]) async { + return rtc.RTCSessionDescription(offer, 'offer'); + } + + @override + Future createAnswer([Map? constraints]) async { + return rtc.RTCSessionDescription(answer, 'answer'); + } + + @override + Future setLocalDescription(rtc.RTCSessionDescription description) async { + if (rejectAll) throw StateError('All local descriptions rejected'); + if (rejectNack && (description.sdp ?? '').contains('a=rtcp-fb:109 nack\r\n')) { + rejectedNackOffers++; + throw StateError('NACK munging rejected'); + } + await super.setLocalDescription(description); + } +} + +void _expectNack(String sdp, {required int audioSections, required bool enabled}) { + final sections = (sdp_transform.parse(sdp)['media'] as List).where((m) => m['type'] == 'audio').toList(); + expect(sections, hasLength(audioSections)); + for (final section in sections) { + final opus = (section['rtp'] as List).where((rtp) => (rtp['codec'] as String).toLowerCase() == 'opus'); + expect(opus, isNotEmpty); + for (final mapping in opus) { + final feedback = section['rtcpFb'] as List? ?? const []; + final nack = feedback.where( + (fb) => + fb['payload'] == mapping['payload'] && + fb['type'] == 'nack' && + (fb['subtype'] == null || fb['subtype'] == ''), + ); + expect(nack, hasLength(enabled ? 1 : 0)); + } + } +} + +Future<(Transport, _AudioPeerConnection, List)> _makeTransport({ + required bool singlePC, + bool requestedSinglePC = false, +}) async { + final pc = _AudioPeerConnection(); + final transport = await Transport.create( + (Map configuration, [Map? constraints]) async => pc, + connectOptions: ConnectOptions(singlePeerConnection: requestedSinglePC), + ); + transport.negotiateOpusNack = singlePC; + addTearDown(transport.dispose); + final offers = []; + transport.onOffer = offers.add; + return (transport, pc, offers); +} + +void main() { + group('Opus NACK SDP', () { + for (final direction in ['sendonly', 'recvonly', 'sendrecv', 'inactive']) { + test('adds plain NACK to all Opus payloads in a $direction section', () { + final media = { + 'type': 'audio', + 'direction': direction, + 'rtp': [ + {'payload': 109, 'codec': 'OPUS'}, + {'payload': 112, 'codec': 'opus'}, + ], + 'rtcpFb': [ + {'payload': 109, 'type': 'nack', 'subtype': 'pli'}, + ], + }; + final expectedFeedback = [ + {'payload': 109, 'type': 'nack', 'subtype': 'pli'}, + {'payload': 109, 'type': 'nack'}, + {'payload': 112, 'type': 'nack'}, + ]; + ensureOpusNack(media); + expect(media['rtcpFb'], expectedFeedback); + ensureOpusNack(media); + expect(media['rtcpFb'], expectedFeedback, reason: 'repeated munging must not duplicate feedback'); + }); + } + + for (final feedback in [ + {'payload': 109, 'type': 'nack'}, + {'payload': '*', 'type': 'nack'}, + {'payload': 109, 'type': 'nack', 'subtype': ''}, + ]) { + test('preserves existing feedback $feedback', () { + final media = { + 'type': 'audio', + 'rtp': [ + {'payload': 109, 'codec': 'opus'}, + ], + 'rtcpFb': [feedback], + }; + ensureOpusNack(media); + expect(media['rtcpFb'], [feedback]); + }); + } + + test('payload types and feedback stay scoped to their audio sections', () { + final media = [ + { + 'type': 'audio', + 'rtp': [ + {'payload': 109, 'codec': 'opus'}, + ], + }, + { + 'type': 'video', + 'rtp': [ + {'payload': 109, 'codec': 'VP8'}, + ], + 'rtcpFb': [ + {'payload': 109, 'type': 'nack'}, + ], + }, + { + 'type': 'audio', + 'rtp': [ + {'payload': 109, 'codec': 'PCMU'}, + ], + }, + {'type': 'audio'}, + ]; + for (final section in media) { + ensureOpusNack(section); + } + expect(media[0]['rtcpFb'], [ + {'payload': 109, 'type': 'nack'}, + ]); + expect(media[1]['rtcpFb'], [ + {'payload': 109, 'type': 'nack'}, + ]); + expect(media[2].containsKey('rtcpFb'), isFalse); + expect(media[3].containsKey('rtcpFb'), isFalse); + }); + + test('parsed wildcard NACK covers every Opus payload without duplication', () { + final parsed = sdp_transform.parse( + _offer.replaceFirst( + 'a=rtpmap:109 opus/48000/2', + 'a=rtpmap:109 opus/48000/2\r\na=rtcp-fb:* nack', + ), + ); + final first = (parsed['media'] as List).first as Map; + ensureOpusNack(first); + expect(first['rtcpFb'], [ + {'payload': '*', 'type': 'nack', 'subtype': null}, + ]); + }); + }); + + group('Opus NACK subscriber answers', () { + final offeredNack = _offer.replaceFirst('a=mid:0\r\n', 'a=mid:0\r\na=rtcp-fb:109 nack\r\n'); + + test('accepts NACK only on the offered MID and returns the applied answer', () async { + final (transport, pc, _) = await _makeTransport(singlePC: false); + final offer = rtc.RTCSessionDescription(offeredNack, 'offer'); + await transport.setRemoteDescription(offer); + final answer = await transport.createAndSetAnswer(offer); + final sections = sdp_transform.parse(answer.sdp!)['media'] as List; + expect( + sections[0]['rtcpFb'], + equals([ + {'payload': 109, 'type': 'nack', 'subtype': null}, + ]), + ); + expect(sections[1]['rtcpFb'] ?? [], isEmpty); + expect(sections[2]['rtcpFb'] ?? [], isEmpty); + expect((await pc.getLocalDescription())!.sdp, answer.sdp); + }); + + test('does not add NACK when the offer lacks it or only offers NACK PLI', () async { + for (final feedback in ['', 'a=rtcp-fb:109 nack pli\r\n']) { + final (transport, pc, _) = await _makeTransport(singlePC: false); + final offer = rtc.RTCSessionDescription(_offer.replaceFirst('a=mid:0\r\n', 'a=mid:0\r\n$feedback'), 'offer'); + await transport.setRemoteDescription(offer); + final answer = await transport.createAndSetAnswer(offer); + expect(answer.sdp, pc.answer); + _expectNack(answer.sdp!, audioSections: 3, enabled: false); + } + }); + + test('rejected NACK falls back and returns the original applied answer', () async { + final (transport, pc, _) = await _makeTransport(singlePC: false); + pc.rejectNack = true; + final offer = rtc.RTCSessionDescription(offeredNack, 'offer'); + await transport.setRemoteDescription(offer); + final answer = await transport.createAndSetAnswer(offer); + expect(pc.rejectedNackOffers, 1); + expect(answer.sdp, pc.answer); + expect((await pc.getLocalDescription())!.sdp, answer.sdp); + }); + + test('propagates failure if the original answer is rejected too', () async { + final (transport, pc, _) = await _makeTransport(singlePC: false); + pc.rejectAll = true; + final offer = rtc.RTCSessionDescription(offeredNack, 'offer'); + await transport.setRemoteDescription(offer); + await expectLater(transport.createAndSetAnswer(offer), throwsStateError); + expect(await pc.getLocalDescription(), isNull); + }); + + test('matches reordered MIDs and resolves the answer payload independently', () { + final answer = + 'v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\n' + 'm=audio 9 UDP/TLS/RTP/SAVPF 108\r\na=mid:1\r\na=rtpmap:108 opus/48000/2\r\n' + 'm=audio 9 UDP/TLS/RTP/SAVPF 110\r\na=mid:0\r\na=rtpmap:110 opus/48000/2\r\n'; + for (final payload in ['109', '*']) { + final offer = offeredNack.replaceFirst('a=rtcp-fb:109 nack', 'a=rtcp-fb:$payload nack'); + final munged = mungeOpusNackForAnswer(answer, offer); + final sections = sdp_transform.parse(munged)['media'] as List; + expect(sections[0]['rtcpFb'] ?? [], isEmpty); + expect( + sections[1]['rtcpFb'], + equals([ + {'payload': 110, 'type': 'nack', 'subtype': null}, + ]), + ); + expect(mungeOpusNackForAnswer(munged, offer), munged); + } + }); + }); + + group('Opus NACK offers', () { + test('single-PC first offer includes NACK in every audio direction', () async { + final (transport, pc, offers) = await _makeTransport(singlePC: true); + await transport.createAndSendOffer(); + expect(offers, hasLength(1)); + _expectNack(offers.single.sdp!, audioSections: 3, enabled: true); + expect((await pc.getLocalDescription())?.sdp, offers.single.sdp); + }); + + test('dual-PC publisher keeps NACK absent', () async { + final (transport, _, offers) = await _makeTransport(singlePC: false); + await transport.createAndSendOffer(); + _expectNack(offers.single.sdp!, audioSections: 3, enabled: false); + }); + + test('does not enable NACK from the requested option when the effective mode is dual-PC', () async { + final (transport, _, offers) = await _makeTransport(singlePC: false, requestedSinglePC: true); + await transport.createAndSendOffer(); + _expectNack(offers.single.sdp!, audioSections: 3, enabled: false); + }); + + test('send-only audio offers NACK before receive sections exist and after renegotiation', () async { + final (transport, pc, offers) = await _makeTransport(singlePC: true); + pc.offer = _offer.split('m=audio 9 UDP/TLS/RTP/SAVPF 111').first; + await transport.createAndSendOffer(); + _expectNack(offers.single.sdp!, audioSections: 1, enabled: true); + await transport.setRemoteDescription(rtc.RTCSessionDescription('v=0\r\n', 'answer')); + pc.offer = _offer; + await transport.createAndSendOffer(); + expect(offers, hasLength(2)); + _expectNack(offers.last.sdp!, audioSections: 3, enabled: true); + }); + + test('falls back to the original offer if NACK munging is rejected', () async { + final (transport, pc, offers) = await _makeTransport(singlePC: true); + pc.rejectNack = true; + await transport.createAndSendOffer(); + expect(pc.rejectedNackOffers, 1); + expect(offers.single.sdp, _offer); + expect((await pc.getLocalDescription())?.sdp, _offer); + _expectNack(offers.single.sdp!, audioSections: 3, enabled: false); + }); + + test('does not signal an offer if both munged and original descriptions are rejected', () async { + final (transport, pc, offers) = await _makeTransport(singlePC: true); + pc.rejectAll = true; + await expectLater(transport.createAndSendOffer(), throwsA(isA())); + expect(offers, isEmpty); + }); + }); +}