Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .changes/single-pc-opus-nack
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
patch type="fixed" "Negotiate Opus NACK on every audio section in single peer connection mode, including send-only offers created before any subscriptions, enabling retransmission of lost audio in both publish and subscribe directions when RED is disabled, such as with end-to-end encryption. Accept server-offered Opus NACK in dual peer connection subscriber answers, with fallback if the modified answer is rejected"
4 changes: 2 additions & 2 deletions lib/src/core/engine.dart
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,7 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
publisher?.onNegotiationError = _onPublisherNegotiationError;
// subscribed video arrives on the publisher's recvonly sections in single PC mode
publisher?.negotiateReceivedVideoDependencyDescriptor = _singlePCMode;
publisher?.negotiateOpusNack = _singlePCMode;

// In single PC mode, server opens data channels on publisher.
// In dual PC subscriber-primary mode, server opens data channels on subscriber.
Expand Down Expand Up @@ -1721,10 +1722,9 @@ class Engine extends Disposable with EventsEmittable<EngineEvent> {
await subscriber!.setRemoteDescription(event.sd);

try {
final answer = await subscriber!.pc.createAnswer();
final answer = await subscriber!.createAndSetAnswer(event.sd);
logger.fine('Created answer');
logger.finer('sdp: ${answer.sdp}');
await subscriber!.pc.setLocalDescription(answer);
signalClient.sendAnswer(answer, offerId: event.offerId);
} catch (_) {
logger.severe('[$objectId] Failed to createAnswer()');
Expand Down
79 changes: 79 additions & 0 deletions lib/src/core/transport.dart
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,69 @@ void applyVideoStartBitrate(Map<String, dynamic> media, int codecPayload, int st
media['fmtp'] = fmtpList;
}

/// Adds plain NACK to every Opus payload in an audio media section.
///
/// The SFU negotiates audio feedback from the first audio section, so single-PC
/// send-only sections must advertise it before subscriptions exist. Existing plain
/// or wildcard NACK is preserved; `nack pli` does not imply plain NACK.
@internal
void ensureOpusNack(Map<String, dynamic> media) {
if (media['type'] != 'audio') return;
final opusPayloads = (media['rtp'] as List? ?? const [])
.where((rtp) => (rtp['codec'] as String?)?.toLowerCase() == 'opus')
.map((rtp) => rtp['payload'] as int)
.toSet();
if (opusPayloads.isEmpty) return;

final feedback = media['rtcpFb'] as List? ?? <dynamic>[];
for (final payload in opusPayloads) {
final hasNack = feedback.any(
(fb) =>
(fb['payload'] == payload || fb['payload'] == '*') &&
fb['type'] == 'nack' &&
(fb['subtype'] == null || fb['subtype'] == ''),
);
if (!hasNack) {
feedback.add({'payload': payload, 'type': 'nack'});
}
}
media['rtcpFb'] = feedback;
}

/// Accepts Opus NACK only on answer sections whose MID offered it.
@internal
String mungeOpusNackForAnswer(String answer, String offer) {
final offeredMedia = sdp_transform.parse(offer)['media'] as List? ?? const [];
final nackMids = <String>{};
for (final media in offeredMedia) {
if (media['type'] != 'audio' || media['mid'] == null) continue;
final opus = (media['rtp'] as List? ?? const [])
.where((rtp) => (rtp['codec'] as String?)?.toLowerCase() == 'opus')
.map((rtp) => rtp['payload'])
.toSet();
final feedback = media['rtcpFb'] as List? ?? const [];
if (opus.isNotEmpty &&
feedback.any(
(fb) =>
(fb['payload'] == '*' || opus.contains(fb['payload'])) &&
fb['type'] == 'nack' &&
(fb['subtype'] == null || fb['subtype'] == ''),
)) {
nackMids.add(media['mid'].toString());
}
}
if (nackMids.isEmpty) return answer;
final document = sdp_transform.parse(answer);
var changed = false;
for (final media in document['media'] as List? ?? const []) {
if (media['mid'] == null || !nackMids.contains(media['mid'].toString())) continue;
final before = (media['rtcpFb'] as List? ?? const []).length;
ensureOpusNack(media);
changed = changed || (media['rtcpFb'] as List? ?? const []).length != before;
}
return changed ? sdp_transform.write(document, null) : answer;
}

typedef TransportOnOffer = void Function(rtc.RTCSessionDescription offer);

/// [offerId] numbers the offer, starting at 1, so its answer can be told apart
Expand Down Expand Up @@ -245,6 +308,11 @@ class Transport extends Disposable {
@internal
bool negotiateReceivedVideoDependencyDescriptor = false;

/// Advertise Opus NACK on local audio offers when this peer connection also receives.
/// Set from the effective single-PC mode, which can differ from the requested mode after fallback.
@internal
bool negotiateOpusNack = false;

/// Id of the last offer created on this transport, 0 before the first one.
@internal
int latestOfferId = 0;
Expand Down Expand Up @@ -467,6 +535,9 @@ class Transport extends Disposable {
}
var appliedVideoStartBitrate = false;
sdpParsed['media']?.forEach((media) {
if (negotiateOpusNack) {
ensureOpusNack(media);
}
if (media['type'] == 'video') {
final codec = (media['rtp']?[0]?['codec'] as String?) ?? '';
final receiveOnly = media['direction'] == 'recvonly';
Expand Down Expand Up @@ -652,6 +723,14 @@ class Transport extends Disposable {
return maxId + 1 == 15 ? 16 : maxId + 1;
}

/// Creates and applies an answer accepting the server's offered Opus NACK.
Future<rtc.RTCSessionDescription> createAndSetAnswer(rtc.RTCSessionDescription offer) async {
final answer = await pc.createAnswer();
final munged = mungeOpusNackForAnswer(answer.sdp ?? '', offer.sdp ?? '');
await setMungedSDP(sd: answer, munged: munged == answer.sdp ? null : munged);
return answer;
}

Future<void> setMungedSDP({required rtc.RTCSessionDescription sd, String? munged, bool? remote}) async {
if (munged != null) {
final originalSdp = sd.sdp;
Expand Down
Loading