diff --git a/README.md b/README.md index 12ff8a3..5e67747 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ remote audio and video, transform it, and publish media back into the call. G.711, and slice it into chunks and sliding windows. - Transform and republish audio or video through local tracks. - Temporarily mute publications, publish screen-share audio, configure local - video bitrate, and control SFU-side noise cancellation. + audio and video encoder settings, and control SFU-side noise cancellation. - Emit structured, secret-redacted diagnostics through `tracing`. ## Crate map @@ -258,19 +258,94 @@ temporary `mute_track` / `unmute_track` preserves the same local track and sender; `stop_publish` remains terminal for that local track handle. The latest SFU view is available synchronously through `Call::call_state`. +Tracks carry 48 kHz signed 16-bit PCM, which is rarely what a model or telephony +API wants. The [`rtc::pcm`](https://docs.rs/getstream/latest/getstream/rtc/pcm/) +module converts the rate and channel count, converts to 32-bit float, raw bytes, +WAV, or G.711, and slices audio into chunks and sliding windows. + +## Encoder settings + +Local tracks encode with defaults that suit a server-side agent. Both the audio +and the video defaults are adjustable per track. + +`LocalAudioTrack::opus()` publishes mono 48 kHz Opus. Its defaults are 32 kbps, +in-band FEC for 10% expected packet loss, and DTX. Use `opus_with_config` to +change them. In-band FEC puts a low-quality copy of the previous frame into each +packet, and the decoder plays that copy when it loses a packet. FEC uses more +bitrate, and it adds nothing while the expected packet loss is 0. DTX stops the +payload during silence. + +```rust,no_run +use getstream::rtc::{LocalAudioTrack, LocalAudioTrackConfig, RtcResult}; + +fn audio_tracks() -> RtcResult<()> { + // Defaults: 32 kbps, FEC for 10% expected packet loss, DTX on. + let default_audio = LocalAudioTrack::opus()?; + + // The network loses more packets. Add more redundancy. + let lossy_network = LocalAudioTrack::opus_with_config( + LocalAudioTrackConfig::default().with_expected_packet_loss_pct(30), + )?; + + // The uplink is small. Use a lower bitrate and no redundancy. + let low_bandwidth = LocalAudioTrack::opus_with_config( + LocalAudioTrackConfig::new(24_000).with_inband_fec(false), + )?; + + // The audio is continuous, for example music. Stop DTX. + let continuous_audio = + LocalAudioTrack::opus_with_config(LocalAudioTrackConfig::new(64_000).with_dtx(false))?; + Ok(()) +} +``` + +Video encodes at 1 Mbps on one layer. `LocalVideoTrackConfig` sets the target +bitrate, and every codec has a `_with_config` constructor that takes it. + Layered publishing is opt-in. `LocalVideoTrack::vp9_svc()` provides camera SVC with up to three spatial and temporal layers on one SSRC. `LocalVideoTrack::h264_simulcast()` supports camera video and `LocalVideoTrack::vp8_simulcast()` supports screen share with a `q`/`h`/`f` RID -ladder on one m-line. All three follow SFU quality updates. Feed raw I420 at the -full resolution announced by the SFU publish option; a mismatch is rejected -instead of advertising dimensions that are not sent. Pre-encoded samples and -forwarded RTP remain single-layer-only. +ladder on one m-line. Each of the three is a shortcut for the matching +`_with_config` call on a `server_managed` config. All three follow SFU quality +updates. To publish fewer layers than the SFU offers, set `VideoLayering` +directly and cap the counts. -Tracks carry 48 kHz signed 16-bit PCM, which is rarely what a model or telephony -API wants. The [`rtc::pcm`](https://docs.rs/getstream/latest/getstream/rtc/pcm/) -module converts the rate and channel count, converts to 32-bit float, raw bytes, -WAV, or G.711, and slices audio into chunks and sliding windows. +```rust,no_run +use std::num::NonZeroU8; + +use getstream::rtc::{LocalVideoTrack, LocalVideoTrackConfig, RtcResult, VideoLayering}; + +fn video_tracks() -> RtcResult<()> { + // Default: VP9 camera video, 1 Mbps, one layer. + let default_video = LocalVideoTrack::vp9()?; + + // The uplink is small. Lower the target bitrate. + let low_bitrate = LocalVideoTrack::vp9_with_config(LocalVideoTrackConfig::new(400_000))?; + + // Camera SVC on one SSRC, up to three spatial and three temporal layers. + let svc = LocalVideoTrack::vp9_svc()?; + + // The same, but the SDK encodes at most two spatial and two temporal layers. + let mut capped = LocalVideoTrackConfig::new(600_000); + capped.layering = VideoLayering::ServerManaged { + max_spatial_layers: NonZeroU8::new(2), + max_temporal_layers: NonZeroU8::new(2), + }; + let capped_svc = LocalVideoTrack::vp9_with_config(capped)?; + + // H264 camera simulcast, and VP8 screen-share simulcast. + let h264_camera = LocalVideoTrack::h264_simulcast()?; + let screen_share = LocalVideoTrack::vp8_simulcast()?; + Ok(()) +} +``` + +Feed raw I420 at the full resolution announced by the SFU publish option; a +mismatch is rejected instead of advertising dimensions that are not sent. +Pre-encoded samples and forwarded RTP remain single-layer-only. Publish a +screen-share track with `Call::publish_screen_share` and a camera track with +`Call::publish_video`. ## Examples diff --git a/src/rtc/error.rs b/src/rtc/error.rs index ec24a13..e4734d4 100644 --- a/src/rtc/error.rs +++ b/src/rtc/error.rs @@ -180,6 +180,12 @@ impl From for RtcError { } } +impl From for RtcError { + fn from(e: opus::Error) -> Self { + RtcError::Media(e.to_string()) + } +} + impl From for RtcError { fn from(e: webrtc::error::Error) -> Self { RtcError::Webrtc(Box::new(e)) diff --git a/src/rtc/local_track.rs b/src/rtc/local_track.rs index a6f709f..a01ac7e 100644 --- a/src/rtc/local_track.rs +++ b/src/rtc/local_track.rs @@ -72,6 +72,10 @@ const MAX_LOCAL_VIDEO_PIXELS: u64 = 3_840 * 2_160; const MAX_LOCAL_VIDEO_I420_BYTES: usize = 3_840 * 2_160 * 3 / 2; const PCM_QUEUE_CAPACITY_SAMPLES: usize = FRAME_SAMPLES_20MS * 10; const MAX_OPUS_PACKET_BYTES: usize = 1_500; + +const AUDIO_BITRATE_BPS: u32 = 32_000; +/// libopus adds no in-band FEC redundancy while the expected loss is 0. +const EXPECTED_PACKET_LOSS_PCT: u8 = 10; /// Force a fresh keyframe at least this often. The backend publisher does not /// answer receiver PLI/FIR (webrtc-rs has no publisher keyframe-request hook), /// and libvpx only emits a keyframe on the first frame or a scene change — so a @@ -328,6 +332,65 @@ struct AudioInner { write_guard: tokio::sync::Mutex<()>, } +/// Encoder settings for a locally encoded Opus track. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct LocalAudioTrackConfig { + /// Target encoder bitrate in bits per second. + pub target_bitrate_bps: u32, + /// In-band forward error correction: carry a low-quality copy of the + /// previous frame so a lost packet can be recovered from the next one. + pub inband_fec: bool, + /// Network loss the encoder should expect, `0..=100`. In-band FEC adds no + /// redundancy at 0. + pub expected_packet_loss_pct: u8, + /// Discontinuous transmission: stop emitting packets during silence. + pub dtx: bool, +} + +impl Default for LocalAudioTrackConfig { + fn default() -> Self { + Self { + target_bitrate_bps: AUDIO_BITRATE_BPS, + inband_fec: true, + expected_packet_loss_pct: EXPECTED_PACKET_LOSS_PCT, + dtx: true, + } + } +} + +impl LocalAudioTrackConfig { + /// Configure a local encoder target bitrate in bits per second. + #[must_use] + pub fn new(target_bitrate_bps: u32) -> Self { + Self { + target_bitrate_bps, + ..Self::default() + } + } + + /// Enable or disable in-band forward error correction. + #[must_use] + pub fn with_inband_fec(mut self, inband_fec: bool) -> Self { + self.inband_fec = inband_fec; + self + } + + /// Set the network loss the encoder should expect, `0..=100`. + #[must_use] + pub fn with_expected_packet_loss_pct(mut self, expected_packet_loss_pct: u8) -> Self { + self.expected_packet_loss_pct = expected_packet_loss_pct; + self + } + + /// Enable or disable discontinuous transmission. + #[must_use] + pub fn with_dtx(mut self, dtx: bool) -> Self { + self.dtx = dtx; + self + } +} + /// An outbound Opus audio track. /// /// Feed it PCM ([`write_pcm`](Self::write_pcm)), pre-encoded Opus @@ -341,6 +404,11 @@ pub struct LocalAudioTrack { impl LocalAudioTrack { /// Build a mono Opus track (48 kHz, matching the SFU/webrtc-rs default codec). pub fn opus() -> Result { + Self::opus_with_config(LocalAudioTrackConfig::default()) + } + + /// Build a mono Opus track with explicit local encoder settings. + pub fn opus_with_config(config: LocalAudioTrackConfig) -> Result { let codec = RTCRtpCodecCapability { mime_type: MIME_TYPE_OPUS.to_owned(), clock_rate: OPUS_SAMPLE_RATE, @@ -350,12 +418,21 @@ impl LocalAudioTrack { }; let track_id = format!("audio-{}", uuid::Uuid::new_v4().simple()); let core = TrackCore::new(codec, track_id, "stream-rust-audio".to_owned())?; - let encoder = opus::Encoder::new( + let bitrate = i32::try_from(config.target_bitrate_bps).map_err(|_| { + RtcError::Media(format!( + "opus bitrate out of range: {} bps", + config.target_bitrate_bps + )) + })?; + let mut encoder = opus::Encoder::new( OPUS_SAMPLE_RATE, opus::Channels::Mono, opus::Application::Voip, - ) - .map_err(|error| RtcError::Media(error.to_string()))?; + )?; + encoder.set_bitrate(opus::Bitrate::Bits(bitrate))?; + encoder.set_inband_fec(config.inband_fec)?; + encoder.set_packet_loss_perc(i32::from(config.expected_packet_loss_pct))?; + encoder.set_dtx(config.dtx)?; Ok(Self { inner: Arc::new(AudioInner { core, @@ -607,9 +684,7 @@ fn push_bounded_pcm(queue: &mut VecDeque, samples: Vec) -> usize { } fn encode_opus_into(encoder: &mut opus::Encoder, pcm: &[i16], output: &mut [u8]) -> Result { - encoder - .encode(pcm, output) - .map_err(|error| RtcError::Media(error.to_string())) + Ok(encoder.encode(pcm, output)?) } // Video @@ -1918,6 +1993,86 @@ impl LocalTrack { mod tests { use super::*; + /// One 20 ms frame of 440 Hz tone: FEC and DTX both key off whether the + /// frame carries signal, so silence would not exercise either. + fn tone_20ms() -> Vec { + (0..FRAME_SAMPLES_20MS) + .map(|index| { + let time = index as f64 / f64::from(OPUS_SAMPLE_RATE); + (12_000.0 * (std::f64::consts::TAU * 440.0 * time).sin()) as i16 + }) + .collect() + } + + /// Encode `frames` copies of `pcm` through a configured track's encoder and + /// return each packet's byte length. + fn encoded_lengths(config: LocalAudioTrackConfig, pcm: &[i16], frames: usize) -> Vec { + let track = LocalAudioTrack::opus_with_config(config).expect("opus track"); + let mut encoder = track + .inner + .encoder + .lock() + .unwrap_or_else(|e| e.into_inner()); + let mut output = vec![0u8; MAX_OPUS_PACKET_BYTES]; + (0..frames) + .map(|_| encode_opus_into(&mut encoder, pcm, &mut output).expect("encode")) + .collect() + } + + #[test] + fn a_higher_configured_bitrate_produces_bigger_packets() { + let pcm = tone_20ms(); + + let low = encoded_lengths(LocalAudioTrackConfig::new(16_000), &pcm, 5); + let high = encoded_lengths(LocalAudioTrackConfig::new(64_000), &pcm, 5); + + let (low, high) = (low[4], high[4]); + assert!( + high > low * 2, + "64 kbps packet ({high} B) should be far larger than 16 kbps ({low} B)" + ); + } + + #[test] + fn dtx_stops_emitting_payload_during_silence() { + let silence = vec![0i16; FRAME_SAMPLES_20MS]; + let without = LocalAudioTrackConfig { + dtx: false, + ..LocalAudioTrackConfig::default() + }; + + let with_dtx = encoded_lengths(LocalAudioTrackConfig::default(), &silence, 20); + let without_dtx = encoded_lengths(without, &silence, 20); + + let last = with_dtx.last().copied().expect("frames"); + assert!(last <= 2, "DTX should collapse silence, got {last} B"); + assert!( + without_dtx.last().copied().expect("frames") > 2, + "silence without DTX still sends a payload" + ); + } + + #[test] + fn inband_fec_adds_redundancy_to_later_packets() { + let pcm = tone_20ms(); + let without = LocalAudioTrackConfig { + inband_fec: false, + ..LocalAudioTrackConfig::default() + }; + + let with_fec = encoded_lengths(LocalAudioTrackConfig::default(), &pcm, 5); + let without_fec = encoded_lengths(without, &pcm, 5); + + // The first packet has no previous frame to protect; redundancy shows up + // from the second onward. + assert!( + with_fec[4] > without_fec[4], + "FEC packet ({} B) should exceed the plain one ({} B)", + with_fec[4], + without_fec[4] + ); + } + #[test] fn encode_opus_into_produces_a_packet_for_a_20ms_mono_frame() { let mut encoder = opus::Encoder::new( diff --git a/src/rtc/mod.rs b/src/rtc/mod.rs index e5d2807..4a576d3 100644 --- a/src/rtc/mod.rs +++ b/src/rtc/mod.rs @@ -61,8 +61,8 @@ pub use error::{ pub use identity::{CLIENT_TYPE, SDK_TYPE, client_details, client_header}; pub use join::{CallEvent, CallStateSnapshot, CallingState, JoinCallData, RtcCore}; pub use local_track::{ - LocalAudioTrack, LocalTrack, LocalVideoTrack, LocalVideoTrackConfig, RtpPacket, VideoLayering, - audio_level_dbov, + LocalAudioTrack, LocalAudioTrackConfig, LocalTrack, LocalVideoTrack, LocalVideoTrackConfig, + RtpPacket, VideoLayering, audio_level_dbov, }; pub use pcm::chunk::Pad; pub use pcm::convert::G711_SAMPLE_RATE;