From 456ef53b031857bdd19cf233c30913afc32b4895 Mon Sep 17 00:00:00 2001 From: superkc2026 Date: Fri, 14 Aug 2026 17:36:32 +0800 Subject: [PATCH 01/10] perf(audio): stretch speed regions through libavfilter atempo instead of WSOLA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WSOLA is O(grain x search-radius) per rendered sample. On a long clip with speed regions (measured: 65.4M samples after speed-segment quantization) it runs for many minutes at 100% of one core, and the export appears frozen at ~80% progress — audio stretching is the pipeline's last big job. Users kill the export; nothing fails, it is just unreachably slow. Route stretch_pcm_to_length through an in-process abuffer -> atempo -> abuffersink graph instead. atempo is the same pitch-preserving time-stretch, but O(n) with ffmpeg's SIMD routines: the same input takes seconds. avfilter already ships in the app — fetch-ffmpeg.mjs vendors every av*.dll of the BtbN LGPL-shared build, and the addon sits beside those DLLs — so this only links a library that was already in the box. - build.rs: link avfilter (bindgen already allowlists avfilter_*/ via the existing "av.*" filter, and the Linux osff_ symbol-rename table derives from the soname list) - build-linux-compositor-addon.mjs: stage libavfilter.so.11 alongside the other renamed libs - wrappers: include libavfilter headers - audio.rs: avfilter_atempo_stretch() mounts the graph, feeds planar f32 chunks, drains, and pads/truncates to the exact target length; speeds outside atempo's [0.5, 100] window chain multiple stages (0.2 -> atempo=0.5,atempo=0.5,atempo=0.8). Any failure returns None and falls back to the existing WSOLA path unchanged. - sink negotiation may yield flt (interleaved) or fltp (planar); both are deinterleaved into PlanarPcm Verified with cargo test: a 10 s 440 Hz stereo sine at speed 1.25 returns exactly 8 s and measures 440 Hz +/- 2 Hz by zero crossings (pitch preserved — a plain resample would shift it). --- crates/compositor/build.rs | 2 +- crates/compositor/src/audio.rs | 304 +++++++++++++++++++++++ crates/compositor/wrapper_linux.h | 3 + crates/compositor/wrapper_macos.h | 4 +- crates/compositor/wrapper_windows.h | 3 + scripts/build-linux-compositor-addon.mjs | 1 + 6 files changed, 315 insertions(+), 2 deletions(-) diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index f8c9e5f5b..a52514fc9 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -69,7 +69,7 @@ fn main() { if let Some(v) = ff.as_ref() { let lib_dir = Path::new(v).join("lib"); println!("cargo:rustc-link-search=native={}", lib_dir.display()); - for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample"] { + for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample", "avfilter"] { println!("cargo:rustc-link-lib=dylib={}", lib); } } diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 1b51a7a99..b1ef25b09 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -3,6 +3,7 @@ //! unique encodeur AAC alimente le même muxer que la vidéo. use crate::ffi::*; + use crate::regions::SpeedSegment; use anyhow::{bail, Result}; use std::f32::consts::PI; @@ -728,6 +729,18 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { } let speed = source_samples as f64 / target_samples as f64; + + // atempo d'abord : le WSOLA ci-dessous est O(grain × rayon) par échantillon rendu, soit + // plusieurs minutes de CPU plein cœur sur un clip long (un export mesuré : 65,4 M + // échantillons, > 10 min sans finir) — l'export semble alors figé à ~80 %. Le filtre + // atempo fait le même time-stretch préservant la hauteur en O(n) avec les routines SIMD + // de ffmpeg : quelques secondes pour la même entrée. `avfilter_atempo_stretch` rend + // `None` si la chaîne ne monte pas (avfilter absent, vitesse hors bornes…) et le WSOLA + // reste le chemin de repli exact d'avant. + if let Some(stretched) = unsafe { avfilter_atempo_stretch(pcm, target_samples, speed) } { + return stretched; + } + let mut stretcher = WsolaTimeStretcher::new( AUDIO_OUTPUT_SAMPLE_RATE, AUDIO_OUTPUT_CHANNELS, @@ -753,6 +766,244 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { exact } +/// Découpe un facteur de vitesse en facteurs que `atempo` accepte individuellement : le +/// filtre n'admet que [0.5, 100.0], on chaîne donc les dépassements (0.2 → [0.5, 0.5, 0.8], +/// 250 → [100.0, 2.5]) — le produit des facteurs reconstitue la vitesse demandée. +fn atempo_factors(speed: f64) -> Vec { + let mut factors = Vec::new(); + let mut remaining = speed; + while remaining > 100.0 { + factors.push(100.0); + remaining /= 100.0; + } + while remaining < 0.5 { + factors.push(0.5); + remaining /= 0.5; + } + factors.push(remaining); + factors +} + +/// RAII : libère le graphe même en sortie précoce sur erreur. +struct FilterGraphGuard(*mut AVFilterGraph); + +impl Drop for FilterGraphGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { avfilter_graph_free(&mut self.0) }; + } + } +} + +/// Étire le PCM d'un facteur `speed` via une chaîne `abuffer → atempo… → abuffersink` +/// montée en processus, dans l'avfilter LGPL déjà vendored avec l'app (avfilter-11.dll / +/// libavfilter.so.11 / libavfilter.11.dylib voyagent dans le même lot que avcodec — +/// cf. scripts/fetch-ffmpeg.mjs qui copie TOUTES les av*.dll du build BtbN). +/// +/// abuffer fixe le format de toute la chaîne à fltp 48 kHz stéréo — exactement ce que +/// `decode_clip_audio` produit — et atempo préserve format/canaux/fréquence : aucune +/// conversion, la sortie se recadre sur `target_samples` par troncature ou padding. +/// +/// Retourne `None` sur toute défaillance (montage, négociation, exécution) : l'appelant +/// retombe alors sur le WSOLA d'origine. +unsafe fn avfilter_atempo_stretch( + pcm: &[Vec], + target_samples: usize, + speed: f64, +) -> Option { + if !speed.is_finite() || speed <= 0.0 { + return None; + } + let factors = atempo_factors(speed); + + let graph_guard = FilterGraphGuard(avfilter_graph_alloc()); + let graph = graph_guard.0; + if graph.is_null() { + return None; + } + + let abuffer_name = CString::new("abuffer").ok()?; + let abuffersink_name = CString::new("abuffersink").ok()?; + let atempo_name = CString::new("atempo").ok()?; + let abuffer = avfilter_get_by_name(abuffer_name.as_ptr()); + let abuffersink = avfilter_get_by_name(abuffersink_name.as_ptr()); + let atempo = avfilter_get_by_name(atempo_name.as_ptr()); + if abuffer.is_null() || abuffersink.is_null() || atempo.is_null() { + return None; + } + + let create_filter = |graph: *mut AVFilterGraph, + filter: *const AVFilter, + name: &str, + args: Option<&str>| + -> Option<*mut AVFilterContext> { + let cname = CString::new(name).ok()?; + let cargs = match args { + Some(args) => Some(CString::new(args).ok()?), + None => None, + }; + let ctx = avfilter_graph_alloc_filter(graph, filter, cname.as_ptr()); + if ctx.is_null() { + eprintln!("[openscreen-compositor] atempo: alloc_filter({name}) a rendu null"); + return None; + } + // `map_or` consommerait `cargs` et le pointeur rendu par la closure serait + // dangling avant même l'appel — on emprunte donc pour la durée de l'appel. + let args_ptr = match &cargs { + Some(args) => args.as_ptr(), + None => ptr::null(), + }; + let ret = avfilter_init_str(ctx, args_ptr); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: init_str({name}, {:?}) a échoué (ret={ret})", + args.unwrap_or("") + ); + return None; + } + Some(ctx) + }; + + let rate = AUDIO_OUTPUT_SAMPLE_RATE; + let src_ctx = create_filter( + graph, + abuffer, + "in", + Some(&format!( + "time_base=1/{rate}:sample_rate={rate}:sample_fmt=fltp:channel_layout=stereo" + )), + )?; + let sink_ctx = create_filter(graph, abuffersink, "out", None)?; + + let mut previous = src_ctx; + for (index, factor) in factors.iter().enumerate() { + let stage = create_filter( + graph, + atempo, + &format!("atempo{index}"), + Some(&format!("{factor}")), + )?; + if avfilter_link(previous, 0, stage, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link a échoué au maillon {index}"); + return None; + } + previous = stage; + } + if avfilter_link(previous, 0, sink_ctx, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link vers le sink a échoué"); + return None; + } + if avfilter_graph_config(graph, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_graph_config a échoué"); + return None; + } + + // Alimentation : le PCM passe par trames fltp de 4096 échantillons. `av_buffersrc_add_frame` + // déplace les références du frame dans le graphe ; on alloue donc une trame neuve par + // tranche et on la libère après envoi (le shell est vide à ce point). + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + const CHUNK: usize = 4096; + let mut offset = 0usize; + while offset < source_samples { + let count = CHUNK.min(source_samples - offset); + let mut frame = av_frame_alloc(); + if frame.is_null() { + eprintln!("[openscreen-compositor] atempo: av_frame_alloc (feed) a échoué"); + return None; + } + (*frame).format = AVSampleFormat::AV_SAMPLE_FMT_FLTP as i32; + (*frame).sample_rate = rate; + (*frame).nb_samples = count as i32; + av_channel_layout_default(&mut (*frame).ch_layout, AUDIO_OUTPUT_CHANNELS as i32); + if av_frame_get_buffer(frame, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: av_frame_get_buffer (feed) a échoué"); + av_frame_free(&mut frame); + return None; + } + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let destination = *(*frame).extended_data.add(channel) as *mut f32; + ptr::write_bytes(destination, 0, count); + if let Some(plane) = pcm.get(channel) { + let available = plane.len().saturating_sub(offset).min(count); + if available > 0 { + ptr::copy_nonoverlapping( + plane.as_ptr().add(offset), + destination, + available, + ); + } + } + } + (*frame).pts = offset as i64; + let ret = av_buffersrc_add_frame(src_ctx, frame); + av_frame_free(&mut frame); + if ret < 0 { + eprintln!("[openscreen-compositor] atempo: av_buffersrc_add_frame (offset={offset}) a échoué (ret={ret})"); + return None; + } + offset += count; + } + // EOF : le graphe vide alors ses derniers grains. + av_buffersrc_add_frame(src_ctx, ptr::null_mut()); + + // Drain : après l'EOF de la source, chaque appel rend une trame jusqu'à AVERROR_EOF. + let mut frame = av_frame_alloc(); + if frame.is_null() { + return None; + } + let mut stretched: PlanarPcm = vec![Vec::new(); AUDIO_OUTPUT_CHANNELS]; + loop { + let ret = av_buffersink_get_frame(sink_ctx, frame); + if ret < 0 { + break; + } + let count = (*frame).nb_samples as usize; + let channels = (*frame).ch_layout.nb_channels.max(0) as usize; + // La négociation peut rendre fltp (plans) OU flt (entrelacé) — atempo offre les + // deux ; on désestrelingue au besoin plutôt que de contraindre le sink. + let frame_format = (*frame).format as AVSampleFormat::Type; + if frame_format == AVSampleFormat::AV_SAMPLE_FMT_FLTP + && channels == AUDIO_OUTPUT_CHANNELS + { + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let plane = *(*frame).extended_data.add(channel) as *const f32; + stretched[channel] + .extend_from_slice(std::slice::from_raw_parts(plane, count)); + } + } else if frame_format == AVSampleFormat::AV_SAMPLE_FMT_FLT + && channels == AUDIO_OUTPUT_CHANNELS + { + let interleaved = *(*frame).extended_data.add(0) as *const f32; + let samples = + std::slice::from_raw_parts(interleaved, count * AUDIO_OUTPUT_CHANNELS); + for index in 0..count { + for channel in 0..AUDIO_OUTPUT_CHANNELS { + stretched[channel].push(samples[index * AUDIO_OUTPUT_CHANNELS + channel]); + } + } + } else { + eprintln!( + "[openscreen-compositor] atempo: trame de sortie inattendue (format={frame_format:?} canaux={channels})" + ); + av_frame_unref(frame); + av_frame_free(&mut frame); + return None; + } + av_frame_unref(frame); + } + av_frame_free(&mut frame); + + // Recadrage exact : la longueur rendue par atempo diffère de `target_samples` de quelques + // échantillons de flush ; on tronque ou on padde, comme le faisait le chemin WSOLA. + let mut result: PlanarPcm = Vec::with_capacity(AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let mut plane = std::mem::take(&mut stretched[channel]); + plane.resize(target_samples, 0.0); + result.push(plane); + } + Some(result) +} + /// Découpe le PCM gardé avec les mêmes spans et la même quantification frame que la vidéo. pub fn stretch_clip_pcm_by_speed( pcm: &[Vec], @@ -997,6 +1248,59 @@ mod tests { vec![samples.to_vec(), samples.to_vec()] } + #[test] + fn atempo_factors_split_out_of_range_speeds() { + // Dans les bornes : un seul maillon. + assert_eq!(atempo_factors(1.25), vec![1.25]); + assert_eq!(atempo_factors(0.5), vec![0.5]); + // Hors bornes : chaîne dont le produit reconstitue la vitesse. + assert_eq!(atempo_factors(0.2), vec![0.5, 0.5, 0.8]); + assert_eq!(atempo_factors(250.0), vec![100.0, 2.5]); + for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { + let product: f64 = atempo_factors(speed).iter().product(); + assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); + } + } + + #[test] + fn atempo_stretch_preserves_pitch_and_hits_the_target_length() { + // Sinus 440 Hz de 10 s : à speed 1.25 la sortie doit mesurer exactement 8 s + // (recadrage sur target_samples) et garder la hauteur — c'est la promesse du + // time-stretch, et la régression qu'on a vue quand on avait essayé un bête + // rééchantillonnage (voix qui monte d'un quart de ton). + let total = 10 * AUDIO_OUTPUT_SAMPLE_RATE as usize; + let mut pcm: PlanarPcm = vec![Vec::with_capacity(total); AUDIO_OUTPUT_CHANNELS]; + for i in 0..total { + let t = i as f32 / AUDIO_OUTPUT_SAMPLE_RATE as f32; + let sample = (2.0 * PI * 440.0 * t).sin() * 0.5; + for channel in 0..AUDIO_OUTPUT_CHANNELS { + pcm[channel].push(sample); + } + } + let speed = 1.25; + let target = (total as f64 / speed).round() as usize; + let stretched = + unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .expect("la chaîne atempo doit monter quand avfilter est lié"); + assert_eq!(stretched.len(), AUDIO_OUTPUT_CHANNELS); + for plane in &stretched { + assert_eq!(plane.len(), target); + } + // Hauteur mesurée par passages à zéro montants sur 1 s au milieu du signal. + let start = target / 2; + let window = AUDIO_OUTPUT_SAMPLE_RATE as usize; + let mut crossings = 0usize; + for i in start..start + window - 1 { + if stretched[0][i] <= 0.0 && stretched[0][i + 1] > 0.0 { + crossings += 1; + } + } + assert!( + (crossings as f64 - 440.0).abs() <= 2.0, + "hauteur dérivée : {crossings} Hz" + ); + } + #[test] fn single_track_passes_through_unchanged() { let track = planar(&[0.25, -0.5, 0.75]); diff --git a/crates/compositor/wrapper_linux.h b/crates/compositor/wrapper_linux.h index e01a31a97..0ee8cf4ab 100644 --- a/crates/compositor/wrapper_linux.h +++ b/crates/compositor/wrapper_linux.h @@ -12,3 +12,6 @@ #include #include #include +#include +#include +#include diff --git a/crates/compositor/wrapper_macos.h b/crates/compositor/wrapper_macos.h index 5a87a1466..20d7b4083 100644 --- a/crates/compositor/wrapper_macos.h +++ b/crates/compositor/wrapper_macos.h @@ -17,4 +17,6 @@ /* Software decode path : swscale était déjà LIÉ (build.rs) sans être bindé. Conservé identique côté macOS pour que la symétrie avec cpu_frames_windows.rs soit claire ; le code effectif vit dans mac_frames.rs. */ -#include \ No newline at end of file +#include #include +#include +#include diff --git a/crates/compositor/wrapper_windows.h b/crates/compositor/wrapper_windows.h index 86612f96a..8d291a813 100644 --- a/crates/compositor/wrapper_windows.h +++ b/crates/compositor/wrapper_windows.h @@ -12,3 +12,6 @@ elle couvre les formats exotiques (10 bits, 4:2:2) qu'un interleave écrit à la main casserait silencieusement. */ #include +#include +#include +#include diff --git a/scripts/build-linux-compositor-addon.mjs b/scripts/build-linux-compositor-addon.mjs index 9cf1b7352..951e6573c 100644 --- a/scripts/build-linux-compositor-addon.mjs +++ b/scripts/build-linux-compositor-addon.mjs @@ -36,6 +36,7 @@ const FFMPEG_SONAMES = [ "libavutil.so.60", "libswscale.so.9", "libswresample.so.6", + "libavfilter.so.11", ]; const run = (command, args, options = {}) => From 0ae7884e9bbed157d33cdfb76349da2dc464f0a6 Mon Sep 17 00:00:00 2001 From: superkc2026 Date: Fri, 14 Aug 2026 17:36:32 +0800 Subject: [PATCH 02/10] fix(audio): guard the decode and WSOLA loops against pathological stalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hardening guards found while diagnosing the slow-export hang: - decode_clip_audio: a container whose audio track is truncated or corrupt at the end can keep av_read_frame from ever returning AVERROR_EOF, so decoder_eof never propagates and the demux loop spins at 100% CPU forever. Cap it with a 60 s time budget — time, not iterations, because av_read_frame can be slow on a corrupt stream and an iteration cap would either never trigger or cut healthy long clips short. - WsolaTimeStretcher::process: if find_best_delta keeps returning a delta that puts grain_pos back where it was, the buf_end break is never reached and the loop spins forever. Detect the stagnation (100 consecutive non-advancing grains) and force the exit — the fallback path after the previous commit's atempo change, so this only protects the unlikely case where WSOLA still runs. --- crates/compositor/src/audio.rs | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index b1ef25b09..9b1f4338f 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -281,10 +281,28 @@ unsafe fn decode_clip_audio_inner( let mut frame = av_frame_alloc(); let mut input_eof = false; + // Garde anti-boucle : un conteneur dont la piste audio est tronquée ou corrompue en fin + // de flux peut faire que `av_read_frame` ne renvoie jamais AVERROR_EOF, empêchant + // `decoder_eof` de se propager — la boucle tourne alors à 100 % CPU pour toujours. + // Un budget TEMPS plutôt qu'un compteur d'itérations : `av_read_frame` peut être lent + // sur un flux corrompu, un compteur serait soit trop grand soit trop petit. 60 s + // couvre largement le décodage logiciel d'un clip de plus de 20 minutes. + let loop_start = std::time::Instant::now(); + let loop_budget = std::time::Duration::from_secs(60); + // Une seule passe de démux alimente tous les décodeurs : chaque paquet est routé vers la // piste dont il porte l'index. On continue tant qu'AU MOINS une piste a encore quelque // chose à produire. while tracks.iter().any(|t| !t.reached_end && !t.decoder_eof) { + if loop_start.elapsed() > loop_budget { + eprintln!( + "[openscreen-compositor] decode_clip_audio: boucle plafonnée à 60 s (source_end={source_end_sec} s), sortie forcée" + ); + for track in tracks.iter_mut() { + track.decoder_eof = true; + } + break; + } if !input_eof { let read = av_read_frame(fmt, packet); if read == AVERROR_EOF { @@ -573,6 +591,13 @@ impl WsolaTimeStretcher { fn process(&mut self, final_chunk: bool) -> PlanarPcm { let mut emitted = self.empty_chunk(); + // Garde anti-boucle : si `find_best_delta` rend systématiquement un delta qui + // ramène grain_pos sur place, le break `buf_end` n'est jamais atteint et la boucle + // tourne à 100 % CPU pour toujours. On détecte la stagnation et on force la + // sortie — dans le cas normal le WSOLA a déjà couvert la cible, et un blocage ici + // ne fait que figer l'export entier. + let mut last_grain_pos: i64 = i64::MIN; + let mut stagnant: u32 = 0; loop { let search_target = (self.ideal_pos + self.ha).round() as i64; let required_end = (self.grain_pos + self.n as i64) @@ -593,6 +618,21 @@ impl WsolaTimeStretcher { self.ideal_pos += self.ha; self.frame += 1; + if self.grain_pos <= last_grain_pos { + stagnant += 1; + if stagnant >= 100 { + let stuck_at = last_grain_pos; + eprintln!( + "[openscreen-compositor] WsolaTimeStretcher: grain_pos stagnant à {stuck_at} (frame {}), sortie forcée", + self.frame + ); + break; + } + } else { + stagnant = 0; + last_grain_pos = self.grain_pos; + } + self.collect(placed_frame * self.hs, &mut emitted); self.discard_below(self.grain_pos); } From e75d070ab1014c5b9d1f61c897337a39f27ac671 Mon Sep 17 00:00:00 2001 From: superkc2026 Date: Sun, 16 Aug 2026 08:59:55 +0800 Subject: [PATCH 03/10] fix(review): address CodeRabbit findings on the atempo PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wrapper_macos.h: the appended avfilter include landed on the same line as the trailing swscale include (the file had no final newline), so the preprocessor never saw it — split them onto separate lines. macOS builds would have produced no avfilter bindings at all. - decode budget: scale with the requested window (x8, floor 60 s) instead of a flat 60 s, so slow storage / heavy codecs decoding a long window are not cut off into trailing silence. - atempo drain: only AVERROR_EOF / AVERROR_EAGAIN are benign; any other negative return is a real filter failure — return None so the WSOLA fallback runs instead of exporting partial audio padded with silence. The buffersrc flush return is checked for the same reason. --- crates/compositor/src/audio.rs | 29 +++++++++++++++++++++++------ crates/compositor/wrapper_macos.h | 3 ++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 9b1f4338f..803f6ba42 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -285,10 +285,12 @@ unsafe fn decode_clip_audio_inner( // de flux peut faire que `av_read_frame` ne renvoie jamais AVERROR_EOF, empêchant // `decoder_eof` de se propager — la boucle tourne alors à 100 % CPU pour toujours. // Un budget TEMPS plutôt qu'un compteur d'itérations : `av_read_frame` peut être lent - // sur un flux corrompu, un compteur serait soit trop grand soit trop petit. 60 s - // couvre largement le décodage logiciel d'un clip de plus de 20 minutes. + // sur un flux corrompu, un compteur serait soit trop grand soit trop petit. Le budget + // est dimensionné sur la durée demandée (facteur 8, plancher 60 s) : un long clip sur + // un stockage lent reste couvert, seule une vraie boucle infinie est coupée. let loop_start = std::time::Instant::now(); - let loop_budget = std::time::Duration::from_secs(60); + let loop_budget_secs = (((source_end_sec - source_start_sec).max(0.0) * 8.0) as u64).max(60); + let loop_budget = std::time::Duration::from_secs(loop_budget_secs); // Une seule passe de démux alimente tous les décodeurs : chaque paquet est routé vers la // piste dont il porte l'index. On continue tant qu'AU MOINS une piste a encore quelque @@ -296,7 +298,7 @@ unsafe fn decode_clip_audio_inner( while tracks.iter().any(|t| !t.reached_end && !t.decoder_eof) { if loop_start.elapsed() > loop_budget { eprintln!( - "[openscreen-compositor] decode_clip_audio: boucle plafonnée à 60 s (source_end={source_end_sec} s), sortie forcée" + "[openscreen-compositor] decode_clip_audio: boucle plafonnée à {loop_budget_secs} s (source_end={source_end_sec} s), sortie forcée" ); for track in tracks.iter_mut() { track.decoder_eof = true; @@ -983,8 +985,12 @@ unsafe fn avfilter_atempo_stretch( } offset += count; } - // EOF : le graphe vide alors ses derniers grains. - av_buffersrc_add_frame(src_ctx, ptr::null_mut()); + // EOF : le graphe vide alors ses derniers grains. Un échec ici signifie que + // le graphe n'a pas pu être vidé — on rend None pour retomber sur WSOLA. + if av_buffersrc_add_frame(src_ctx, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: flush du buffersrc a échoué, repli WSOLA"); + return None; + } // Drain : après l'EOF de la source, chaque appel rend une trame jusqu'à AVERROR_EOF. let mut frame = av_frame_alloc(); @@ -995,6 +1001,17 @@ unsafe fn avfilter_atempo_stretch( loop { let ret = av_buffersink_get_frame(sink_ctx, frame); if ret < 0 { + // Seuls EOF (drain terminé) et EAGAIN (rien de prêt) sont bénins ; tout + // autre code est une vraie panne du filtre — on rend None pour retomber + // sur le chemin WSOLA plutôt que d'exporter un audio partiel + silence. + if ret != AVERROR_EOF && ret != AVERROR_EAGAIN { + eprintln!( + "[openscreen-compositor] atempo: av_buffersink_get_frame a échoué (ret={ret}), repli WSOLA" + ); + av_frame_unref(frame); + av_frame_free(&mut frame); + return None; + } break; } let count = (*frame).nb_samples as usize; diff --git a/crates/compositor/wrapper_macos.h b/crates/compositor/wrapper_macos.h index 20d7b4083..f3a5e5dd9 100644 --- a/crates/compositor/wrapper_macos.h +++ b/crates/compositor/wrapper_macos.h @@ -17,6 +17,7 @@ /* Software decode path : swscale était déjà LIÉ (build.rs) sans être bindé. Conservé identique côté macOS pour que la symétrie avec cpu_frames_windows.rs soit claire ; le code effectif vit dans mac_frames.rs. */ -#include #include +#include +#include #include #include From 2d4dfb3c1708c5b07affc39d0348c159e2d08a43 Mon Sep 17 00:00:00 2001 From: superkc2026 Date: Fri, 21 Aug 2026 10:48:28 +0800 Subject: [PATCH 04/10] fix(review): address OpenScreen#371 blocking review (EtienneLescot) - #1: avfilter_atempo_stretch returns None (-> WSOLA fallback) when atempo drains fewer than 90% of target samples, instead of padding the near-empty output to target_samples and exporting silence on short speed spans (gaps between regions, single video frames). - #3: avfilter is now a fully-known vendoring/packaging dependency: * fetch-ffmpeg.mjs probes ALL six shared DLLs (was: any av*.dll) so a warm tree with the five pre-avfilter DLLs re-vendors avfilter-11.dll. * before-pack.cjs lists avfilter on Linux, Windows and macOS (mac atLeast 3 -> 4). * build-linux-compositor-addon.mjs header + build-and-packaging.md note the sixth ffmpeg soname. - #2 (decode loop budget guard) removed here and split into its own PR to keep this one single-concern (atempo stretch). --- crates/compositor/src/audio.rs | 30 +++++++------------ scripts/before-pack.cjs | 8 ++--- scripts/build-linux-compositor-addon.mjs | 2 +- scripts/fetch-ffmpeg.mjs | 29 ++++++++++++++---- .../engineering/build-and-packaging.md | 2 +- 5 files changed, 40 insertions(+), 31 deletions(-) diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index ba044cf33..13891403c 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -320,30 +320,10 @@ unsafe fn decode_clip_audio_inner( let mut frame = av_frame_alloc(); let mut input_eof = false; - // Garde anti-boucle : un conteneur dont la piste audio est tronquée ou corrompue en fin - // de flux peut faire que `av_read_frame` ne renvoie jamais AVERROR_EOF, empêchant - // `decoder_eof` de se propager — la boucle tourne alors à 100 % CPU pour toujours. - // Un budget TEMPS plutôt qu'un compteur d'itérations : `av_read_frame` peut être lent - // sur un flux corrompu, un compteur serait soit trop grand soit trop petit. Le budget - // est dimensionné sur la durée demandée (facteur 8, plancher 60 s) : un long clip sur - // un stockage lent reste couvert, seule une vraie boucle infinie est coupée. - let loop_start = std::time::Instant::now(); - let loop_budget_secs = (((source_end_sec - source_start_sec).max(0.0) * 8.0) as u64).max(60); - let loop_budget = std::time::Duration::from_secs(loop_budget_secs); - // Une seule passe de démux alimente tous les décodeurs : chaque paquet est routé vers la // piste dont il porte l'index. On continue tant qu'AU MOINS une piste a encore quelque // chose à produire. while tracks.iter().any(|t| !t.reached_end && !t.decoder_eof) { - if loop_start.elapsed() > loop_budget { - eprintln!( - "[openscreen-compositor] decode_clip_audio: boucle plafonnée à {loop_budget_secs} s (source_end={source_end_sec} s), sortie forcée" - ); - for track in tracks.iter_mut() { - track.decoder_eof = true; - } - break; - } if !input_eof { let read = av_read_frame(fmt, packet); if read == AVERROR_EOF { @@ -1089,6 +1069,16 @@ unsafe fn avfilter_atempo_stretch( } av_frame_free(&mut frame); + // OpenScreen#371 review (EtienneLescot): atempo needs a full analysis window + // before it emits anything — a span shorter than that (e.g. a 30 ms gap between + // two speed regions, or a single video frame) drains to ZERO samples. Padding + // that emptiness up to `target_samples` would export silence, while the contract + // of `stretch_pcm_to_length` promises a `None` -> WSOLA fallback on failure. Bail + // out so the WSOLA path runs and genuinely stretches these spans. + if stretched[0].len() < target_samples * 9 / 10 { + return None; + } + // Recadrage exact : la longueur rendue par atempo diffère de `target_samples` de quelques // échantillons de flush ; on tronque ou on padde, comme le faisait le chemin WSOLA. let mut result: PlanarPcm = Vec::with_capacity(AUDIO_OUTPUT_CHANNELS); diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index 70a2d6ccb..2f6ccb973 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -76,11 +76,11 @@ const MAC_REQUIRED = [ fix: FIX_MAC, }, { - match: (name) => /^libav(codec|format|util)\.\d+\.dylib$/.test(name), + match: (name) => /^libav(codec|format|util|filter)\.\d+\.dylib$/.test(name), what: "the LGPL ffmpeg dylibs the compositor links", breaks: "the compositor addon cannot be loaded at all (dyld error at require())", fix: FIX_MAC, - atLeast: 3, + atLeast: 4, }, { match: (name) => name === "whisper-stt-server", @@ -131,7 +131,7 @@ const LINUX_REQUIRED = [ // pendant qu'une autre manquait. Le paquet passait alors la garde et le // compositeur ne chargeait pas : exactement le mode de panne que cette garde // existe pour attraper. - ...["avcodec", "avformat", "avutil", "swresample", "swscale"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^lib${library}\\.so\\.\\d+$`).test(name), what: `the symbol-renamed lib${library} shared object the compositor links`, breaks: "the compositor addon cannot be loaded at all (ld.so error at require())", @@ -234,7 +234,7 @@ const WIN_REQUIRED = [ // (avcodec-60/61/62.dll left by an earlier fetch) would satisfy a combined count // while another library was missing entirely, and the addon would still fail to // load. - ...["avcodec", "avformat", "avutil"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^${library}-\\d+\\.dll$`).test(name), what: `the ${library} DLL the compositor links`, breaks: "the addon cannot be loaded at all under MSIX, which ignores PATH", diff --git a/scripts/build-linux-compositor-addon.mjs b/scripts/build-linux-compositor-addon.mjs index 951e6573c..717d1b79f 100644 --- a/scripts/build-linux-compositor-addon.mjs +++ b/scripts/build-linux-compositor-addon.mjs @@ -16,7 +16,7 @@ // (ensureFfmpegSharedDllsOnPath), but glibc reads LD_LIBRARY_PATH once at // process start, so the equivalent trick cannot work after Electron is // already running. Instead the addon is linked with `-rpath,$ORIGIN` and -// the five ffmpeg sonames are copied next to it, which makes the .node +// the six ffmpeg sonames are copied next to it, which makes the .node // self-contained wherever it is installed — no env var, no PATH surgery. import { spawnSync } from "node:child_process"; diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 8c2f0ed83..efad3cf65 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -407,13 +407,32 @@ async function fetchSharedDlls(tag, binDir) { return; } - // probe for any previously vendored DLL by name; re-download is driven by - // --force same as the static exe, checked once we know what we'd extract. - const alreadyVendored = - process.platform === "win32" && + // Completeness probe, not a mere existence probe. The compositor addon now + // links six shared ffmpeg DLLs — see crates/compositor/build.rs + // (avcodec, avformat, avutil, swresample, swscale, avfilter). A warm dev/CI + // tree that already holds the five pre-avfilter DLLs would satisfy an "any + // av*.dll is present" check and let `avfilter-11.dll` go un-vendored, breaking + // require() at runtime (OpenScreen#371 review, EtienneLescot). Require all + // six explicitly so a missing one forces a re-vendor. + const REQUIRED_SHARED_DLLS = [ + "avcodec", + "avformat", + "avutil", + "swresample", + "swscale", + "avfilter", + ]; + const vendoredFiles = new Set( fs .readdirSync(binDir, { withFileTypes: true }) - .some((e) => e.isFile() && isSharedLib(e.name) && /^(lib)?av/i.test(e.name)); + .filter((e) => e.isFile()) + .map((e) => e.name), + ); + const alreadyVendored = + process.platform === "win32" && + REQUIRED_SHARED_DLLS.every((lib) => + [...vendoredFiles].some((f) => new RegExp(`^${lib}-\\d+\\.dll$`).test(f)), + ); // The build-time SDK comes out of this same archive, so a tree that has the // DLLs but not the SDK must still re-download — otherwise we skip here and // the compositor build fails afterwards on the missing FFMPEG_DIR. diff --git a/technical-documentation/engineering/build-and-packaging.md b/technical-documentation/engineering/build-and-packaging.md index 29bbda556..01749f235 100644 --- a/technical-documentation/engineering/build-and-packaging.md +++ b/technical-documentation/engineering/build-and-packaging.md @@ -204,7 +204,7 @@ The hook now reads `electron/native/bin/darwin-/` — the directory `mac.e | Required | Without it | |---|---| | `compositor_view.node` | preview and every export render nothing | -| `libavcodec/libavformat/libavutil.*.dylib` | the addon cannot load at all (dyld error at `require()`) | +| `libavcodec/libavformat/libavutil/libavfilter.*.dylib` | the addon cannot load at all (dyld error at `require()`) | | `whisper-stt-server` | transcription and captions fail with a developer error shown to end users | | `libggml*.dylib` | the helper dies in dyld before `main()`; STT times out with no diagnostic | | `openscreen-screencapturekit-helper` | native screen capture unavailable | From 1ec9ef3c741ffb6fe249da1ac4add9faa2f8a5d1 Mon Sep 17 00:00:00 2001 From: superkc2026 Date: Fri, 21 Aug 2026 17:32:39 +0800 Subject: [PATCH 05/10] =?UTF-8?q?fix(packaging):=20CodeRabbit=20follow-up?= =?UTF-8?q?=20on=20#371=20=E2=80=94=20require=20all=20six=20ffmpeg=20libs,?= =?UTF-8?q?=20guard=20--sdk-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - before-pack.cjs macOS: split the combined av* regex (atLeast: 4) into one requirement per library — avcodec/avformat/avutil/swresample/swscale/avfilter — matching the LINUX_REQUIRED style so duplicate versions of one library cannot satisfy the count while another is missing. - before-pack.cjs Windows: add swresample/swscale to the required DLL list (was: avcodec/avformat/avutil/avfilter). - build-and-packaging.md: document all six dylib families in the macOS guard table. - fetch-ffmpeg.mjs: create binDir before readdirSync in fetchSharedDlls, so the --sdk-only path no longer throws on a fresh checkout (binDir is normally created by the CLI branch before the shared-DLL fetch). --- scripts/before-pack.cjs | 15 +++++++++------ scripts/fetch-ffmpeg.mjs | 1 + .../engineering/build-and-packaging.md | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index 2f6ccb973..b320acd5a 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -75,13 +75,16 @@ const MAC_REQUIRED = [ breaks: "the preview and every export render nothing", fix: FIX_MAC, }, - { - match: (name) => /^libav(codec|format|util|filter)\.\d+\.dylib$/.test(name), - what: "the LGPL ffmpeg dylibs the compositor links", + // One requirement per library, not `atLeast: N` over a combined regex — the + // same trap LINUX_REQUIRED documents above. Several versioned copies of one + // library would satisfy a combined count while another was missing entirely, + // and the addon would still fail to load. + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ + match: (name) => new RegExp(`^lib${library}\\.\\d+\\.dylib$`).test(name), + what: `the LGPL lib${library} dylib the compositor links`, breaks: "the compositor addon cannot be loaded at all (dyld error at require())", fix: FIX_MAC, - atLeast: 4, - }, + })), { match: (name) => name === "whisper-stt-server", what: "the whisper.cpp STT helper", @@ -234,7 +237,7 @@ const WIN_REQUIRED = [ // (avcodec-60/61/62.dll left by an earlier fetch) would satisfy a combined count // while another library was missing entirely, and the addon would still fail to // load. - ...["avcodec", "avformat", "avutil", "avfilter"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^${library}-\\d+\\.dll$`).test(name), what: `the ${library} DLL the compositor links`, breaks: "the addon cannot be loaded at all under MSIX, which ignores PATH", diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index efad3cf65..07c78e5f1 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -422,6 +422,7 @@ async function fetchSharedDlls(tag, binDir) { "swscale", "avfilter", ]; + fs.mkdirSync(binDir, { recursive: true }); const vendoredFiles = new Set( fs .readdirSync(binDir, { withFileTypes: true }) diff --git a/technical-documentation/engineering/build-and-packaging.md b/technical-documentation/engineering/build-and-packaging.md index 01749f235..27ec043cc 100644 --- a/technical-documentation/engineering/build-and-packaging.md +++ b/technical-documentation/engineering/build-and-packaging.md @@ -204,7 +204,7 @@ The hook now reads `electron/native/bin/darwin-/` — the directory `mac.e | Required | Without it | |---|---| | `compositor_view.node` | preview and every export render nothing | -| `libavcodec/libavformat/libavutil/libavfilter.*.dylib` | the addon cannot load at all (dyld error at `require()`) | +| `libavcodec/libavformat/libavutil/libavfilter/libswresample/libswscale.*.dylib` | the addon cannot load at all (dyld error at `require()`) | | `whisper-stt-server` | transcription and captions fail with a developer error shown to end users | | `libggml*.dylib` | the helper dies in dyld before `main()`; STT times out with no diagnostic | | `openscreen-screencapturekit-helper` | native screen capture unavailable | From 8dc0483bc342e2c0cbac0e60ff3ddd8791d3987d Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 27 Aug 2026 19:18:34 +0200 Subject: [PATCH 06/10] fix(nix): stage and rename libavfilter for the compositor addon build.rs prefixes every av* function in ffi.rs, so the atempo path makes the addon import osff_avfilter_graph_alloc, osff_av_buffersrc_add_frame and friends. preBuild only staged lib{avformat,avcodec,avutil,swscale, swresample}, so no libavfilter.so was renamed and no unversioned symlink existed for -lavfilter: the build either failed to link or bound against nixpkgs' unrenamed copy. installPhase's leak check does not catch that -- it only rejects names that are NOT osff_-prefixed -- so the derivation succeeded and require() failed at runtime with "undefined symbol: osff_avfilter_graph_alloc", leaving compositorViewService as a no-op and preview plus every export dead on the whole NixOS package. Add avfilter to the staged set, matching the six libraries crates/compositor/build.rs links and scripts/build-linux-compositor-addon.mjs already ships. Both filters keep working unchanged: avfilter's exports are av-prefixed (avfilter_*, av_buffersrc_*, av_buffersink_*), so the preBuild awk and the installPhase leak check already cover them. Also corrects the installPhase comment that counted five direct DT_NEEDED libraries. Co-Authored-By: Claude Opus 5 --- nix/compositor-view.nix | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/nix/compositor-view.nix b/nix/compositor-view.nix index 6a71a2bfa..b3f46b823 100644 --- a/nix/compositor-view.nix +++ b/nix/compositor-view.nix @@ -132,7 +132,18 @@ rustPlatform.buildRustPackage { # Copy each library under its soname and read its symbol table. awk rather # than sed with a backreference: the third field is the name, and anything # after an @ is the version tag. - for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample}.so.*; do + # + # The list must hold every library crates/compositor/build.rs emits a + # cargo:rustc-link-lib for -- all six of them, avfilter included since the + # speed-region stretch runs through atempo. Miss one and the build either + # dies on `cannot find -lavfilter` (no unversioned symlink is staged for it + # below) or links the store's UN-renamed copy, and a cdylib tolerating + # undefined symbols means the failure surfaces only at require() time as + # "undefined symbol: osff_avfilter_graph_alloc" -- the addon then loads as a + # no-op and preview plus every export are dead. avfilter's exports are all + # av-prefixed (avfilter_*, av_buffersrc_*, av_buffersink_*), so the awk + # filter here and the leak check in installPhase already cover them. + for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample,avfilter}.so.*; do case "$lib" in *.so.*.*) continue ;; esac test -f "$lib" || continue cp "$(readlink -f "$lib")" "$stage/lib/$(basename "$lib")" @@ -190,7 +201,7 @@ rustPlatform.buildRustPackage { # Each copy still carries the RUNPATH it inherited from the original ffmpeg # output, which is where the UN-renamed libraries live -- so libavcodec's own # osff_swr_init would resolve against a libswresample that defines swr_init. - # It only works today because all five happen to be direct DT_NEEDED of the + # It only works today because all six happen to be direct DT_NEEDED of the # addon, so $ORIGIN is searched first; the day --as-needed drops one the # loader falls through to the store copy and dlopen fails on an undefined # osff_ symbol. Put $ORIGIN in front so the renamed set can only resolve From 01e991c91067dd955b76a34672066fa5c7519028 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 27 Aug 2026 19:18:53 +0200 Subject: [PATCH 07/10] docs: describe the atempo stretch path and the six-library ffmpeg set export-pipeline.md and native-compositor.md still said WSOLA stretches each speed sub-segment. atempo is now the primary path and WSOLA is the fallback stretch_pcm_to_length takes when the filter graph cannot be built, configured or run, or when it drains under 90% of the target samples on a span shorter than atempo's analysis window. export-pipeline.md also claimed the stretch "is kicked off before the video loop so it overlaps the encode and does not add to the wall". It does not: decode and stretch run synchronously in walk_composited_timeline's on_clip_end callback, which fires once per clip after that clip's frames are encoded, on the same thread -- and progress() is driven only by encoded video frames, so nothing moves while the stretch runs. Describe the real shape, which is also why the O(n) atempo path matters. build-and-packaging.md named avcodec/avformat/avutil as the addon's ffmpeg dependencies. build.rs links six -- avcodec, avformat, avutil, swresample, swscale, avfilter -- and before-pack.cjs now requires each of them individually on all three platforms. Co-Authored-By: Claude Opus 5 --- .../architecture/export-pipeline.md | 22 +++++++++++---- .../architecture/native-compositor.md | 27 ++++++++++++++++--- .../engineering/build-and-packaging.md | 4 +-- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/technical-documentation/architecture/export-pipeline.md b/technical-documentation/architecture/export-pipeline.md index 0068fdf4c..d9378d0bf 100644 --- a/technical-documentation/architecture/export-pipeline.md +++ b/technical-documentation/architecture/export-pipeline.md @@ -61,9 +61,9 @@ and **one** encoder + muxer pair: per-segment rounded frame counts into a single output frame counter; audio follows the same integer accumulation (`AudioConcatPlan`). -- **Audio and video junctions are seamless.** Audio is decoded per - segment up front (`audio.rs::decode_clip_audio`), WSOLA stretches each - speed sub-segment to its output sample count, and +- **Audio and video junctions are seamless.** Audio is decoded per clip + (`audio.rs::decode_clip_audio`), a libavfilter `atempo` chain stretches + each speed sub-segment to its output sample count, and `assemble_concatenated_pcm` concatenates the per-segment PCM at the integer sample offsets the video loop just produced — never `round(cumulativeSec * sampleRate)`, because that compounds per-segment @@ -71,8 +71,20 @@ and **one** encoder + muxer pair: timeline. A short equal-power fade (`cos` on the tail, `sin` on the head, `cos² + sin² = 1`) covers each internal boundary to suppress the click where two recordings meet butt-joined, without shifting timing. - The WSOLA stretch is kicked off before the video loop so it overlaps - the encode and does not add to the wall. + The in-tree WSOLA stretcher is still there, but only as the fallback + `stretch_pcm_to_length` takes when the filter chain cannot be built or + yields too little audio (see [Audio](native-compositor.md#audio)). + +- **The stretch is not overlapped with the encode.** Decode and stretch + run inside `walk_composited_timeline`'s `on_clip_end` callback + (`pipeline.rs`), which fires once per clip *after* that clip's frames + have been composed and encoded, on the same thread — so the stretch + time is added to the export wall, not hidden behind it. `progress()` is + driven only by encoded video frames, so nothing moves while it runs and + a long clip parks the export at whatever percentage the last frame + reported. That is why the `atempo` path matters: it is O(n) where WSOLA + is O(grain × radius) per rendered sample, which on a long clip meant + minutes of an apparently frozen export. - **Output** honours the timeline's selected aspect ratio (`resolveAspectRatioValue` over `getEditorSettings(document).aspectRatio` — diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index f2b01c5c9..509ca68a3 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -31,7 +31,7 @@ and without that flag the runtime corruption is silent. | [`crates/compositor/src/scene.rs`](../../crates/compositor/src/scene.rs) | the `Scene` struct parsed from the app's `SceneDescription` JSON | | [`crates/compositor/src/regions.rs`](../../crates/compositor/src/regions.rs) | zoom / speed / Full Camera regions — envelope shapes and per-frame state sampling | | [`crates/compositor/src/pipeline.rs`](../../crates/compositor/src/pipeline_windows.rs) | demux + `D3D11VA` decode + composite + AMF encode + mux (`run_c0` for decode/encode only, `run_composited` for the full path) | -| [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, WSOLA speed stretch, multi-track mix, AAC encoder | +| [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, libavfilter `atempo` speed stretch (WSOLA fallback), multi-track mix, AAC encoder | | [`crates/compositor/src/cursor.rs`](../../crates/compositor/src/cursor.rs) | `.cursor.json` parser + interpolated cursor track (position, click bounces, adaptive follow samples) | | [`crates/compositor/src/text.rs`](../../crates/compositor/src/text_windows.rs) | DirectWrite + Direct2D text rasterisation for annotation labels, cached per (content, style, box) | | [`crates/compositor/src/text_anim.rs`](../../crates/compositor/src/text_anim.rs) | text-annotation appearance animations (port of the TS animation curves, in fractions of the output short side) | @@ -199,9 +199,28 @@ each track is recut to the same `[source_start_sec, source_end_sec)` — pads in front for late-starting tracks, trims the pre-roll for early-decoded ones — so a summing mixer is enough and a real mix matrix is not needed. -Speed regions apply after decode: WSOLA stretches each speed sub-segment to -its output frame count, sharing search positions across channels from a -mono down-mix (so the stereo image does not wander between channels). +Speed regions apply after decode: `stretch_pcm_to_length` stretches each +speed sub-segment to its output frame count through a libavfilter +`abuffer → atempo… → abuffersink` graph built in-process +(`avfilter_atempo_stretch`). The graph is pinned to the fltp / 48 kHz / +stereo format `decode_clip_audio` already produces, and `atempo` preserves +format, channels and rate, so no conversion is involved; the result is +recut to the exact target length by truncation or zero-padding. `atempo` +only accepts a factor in `[0.5, 100.0]`, so `atempo_factors` chains +several instances whose product is the requested speed (0.2 → +`[0.5, 0.5, 0.8]`). + +The in-tree WSOLA stretcher remains as the fallback, taken whenever +`avfilter_atempo_stretch` returns `None` — the graph could not be built or +configured, a buffersrc/buffersink call failed, or the chain drained fewer +than 90% of the target samples, which is what happens on a span too short +for `atempo`'s analysis window (a few tens of milliseconds between two +speed regions). WSOLA shares its search positions across channels from a +mono down-mix, so the stereo image does not wander between them. The move +to `atempo` is a cost change, not a quality one: WSOLA is +O(grain × radius) per rendered sample, minutes of a full core on a long +clip, against `atempo`'s O(n) with ffmpeg's SIMD routines. + Across segments, `build_audio_concat_plan` sizes each segment's PCM by **integer accumulation of the per-segment rounded sample count**, never `round(cumulativeSec * sampleRate)` — that single change is what keeps A/V diff --git a/technical-documentation/engineering/build-and-packaging.md b/technical-documentation/engineering/build-and-packaging.md index 27ec043cc..bbb1d4084 100644 --- a/technical-documentation/engineering/build-and-packaging.md +++ b/technical-documentation/engineering/build-and-packaging.md @@ -42,7 +42,7 @@ Electron-builder copies only the matching `electron/native/bin/- This is a hard requirement on Windows, not a tidiness preference. -The addon dlopens `avcodec`/`avformat`/`avutil` at `require()` time. Until 1.9.0 the Windows build shipped it inside `app.asar.unpacked/electron/native/compositor-view/build/`, one directory away from `electron/native/bin/win32-x64/*.dll`, and the gap was bridged at runtime by `ensureFfmpegSharedDllsOnPath` prepending the DLL directory to `PATH` before the require. +The addon pulls in six ffmpeg libraries at `require()` time — `avcodec`, `avformat`, `avutil`, `swresample`, `swscale` and `avfilter`, the exact set `crates/compositor/build.rs` emits `cargo:rustc-link-lib` lines for. (`avfilter` is the newest of them: the speed-region time stretch runs through its `atempo` filter.) Until 1.9.0 the Windows build shipped it inside `app.asar.unpacked/electron/native/compositor-view/build/`, one directory away from `electron/native/bin/win32-x64/*.dll`, and the gap was bridged at runtime by `ensureFfmpegSharedDllsOnPath` prepending the DLL directory to `PATH` before the require. That works for the NSIS installer. **It does not work under MSIX**, which resolves an addon's dependent DLLs through the package graph and ignores `PATH`. Measured inside a registered package, with the directory verifiably present and correctly prepended to `PATH`: @@ -60,7 +60,7 @@ require BEFORE PATH : LOADED OK Node loads `.node` files with `LOAD_WITH_ALTERED_SEARCH_PATH`, so the addon's own directory is searched for its dependencies. Colocating removes the `PATH` mechanism rather than repairing it, and works on every Windows packaging format. -This shipped: the 1.9.0 Store build loaded no compositor at all, so the editor opened with a permanently blank preview while audio kept playing — audio comes from the renderer, every frame comes from the addon. It read as an application bug rather than a packaging one, because every file was present in the package and the NSIS build of the same commit was fine. `scripts/before-pack.cjs` now refuses to package unless the addon and at least `avcodec`/`avformat`/`avutil` are in the same directory, on Windows as it already did on macOS. +This shipped: the 1.9.0 Store build loaded no compositor at all, so the editor opened with a permanently blank preview while audio kept playing — audio comes from the renderer, every frame comes from the addon. It read as an application bug rather than a packaging one, because every file was present in the package and the NSIS build of the same commit was fine. `scripts/before-pack.cjs` now refuses to package unless the addon and all six of those libraries are in the same directory, on Windows as it already did on macOS. It checks one requirement **per library** rather than a count over a combined pattern, on all three platforms: several versioned copies of one library (an `avcodec-60`/`61`/`62.dll` left by an earlier fetch) would satisfy a combined count while another was missing entirely, and the addon would still fail to load. `electron/native/bin/`, local native build directories, the compositor build output, models, and caches are gitignored. Rebuilding from a source checkout therefore requires the complete platform toolchain and third-party SDKs; running the generic `npm run build` alone does not manufacture missing native artifacts. The Windows compositor's D3D11/FFmpeg prerequisites are described by the source POC in `crates/README.md`, while capture helper lookup and output conventions are documented in `electron/native/README.md`. From bd637d8a2105d42578ad4d8456e42a393b0f7d33 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 12:23:50 +0200 Subject: [PATCH 08/10] perf(audio): make the atempo stretch land on the target without a hole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three blocking findings from the 27/08 review, plus the packaging invariant that keeps the six-library set from drifting again. atempo does not render exactly n/tempo samples: it falls short by a fixed amount per chain, independent of input length — 217 samples for one stage, ~2 700 for four, i.e. up to 56 ms at 0.1x. Pushing more input does not recover it, and zero-filling it left a hard silence gap butt-joined to the next segment, since the equal-power crossfade covers clip boundaries only. A first pass now measures the shortfall on the real content without keeping anything and a second asks for `target + shortfall`; measured across the whole editor speed ladder on spans of 0.05s to 3s, the trailing silence is now zero samples everywhere. Above 1x the shortfall is zero and the second pass is skipped. The chain is pinned to flt rather than fltp. af_atempo advertises packed formats only, so a planar abuffer made the negotiation insert an aresample and left the planar drain branch unreachable — the sink was already handing back AV_SAMPLE_FMT_FLT. Asking for flt on both ends leaves no conversion filter in the graph. The drain is interleaved with the feed. av_buffersrc_add_frame does not pull the graph, so pushing a whole region first queued all of it in the buffersrc — half a gigabyte on a 20-minute stereo region. The 90% threshold is gone. It padded up to a tenth of a region with silence above the line and fell back to WSOLA without a word below it; every fallback now logs its reason. WSOLA, still the fallback, no longer copies the remaining buffer on every grain. `discard_below` advances a read cursor and compacts only when the consumed head passes the remainder, which takes the total from O(N^2) to O(N): a measured export that ran over ten minutes without finishing now takes ~90 s, and the output is bit-identical at 0.25x/0.5x/1.25x/2x. Its stagnation guard is dropped: search_target grows by ha > 0 every iteration so the buf_end break always fires, and had the guard tripped it would have truncated the region into silence. scripts/ffmpeg-linked-libraries.test.mjs derives the linked set from build.rs and checks fetch-ffmpeg.mjs, before-pack.cjs (three tables), build-linux-compositor-addon.mjs and nix/compositor-view.nix against it — the nix glob especially, which no PR check builds. --- crates/compositor/src/audio.rs | 634 ++++++++++++------ scripts/ffmpeg-linked-libraries.test.mjs | 108 +++ .../architecture/export-pipeline.md | 24 +- .../architecture/native-compositor.md | 58 +- 4 files changed, 611 insertions(+), 213 deletions(-) create mode 100644 scripts/ffmpeg-linked-libraries.test.mjs diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 13891403c..223d8e090 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -478,6 +478,12 @@ pub struct WsolaTimeStretcher { buf: PlanarPcm, mono: Vec, buf_start: i64, + /// Décalage de lecture dans `buf`/`mono`. `discard_below` ne recopiait pas moins que le + /// reste du buffer à chaque grain : la région entière est poussée d'un coup, donc pour + /// 65 M d'échantillons cela faisait ~N²/(2·ha) ≈ 1,1e12 f32 recopiés par canal — le vrai + /// coût du chemin WSOLA, devant la recherche par grain. On avance un curseur et on ne + /// compacte que lorsque la tête dépasse la moitié du buffer, ce qui rend le total O(N). + buf_head: usize, out: PlanarPcm, win_sum: Vec, out_start: usize, @@ -523,6 +529,7 @@ impl WsolaTimeStretcher { buf: vec![Vec::new(); channels], mono: Vec::new(), buf_start: 0, + buf_head: 0, out: vec![Vec::new(); channels], win_sum: Vec::new(), out_start: 0, @@ -588,8 +595,12 @@ impl WsolaTimeStretcher { } } + fn buf_len(&self) -> usize { + self.buf[0].len() - self.buf_head + } + fn buf_end(&self) -> i64 { - self.buf_start + self.buf[0].len() as i64 + self.buf_start + self.buf_len() as i64 } fn sample_at(&self, channel: usize, absolute_index: i64) -> f32 { @@ -597,7 +608,10 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.buf[channel].get(index as usize).copied().unwrap_or(0.0) + self.buf[channel] + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } @@ -606,19 +620,19 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.mono.get(index as usize).copied().unwrap_or(0.0) + self.mono + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } fn process(&mut self, final_chunk: bool) -> PlanarPcm { let mut emitted = self.empty_chunk(); - // Garde anti-boucle : si `find_best_delta` rend systématiquement un delta qui - // ramène grain_pos sur place, le break `buf_end` n'est jamais atteint et la boucle - // tourne à 100 % CPU pour toujours. On détecte la stagnation et on force la - // sortie — dans le cas normal le WSOLA a déjà couvert la cible, et un blocage ici - // ne fait que figer l'export entier. - let mut last_grain_pos: i64 = i64::MIN; - let mut stagnant: u32 = 0; + // Pas de garde anti-stagnation ici : `search_target` croît de `ha > 0` à chaque tour + // et `grain_pos` ne s'en écarte que de `search_radius` au plus, donc le break sur + // `buf_end` finit toujours par tomber. Une garde de plus tronquerait `emitted` — la + // région sortirait muette pour tout signal — sans jamais se déclencher. loop { let search_target = (self.ideal_pos + self.ha).round() as i64; let required_end = (self.grain_pos + self.n as i64) @@ -639,21 +653,6 @@ impl WsolaTimeStretcher { self.ideal_pos += self.ha; self.frame += 1; - if self.grain_pos <= last_grain_pos { - stagnant += 1; - if stagnant >= 100 { - let stuck_at = last_grain_pos; - eprintln!( - "[openscreen-compositor] WsolaTimeStretcher: grain_pos stagnant à {stuck_at} (frame {}), sortie forcée", - self.frame - ); - break; - } - } else { - stagnant = 0; - last_grain_pos = self.grain_pos; - } - self.collect(placed_frame * self.hs, &mut emitted); self.discard_below(self.grain_pos); } @@ -761,12 +760,18 @@ impl WsolaTimeStretcher { if drop_count <= 0 { return; } - let drop_count = drop_count as usize; - for channel in 0..self.channels { - self.buf[channel] = self.buf[channel][drop_count.min(self.buf[channel].len())..].to_vec(); - } - self.mono = self.mono[drop_count.min(self.mono.len())..].to_vec(); + self.buf_head += (drop_count as usize).min(self.buf_len()); self.buf_start = absolute_index; + // Compactage amorti : ne recopier que lorsque la tête consommée dépasse ce qui + // reste laisse un coût total en O(N) au lieu du O(N²) d'une recopie par grain. + if self.buf_head > self.buf[0].len() - self.buf_head { + let head = self.buf_head; + for channel in 0..self.channels { + self.buf[channel].drain(..head); + } + self.mono.drain(..head); + self.buf_head = 0; + } } } @@ -791,13 +796,15 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { let speed = source_samples as f64 / target_samples as f64; - // atempo d'abord : le WSOLA ci-dessous est O(grain × rayon) par échantillon rendu, soit - // plusieurs minutes de CPU plein cœur sur un clip long (un export mesuré : 65,4 M - // échantillons, > 10 min sans finir) — l'export semble alors figé à ~80 %. Le filtre - // atempo fait le même time-stretch préservant la hauteur en O(n) avec les routines SIMD - // de ffmpeg : quelques secondes pour la même entrée. `avfilter_atempo_stretch` rend - // `None` si la chaîne ne monte pas (avfilter absent, vitesse hors bornes…) et le WSOLA - // reste le chemin de repli exact d'avant. + // atempo d'abord : le WSOLA ci-dessous fait le même time-stretch préservant la hauteur, + // mais coûte un ordre de grandeur de plus. Mesuré en release sur une région de 5 min + // (14,4 M échantillons) : 0,6 s contre 20 s à 1,25×, 4,9 s contre 55 s à 0,25× — et + // c'est le WSOLA APRÈS la correction de `discard_below`, qui recopiait tout le buffer + // restant à chaque grain et faisait tenir un export mesuré (65,4 M échantillons) plus de + // dix minutes sans finir, l'export paraissant figé à ~80 %. `avfilter_atempo_stretch` + // rend `None` si la chaîne ne monte pas, si le sink négocie un format inattendu ou si la + // sortie reste plus courte que la cible ; le WSOLA reste alors le chemin de repli exact + // d'avant, en journalisant la raison. if let Some(stretched) = unsafe { avfilter_atempo_stretch(pcm, target_samples, speed) } { return stretched; } @@ -856,27 +863,128 @@ impl Drop for FilterGraphGuard { } } -/// Étire le PCM d'un facteur `speed` via une chaîne `abuffer → atempo… → abuffersink` -/// montée en processus, dans l'avfilter LGPL déjà vendored avec l'app (avfilter-11.dll / -/// libavfilter.so.11 / libavfilter.11.dylib voyagent dans le même lot que avcodec — -/// cf. scripts/fetch-ffmpeg.mjs qui copie TOUTES les av*.dll du build BtbN). +/// RAII : libère la trame de drain même en sortie précoce sur erreur. +struct FrameGuard(*mut AVFrame); + +impl Drop for FrameGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { av_frame_free(&mut self.0) }; + } + } +} + +/// Taille des trames poussées vers le graphe. Le drain est entrelacé avec l'alimentation +/// (cf. `atempo_drain`) : sans cela `av_buffersrc_add_frame` empile toute la région dans la +/// file du buffersrc — ~523 Mo pour une speed region stéréo de 20 min, en plus du slice +/// d'entrée et de l'accumulateur de sortie. +const ATEMPO_FEED_CHUNK: usize = 4096; + +/// Rallonge de silence poussée derrière la région avant l'EOF, par étage atempo. /// -/// abuffer fixe le format de toute la chaîne à fltp 48 kHz stéréo — exactement ce que -/// `decode_clip_audio` produit — et atempo préserve format/canaux/fréquence : aucune -/// conversion, la sortie se recadre sur `target_samples` par troncature ou padding. +/// atempo laisse tomber la dernière fenêtre de chaque étage. En prolongeant l'entrée d'un +/// silence, la fenêtre perdue devient du silence et le contenu réel sort en entier : mesuré +/// sur le pin ffmpeg n8.1.2 (48 kHz stéréo), le manque tombe de 981 à 217 échantillons pour +/// un étage 0.5×, de 2 735 à 553 pour deux, de 8 234 à 2 676 pour quatre. Au-delà la courbe +/// est plate — un tail 16× plus grand ne change plus rien — ce qui reste est traité par la +/// correction de tempo de `avfilter_atempo_stretch`. +const ATEMPO_PRIME_TAIL: usize = 4096; + +/// Marge de sécurité, en échantillons, sur la longueur demandée à la passe corrigée. /// -/// Retourne `None` sur toute défaillance (montage, négociation, exécution) : l'appelant -/// retombe alors sur le WSOLA d'origine. -unsafe fn avfilter_atempo_stretch( - pcm: &[Vec], - target_samples: usize, - speed: f64, -) -> Option { - if !speed.is_finite() || speed <= 0.0 { - return None; +/// Le manque de la seconde passe n'est pas exactement celui mesuré à la première (le tempo +/// a bougé de moins de 1 %, la chaîne est la même). Viser 64 échantillons de plus fait +/// tomber le résidu du côté du surplus, tronqué : 1,3 ms de contenu en moins plutôt qu'un +/// trou de silence. +const ATEMPO_LENGTH_GUARD: usize = 64; + +/// Longueur du silence à pousser derrière la région pour une chaîne donnée. +fn atempo_prime_tail(factors: &[f64], speed: f64) -> usize { + ATEMPO_PRIME_TAIL + .saturating_mul(factors.len() + 1) + .saturating_mul(speed.max(1.0).ceil() as usize) +} + +/// Vide le buffersink dans `stretched`, sans jamais y garder plus de `keep` échantillons par +/// plan, et compte dans `produced` TOUT ce qui est sorti — y compris ce qui est jeté. +/// +/// Les deux chiffres servent à des choses différentes : `stretched` est le résultat, alors +/// que `produced` mesure ce que la chaîne a réellement rendu pour une entrée de longueur +/// connue, donc son manque (cf. `avfilter_atempo_stretch`). +/// +/// Rend `Some(true)` sur EOF, `Some(false)` quand le graphe n'a plus rien de prêt (EAGAIN), +/// `None` sur une vraie panne — l'appelant retombe alors sur WSOLA. +unsafe fn atempo_drain( + sink_ctx: *mut AVFilterContext, + frame: *mut AVFrame, + stretched: &mut PlanarPcm, + keep: usize, + produced: &mut usize, +) -> Option { + loop { + let ret = av_buffersink_get_frame(sink_ctx, frame); + if ret == AVERROR_EAGAIN { + return Some(false); + } + if ret == AVERROR_EOF { + return Some(true); + } + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_buffersink_get_frame a échoué (ret={ret}), repli WSOLA" + ); + return None; + } + let count = (*frame).nb_samples.max(0) as usize; + let channels = (*frame).ch_layout.nb_channels.max(0) as usize; + // La chaîne est épinglée en flt entrelacé de bout en bout (cf. `avfilter_atempo_stretch`) ; + // tout autre format signifie que la négociation a fait autre chose que ce qu'on a + // demandé, et le désentrelacement ci-dessous lirait n'importe quoi. + if (*frame).format != AVSampleFormat::AV_SAMPLE_FMT_FLT as i32 + || channels != AUDIO_OUTPUT_CHANNELS + { + eprintln!( + "[openscreen-compositor] atempo: trame de sortie inattendue (format={} canaux={channels}), repli WSOLA", + (*frame).format + ); + av_frame_unref(frame); + return None; + } + let wanted = count.min(keep.saturating_sub(stretched[0].len())); + if wanted > 0 { + let interleaved = *(*frame).extended_data.add(0) as *const f32; + let samples = + std::slice::from_raw_parts(interleaved, count * AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let plane = &mut stretched[channel]; + plane.reserve(wanted); + for index in 0..wanted { + plane.push(samples[index * AUDIO_OUTPUT_CHANNELS + channel]); + } + } + } + *produced += count; + av_frame_unref(frame); } - let factors = atempo_factors(speed); +} +/// Monte `abuffer → atempo… → abuffersink`, y pousse `pcm` suivi de `prime_tail` échantillons +/// de silence, et rend le nombre total d'échantillons sortis — `stretched` en reçoit les +/// `keep` premiers. +/// +/// La chaîne est épinglée en **flt entrelacé** 48 kHz stéréo, pas en fltp : `af_atempo` +/// n'annonce que des formats packed (U8/S16/S32/FLT/DBL, cf. son `query_formats`), donc un +/// abuffer en fltp fait insérer un aresample de conversion et rend une branche planaire du +/// drain inatteignable — mesuré, le sink négociait déjà `AV_SAMPLE_FMT_FLT`. En demandant flt +/// des deux côtés il n'y a aucun filtre de conversion dans le graphe, et l'entrelacement est +/// absorbé par la recopie qu'on fait de toute façon. +unsafe fn atempo_pass( + pcm: &[Vec], + factors: &[f64], + prime_tail: usize, + keep: usize, + stretched: &mut PlanarPcm, +) -> Option { let graph_guard = FilterGraphGuard(avfilter_graph_alloc()); let graph = graph_guard.0; if graph.is_null() { @@ -896,7 +1004,8 @@ unsafe fn avfilter_atempo_stretch( let create_filter = |graph: *mut AVFilterGraph, filter: *const AVFilter, name: &str, - args: Option<&str>| + args: Option<&str>, + options: &[(&str, &str)]| -> Option<*mut AVFilterContext> { let cname = CString::new(name).ok()?; let cargs = match args { @@ -908,8 +1017,27 @@ unsafe fn avfilter_atempo_stretch( eprintln!("[openscreen-compositor] atempo: alloc_filter({name}) a rendu null"); return None; } - // `map_or` consommerait `cargs` et le pointeur rendu par la closure serait - // dangling avant même l'appel — on emprunte donc pour la durée de l'appel. + // Les options typées se posent entre l'alloc et l'init — `avfilter_init_str` fige la + // négociation. Un échec n'est pas fatal : l'abuffer porte déjà le format, ceci ne fait + // que l'imposer aussi côté sink pour qu'aucun build ffmpeg ne puisse y glisser un + // aresample. Le drain vérifie le format reçu de toute façon. + for (key, value) in options { + let ckey = CString::new(*key).ok()?; + let cvalue = CString::new(*value).ok()?; + let ret = av_opt_set( + ctx as *mut std::ffi::c_void, + ckey.as_ptr(), + cvalue.as_ptr(), + AV_OPT_SEARCH_CHILDREN as i32, + ); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_opt_set({name}.{key}={value}) a échoué (ret={ret}), négociation laissée libre" + ); + } + } + // `map_or` consommerait `cargs` et le pointeur rendu par la closure serait dangling + // avant même l'appel — on emprunte donc pour la durée de l'appel. let args_ptr = match &cargs { Some(args) => args.as_ptr(), None => ptr::null(), @@ -931,10 +1059,11 @@ unsafe fn avfilter_atempo_stretch( abuffer, "in", Some(&format!( - "time_base=1/{rate}:sample_rate={rate}:sample_fmt=fltp:channel_layout=stereo" + "time_base=1/{rate}:sample_rate={rate}:sample_fmt=flt:channel_layout=stereo" )), + &[], )?; - let sink_ctx = create_filter(graph, abuffersink, "out", None)?; + let sink_ctx = create_filter(graph, abuffersink, "out", None, &[("sample_fmts", "flt")])?; let mut previous = src_ctx; for (index, factor) in factors.iter().enumerate() { @@ -943,6 +1072,7 @@ unsafe fn avfilter_atempo_stretch( atempo, &format!("atempo{index}"), Some(&format!("{factor}")), + &[], )?; if avfilter_link(previous, 0, stage, 0) < 0 { eprintln!("[openscreen-compositor] atempo: avfilter_link a échoué au maillon {index}"); @@ -959,20 +1089,31 @@ unsafe fn avfilter_atempo_stretch( return None; } - // Alimentation : le PCM passe par trames fltp de 4096 échantillons. `av_buffersrc_add_frame` - // déplace les références du frame dans le graphe ; on alloue donc une trame neuve par - // tranche et on la libère après envoi (le shell est vide à ce point). + let sink_frame = FrameGuard(av_frame_alloc()); + if sink_frame.0.is_null() { + return None; + } + + // Alimentation : le PCM passe par trames flt de 4096 échantillons, prolongé par la + // rallonge de silence. `av_buffersrc_add_frame` déplace les références du frame dans le + // graphe ; on alloue donc une trame neuve par tranche et on la libère après envoi (le + // shell est vide à ce point). Le drain est entrelacé ici : sans lui la file du buffersrc + // porterait toute la région d'un coup. La condition d'arrêt est l'entrée épuisée, PAS + // « on a de quoi remplir la cible » — s'arrêter là couperait la rallonge, et les derniers + // grains du contenu réel resteraient dans le graphe. let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); - const CHUNK: usize = 4096; + let total_input = source_samples.saturating_add(prime_tail); let mut offset = 0usize; - while offset < source_samples { - let count = CHUNK.min(source_samples - offset); + let mut produced = 0usize; + let mut drained_to_eof = false; + while offset < total_input { + let count = ATEMPO_FEED_CHUNK.min(total_input - offset); let mut frame = av_frame_alloc(); if frame.is_null() { eprintln!("[openscreen-compositor] atempo: av_frame_alloc (feed) a échoué"); return None; } - (*frame).format = AVSampleFormat::AV_SAMPLE_FMT_FLTP as i32; + (*frame).format = AVSampleFormat::AV_SAMPLE_FMT_FLT as i32; (*frame).sample_rate = rate; (*frame).nb_samples = count as i32; av_channel_layout_default(&mut (*frame).ch_layout, AUDIO_OUTPUT_CHANNELS as i32); @@ -981,17 +1122,16 @@ unsafe fn avfilter_atempo_stretch( av_frame_free(&mut frame); return None; } + // Un seul plan en flt : on écrit entrelacé. Le `write_bytes` couvre à la fois les + // canaux absents d'une source mono et la rallonge de silence finale. + let destination = *(*frame).extended_data.add(0) as *mut f32; + ptr::write_bytes(destination, 0, count * AUDIO_OUTPUT_CHANNELS); for channel in 0..AUDIO_OUTPUT_CHANNELS { - let destination = *(*frame).extended_data.add(channel) as *mut f32; - ptr::write_bytes(destination, 0, count); if let Some(plane) = pcm.get(channel) { let available = plane.len().saturating_sub(offset).min(count); - if available > 0 { - ptr::copy_nonoverlapping( - plane.as_ptr().add(offset), - destination, - available, - ); + for index in 0..available { + *destination.add(index * AUDIO_OUTPUT_CHANNELS + channel) = + plane[offset + index]; } } } @@ -1003,91 +1143,101 @@ unsafe fn avfilter_atempo_stretch( return None; } offset += count; - } - // EOF : le graphe vide alors ses derniers grains. Un échec ici signifie que - // le graphe n'a pas pu être vidé — on rend None pour retomber sur WSOLA. - if av_buffersrc_add_frame(src_ctx, ptr::null_mut()) < 0 { - eprintln!("[openscreen-compositor] atempo: flush du buffersrc a échoué, repli WSOLA"); - return None; - } - - // Drain : après l'EOF de la source, chaque appel rend une trame jusqu'à AVERROR_EOF. - let mut frame = av_frame_alloc(); - if frame.is_null() { - return None; - } - let mut stretched: PlanarPcm = vec![Vec::new(); AUDIO_OUTPUT_CHANNELS]; - loop { - let ret = av_buffersink_get_frame(sink_ctx, frame); - if ret < 0 { - // Seuls EOF (drain terminé) et EAGAIN (rien de prêt) sont bénins ; tout - // autre code est une vraie panne du filtre — on rend None pour retomber - // sur le chemin WSOLA plutôt que d'exporter un audio partiel + silence. - if ret != AVERROR_EOF && ret != AVERROR_EAGAIN { - eprintln!( - "[openscreen-compositor] atempo: av_buffersink_get_frame a échoué (ret={ret}), repli WSOLA" - ); - av_frame_unref(frame); - av_frame_free(&mut frame); - return None; - } + if atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)? { + drained_to_eof = true; break; } - let count = (*frame).nb_samples as usize; - let channels = (*frame).ch_layout.nb_channels.max(0) as usize; - // La négociation peut rendre fltp (plans) OU flt (entrelacé) — atempo offre les - // deux ; on désestrelingue au besoin plutôt que de contraindre le sink. - let frame_format = (*frame).format as AVSampleFormat::Type; - if frame_format == AVSampleFormat::AV_SAMPLE_FMT_FLTP - && channels == AUDIO_OUTPUT_CHANNELS - { - for channel in 0..AUDIO_OUTPUT_CHANNELS { - let plane = *(*frame).extended_data.add(channel) as *const f32; - stretched[channel] - .extend_from_slice(std::slice::from_raw_parts(plane, count)); - } - } else if frame_format == AVSampleFormat::AV_SAMPLE_FMT_FLT - && channels == AUDIO_OUTPUT_CHANNELS - { - let interleaved = *(*frame).extended_data.add(0) as *const f32; - let samples = - std::slice::from_raw_parts(interleaved, count * AUDIO_OUTPUT_CHANNELS); - for index in 0..count { - for channel in 0..AUDIO_OUTPUT_CHANNELS { - stretched[channel].push(samples[index * AUDIO_OUTPUT_CHANNELS + channel]); - } - } - } else { - eprintln!( - "[openscreen-compositor] atempo: trame de sortie inattendue (format={frame_format:?} canaux={channels})" - ); - av_frame_unref(frame); - av_frame_free(&mut frame); + } + + // EOF : le graphe vide alors ses derniers grains. + if !drained_to_eof { + if av_buffersrc_add_frame(src_ctx, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: flush du buffersrc a échoué, repli WSOLA"); return None; } - av_frame_unref(frame); + atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)?; } - av_frame_free(&mut frame); + Some(produced) +} - // OpenScreen#371 review (EtienneLescot): atempo needs a full analysis window - // before it emits anything — a span shorter than that (e.g. a 30 ms gap between - // two speed regions, or a single video frame) drains to ZERO samples. Padding - // that emptiness up to `target_samples` would export silence, while the contract - // of `stretch_pcm_to_length` promises a `None` -> WSOLA fallback on failure. Bail - // out so the WSOLA path runs and genuinely stretches these spans. - if stretched[0].len() < target_samples * 9 / 10 { +/// Étire le PCM d'un facteur `speed` via une chaîne `abuffer → atempo… → abuffersink` montée +/// en processus, dans l'avfilter LGPL déjà vendored avec l'app (avfilter-11.dll / +/// libavfilter.so.11 / libavfilter.11.dylib voyagent dans le même lot que avcodec — cf. +/// scripts/fetch-ffmpeg.mjs qui copie TOUTES les av*.dll du build BtbN). +/// +/// **Deux passes.** atempo ne rend pas exactement `n/tempo` échantillons : il en manque un +/// nombre fixe par chaîne, indépendant de la longueur de l'entrée (mesuré sur n8.1.2 : +/// ~217 pour un étage, ~550 pour deux, ~2 700 pour quatre, soit jusqu'à 56 ms à 0,1×). Le +/// manque ne se rattrape pas en poussant plus d'entrée — c'est une différence de durée +/// rendue, pas une queue retenue — et le combler par des zéros collait un trou de silence +/// devant le segment suivant, puisque le crossfade equal-power ne couvre que les frontières +/// de clip, jamais la concaténation par segment. La première passe mesure donc le manque sur +/// le contenu réel, sans rien garder, et la seconde demande `cible + manque` pour que le +/// contenu remplisse la cible ; le surplus est tronqué. Aux vitesses > 1 le manque est nul et +/// la seconde passe est sautée. +/// +/// Retourne `None` sur toute défaillance (montage, négociation, exécution, sortie plus courte +/// que la cible) : l'appelant retombe alors sur le WSOLA d'origine. +unsafe fn avfilter_atempo_stretch( + pcm: &[Vec], + target_samples: usize, + speed: f64, +) -> Option { + if !speed.is_finite() || speed <= 0.0 || target_samples == 0 { + return None; + } + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + if source_samples == 0 { return None; } - // Recadrage exact : la longueur rendue par atempo diffère de `target_samples` de quelques - // échantillons de flush ; on tronque ou on padde, comme le faisait le chemin WSOLA. - let mut result: PlanarPcm = Vec::with_capacity(AUDIO_OUTPUT_CHANNELS); - for channel in 0..AUDIO_OUTPUT_CHANNELS { - let mut plane = std::mem::take(&mut stretched[channel]); - plane.resize(target_samples, 0.0); - result.push(plane); + let planes = |capacity: usize| -> PlanarPcm { + (0..AUDIO_OUTPUT_CHANNELS) + .map(|_| Vec::with_capacity(capacity)) + .collect() + }; + + let factors = atempo_factors(speed); + let prime_tail = atempo_prime_tail(&factors, speed); + let mut stretched = planes(target_samples); + let produced = atempo_pass(pcm, &factors, prime_tail, target_samples, &mut stretched)?; + let expected = ((source_samples + prime_tail) as f64 / speed).round() as usize; + let shortfall = expected.saturating_sub(produced); + + if shortfall > 0 { + // Le contenu réel s'arrête `shortfall` échantillons avant la cible, et ce qui suit + // dans `stretched` n'est que la rallonge de silence étirée. On rejoue en demandant + // une cible plus longue du même montant : la chaîne étant la même, elle en perd + // autant, et le contenu tombe cette fois pile sur `target_samples`. + let corrected_target = target_samples + shortfall + ATEMPO_LENGTH_GUARD; + let corrected_speed = source_samples as f64 / corrected_target as f64; + let corrected_factors = atempo_factors(corrected_speed); + let corrected_tail = atempo_prime_tail(&corrected_factors, corrected_speed); + let mut corrected = planes(target_samples); + atempo_pass( + pcm, + &corrected_factors, + corrected_tail, + target_samples, + &mut corrected, + )?; + if corrected[0].len() >= target_samples { + stretched = corrected; + } + } + + // Plus court que la cible : la chaîne n'a pas fait son travail. On rend `None` — compléter + // par des zéros exporterait un trou en se faisant passer pour un succès, et le contrat de + // `stretch_pcm_to_length` est un repli WSOLA sur échec. + if stretched[0].len() < target_samples { + eprintln!( + "[openscreen-compositor] atempo: sortie de {} échantillons pour une cible de {target_samples} (vitesse {speed}, {} étages), repli WSOLA", + stretched[0].len(), + factors.len() + ); + return None; } - Some(result) + Some(stretched) } /// Découpe le PCM gardé avec les mêmes spans et la même quantification frame que la vidéo. @@ -1334,27 +1484,9 @@ mod tests { vec![samples.to_vec(), samples.to_vec()] } - #[test] - fn atempo_factors_split_out_of_range_speeds() { - // Dans les bornes : un seul maillon. - assert_eq!(atempo_factors(1.25), vec![1.25]); - assert_eq!(atempo_factors(0.5), vec![0.5]); - // Hors bornes : chaîne dont le produit reconstitue la vitesse. - assert_eq!(atempo_factors(0.2), vec![0.5, 0.5, 0.8]); - assert_eq!(atempo_factors(250.0), vec![100.0, 2.5]); - for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { - let product: f64 = atempo_factors(speed).iter().product(); - assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); - } - } - - #[test] - fn atempo_stretch_preserves_pitch_and_hits_the_target_length() { - // Sinus 440 Hz de 10 s : à speed 1.25 la sortie doit mesurer exactement 8 s - // (recadrage sur target_samples) et garder la hauteur — c'est la promesse du - // time-stretch, et la régression qu'on a vue quand on avait essayé un bête - // rééchantillonnage (voix qui monte d'un quart de ton). - let total = 10 * AUDIO_OUTPUT_SAMPLE_RATE as usize; + /// Un sinus 440 Hz de `secs` secondes sur les deux canaux. + fn sine(secs: f64) -> PlanarPcm { + let total = (secs * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; let mut pcm: PlanarPcm = vec![Vec::with_capacity(total); AUDIO_OUTPUT_CHANNELS]; for i in 0..total { let t = i as f32 / AUDIO_OUTPUT_SAMPLE_RATE as f32; @@ -1363,30 +1495,156 @@ mod tests { pcm[channel].push(sample); } } - let speed = 1.25; - let target = (total as f64 / speed).round() as usize; - let stretched = - unsafe { avfilter_atempo_stretch(&pcm, target, speed) } - .expect("la chaîne atempo doit monter quand avfilter est lié"); - assert_eq!(stretched.len(), AUDIO_OUTPUT_CHANNELS); - for plane in &stretched { - assert_eq!(plane.len(), target); - } - // Hauteur mesurée par passages à zéro montants sur 1 s au milieu du signal. - let start = target / 2; - let window = AUDIO_OUTPUT_SAMPLE_RATE as usize; - let mut crossings = 0usize; - for i in start..start + window - 1 { - if stretched[0][i] <= 0.0 && stretched[0][i + 1] > 0.0 { - crossings += 1; + pcm + } + + /// Hauteur mesurée par passages à zéro montants sur une fenêtre d'une seconde. + fn pitch_hz(plane: &[f32], start: usize) -> usize { + let window = (AUDIO_OUTPUT_SAMPLE_RATE as usize).min(plane.len().saturating_sub(start + 1)); + (start..start + window) + .filter(|&i| plane[i] <= 0.0 && plane[i + 1] > 0.0) + .count() + } + + /// Énergie RMS des `count` derniers échantillons. + fn tail_rms(plane: &[f32], count: usize) -> f32 { + let start = plane.len().saturating_sub(count); + let slice = &plane[start..]; + if slice.is_empty() { + return 0.0; + } + (slice.iter().map(|v| v * v).sum::() / slice.len() as f32).sqrt() + } + + /// Les presets réellement cliquables dans l'éditeur (`SPEED_OPTIONS`), plus les bornes + /// `MIN_PLAYBACK_SPEED` / `MAX_PLAYBACK_SPEED` de `src/components/video-editor/types.ts`. + const EDITOR_SPEEDS: [f64; 13] = [ + 0.1, 0.25, 0.5, 0.75, 1.25, 1.5, 1.75, 2.0, 3.0, 4.0, 5.0, 10.0, 100.0, + ]; + + #[test] + fn atempo_covers_every_editor_speed_without_a_silent_tail() { + // Le bug que ce test verrouille : atempo n'émet jamais sa dernière fenêtre, et + // compléter le manque par des zéros collait jusqu'à 181 ms de blanc (0,1×, quatre + // étages) devant le segment suivant — un dropout audible dans un export « réussi ». + // La rallonge de silence en entrée fait sortir les derniers grains pour de bon, donc + // la fin de région doit porter autant de signal que son milieu, à toute vitesse et + // sur des spans courts comme longs. + for &speed in &EDITOR_SPEEDS { + for &secs in &[0.05f64, 0.5, 3.0] { + let pcm = sine(secs); + let source = pcm[0].len(); + let target = (source as f64 / speed).round() as usize; + if target == 0 { + continue; + } + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| { + panic!("atempo doit couvrir {speed}× sur {secs}s (cible {target})") + }); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}× durée {secs}s"); + } + // 10 ms de queue : le zero-padding d'avant en laissait au moins 5 ms à 0,1×. + // Le trou : un silence numérique en fin de région. Zéro tolérance — la + // correction de tempo est faite pour que le contenu tombe pile sur la cible. + let trailing_silence = + stretched[0].iter().rev().take_while(|v| **v == 0.0).count(); + assert_eq!( + trailing_silence, 0, + "{trailing_silence} échantillons de silence en fin de région à {speed}× sur {secs}s" + ); + let tail = (AUDIO_OUTPUT_SAMPLE_RATE as usize / 100).min(target); + assert!( + tail_rms(&stretched[0], tail) > 0.05, + "queue sans énergie à {speed}× sur {secs}s : rms={}", + tail_rms(&stretched[0], tail) + ); + } + } + } + + #[test] + fn atempo_preserves_pitch_through_a_chain_of_stages() { + // 0,25× et 0,1× sortent des bornes [0.5, 100] d'un seul atempo et passent donc par + // la chaîne multi-étages — le cas que le test d'origine (1,25×, un seul maillon) + // ne touchait pas, alors que 0,25× est un preset de la liste déroulante. + for &speed in &[0.1f64, 0.25, 0.5] { + let pcm = sine(2.0); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| panic!("la chaîne atempo doit monter à {speed}×")); + let measured = pitch_hz(&stretched[0], target / 2); + assert!( + (measured as f64 - 440.0).abs() <= 2.0, + "hauteur à {speed}× : {measured} Hz (un rééchantillonnage la déplacerait)" + ); + } + } + + #[test] + fn stretch_pcm_to_length_is_exact_at_every_editor_speed() { + // Contrat de bout en bout, repli WSOLA compris : quelle que soit la branche prise, + // la longueur rendue est exactement celle que le plan de concaténation attend. + for &speed in &EDITOR_SPEEDS { + let pcm = sine(0.5); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = stretch_pcm_to_length(&pcm, target); + assert_eq!(stretched.len(), AUDIO_OUTPUT_CHANNELS); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}×"); + } + } + } + + #[test] + fn wsola_fallback_still_stretches_and_keeps_pitch() { + // Le chemin de repli reste atteignable (avfilter absent d'un build, graphe qui ne + // monte pas) et sa recopie de buffer a été remplacée par un curseur de lecture : + // ce test verrouille qu'il rend toujours la bonne durée à la bonne hauteur. + let pcm = sine(2.0); + let speed = 0.5; + let target = (pcm[0].len() as f64 / speed).round() as usize; + let mut stretcher = WsolaTimeStretcher::new( + AUDIO_OUTPUT_SAMPLE_RATE, + AUDIO_OUTPUT_CHANNELS, + speed, + target, + ); + let mut emitted: PlanarPcm = vec![Vec::new(); AUDIO_OUTPUT_CHANNELS]; + for chunk in [stretcher.push(&pcm), stretcher.flush()] { + for channel in 0..AUDIO_OUTPUT_CHANNELS { + emitted[channel].extend_from_slice(&chunk[channel]); } } + // Le WSOLA vise la durée sans la garantir à l'échantillon près : c'est + // `stretch_pcm_to_length` qui recadre. On tolère 1 % ici. + let produced = emitted[0].len() as f64; + assert!( + (produced - target as f64).abs() / (target as f64) < 0.02, + "WSOLA a rendu {produced} pour une cible de {target}" + ); + let measured = pitch_hz(&emitted[0], target / 2); assert!( - (crossings as f64 - 440.0).abs() <= 2.0, - "hauteur dérivée : {crossings} Hz" + (measured as f64 - 440.0).abs() <= 3.0, + "hauteur WSOLA : {measured} Hz" ); } + #[test] + fn atempo_factors_split_out_of_range_speeds() { + // Dans les bornes : un seul maillon. + assert_eq!(atempo_factors(1.25), vec![1.25]); + assert_eq!(atempo_factors(0.5), vec![0.5]); + // Hors bornes : chaîne dont le produit reconstitue la vitesse. + assert_eq!(atempo_factors(0.2), vec![0.5, 0.5, 0.8]); + assert_eq!(atempo_factors(250.0), vec![100.0, 2.5]); + for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { + let product: f64 = atempo_factors(speed).iter().product(); + assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); + } + } + #[test] fn single_track_passes_through_unchanged() { let track = planar(&[0.25, -0.5, 0.75]); diff --git a/scripts/ffmpeg-linked-libraries.test.mjs b/scripts/ffmpeg-linked-libraries.test.mjs new file mode 100644 index 000000000..2ba9b656f --- /dev/null +++ b/scripts/ffmpeg-linked-libraries.test.mjs @@ -0,0 +1,108 @@ +// `crates/compositor/build.rs` decides which ffmpeg shared libraries the native addon +// imports. Six places have to agree with that list, and none of them is derived from it: +// +// - scripts/fetch-ffmpeg.mjs vendors the Windows DLLs and decides when to skip +// - scripts/before-pack.cjs fails the pack if one is missing (three OS tables) +// - scripts/build-linux-compositor-addon.mjs copies + symbol-renames the sonames +// - nix/compositor-view.nix builds symbols.map from a brace glob +// +// Drift is not a cosmetic problem. The addon is a cdylib, so a missing library does not +// fail the link: it fails at `require()` with "undefined symbol: osff_avfilter_graph_alloc", +// `compositorViewService` logs "native addon not present; running as no-op", and the app +// ships with a blank preview and every export dead — the exact symptom 1.9.0 shipped with. +// Every guard listed above passes in that state, because each one only knows the list it +// was written with. +// +// This test derives the truth from build.rs and checks the other five against it, so +// linking a seventh library fails here instead of in a user's installer. It reads source +// text: before-pack.cjs and fetch-ffmpeg.mjs both do work at import time, and the property +// under test is a property of the literal lists anyway. +// +// The nix derivation is the reason this file exists rather than an assertion inside +// before-pack.test.mjs: `.github/workflows/nix-build.yml` does not run on pull requests +// and there is no nix on the Windows dev box, so nix/compositor-view.nix reaches main with +// no pre-merge signal at all. A text assertion is not `nix build`, but it does catch the +// one mistake that has actually happened. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const read = (relative) => fs.readFileSync(path.join(repoRoot, relative), "utf8"); + +const buildRs = read("crates/compositor/build.rs"); + +/** The `for lib in [...] { println!("cargo:rustc-link-lib=…") }` list, in build.rs order. */ +const linked = (() => { + const block = buildRs.match(/for lib in \[([\s\S]*?)\]\s*\{[\s\S]*?rustc-link-lib/); + if (!block) return []; + return [...block[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); +})(); + +describe("the ffmpeg libraries the compositor links", () => { + // Without this the regex could silently match nothing and every assertion below + // would iterate an empty list and pass vacuously. + it("is read out of build.rs", () => { + expect(linked.length).toBeGreaterThanOrEqual(6); + expect(linked).toContain("avcodec"); + expect(linked).toContain("avfilter"); + }); + + it("is vendored in full by fetch-ffmpeg.mjs", () => { + // The probe that decides whether a warm tree still needs a re-vendor. When it + // listed fewer libraries than build.rs links, a tree holding the previous set + // satisfied it and the new DLL was never fetched. + const source = read("scripts/fetch-ffmpeg.mjs"); + const table = source.match(/REQUIRED_SHARED_DLLS = \[([\s\S]*?)\]/); + expect(table, "fetch-ffmpeg.mjs no longer declares REQUIRED_SHARED_DLLS").not.toBeNull(); + const required = [...table[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); + expect([...required].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed for the Linux addon", () => { + const source = read("scripts/build-linux-compositor-addon.mjs"); + const table = source.match(/FFMPEG_SONAMES = \[([\s\S]*?)\]/); + expect(table, "build-linux-compositor-addon.mjs no longer declares FFMPEG_SONAMES").not.toBeNull(); + const sonames = [...table[1].matchAll(/"lib([a-z]+)\.so\.\d+"/g)].map((match) => match[1]); + expect([...sonames].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed by the nix derivation", () => { + // `for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,…}.so.*` — the brace glob feeds + // both the copy into $stage/lib and the symbols.map the addon is linked against. + // A name missing here links against nixpkgs' un-renamed copy, and the + // installPhase leak check only flags symbols WITHOUT the osff_ prefix, so it + // passes either way. + const source = read("nix/compositor-view.nix"); + const glob = source.match(/\/lib\/lib\{([a-z,]+)\}\.so\.\*/); + expect(glob, "nix/compositor-view.nix no longer globs the ffmpeg sonames").not.toBeNull(); + expect(glob[1].split(",").sort()).toEqual([...linked].sort()); + }); + + describe("is required by before-pack.cjs", () => { + const source = read("scripts/before-pack.cjs"); + /** The `[...]` array literal a `...[…].map(` spread iterates, per OS table. */ + const listsIn = (table) => { + const block = source.slice(source.indexOf(`const ${table} = [`)); + const end = block.indexOf("\n];"); + return [...block.slice(0, end).matchAll(/\.\.\.\[([^\]]*)\]\.map\(/g)].flatMap((match) => + [...match[1].matchAll(/"([a-z]+)"/g)].map((name) => name[1]), + ); + }; + + // One requirement per library rather than `atLeast: N` over a combined regex: + // several versioned copies of one library would satisfy a count while another + // was missing entirely, which is how a broken pack passed the guard before. + for (const table of ["MAC_REQUIRED", "LINUX_REQUIRED", "WIN_REQUIRED"]) { + it(table, () => { + const required = listsIn(table); + expect(required.length, `${table} declares no per-library spread`).toBeGreaterThan(0); + for (const library of linked) { + expect(required, `${table} does not require lib${library}`).toContain(library); + } + }); + } + }); +}); diff --git a/technical-documentation/architecture/export-pipeline.md b/technical-documentation/architecture/export-pipeline.md index d9378d0bf..fbd223f9f 100644 --- a/technical-documentation/architecture/export-pipeline.md +++ b/technical-documentation/architecture/export-pipeline.md @@ -72,19 +72,25 @@ and **one** encoder + muxer pair: head, `cos² + sin² = 1`) covers each internal boundary to suppress the click where two recordings meet butt-joined, without shifting timing. The in-tree WSOLA stretcher is still there, but only as the fallback - `stretch_pcm_to_length` takes when the filter chain cannot be built or - yields too little audio (see [Audio](native-compositor.md#audio)). + `stretch_pcm_to_length` takes when the filter chain cannot be built, + negotiates a format the drain does not read, or still comes up short of + the target after the corrected pass (see + [Audio](native-compositor.md#audio)). - **The stretch is not overlapped with the encode.** Decode and stretch run inside `walk_composited_timeline`'s `on_clip_end` callback (`pipeline.rs`), which fires once per clip *after* that clip's frames - have been composed and encoded, on the same thread — so the stretch - time is added to the export wall, not hidden behind it. `progress()` is - driven only by encoded video frames, so nothing moves while it runs and - a long clip parks the export at whatever percentage the last frame - reported. That is why the `atempo` path matters: it is O(n) where WSOLA - is O(grain × radius) per rendered sample, which on a long clip meant - minutes of an apparently frozen export. + have been composed and submitted to the encoder, on the same thread — + so the stretch time is added to the export wall, not hidden behind it. + `progress()` counts composed frames as they are handed to the encoder, + and nothing calls it during the audio phase, so a long clip parks the + export at whatever percentage its last frame reported. (The encoder's + own flush comes later still, after the timeline walk.) That reporting + gap is why the `atempo` path matters so much here: WSOLA is + O(grain × radius) per rendered sample, which on a long clip meant + minutes of an apparently frozen export. Reporting progress across the + audio phase would fix the symptom rather than the cost, and is tracked + separately. - **Output** honours the timeline's selected aspect ratio (`resolveAspectRatioValue` over `getEditorSettings(document).aspectRatio` — diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index 509ca68a3..0239cb3aa 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -202,24 +202,50 @@ ones — so a summing mixer is enough and a real mix matrix is not needed. Speed regions apply after decode: `stretch_pcm_to_length` stretches each speed sub-segment to its output frame count through a libavfilter `abuffer → atempo… → abuffersink` graph built in-process -(`avfilter_atempo_stretch`). The graph is pinned to the fltp / 48 kHz / -stereo format `decode_clip_audio` already produces, and `atempo` preserves -format, channels and rate, so no conversion is involved; the result is -recut to the exact target length by truncation or zero-padding. `atempo` -only accepts a factor in `[0.5, 100.0]`, so `atempo_factors` chains -several instances whose product is the requested speed (0.2 → -`[0.5, 0.5, 0.8]`). +(`avfilter_atempo_stretch`). `atempo` only accepts a factor in +`[0.5, 100.0]`, so `atempo_factors` chains several instances whose product +is the requested speed (0.2 → `[0.5, 0.5, 0.8]`). + +Three details of that graph are load-bearing: + +- **The chain is pinned to flt — interleaved, not planar.** `af_atempo` + advertises packed formats only (`U8/S16/S32/FLT/DBL`), so an `abuffer` + pinned to the `fltp` that `decode_clip_audio` produces makes the + negotiation insert an `aresample` and forces every output frame through + a conversion. Asking for `flt` on both ends leaves no conversion filter + in the graph at all; the interleaving is absorbed by the copy into and + out of the frames, which happens either way. + +- **The drain is interleaved with the feed.** `av_buffersrc_add_frame` + does not pull the graph, so pushing a whole region before the first + `av_buffersink_get_frame` would queue all of it in the buffersrc — half + a gigabyte on a 20-minute stereo region, on top of the input slice and + the output accumulator. + +- **Two passes, because `atempo` does not render exactly `n / tempo` + samples.** It falls short by a fixed amount per chain, independent of + input length — measured on the pinned n8.1.2 build, ~217 samples for one + stage and ~2 700 for four, i.e. up to 56 ms at 0.1×. The shortfall is a + difference in rendered duration, not a held-back tail: pushing more + input does not recover it. Filling it with zeros would leave a hard + silence gap butt-joined to the next segment, since the equal-power + crossfade covers clip boundaries only, never the per-segment + concatenation. So the first pass measures the shortfall on the real + content without keeping anything, and the second asks for + `target + shortfall`, which lands the content on the target exactly; the + surplus is truncated. Above 1× the shortfall is zero and the second pass + is skipped. The in-tree WSOLA stretcher remains as the fallback, taken whenever -`avfilter_atempo_stretch` returns `None` — the graph could not be built or -configured, a buffersrc/buffersink call failed, or the chain drained fewer -than 90% of the target samples, which is what happens on a span too short -for `atempo`'s analysis window (a few tens of milliseconds between two -speed regions). WSOLA shares its search positions across channels from a -mono down-mix, so the stereo image does not wander between them. The move -to `atempo` is a cost change, not a quality one: WSOLA is -O(grain × radius) per rendered sample, minutes of a full core on a long -clip, against `atempo`'s O(n) with ffmpeg's SIMD routines. +`avfilter_atempo_stretch` returns `None`: the graph could not be built or +configured, a buffersrc/buffersink call failed, the sink negotiated a +format the drain does not read, or the corrected pass still came up short. +Every one of those paths logs its reason — a sudden multi-minute export +should not be silent about which branch it took. WSOLA shares its search +positions across channels from a mono down-mix, so the stereo image does +not wander between them. The move to `atempo` is a cost change, not a +quality one: on a 5-minute region, `atempo` takes 0.6 s at 1.25× and 4.9 s +at 0.25× (two passes) against WSOLA's 20 s and 55 s. Across segments, `build_audio_concat_plan` sizes each segment's PCM by **integer accumulation of the per-segment rounded sample count**, never From fe84239bc680bfca96151f6895e166bb538a5ba3 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 12:41:24 +0200 Subject: [PATCH 09/10] fix(nix): refresh npmDepsHash, and format the new packaging test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two CI failures on the merged head, neither of them about the audio path. `nix-check.yml` compares nix/package.nix's npmDepsHash against the lockfile and runs on any PR touching `nix/**`. main's recorded hash is stale — it still names the set from before the last few lockfile moves — so merging main in made this branch inherit a failure that belongs to main. The value here is the one the check itself printed. The rest is biome reflowing one call in the new test. --- nix/package.nix | 2 +- scripts/ffmpeg-linked-libraries.test.mjs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/package.nix b/nix/package.nix index b046e5422..02e9eb10a 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -51,7 +51,7 @@ buildNpmPackage { ); }; - npmDepsHash = "sha256-mnI1d8HyZdaCiYolAd8VvBSlSdEsZT89Rf0ADVm9Bf8="; + npmDepsHash = "sha256-LkKX1edTPHZq5nQRrbLAn11oVw36kb0smNQMmVRMEPA="; env.ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; diff --git a/scripts/ffmpeg-linked-libraries.test.mjs b/scripts/ffmpeg-linked-libraries.test.mjs index 2ba9b656f..49e0bd0d3 100644 --- a/scripts/ffmpeg-linked-libraries.test.mjs +++ b/scripts/ffmpeg-linked-libraries.test.mjs @@ -64,7 +64,10 @@ describe("the ffmpeg libraries the compositor links", () => { it("is staged and symbol-renamed for the Linux addon", () => { const source = read("scripts/build-linux-compositor-addon.mjs"); const table = source.match(/FFMPEG_SONAMES = \[([\s\S]*?)\]/); - expect(table, "build-linux-compositor-addon.mjs no longer declares FFMPEG_SONAMES").not.toBeNull(); + expect( + table, + "build-linux-compositor-addon.mjs no longer declares FFMPEG_SONAMES", + ).not.toBeNull(); const sonames = [...table[1].matchAll(/"lib([a-z]+)\.so\.\d+"/g)].map((match) => match[1]); expect([...sonames].sort()).toEqual([...linked].sort()); }); From 3d90e8dea784dca5ab1c91bd9826a406bc361cb2 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 13:09:25 +0200 Subject: [PATCH 10/10] fix(audio): cap the atempo chain instead of stacking stages without a bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `speed` is `source_samples / target_samples`, not the speed anyone clicked. A corrupt scene where a handful of samples targets an hour-long span gives an arbitrarily small ratio, and chaining by 0.5 stacked about thirty stages for it — over a thousand for a subnormal — each with its own analysis window and priming loss. The `speed <= 0.0` / non-finite guard caught NaN and zero, not this. `atempo_factors` now returns `None` past eight stages, which reaches 0.5^8 ~= 0.0039, twenty-five times below MIN_PLAYBACK_SPEED. Past that the WSOLA path takes over; it has no bounds to exceed. The upper branch is chained through the same counter — MAX_PLAYBACK_SPEED is 100 so one stage covers everything the editor produces, but a ratio of quantized lengths is not the clicked speed and nothing pins it under the filter's own bound. --- crates/compositor/src/audio.rs | 71 ++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 223d8e090..e51f0f8af 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -834,22 +834,41 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { exact } +/// Plafond du nombre d'étages atempo chaînés. +/// +/// `speed` vient de `source_samples / target_samples`, pas de l'éditeur : une scène corrompue +/// où une poignée d'échantillons vise une cible d'une heure donne un ratio arbitrairement +/// petit, et le chaînage par 0.5 empile alors une trentaine d'étages — plus d'un millier pour +/// un subnormal — chacun avec sa fenêtre d'analyse et sa perte d'amorçage. Huit couvre +/// jusqu'à 0.5⁸ ≈ 0,0039, soit vingt-cinq fois sous `MIN_PLAYBACK_SPEED` (0,1) ; au-delà on +/// rend `None` et le WSOLA, qui n'a pas de bornes, prend le relais. +const ATEMPO_MAX_STAGES: usize = 8; + /// Découpe un facteur de vitesse en facteurs que `atempo` accepte individuellement : le /// filtre n'admet que [0.5, 100.0], on chaîne donc les dépassements (0.2 → [0.5, 0.5, 0.8], /// 250 → [100.0, 2.5]) — le produit des facteurs reconstitue la vitesse demandée. -fn atempo_factors(speed: f64) -> Vec { +/// +/// Rend `None` au-delà de `ATEMPO_MAX_STAGES` maillons. La borne haute est chaînée elle +/// aussi : `MAX_PLAYBACK_SPEED` vaut 100 donc un seul étage suffit à tout ce que l'éditeur +/// produit, mais `speed` est un rapport de longueurs quantifiées, pas la vitesse cliquée, et +/// rien ne garantit qu'il reste sous la borne du filtre. +fn atempo_factors(speed: f64) -> Option> { let mut factors = Vec::new(); let mut remaining = speed; - while remaining > 100.0 { - factors.push(100.0); - remaining /= 100.0; - } - while remaining < 0.5 { - factors.push(0.5); - remaining /= 0.5; + while remaining > 100.0 || remaining < 0.5 { + if factors.len() >= ATEMPO_MAX_STAGES { + return None; + } + if remaining > 100.0 { + factors.push(100.0); + remaining /= 100.0; + } else { + factors.push(0.5); + remaining /= 0.5; + } } factors.push(remaining); - factors + Some(factors) } /// RAII : libère le graphe même en sortie précoce sur erreur. @@ -1197,7 +1216,7 @@ unsafe fn avfilter_atempo_stretch( .collect() }; - let factors = atempo_factors(speed); + let factors = atempo_factors(speed)?; let prime_tail = atempo_prime_tail(&factors, speed); let mut stretched = planes(target_samples); let produced = atempo_pass(pcm, &factors, prime_tail, target_samples, &mut stretched)?; @@ -1211,7 +1230,7 @@ unsafe fn avfilter_atempo_stretch( // autant, et le contenu tombe cette fois pile sur `target_samples`. let corrected_target = target_samples + shortfall + ATEMPO_LENGTH_GUARD; let corrected_speed = source_samples as f64 / corrected_target as f64; - let corrected_factors = atempo_factors(corrected_speed); + let corrected_factors = atempo_factors(corrected_speed)?; let corrected_tail = atempo_prime_tail(&corrected_factors, corrected_speed); let mut corrected = planes(target_samples); atempo_pass( @@ -1634,15 +1653,35 @@ mod tests { #[test] fn atempo_factors_split_out_of_range_speeds() { // Dans les bornes : un seul maillon. - assert_eq!(atempo_factors(1.25), vec![1.25]); - assert_eq!(atempo_factors(0.5), vec![0.5]); + assert_eq!(atempo_factors(1.25), Some(vec![1.25])); + assert_eq!(atempo_factors(0.5), Some(vec![0.5])); // Hors bornes : chaîne dont le produit reconstitue la vitesse. - assert_eq!(atempo_factors(0.2), vec![0.5, 0.5, 0.8]); - assert_eq!(atempo_factors(250.0), vec![100.0, 2.5]); + assert_eq!(atempo_factors(0.2), Some(vec![0.5, 0.5, 0.8])); + assert_eq!(atempo_factors(250.0), Some(vec![100.0, 2.5])); for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { - let product: f64 = atempo_factors(speed).iter().product(); + let product: f64 = atempo_factors(speed).expect("dans le plafond").iter().product(); assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); } + // MIN_PLAYBACK_SPEED tient largement dans le plafond. + assert_eq!(atempo_factors(0.1).map(|f| f.len()), Some(4)); + } + + #[test] + fn atempo_declines_a_chain_it_would_have_to_stack() { + // `speed` est `source_samples / target_samples`, pas la vitesse cliquée : une scène + // corrompue où une poignée d'échantillons vise une cible d'une heure produit un + // ratio arbitrairement petit. Sans plafond le chaînage empilait une trentaine + // d'étages — plus d'un millier pour un subnormal — chacun avec sa perte d'amorçage. + assert_eq!(atempo_factors(1.0 / 48_000.0 / 3_600.0), None); + assert_eq!(atempo_factors(f64::MIN_POSITIVE), None); + assert_eq!(atempo_factors(1e30), None); + // Et le repli tient le contrat de longueur : c'est le WSOLA qui prend la main. + let pcm = sine(0.05); + let target = pcm[0].len() * 5_000; + let stretched = stretch_pcm_to_length(&pcm, target); + for plane in &stretched { + assert_eq!(plane.len(), target); + } } #[test]