diff --git a/crates/compositor/build.rs b/crates/compositor/build.rs index 2dc9ec53..99b11e2f 100644 --- a/crates/compositor/build.rs +++ b/crates/compositor/build.rs @@ -71,7 +71,7 @@ fn main() { if let Some(v) = ff.as_ref() { let lib_dir = Path::new(v).join("lib"); println!("cargo:rustc-link-search=native={}", lib_dir.display()); - for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample"] { + for lib in ["avformat", "avcodec", "avutil", "swscale", "swresample", "avfilter"] { println!("cargo:rustc-link-lib=dylib={}", lib); } } diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index db9b46e1..e51f0f8a 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -3,6 +3,7 @@ //! unique encodeur AAC alimente le même muxer que la vidéo. use crate::ffi::*; + use crate::regions::SpeedSegment; use crate::scene::SceneAudio; use anyhow::{bail, Result}; @@ -477,6 +478,12 @@ pub struct WsolaTimeStretcher { buf: PlanarPcm, mono: Vec, buf_start: i64, + /// Décalage de lecture dans `buf`/`mono`. `discard_below` ne recopiait pas moins que le + /// reste du buffer à chaque grain : la région entière est poussée d'un coup, donc pour + /// 65 M d'échantillons cela faisait ~N²/(2·ha) ≈ 1,1e12 f32 recopiés par canal — le vrai + /// coût du chemin WSOLA, devant la recherche par grain. On avance un curseur et on ne + /// compacte que lorsque la tête dépasse la moitié du buffer, ce qui rend le total O(N). + buf_head: usize, out: PlanarPcm, win_sum: Vec, out_start: usize, @@ -522,6 +529,7 @@ impl WsolaTimeStretcher { buf: vec![Vec::new(); channels], mono: Vec::new(), buf_start: 0, + buf_head: 0, out: vec![Vec::new(); channels], win_sum: Vec::new(), out_start: 0, @@ -587,8 +595,12 @@ impl WsolaTimeStretcher { } } + fn buf_len(&self) -> usize { + self.buf[0].len() - self.buf_head + } + fn buf_end(&self) -> i64 { - self.buf_start + self.buf[0].len() as i64 + self.buf_start + self.buf_len() as i64 } fn sample_at(&self, channel: usize, absolute_index: i64) -> f32 { @@ -596,7 +608,10 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.buf[channel].get(index as usize).copied().unwrap_or(0.0) + self.buf[channel] + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } @@ -605,12 +620,19 @@ impl WsolaTimeStretcher { if index < 0 { 0.0 } else { - self.mono.get(index as usize).copied().unwrap_or(0.0) + self.mono + .get(self.buf_head + index as usize) + .copied() + .unwrap_or(0.0) } } fn process(&mut self, final_chunk: bool) -> PlanarPcm { let mut emitted = self.empty_chunk(); + // Pas de garde anti-stagnation ici : `search_target` croît de `ha > 0` à chaque tour + // et `grain_pos` ne s'en écarte que de `search_radius` au plus, donc le break sur + // `buf_end` finit toujours par tomber. Une garde de plus tronquerait `emitted` — la + // région sortirait muette pour tout signal — sans jamais se déclencher. loop { let search_target = (self.ideal_pos + self.ha).round() as i64; let required_end = (self.grain_pos + self.n as i64) @@ -738,12 +760,18 @@ impl WsolaTimeStretcher { if drop_count <= 0 { return; } - let drop_count = drop_count as usize; - for channel in 0..self.channels { - self.buf[channel] = self.buf[channel][drop_count.min(self.buf[channel].len())..].to_vec(); - } - self.mono = self.mono[drop_count.min(self.mono.len())..].to_vec(); + self.buf_head += (drop_count as usize).min(self.buf_len()); self.buf_start = absolute_index; + // Compactage amorti : ne recopier que lorsque la tête consommée dépasse ce qui + // reste laisse un coût total en O(N) au lieu du O(N²) d'une recopie par grain. + if self.buf_head > self.buf[0].len() - self.buf_head { + let head = self.buf_head; + for channel in 0..self.channels { + self.buf[channel].drain(..head); + } + self.mono.drain(..head); + self.buf_head = 0; + } } } @@ -767,6 +795,20 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { } let speed = source_samples as f64 / target_samples as f64; + + // atempo d'abord : le WSOLA ci-dessous fait le même time-stretch préservant la hauteur, + // mais coûte un ordre de grandeur de plus. Mesuré en release sur une région de 5 min + // (14,4 M échantillons) : 0,6 s contre 20 s à 1,25×, 4,9 s contre 55 s à 0,25× — et + // c'est le WSOLA APRÈS la correction de `discard_below`, qui recopiait tout le buffer + // restant à chaque grain et faisait tenir un export mesuré (65,4 M échantillons) plus de + // dix minutes sans finir, l'export paraissant figé à ~80 %. `avfilter_atempo_stretch` + // rend `None` si la chaîne ne monte pas, si le sink négocie un format inattendu ou si la + // sortie reste plus courte que la cible ; le WSOLA reste alors le chemin de repli exact + // d'avant, en journalisant la raison. + if let Some(stretched) = unsafe { avfilter_atempo_stretch(pcm, target_samples, speed) } { + return stretched; + } + let mut stretcher = WsolaTimeStretcher::new( AUDIO_OUTPUT_SAMPLE_RATE, AUDIO_OUTPUT_CHANNELS, @@ -792,6 +834,431 @@ fn stretch_pcm_to_length(pcm: &[Vec], target_samples: usize) -> PlanarPcm { exact } +/// Plafond du nombre d'étages atempo chaînés. +/// +/// `speed` vient de `source_samples / target_samples`, pas de l'éditeur : une scène corrompue +/// où une poignée d'échantillons vise une cible d'une heure donne un ratio arbitrairement +/// petit, et le chaînage par 0.5 empile alors une trentaine d'étages — plus d'un millier pour +/// un subnormal — chacun avec sa fenêtre d'analyse et sa perte d'amorçage. Huit couvre +/// jusqu'à 0.5⁸ ≈ 0,0039, soit vingt-cinq fois sous `MIN_PLAYBACK_SPEED` (0,1) ; au-delà on +/// rend `None` et le WSOLA, qui n'a pas de bornes, prend le relais. +const ATEMPO_MAX_STAGES: usize = 8; + +/// Découpe un facteur de vitesse en facteurs que `atempo` accepte individuellement : le +/// filtre n'admet que [0.5, 100.0], on chaîne donc les dépassements (0.2 → [0.5, 0.5, 0.8], +/// 250 → [100.0, 2.5]) — le produit des facteurs reconstitue la vitesse demandée. +/// +/// Rend `None` au-delà de `ATEMPO_MAX_STAGES` maillons. La borne haute est chaînée elle +/// aussi : `MAX_PLAYBACK_SPEED` vaut 100 donc un seul étage suffit à tout ce que l'éditeur +/// produit, mais `speed` est un rapport de longueurs quantifiées, pas la vitesse cliquée, et +/// rien ne garantit qu'il reste sous la borne du filtre. +fn atempo_factors(speed: f64) -> Option> { + let mut factors = Vec::new(); + let mut remaining = speed; + while remaining > 100.0 || remaining < 0.5 { + if factors.len() >= ATEMPO_MAX_STAGES { + return None; + } + if remaining > 100.0 { + factors.push(100.0); + remaining /= 100.0; + } else { + factors.push(0.5); + remaining /= 0.5; + } + } + factors.push(remaining); + Some(factors) +} + +/// RAII : libère le graphe même en sortie précoce sur erreur. +struct FilterGraphGuard(*mut AVFilterGraph); + +impl Drop for FilterGraphGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { avfilter_graph_free(&mut self.0) }; + } + } +} + +/// RAII : libère la trame de drain même en sortie précoce sur erreur. +struct FrameGuard(*mut AVFrame); + +impl Drop for FrameGuard { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { av_frame_free(&mut self.0) }; + } + } +} + +/// Taille des trames poussées vers le graphe. Le drain est entrelacé avec l'alimentation +/// (cf. `atempo_drain`) : sans cela `av_buffersrc_add_frame` empile toute la région dans la +/// file du buffersrc — ~523 Mo pour une speed region stéréo de 20 min, en plus du slice +/// d'entrée et de l'accumulateur de sortie. +const ATEMPO_FEED_CHUNK: usize = 4096; + +/// Rallonge de silence poussée derrière la région avant l'EOF, par étage atempo. +/// +/// atempo laisse tomber la dernière fenêtre de chaque étage. En prolongeant l'entrée d'un +/// silence, la fenêtre perdue devient du silence et le contenu réel sort en entier : mesuré +/// sur le pin ffmpeg n8.1.2 (48 kHz stéréo), le manque tombe de 981 à 217 échantillons pour +/// un étage 0.5×, de 2 735 à 553 pour deux, de 8 234 à 2 676 pour quatre. Au-delà la courbe +/// est plate — un tail 16× plus grand ne change plus rien — ce qui reste est traité par la +/// correction de tempo de `avfilter_atempo_stretch`. +const ATEMPO_PRIME_TAIL: usize = 4096; + +/// Marge de sécurité, en échantillons, sur la longueur demandée à la passe corrigée. +/// +/// Le manque de la seconde passe n'est pas exactement celui mesuré à la première (le tempo +/// a bougé de moins de 1 %, la chaîne est la même). Viser 64 échantillons de plus fait +/// tomber le résidu du côté du surplus, tronqué : 1,3 ms de contenu en moins plutôt qu'un +/// trou de silence. +const ATEMPO_LENGTH_GUARD: usize = 64; + +/// Longueur du silence à pousser derrière la région pour une chaîne donnée. +fn atempo_prime_tail(factors: &[f64], speed: f64) -> usize { + ATEMPO_PRIME_TAIL + .saturating_mul(factors.len() + 1) + .saturating_mul(speed.max(1.0).ceil() as usize) +} + +/// Vide le buffersink dans `stretched`, sans jamais y garder plus de `keep` échantillons par +/// plan, et compte dans `produced` TOUT ce qui est sorti — y compris ce qui est jeté. +/// +/// Les deux chiffres servent à des choses différentes : `stretched` est le résultat, alors +/// que `produced` mesure ce que la chaîne a réellement rendu pour une entrée de longueur +/// connue, donc son manque (cf. `avfilter_atempo_stretch`). +/// +/// Rend `Some(true)` sur EOF, `Some(false)` quand le graphe n'a plus rien de prêt (EAGAIN), +/// `None` sur une vraie panne — l'appelant retombe alors sur WSOLA. +unsafe fn atempo_drain( + sink_ctx: *mut AVFilterContext, + frame: *mut AVFrame, + stretched: &mut PlanarPcm, + keep: usize, + produced: &mut usize, +) -> Option { + loop { + let ret = av_buffersink_get_frame(sink_ctx, frame); + if ret == AVERROR_EAGAIN { + return Some(false); + } + if ret == AVERROR_EOF { + return Some(true); + } + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_buffersink_get_frame a échoué (ret={ret}), repli WSOLA" + ); + return None; + } + let count = (*frame).nb_samples.max(0) as usize; + let channels = (*frame).ch_layout.nb_channels.max(0) as usize; + // La chaîne est épinglée en flt entrelacé de bout en bout (cf. `avfilter_atempo_stretch`) ; + // tout autre format signifie que la négociation a fait autre chose que ce qu'on a + // demandé, et le désentrelacement ci-dessous lirait n'importe quoi. + if (*frame).format != AVSampleFormat::AV_SAMPLE_FMT_FLT as i32 + || channels != AUDIO_OUTPUT_CHANNELS + { + eprintln!( + "[openscreen-compositor] atempo: trame de sortie inattendue (format={} canaux={channels}), repli WSOLA", + (*frame).format + ); + av_frame_unref(frame); + return None; + } + let wanted = count.min(keep.saturating_sub(stretched[0].len())); + if wanted > 0 { + let interleaved = *(*frame).extended_data.add(0) as *const f32; + let samples = + std::slice::from_raw_parts(interleaved, count * AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let plane = &mut stretched[channel]; + plane.reserve(wanted); + for index in 0..wanted { + plane.push(samples[index * AUDIO_OUTPUT_CHANNELS + channel]); + } + } + } + *produced += count; + av_frame_unref(frame); + } +} + +/// Monte `abuffer → atempo… → abuffersink`, y pousse `pcm` suivi de `prime_tail` échantillons +/// de silence, et rend le nombre total d'échantillons sortis — `stretched` en reçoit les +/// `keep` premiers. +/// +/// La chaîne est épinglée en **flt entrelacé** 48 kHz stéréo, pas en fltp : `af_atempo` +/// n'annonce que des formats packed (U8/S16/S32/FLT/DBL, cf. son `query_formats`), donc un +/// abuffer en fltp fait insérer un aresample de conversion et rend une branche planaire du +/// drain inatteignable — mesuré, le sink négociait déjà `AV_SAMPLE_FMT_FLT`. En demandant flt +/// des deux côtés il n'y a aucun filtre de conversion dans le graphe, et l'entrelacement est +/// absorbé par la recopie qu'on fait de toute façon. +unsafe fn atempo_pass( + pcm: &[Vec], + factors: &[f64], + prime_tail: usize, + keep: usize, + stretched: &mut PlanarPcm, +) -> Option { + let graph_guard = FilterGraphGuard(avfilter_graph_alloc()); + let graph = graph_guard.0; + if graph.is_null() { + return None; + } + + let abuffer_name = CString::new("abuffer").ok()?; + let abuffersink_name = CString::new("abuffersink").ok()?; + let atempo_name = CString::new("atempo").ok()?; + let abuffer = avfilter_get_by_name(abuffer_name.as_ptr()); + let abuffersink = avfilter_get_by_name(abuffersink_name.as_ptr()); + let atempo = avfilter_get_by_name(atempo_name.as_ptr()); + if abuffer.is_null() || abuffersink.is_null() || atempo.is_null() { + return None; + } + + let create_filter = |graph: *mut AVFilterGraph, + filter: *const AVFilter, + name: &str, + args: Option<&str>, + options: &[(&str, &str)]| + -> Option<*mut AVFilterContext> { + let cname = CString::new(name).ok()?; + let cargs = match args { + Some(args) => Some(CString::new(args).ok()?), + None => None, + }; + let ctx = avfilter_graph_alloc_filter(graph, filter, cname.as_ptr()); + if ctx.is_null() { + eprintln!("[openscreen-compositor] atempo: alloc_filter({name}) a rendu null"); + return None; + } + // Les options typées se posent entre l'alloc et l'init — `avfilter_init_str` fige la + // négociation. Un échec n'est pas fatal : l'abuffer porte déjà le format, ceci ne fait + // que l'imposer aussi côté sink pour qu'aucun build ffmpeg ne puisse y glisser un + // aresample. Le drain vérifie le format reçu de toute façon. + for (key, value) in options { + let ckey = CString::new(*key).ok()?; + let cvalue = CString::new(*value).ok()?; + let ret = av_opt_set( + ctx as *mut std::ffi::c_void, + ckey.as_ptr(), + cvalue.as_ptr(), + AV_OPT_SEARCH_CHILDREN as i32, + ); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: av_opt_set({name}.{key}={value}) a échoué (ret={ret}), négociation laissée libre" + ); + } + } + // `map_or` consommerait `cargs` et le pointeur rendu par la closure serait dangling + // avant même l'appel — on emprunte donc pour la durée de l'appel. + let args_ptr = match &cargs { + Some(args) => args.as_ptr(), + None => ptr::null(), + }; + let ret = avfilter_init_str(ctx, args_ptr); + if ret < 0 { + eprintln!( + "[openscreen-compositor] atempo: init_str({name}, {:?}) a échoué (ret={ret})", + args.unwrap_or("") + ); + return None; + } + Some(ctx) + }; + + let rate = AUDIO_OUTPUT_SAMPLE_RATE; + let src_ctx = create_filter( + graph, + abuffer, + "in", + Some(&format!( + "time_base=1/{rate}:sample_rate={rate}:sample_fmt=flt:channel_layout=stereo" + )), + &[], + )?; + let sink_ctx = create_filter(graph, abuffersink, "out", None, &[("sample_fmts", "flt")])?; + + let mut previous = src_ctx; + for (index, factor) in factors.iter().enumerate() { + let stage = create_filter( + graph, + atempo, + &format!("atempo{index}"), + Some(&format!("{factor}")), + &[], + )?; + if avfilter_link(previous, 0, stage, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link a échoué au maillon {index}"); + return None; + } + previous = stage; + } + if avfilter_link(previous, 0, sink_ctx, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_link vers le sink a échoué"); + return None; + } + if avfilter_graph_config(graph, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: avfilter_graph_config a échoué"); + return None; + } + + let sink_frame = FrameGuard(av_frame_alloc()); + if sink_frame.0.is_null() { + return None; + } + + // Alimentation : le PCM passe par trames flt de 4096 échantillons, prolongé par la + // rallonge de silence. `av_buffersrc_add_frame` déplace les références du frame dans le + // graphe ; on alloue donc une trame neuve par tranche et on la libère après envoi (le + // shell est vide à ce point). Le drain est entrelacé ici : sans lui la file du buffersrc + // porterait toute la région d'un coup. La condition d'arrêt est l'entrée épuisée, PAS + // « on a de quoi remplir la cible » — s'arrêter là couperait la rallonge, et les derniers + // grains du contenu réel resteraient dans le graphe. + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + let total_input = source_samples.saturating_add(prime_tail); + let mut offset = 0usize; + let mut produced = 0usize; + let mut drained_to_eof = false; + while offset < total_input { + let count = ATEMPO_FEED_CHUNK.min(total_input - offset); + let mut frame = av_frame_alloc(); + if frame.is_null() { + eprintln!("[openscreen-compositor] atempo: av_frame_alloc (feed) a échoué"); + return None; + } + (*frame).format = AVSampleFormat::AV_SAMPLE_FMT_FLT as i32; + (*frame).sample_rate = rate; + (*frame).nb_samples = count as i32; + av_channel_layout_default(&mut (*frame).ch_layout, AUDIO_OUTPUT_CHANNELS as i32); + if av_frame_get_buffer(frame, 0) < 0 { + eprintln!("[openscreen-compositor] atempo: av_frame_get_buffer (feed) a échoué"); + av_frame_free(&mut frame); + return None; + } + // Un seul plan en flt : on écrit entrelacé. Le `write_bytes` couvre à la fois les + // canaux absents d'une source mono et la rallonge de silence finale. + let destination = *(*frame).extended_data.add(0) as *mut f32; + ptr::write_bytes(destination, 0, count * AUDIO_OUTPUT_CHANNELS); + for channel in 0..AUDIO_OUTPUT_CHANNELS { + if let Some(plane) = pcm.get(channel) { + let available = plane.len().saturating_sub(offset).min(count); + for index in 0..available { + *destination.add(index * AUDIO_OUTPUT_CHANNELS + channel) = + plane[offset + index]; + } + } + } + (*frame).pts = offset as i64; + let ret = av_buffersrc_add_frame(src_ctx, frame); + av_frame_free(&mut frame); + if ret < 0 { + eprintln!("[openscreen-compositor] atempo: av_buffersrc_add_frame (offset={offset}) a échoué (ret={ret})"); + return None; + } + offset += count; + if atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)? { + drained_to_eof = true; + break; + } + } + + // EOF : le graphe vide alors ses derniers grains. + if !drained_to_eof { + if av_buffersrc_add_frame(src_ctx, ptr::null_mut()) < 0 { + eprintln!("[openscreen-compositor] atempo: flush du buffersrc a échoué, repli WSOLA"); + return None; + } + atempo_drain(sink_ctx, sink_frame.0, stretched, keep, &mut produced)?; + } + Some(produced) +} + +/// Étire le PCM d'un facteur `speed` via une chaîne `abuffer → atempo… → abuffersink` montée +/// en processus, dans l'avfilter LGPL déjà vendored avec l'app (avfilter-11.dll / +/// libavfilter.so.11 / libavfilter.11.dylib voyagent dans le même lot que avcodec — cf. +/// scripts/fetch-ffmpeg.mjs qui copie TOUTES les av*.dll du build BtbN). +/// +/// **Deux passes.** atempo ne rend pas exactement `n/tempo` échantillons : il en manque un +/// nombre fixe par chaîne, indépendant de la longueur de l'entrée (mesuré sur n8.1.2 : +/// ~217 pour un étage, ~550 pour deux, ~2 700 pour quatre, soit jusqu'à 56 ms à 0,1×). Le +/// manque ne se rattrape pas en poussant plus d'entrée — c'est une différence de durée +/// rendue, pas une queue retenue — et le combler par des zéros collait un trou de silence +/// devant le segment suivant, puisque le crossfade equal-power ne couvre que les frontières +/// de clip, jamais la concaténation par segment. La première passe mesure donc le manque sur +/// le contenu réel, sans rien garder, et la seconde demande `cible + manque` pour que le +/// contenu remplisse la cible ; le surplus est tronqué. Aux vitesses > 1 le manque est nul et +/// la seconde passe est sautée. +/// +/// Retourne `None` sur toute défaillance (montage, négociation, exécution, sortie plus courte +/// que la cible) : l'appelant retombe alors sur le WSOLA d'origine. +unsafe fn avfilter_atempo_stretch( + pcm: &[Vec], + target_samples: usize, + speed: f64, +) -> Option { + if !speed.is_finite() || speed <= 0.0 || target_samples == 0 { + return None; + } + let source_samples = pcm.first().map(|plane| plane.len()).unwrap_or(0); + if source_samples == 0 { + return None; + } + + let planes = |capacity: usize| -> PlanarPcm { + (0..AUDIO_OUTPUT_CHANNELS) + .map(|_| Vec::with_capacity(capacity)) + .collect() + }; + + let factors = atempo_factors(speed)?; + let prime_tail = atempo_prime_tail(&factors, speed); + let mut stretched = planes(target_samples); + let produced = atempo_pass(pcm, &factors, prime_tail, target_samples, &mut stretched)?; + let expected = ((source_samples + prime_tail) as f64 / speed).round() as usize; + let shortfall = expected.saturating_sub(produced); + + if shortfall > 0 { + // Le contenu réel s'arrête `shortfall` échantillons avant la cible, et ce qui suit + // dans `stretched` n'est que la rallonge de silence étirée. On rejoue en demandant + // une cible plus longue du même montant : la chaîne étant la même, elle en perd + // autant, et le contenu tombe cette fois pile sur `target_samples`. + let corrected_target = target_samples + shortfall + ATEMPO_LENGTH_GUARD; + let corrected_speed = source_samples as f64 / corrected_target as f64; + let corrected_factors = atempo_factors(corrected_speed)?; + let corrected_tail = atempo_prime_tail(&corrected_factors, corrected_speed); + let mut corrected = planes(target_samples); + atempo_pass( + pcm, + &corrected_factors, + corrected_tail, + target_samples, + &mut corrected, + )?; + if corrected[0].len() >= target_samples { + stretched = corrected; + } + } + + // Plus court que la cible : la chaîne n'a pas fait son travail. On rend `None` — compléter + // par des zéros exporterait un trou en se faisant passer pour un succès, et le contrat de + // `stretch_pcm_to_length` est un repli WSOLA sur échec. + if stretched[0].len() < target_samples { + eprintln!( + "[openscreen-compositor] atempo: sortie de {} échantillons pour une cible de {target_samples} (vitesse {speed}, {} étages), repli WSOLA", + stretched[0].len(), + factors.len() + ); + return None; + } + Some(stretched) +} + /// Découpe le PCM gardé avec les mêmes spans et la même quantification frame que la vidéo. pub fn stretch_clip_pcm_by_speed( pcm: &[Vec], @@ -1036,6 +1503,187 @@ mod tests { vec![samples.to_vec(), samples.to_vec()] } + /// Un sinus 440 Hz de `secs` secondes sur les deux canaux. + fn sine(secs: f64) -> PlanarPcm { + let total = (secs * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + let mut pcm: PlanarPcm = vec![Vec::with_capacity(total); AUDIO_OUTPUT_CHANNELS]; + for i in 0..total { + let t = i as f32 / AUDIO_OUTPUT_SAMPLE_RATE as f32; + let sample = (2.0 * PI * 440.0 * t).sin() * 0.5; + for channel in 0..AUDIO_OUTPUT_CHANNELS { + pcm[channel].push(sample); + } + } + pcm + } + + /// Hauteur mesurée par passages à zéro montants sur une fenêtre d'une seconde. + fn pitch_hz(plane: &[f32], start: usize) -> usize { + let window = (AUDIO_OUTPUT_SAMPLE_RATE as usize).min(plane.len().saturating_sub(start + 1)); + (start..start + window) + .filter(|&i| plane[i] <= 0.0 && plane[i + 1] > 0.0) + .count() + } + + /// Énergie RMS des `count` derniers échantillons. + fn tail_rms(plane: &[f32], count: usize) -> f32 { + let start = plane.len().saturating_sub(count); + let slice = &plane[start..]; + if slice.is_empty() { + return 0.0; + } + (slice.iter().map(|v| v * v).sum::() / slice.len() as f32).sqrt() + } + + /// Les presets réellement cliquables dans l'éditeur (`SPEED_OPTIONS`), plus les bornes + /// `MIN_PLAYBACK_SPEED` / `MAX_PLAYBACK_SPEED` de `src/components/video-editor/types.ts`. + const EDITOR_SPEEDS: [f64; 13] = [ + 0.1, 0.25, 0.5, 0.75, 1.25, 1.5, 1.75, 2.0, 3.0, 4.0, 5.0, 10.0, 100.0, + ]; + + #[test] + fn atempo_covers_every_editor_speed_without_a_silent_tail() { + // Le bug que ce test verrouille : atempo n'émet jamais sa dernière fenêtre, et + // compléter le manque par des zéros collait jusqu'à 181 ms de blanc (0,1×, quatre + // étages) devant le segment suivant — un dropout audible dans un export « réussi ». + // La rallonge de silence en entrée fait sortir les derniers grains pour de bon, donc + // la fin de région doit porter autant de signal que son milieu, à toute vitesse et + // sur des spans courts comme longs. + for &speed in &EDITOR_SPEEDS { + for &secs in &[0.05f64, 0.5, 3.0] { + let pcm = sine(secs); + let source = pcm[0].len(); + let target = (source as f64 / speed).round() as usize; + if target == 0 { + continue; + } + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| { + panic!("atempo doit couvrir {speed}× sur {secs}s (cible {target})") + }); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}× durée {secs}s"); + } + // 10 ms de queue : le zero-padding d'avant en laissait au moins 5 ms à 0,1×. + // Le trou : un silence numérique en fin de région. Zéro tolérance — la + // correction de tempo est faite pour que le contenu tombe pile sur la cible. + let trailing_silence = + stretched[0].iter().rev().take_while(|v| **v == 0.0).count(); + assert_eq!( + trailing_silence, 0, + "{trailing_silence} échantillons de silence en fin de région à {speed}× sur {secs}s" + ); + let tail = (AUDIO_OUTPUT_SAMPLE_RATE as usize / 100).min(target); + assert!( + tail_rms(&stretched[0], tail) > 0.05, + "queue sans énergie à {speed}× sur {secs}s : rms={}", + tail_rms(&stretched[0], tail) + ); + } + } + } + + #[test] + fn atempo_preserves_pitch_through_a_chain_of_stages() { + // 0,25× et 0,1× sortent des bornes [0.5, 100] d'un seul atempo et passent donc par + // la chaîne multi-étages — le cas que le test d'origine (1,25×, un seul maillon) + // ne touchait pas, alors que 0,25× est un preset de la liste déroulante. + for &speed in &[0.1f64, 0.25, 0.5] { + let pcm = sine(2.0); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = unsafe { avfilter_atempo_stretch(&pcm, target, speed) } + .unwrap_or_else(|| panic!("la chaîne atempo doit monter à {speed}×")); + let measured = pitch_hz(&stretched[0], target / 2); + assert!( + (measured as f64 - 440.0).abs() <= 2.0, + "hauteur à {speed}× : {measured} Hz (un rééchantillonnage la déplacerait)" + ); + } + } + + #[test] + fn stretch_pcm_to_length_is_exact_at_every_editor_speed() { + // Contrat de bout en bout, repli WSOLA compris : quelle que soit la branche prise, + // la longueur rendue est exactement celle que le plan de concaténation attend. + for &speed in &EDITOR_SPEEDS { + let pcm = sine(0.5); + let target = (pcm[0].len() as f64 / speed).round() as usize; + let stretched = stretch_pcm_to_length(&pcm, target); + assert_eq!(stretched.len(), AUDIO_OUTPUT_CHANNELS); + for plane in &stretched { + assert_eq!(plane.len(), target, "vitesse {speed}×"); + } + } + } + + #[test] + fn wsola_fallback_still_stretches_and_keeps_pitch() { + // Le chemin de repli reste atteignable (avfilter absent d'un build, graphe qui ne + // monte pas) et sa recopie de buffer a été remplacée par un curseur de lecture : + // ce test verrouille qu'il rend toujours la bonne durée à la bonne hauteur. + let pcm = sine(2.0); + let speed = 0.5; + let target = (pcm[0].len() as f64 / speed).round() as usize; + let mut stretcher = WsolaTimeStretcher::new( + AUDIO_OUTPUT_SAMPLE_RATE, + AUDIO_OUTPUT_CHANNELS, + speed, + target, + ); + let mut emitted: PlanarPcm = vec![Vec::new(); AUDIO_OUTPUT_CHANNELS]; + for chunk in [stretcher.push(&pcm), stretcher.flush()] { + for channel in 0..AUDIO_OUTPUT_CHANNELS { + emitted[channel].extend_from_slice(&chunk[channel]); + } + } + // Le WSOLA vise la durée sans la garantir à l'échantillon près : c'est + // `stretch_pcm_to_length` qui recadre. On tolère 1 % ici. + let produced = emitted[0].len() as f64; + assert!( + (produced - target as f64).abs() / (target as f64) < 0.02, + "WSOLA a rendu {produced} pour une cible de {target}" + ); + let measured = pitch_hz(&emitted[0], target / 2); + assert!( + (measured as f64 - 440.0).abs() <= 3.0, + "hauteur WSOLA : {measured} Hz" + ); + } + + #[test] + fn atempo_factors_split_out_of_range_speeds() { + // Dans les bornes : un seul maillon. + assert_eq!(atempo_factors(1.25), Some(vec![1.25])); + assert_eq!(atempo_factors(0.5), Some(vec![0.5])); + // Hors bornes : chaîne dont le produit reconstitue la vitesse. + assert_eq!(atempo_factors(0.2), Some(vec![0.5, 0.5, 0.8])); + assert_eq!(atempo_factors(250.0), Some(vec![100.0, 2.5])); + for speed in [0.07f64, 0.3, 1.0, 3.7, 4_000.0] { + let product: f64 = atempo_factors(speed).expect("dans le plafond").iter().product(); + assert!((product - speed).abs() < 1e-9, "produit={product} attendu={speed}"); + } + // MIN_PLAYBACK_SPEED tient largement dans le plafond. + assert_eq!(atempo_factors(0.1).map(|f| f.len()), Some(4)); + } + + #[test] + fn atempo_declines_a_chain_it_would_have_to_stack() { + // `speed` est `source_samples / target_samples`, pas la vitesse cliquée : une scène + // corrompue où une poignée d'échantillons vise une cible d'une heure produit un + // ratio arbitrairement petit. Sans plafond le chaînage empilait une trentaine + // d'étages — plus d'un millier pour un subnormal — chacun avec sa perte d'amorçage. + assert_eq!(atempo_factors(1.0 / 48_000.0 / 3_600.0), None); + assert_eq!(atempo_factors(f64::MIN_POSITIVE), None); + assert_eq!(atempo_factors(1e30), None); + // Et le repli tient le contrat de longueur : c'est le WSOLA qui prend la main. + let pcm = sine(0.05); + let target = pcm[0].len() * 5_000; + let stretched = stretch_pcm_to_length(&pcm, target); + for plane in &stretched { + assert_eq!(plane.len(), target); + } + } + #[test] fn single_track_passes_through_unchanged() { let track = planar(&[0.25, -0.5, 0.75]); diff --git a/crates/compositor/wrapper_linux.h b/crates/compositor/wrapper_linux.h index e01a31a9..0ee8cf4a 100644 --- a/crates/compositor/wrapper_linux.h +++ b/crates/compositor/wrapper_linux.h @@ -12,3 +12,6 @@ #include #include #include +#include +#include +#include diff --git a/crates/compositor/wrapper_macos.h b/crates/compositor/wrapper_macos.h index 5a87a146..f3a5e5dd 100644 --- a/crates/compositor/wrapper_macos.h +++ b/crates/compositor/wrapper_macos.h @@ -17,4 +17,7 @@ /* Software decode path : swscale était déjà LIÉ (build.rs) sans être bindé. Conservé identique côté macOS pour que la symétrie avec cpu_frames_windows.rs soit claire ; le code effectif vit dans mac_frames.rs. */ -#include \ No newline at end of file +#include +#include +#include +#include diff --git a/crates/compositor/wrapper_windows.h b/crates/compositor/wrapper_windows.h index 86612f96..8d291a81 100644 --- a/crates/compositor/wrapper_windows.h +++ b/crates/compositor/wrapper_windows.h @@ -12,3 +12,6 @@ elle couvre les formats exotiques (10 bits, 4:2:2) qu'un interleave écrit à la main casserait silencieusement. */ #include +#include +#include +#include diff --git a/nix/compositor-view.nix b/nix/compositor-view.nix index ecf8b077..0b42a1bc 100644 --- a/nix/compositor-view.nix +++ b/nix/compositor-view.nix @@ -120,7 +120,18 @@ rustPlatform.buildRustPackage { # Copy each library under its soname and read its symbol table. awk rather # than sed with a backreference: the third field is the name, and anything # after an @ is the version tag. - for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample}.so.*; do + # + # The list must hold every library crates/compositor/build.rs emits a + # cargo:rustc-link-lib for -- all six of them, avfilter included since the + # speed-region stretch runs through atempo. Miss one and the build either + # dies on `cannot find -lavfilter` (no unversioned symlink is staged for it + # below) or links the store's UN-renamed copy, and a cdylib tolerating + # undefined symbols means the failure surfaces only at require() time as + # "undefined symbol: osff_avfilter_graph_alloc" -- the addon then loads as a + # no-op and preview plus every export are dead. avfilter's exports are all + # av-prefixed (avfilter_*, av_buffersrc_*, av_buffersink_*), so the awk + # filter here and the leak check in installPhase already cover them. + for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,avcodec,avutil,swscale,swresample,avfilter}.so.*; do case "$lib" in *.so.*.*) continue ;; esac test -f "$lib" || continue cp "$(readlink -f "$lib")" "$stage/lib/$(basename "$lib")" @@ -178,7 +189,7 @@ rustPlatform.buildRustPackage { # Each copy still carries the RUNPATH it inherited from the original ffmpeg # output, which is where the UN-renamed libraries live -- so libavcodec's own # osff_swr_init would resolve against a libswresample that defines swr_init. - # It only works today because all five happen to be direct DT_NEEDED of the + # It only works today because all six happen to be direct DT_NEEDED of the # addon, so $ORIGIN is searched first; the day --as-needed drops one the # loader falls through to the store copy and dlopen fails on an undefined # osff_ symbol. Put $ORIGIN in front so the renamed set can only resolve diff --git a/nix/package.nix b/nix/package.nix index b046e542..02e9eb10 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -51,7 +51,7 @@ buildNpmPackage { ); }; - npmDepsHash = "sha256-mnI1d8HyZdaCiYolAd8VvBSlSdEsZT89Rf0ADVm9Bf8="; + npmDepsHash = "sha256-LkKX1edTPHZq5nQRrbLAn11oVw36kb0smNQMmVRMEPA="; env.ELECTRON_SKIP_BINARY_DOWNLOAD = "1"; diff --git a/scripts/before-pack.cjs b/scripts/before-pack.cjs index 4b97a35a..e978ca4c 100644 --- a/scripts/before-pack.cjs +++ b/scripts/before-pack.cjs @@ -95,13 +95,16 @@ const MAC_REQUIRED = [ breaks: "the preview and every export render nothing", fix: FIX_MAC, }, - { - match: (name) => /^libav(codec|format|util)\.\d+\.dylib$/.test(name), - what: "the LGPL ffmpeg dylibs the compositor links", + // One requirement per library, not `atLeast: N` over a combined regex — the + // same trap LINUX_REQUIRED documents above. Several versioned copies of one + // library would satisfy a combined count while another was missing entirely, + // and the addon would still fail to load. + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ + match: (name) => new RegExp(`^lib${library}\\.\\d+\\.dylib$`).test(name), + what: `the LGPL lib${library} dylib the compositor links`, breaks: "the compositor addon cannot be loaded at all (dyld error at require())", fix: FIX_MAC, - atLeast: 3, - }, + })), { match: (name) => name === "whisper-stt-server", what: "the whisper.cpp STT helper", @@ -162,7 +165,7 @@ const LINUX_REQUIRED = [ // pendant qu'une autre manquait. Le paquet passait alors la garde et le // compositeur ne chargeait pas : exactement le mode de panne que cette garde // existe pour attraper. - ...["avcodec", "avformat", "avutil", "swresample", "swscale"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^lib${library}\\.so\\.\\d+$`).test(name), what: `the symbol-renamed lib${library} shared object the compositor links`, breaks: "the compositor addon cannot be loaded at all (ld.so error at require())", @@ -276,7 +279,7 @@ const WIN_REQUIRED = [ // (avcodec-60/61/62.dll left by an earlier fetch) would satisfy a combined count // while another library was missing entirely, and the addon would still fail to // load. - ...["avcodec", "avformat", "avutil"].map((library) => ({ + ...["avcodec", "avformat", "avutil", "swresample", "swscale", "avfilter"].map((library) => ({ match: (name) => new RegExp(`^${library}-\\d+\\.dll$`).test(name), what: `the ${library} DLL the compositor links`, breaks: "the addon cannot be loaded at all under MSIX, which ignores PATH", diff --git a/scripts/build-linux-compositor-addon.mjs b/scripts/build-linux-compositor-addon.mjs index 9cf1b735..717d1b79 100644 --- a/scripts/build-linux-compositor-addon.mjs +++ b/scripts/build-linux-compositor-addon.mjs @@ -16,7 +16,7 @@ // (ensureFfmpegSharedDllsOnPath), but glibc reads LD_LIBRARY_PATH once at // process start, so the equivalent trick cannot work after Electron is // already running. Instead the addon is linked with `-rpath,$ORIGIN` and -// the five ffmpeg sonames are copied next to it, which makes the .node +// the six ffmpeg sonames are copied next to it, which makes the .node // self-contained wherever it is installed — no env var, no PATH surgery. import { spawnSync } from "node:child_process"; @@ -36,6 +36,7 @@ const FFMPEG_SONAMES = [ "libavutil.so.60", "libswscale.so.9", "libswresample.so.6", + "libavfilter.so.11", ]; const run = (command, args, options = {}) => diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 8c2f0ed8..07c78e5f 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -407,13 +407,33 @@ async function fetchSharedDlls(tag, binDir) { return; } - // probe for any previously vendored DLL by name; re-download is driven by - // --force same as the static exe, checked once we know what we'd extract. - const alreadyVendored = - process.platform === "win32" && + // Completeness probe, not a mere existence probe. The compositor addon now + // links six shared ffmpeg DLLs — see crates/compositor/build.rs + // (avcodec, avformat, avutil, swresample, swscale, avfilter). A warm dev/CI + // tree that already holds the five pre-avfilter DLLs would satisfy an "any + // av*.dll is present" check and let `avfilter-11.dll` go un-vendored, breaking + // require() at runtime (OpenScreen#371 review, EtienneLescot). Require all + // six explicitly so a missing one forces a re-vendor. + const REQUIRED_SHARED_DLLS = [ + "avcodec", + "avformat", + "avutil", + "swresample", + "swscale", + "avfilter", + ]; + fs.mkdirSync(binDir, { recursive: true }); + const vendoredFiles = new Set( fs .readdirSync(binDir, { withFileTypes: true }) - .some((e) => e.isFile() && isSharedLib(e.name) && /^(lib)?av/i.test(e.name)); + .filter((e) => e.isFile()) + .map((e) => e.name), + ); + const alreadyVendored = + process.platform === "win32" && + REQUIRED_SHARED_DLLS.every((lib) => + [...vendoredFiles].some((f) => new RegExp(`^${lib}-\\d+\\.dll$`).test(f)), + ); // The build-time SDK comes out of this same archive, so a tree that has the // DLLs but not the SDK must still re-download — otherwise we skip here and // the compositor build fails afterwards on the missing FFMPEG_DIR. diff --git a/scripts/ffmpeg-linked-libraries.test.mjs b/scripts/ffmpeg-linked-libraries.test.mjs new file mode 100644 index 00000000..49e0bd0d --- /dev/null +++ b/scripts/ffmpeg-linked-libraries.test.mjs @@ -0,0 +1,111 @@ +// `crates/compositor/build.rs` decides which ffmpeg shared libraries the native addon +// imports. Six places have to agree with that list, and none of them is derived from it: +// +// - scripts/fetch-ffmpeg.mjs vendors the Windows DLLs and decides when to skip +// - scripts/before-pack.cjs fails the pack if one is missing (three OS tables) +// - scripts/build-linux-compositor-addon.mjs copies + symbol-renames the sonames +// - nix/compositor-view.nix builds symbols.map from a brace glob +// +// Drift is not a cosmetic problem. The addon is a cdylib, so a missing library does not +// fail the link: it fails at `require()` with "undefined symbol: osff_avfilter_graph_alloc", +// `compositorViewService` logs "native addon not present; running as no-op", and the app +// ships with a blank preview and every export dead — the exact symptom 1.9.0 shipped with. +// Every guard listed above passes in that state, because each one only knows the list it +// was written with. +// +// This test derives the truth from build.rs and checks the other five against it, so +// linking a seventh library fails here instead of in a user's installer. It reads source +// text: before-pack.cjs and fetch-ffmpeg.mjs both do work at import time, and the property +// under test is a property of the literal lists anyway. +// +// The nix derivation is the reason this file exists rather than an assertion inside +// before-pack.test.mjs: `.github/workflows/nix-build.yml` does not run on pull requests +// and there is no nix on the Windows dev box, so nix/compositor-view.nix reaches main with +// no pre-merge signal at all. A text assertion is not `nix build`, but it does catch the +// one mistake that has actually happened. + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +const repoRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); +const read = (relative) => fs.readFileSync(path.join(repoRoot, relative), "utf8"); + +const buildRs = read("crates/compositor/build.rs"); + +/** The `for lib in [...] { println!("cargo:rustc-link-lib=…") }` list, in build.rs order. */ +const linked = (() => { + const block = buildRs.match(/for lib in \[([\s\S]*?)\]\s*\{[\s\S]*?rustc-link-lib/); + if (!block) return []; + return [...block[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); +})(); + +describe("the ffmpeg libraries the compositor links", () => { + // Without this the regex could silently match nothing and every assertion below + // would iterate an empty list and pass vacuously. + it("is read out of build.rs", () => { + expect(linked.length).toBeGreaterThanOrEqual(6); + expect(linked).toContain("avcodec"); + expect(linked).toContain("avfilter"); + }); + + it("is vendored in full by fetch-ffmpeg.mjs", () => { + // The probe that decides whether a warm tree still needs a re-vendor. When it + // listed fewer libraries than build.rs links, a tree holding the previous set + // satisfied it and the new DLL was never fetched. + const source = read("scripts/fetch-ffmpeg.mjs"); + const table = source.match(/REQUIRED_SHARED_DLLS = \[([\s\S]*?)\]/); + expect(table, "fetch-ffmpeg.mjs no longer declares REQUIRED_SHARED_DLLS").not.toBeNull(); + const required = [...table[1].matchAll(/"([a-z]+)"/g)].map((match) => match[1]); + expect([...required].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed for the Linux addon", () => { + const source = read("scripts/build-linux-compositor-addon.mjs"); + const table = source.match(/FFMPEG_SONAMES = \[([\s\S]*?)\]/); + expect( + table, + "build-linux-compositor-addon.mjs no longer declares FFMPEG_SONAMES", + ).not.toBeNull(); + const sonames = [...table[1].matchAll(/"lib([a-z]+)\.so\.\d+"/g)].map((match) => match[1]); + expect([...sonames].sort()).toEqual([...linked].sort()); + }); + + it("is staged and symbol-renamed by the nix derivation", () => { + // `for lib in ${ffmpegLgpl.lib}/lib/lib{avformat,…}.so.*` — the brace glob feeds + // both the copy into $stage/lib and the symbols.map the addon is linked against. + // A name missing here links against nixpkgs' un-renamed copy, and the + // installPhase leak check only flags symbols WITHOUT the osff_ prefix, so it + // passes either way. + const source = read("nix/compositor-view.nix"); + const glob = source.match(/\/lib\/lib\{([a-z,]+)\}\.so\.\*/); + expect(glob, "nix/compositor-view.nix no longer globs the ffmpeg sonames").not.toBeNull(); + expect(glob[1].split(",").sort()).toEqual([...linked].sort()); + }); + + describe("is required by before-pack.cjs", () => { + const source = read("scripts/before-pack.cjs"); + /** The `[...]` array literal a `...[…].map(` spread iterates, per OS table. */ + const listsIn = (table) => { + const block = source.slice(source.indexOf(`const ${table} = [`)); + const end = block.indexOf("\n];"); + return [...block.slice(0, end).matchAll(/\.\.\.\[([^\]]*)\]\.map\(/g)].flatMap((match) => + [...match[1].matchAll(/"([a-z]+)"/g)].map((name) => name[1]), + ); + }; + + // One requirement per library rather than `atLeast: N` over a combined regex: + // several versioned copies of one library would satisfy a count while another + // was missing entirely, which is how a broken pack passed the guard before. + for (const table of ["MAC_REQUIRED", "LINUX_REQUIRED", "WIN_REQUIRED"]) { + it(table, () => { + const required = listsIn(table); + expect(required.length, `${table} declares no per-library spread`).toBeGreaterThan(0); + for (const library of linked) { + expect(required, `${table} does not require lib${library}`).toContain(library); + } + }); + } + }); +}); diff --git a/technical-documentation/architecture/export-pipeline.md b/technical-documentation/architecture/export-pipeline.md index 0068fdf4..fbd223f9 100644 --- a/technical-documentation/architecture/export-pipeline.md +++ b/technical-documentation/architecture/export-pipeline.md @@ -61,9 +61,9 @@ and **one** encoder + muxer pair: per-segment rounded frame counts into a single output frame counter; audio follows the same integer accumulation (`AudioConcatPlan`). -- **Audio and video junctions are seamless.** Audio is decoded per - segment up front (`audio.rs::decode_clip_audio`), WSOLA stretches each - speed sub-segment to its output sample count, and +- **Audio and video junctions are seamless.** Audio is decoded per clip + (`audio.rs::decode_clip_audio`), a libavfilter `atempo` chain stretches + each speed sub-segment to its output sample count, and `assemble_concatenated_pcm` concatenates the per-segment PCM at the integer sample offsets the video loop just produced — never `round(cumulativeSec * sampleRate)`, because that compounds per-segment @@ -71,8 +71,26 @@ and **one** encoder + muxer pair: timeline. A short equal-power fade (`cos` on the tail, `sin` on the head, `cos² + sin² = 1`) covers each internal boundary to suppress the click where two recordings meet butt-joined, without shifting timing. - The WSOLA stretch is kicked off before the video loop so it overlaps - the encode and does not add to the wall. + The in-tree WSOLA stretcher is still there, but only as the fallback + `stretch_pcm_to_length` takes when the filter chain cannot be built, + negotiates a format the drain does not read, or still comes up short of + the target after the corrected pass (see + [Audio](native-compositor.md#audio)). + +- **The stretch is not overlapped with the encode.** Decode and stretch + run inside `walk_composited_timeline`'s `on_clip_end` callback + (`pipeline.rs`), which fires once per clip *after* that clip's frames + have been composed and submitted to the encoder, on the same thread — + so the stretch time is added to the export wall, not hidden behind it. + `progress()` counts composed frames as they are handed to the encoder, + and nothing calls it during the audio phase, so a long clip parks the + export at whatever percentage its last frame reported. (The encoder's + own flush comes later still, after the timeline walk.) That reporting + gap is why the `atempo` path matters so much here: WSOLA is + O(grain × radius) per rendered sample, which on a long clip meant + minutes of an apparently frozen export. Reporting progress across the + audio phase would fix the symptom rather than the cost, and is tracked + separately. - **Output** honours the timeline's selected aspect ratio (`resolveAspectRatioValue` over `getEditorSettings(document).aspectRatio` — diff --git a/technical-documentation/architecture/native-compositor.md b/technical-documentation/architecture/native-compositor.md index f2b01c5c..0239cb3a 100644 --- a/technical-documentation/architecture/native-compositor.md +++ b/technical-documentation/architecture/native-compositor.md @@ -31,7 +31,7 @@ and without that flag the runtime corruption is silent. | [`crates/compositor/src/scene.rs`](../../crates/compositor/src/scene.rs) | the `Scene` struct parsed from the app's `SceneDescription` JSON | | [`crates/compositor/src/regions.rs`](../../crates/compositor/src/regions.rs) | zoom / speed / Full Camera regions — envelope shapes and per-frame state sampling | | [`crates/compositor/src/pipeline.rs`](../../crates/compositor/src/pipeline_windows.rs) | demux + `D3D11VA` decode + composite + AMF encode + mux (`run_c0` for decode/encode only, `run_composited` for the full path) | -| [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, WSOLA speed stretch, multi-track mix, AAC encoder | +| [`crates/compositor/src/audio.rs`](../../crates/compositor/src/audio.rs) | per-clip audio decode, swresample → f32 planar 48 kHz stereo, libavfilter `atempo` speed stretch (WSOLA fallback), multi-track mix, AAC encoder | | [`crates/compositor/src/cursor.rs`](../../crates/compositor/src/cursor.rs) | `.cursor.json` parser + interpolated cursor track (position, click bounces, adaptive follow samples) | | [`crates/compositor/src/text.rs`](../../crates/compositor/src/text_windows.rs) | DirectWrite + Direct2D text rasterisation for annotation labels, cached per (content, style, box) | | [`crates/compositor/src/text_anim.rs`](../../crates/compositor/src/text_anim.rs) | text-annotation appearance animations (port of the TS animation curves, in fractions of the output short side) | @@ -199,9 +199,54 @@ each track is recut to the same `[source_start_sec, source_end_sec)` — pads in front for late-starting tracks, trims the pre-roll for early-decoded ones — so a summing mixer is enough and a real mix matrix is not needed. -Speed regions apply after decode: WSOLA stretches each speed sub-segment to -its output frame count, sharing search positions across channels from a -mono down-mix (so the stereo image does not wander between channels). +Speed regions apply after decode: `stretch_pcm_to_length` stretches each +speed sub-segment to its output frame count through a libavfilter +`abuffer → atempo… → abuffersink` graph built in-process +(`avfilter_atempo_stretch`). `atempo` only accepts a factor in +`[0.5, 100.0]`, so `atempo_factors` chains several instances whose product +is the requested speed (0.2 → `[0.5, 0.5, 0.8]`). + +Three details of that graph are load-bearing: + +- **The chain is pinned to flt — interleaved, not planar.** `af_atempo` + advertises packed formats only (`U8/S16/S32/FLT/DBL`), so an `abuffer` + pinned to the `fltp` that `decode_clip_audio` produces makes the + negotiation insert an `aresample` and forces every output frame through + a conversion. Asking for `flt` on both ends leaves no conversion filter + in the graph at all; the interleaving is absorbed by the copy into and + out of the frames, which happens either way. + +- **The drain is interleaved with the feed.** `av_buffersrc_add_frame` + does not pull the graph, so pushing a whole region before the first + `av_buffersink_get_frame` would queue all of it in the buffersrc — half + a gigabyte on a 20-minute stereo region, on top of the input slice and + the output accumulator. + +- **Two passes, because `atempo` does not render exactly `n / tempo` + samples.** It falls short by a fixed amount per chain, independent of + input length — measured on the pinned n8.1.2 build, ~217 samples for one + stage and ~2 700 for four, i.e. up to 56 ms at 0.1×. The shortfall is a + difference in rendered duration, not a held-back tail: pushing more + input does not recover it. Filling it with zeros would leave a hard + silence gap butt-joined to the next segment, since the equal-power + crossfade covers clip boundaries only, never the per-segment + concatenation. So the first pass measures the shortfall on the real + content without keeping anything, and the second asks for + `target + shortfall`, which lands the content on the target exactly; the + surplus is truncated. Above 1× the shortfall is zero and the second pass + is skipped. + +The in-tree WSOLA stretcher remains as the fallback, taken whenever +`avfilter_atempo_stretch` returns `None`: the graph could not be built or +configured, a buffersrc/buffersink call failed, the sink negotiated a +format the drain does not read, or the corrected pass still came up short. +Every one of those paths logs its reason — a sudden multi-minute export +should not be silent about which branch it took. WSOLA shares its search +positions across channels from a mono down-mix, so the stereo image does +not wander between them. The move to `atempo` is a cost change, not a +quality one: on a 5-minute region, `atempo` takes 0.6 s at 1.25× and 4.9 s +at 0.25× (two passes) against WSOLA's 20 s and 55 s. + Across segments, `build_audio_concat_plan` sizes each segment's PCM by **integer accumulation of the per-segment rounded sample count**, never `round(cumulativeSec * sampleRate)` — that single change is what keeps A/V diff --git a/technical-documentation/engineering/build-and-packaging.md b/technical-documentation/engineering/build-and-packaging.md index 29bbda55..bbb1d408 100644 --- a/technical-documentation/engineering/build-and-packaging.md +++ b/technical-documentation/engineering/build-and-packaging.md @@ -42,7 +42,7 @@ Electron-builder copies only the matching `electron/native/bin/- This is a hard requirement on Windows, not a tidiness preference. -The addon dlopens `avcodec`/`avformat`/`avutil` at `require()` time. Until 1.9.0 the Windows build shipped it inside `app.asar.unpacked/electron/native/compositor-view/build/`, one directory away from `electron/native/bin/win32-x64/*.dll`, and the gap was bridged at runtime by `ensureFfmpegSharedDllsOnPath` prepending the DLL directory to `PATH` before the require. +The addon pulls in six ffmpeg libraries at `require()` time — `avcodec`, `avformat`, `avutil`, `swresample`, `swscale` and `avfilter`, the exact set `crates/compositor/build.rs` emits `cargo:rustc-link-lib` lines for. (`avfilter` is the newest of them: the speed-region time stretch runs through its `atempo` filter.) Until 1.9.0 the Windows build shipped it inside `app.asar.unpacked/electron/native/compositor-view/build/`, one directory away from `electron/native/bin/win32-x64/*.dll`, and the gap was bridged at runtime by `ensureFfmpegSharedDllsOnPath` prepending the DLL directory to `PATH` before the require. That works for the NSIS installer. **It does not work under MSIX**, which resolves an addon's dependent DLLs through the package graph and ignores `PATH`. Measured inside a registered package, with the directory verifiably present and correctly prepended to `PATH`: @@ -60,7 +60,7 @@ require BEFORE PATH : LOADED OK Node loads `.node` files with `LOAD_WITH_ALTERED_SEARCH_PATH`, so the addon's own directory is searched for its dependencies. Colocating removes the `PATH` mechanism rather than repairing it, and works on every Windows packaging format. -This shipped: the 1.9.0 Store build loaded no compositor at all, so the editor opened with a permanently blank preview while audio kept playing — audio comes from the renderer, every frame comes from the addon. It read as an application bug rather than a packaging one, because every file was present in the package and the NSIS build of the same commit was fine. `scripts/before-pack.cjs` now refuses to package unless the addon and at least `avcodec`/`avformat`/`avutil` are in the same directory, on Windows as it already did on macOS. +This shipped: the 1.9.0 Store build loaded no compositor at all, so the editor opened with a permanently blank preview while audio kept playing — audio comes from the renderer, every frame comes from the addon. It read as an application bug rather than a packaging one, because every file was present in the package and the NSIS build of the same commit was fine. `scripts/before-pack.cjs` now refuses to package unless the addon and all six of those libraries are in the same directory, on Windows as it already did on macOS. It checks one requirement **per library** rather than a count over a combined pattern, on all three platforms: several versioned copies of one library (an `avcodec-60`/`61`/`62.dll` left by an earlier fetch) would satisfy a combined count while another was missing entirely, and the addon would still fail to load. `electron/native/bin/`, local native build directories, the compositor build output, models, and caches are gitignored. Rebuilding from a source checkout therefore requires the complete platform toolchain and third-party SDKs; running the generic `npm run build` alone does not manufacture missing native artifacts. The Windows compositor's D3D11/FFmpeg prerequisites are described by the source POC in `crates/README.md`, while capture helper lookup and output conventions are documented in `electron/native/README.md`. @@ -204,7 +204,7 @@ The hook now reads `electron/native/bin/darwin-/` — the directory `mac.e | Required | Without it | |---|---| | `compositor_view.node` | preview and every export render nothing | -| `libavcodec/libavformat/libavutil.*.dylib` | the addon cannot load at all (dyld error at `require()`) | +| `libavcodec/libavformat/libavutil/libavfilter/libswresample/libswscale.*.dylib` | the addon cannot load at all (dyld error at `require()`) | | `whisper-stt-server` | transcription and captions fail with a developer error shown to end users | | `libggml*.dylib` | the helper dies in dyld before `main()`; STT times out with no diagnostic | | `openscreen-screencapturekit-helper` | native screen capture unavailable |