diff --git a/crates/compositor/src/audio_jobs.rs b/crates/compositor/src/audio_jobs.rs new file mode 100644 index 000000000..8d97be40c --- /dev/null +++ b/crates/compositor/src/audio_jobs.rs @@ -0,0 +1,240 @@ +//! Décodage et étirement de l'audio d'un clip, en parallèle du parcours vidéo. +//! +//! Les trois pipelines faisaient ce travail **dans** le callback `on_clip_end` de +//! `walk_composited_timeline`, donc sur le thread de rendu et entre deux clips. Rien +//! n'appelle `progress()` pendant ce temps : la barre d'export s'arrêtait sur le +//! pourcentage de la dernière frame du clip et y restait pour toute la durée du décodage +//! et de l'étirement. C'est la moitié « reporting » du « figé à ~80 % » — la moitié +//! « coût » a été traitée par le passage à atempo, mais un clip long, un repli WSOLA ou +//! n'importe quelle étape audio future reproduisent le symptôme à l'identique. +//! +//! Y répondre en publiant une progression pendant cette phase aurait demandé de changer le +//! protocole natif → JS (il ne transporte qu'un compteur de frames absolu) et de répartir +//! un total que les deux côtés calculent séparément. Déplacer le travail est plus simple et +//! strictement meilleur : l'audio d'un clip ne dépend que de ce clip, il n'y a donc aucune +//! raison qu'il occupe le thread qui compose les frames du clip suivant. Le parcours vidéo +//! continue de rapporter sa progression sans interruption, et le temps audio disparaît du +//! mur d'export au lieu d'y être seulement mieux affiché — ce que +//! `export-pipeline.md` prétendait déjà. +//! +//! Chaque job ouvre son propre `AVFormatContext` sur le fichier du clip : libavformat +//! n'a pas d'état partagé entre contextes, et le décodeur vidéo du parcours en a un autre +//! sur le même chemin, en lecture seule lui aussi. + +use crate::audio::{decode_clip_audio, stretch_clip_pcm_by_speed, PlanarPcm}; +use crate::regions::SpeedSegment; +use std::collections::VecDeque; +use std::thread::JoinHandle; + +/// Nombre de jobs audio en vol. +/// +/// Un thread par clip serait sans plafond : une timeline de deux cents clips décoderait +/// deux cents pistes à la fois, chacune avec son contexte ffmpeg et son PCM complet en +/// mémoire. Quatre suffisent à couvrir le décodage d'un clip par le rendu du suivant, qui +/// est tout ce qu'on cherche ici. +const MAX_INFLIGHT_AUDIO_JOBS: usize = 4; + +/// Le corps d'un job : décode la fenêtre gardée du clip et l'étire sur ses spans de vitesse. +/// +/// Rend `None` quand le clip se déclare audio mais n'a pas de flux décodable, ou quand le +/// décodage échoue — dans les deux cas l'export continue et le clip sort muet, comme avant +/// que ce travail passe sur un thread. Les deux messages sont les mêmes qu'alors ; ils +/// sortent seulement d'un autre thread. +pub fn decode_and_stretch_clip_audio( + clip_index: usize, + screen_path: &str, + source_start_sec: f64, + source_end_sec: f64, + speed_segments: &[SpeedSegment], + out_fps: f64, +) -> Option { + match decode_clip_audio(screen_path, source_start_sec, source_end_sec) { + Ok(Some(pcm)) => Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps)), + Ok(None) => { + eprintln!( + "[pipeline] warning: clip #{clip_index} déclaré audio mais sans flux décodable; silence conservé" + ); + None + } + Err(error) => { + eprintln!( + "[pipeline] warning: décodage audio du clip #{clip_index} échoué ({error:#}); silence conservé" + ); + None + } + } +} + +/// Collecte les résultats de jobs indexés lancés au fil du parcours, en bornant le nombre +/// de threads simultanés. +/// +/// L'ordre de restitution est celui des index, pas celui d'achèvement : `into_results` rend +/// un `Vec` de la taille annoncée où chaque case porte le résultat de son clip. +pub struct ClipAudioJobs { + inflight: VecDeque<(usize, JoinHandle)>, + results: Vec>, +} + +impl ClipAudioJobs { + pub fn new(clip_count: usize) -> Self { + Self { + inflight: VecDeque::new(), + results: (0..clip_count).map(|_| None).collect(), + } + } + + /// Lance `job` pour `clip_index`. Si le plafond est atteint, attend d'abord le plus + /// ancien job en vol — celui qui a eu le plus de temps pour finir. + pub fn spawn(&mut self, clip_index: usize, job: impl FnOnce() -> T + Send + 'static) { + while self.inflight.len() >= MAX_INFLIGHT_AUDIO_JOBS { + self.collect_oldest(); + } + self.inflight + .push_back((clip_index, std::thread::spawn(job))); + } + + /// Attend tous les jobs restants et rend les résultats rangés par index de clip. + pub fn into_results(mut self) -> Vec> { + while !self.inflight.is_empty() { + self.collect_oldest(); + } + // `mem::take` et pas un move : le `Drop` ci-dessous interdit de sortir un champ de + // `self`. Il ne trouvera plus rien à joindre, la file étant vide. + std::mem::take(&mut self.results) + } + + fn collect_oldest(&mut self) { + let Some((clip_index, handle)) = self.inflight.pop_front() else { + return; + }; + match handle.join() { + Ok(value) => { + if let Some(slot) = self.results.get_mut(clip_index) { + *slot = Some(value); + } + } + // Un panic dans un job audio ne doit pas emporter l'export : le clip sort + // muet, comme il le faisait déjà quand `decode_clip_audio` échouait. + Err(_) => eprintln!( + "[pipeline] warning: le job audio du clip #{clip_index} a paniqué; silence conservé" + ), + } + } +} + +/// Un `JoinHandle` droppé **détache** son thread. Entre le premier `spawn` et +/// `into_results` il y a des `?` — le parcours lui-même, le flush de l'encodeur — et sur +/// l'un d'eux la collection partait en fumée en laissant jusqu'à quatre décodages en vol +/// dans un addon natif que l'hôte peut décharger. On joint donc à la destruction : rien ne +/// survit à la portée, chemin d'erreur compris. +/// +/// Ce n'est pas une annulation : `decode_clip_audio` est un appel opaque et long, et +/// l'interrompre demanderait de lui passer un `AVIOInterruptCB` — un autre changement, dans +/// un autre fichier. L'attente est bornée par le plus lent des quatre, soit quelques +/// secondes depuis que le stretch passe par atempo, et elle ne coûte que sur un export qui +/// a déjà échoué. +impl Drop for ClipAudioJobs { + fn drop(&mut self) { + for (clip_index, handle) in std::mem::take(&mut self.inflight) { + if handle.join().is_err() { + eprintln!( + "[pipeline] warning: le job audio du clip #{clip_index} a paniqué pendant l'abandon de l'export" + ); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[test] + fn results_are_indexed_by_clip_not_by_completion_order() { + // Le premier job est le plus lent : si on rangeait par ordre d'achèvement, le PCM + // du clip 0 atterrirait sur le clip 2 et l'export monterait l'audio dans le + // désordre sans rien signaler. + let mut jobs = ClipAudioJobs::new(3); + jobs.spawn(0, || { + std::thread::sleep(std::time::Duration::from_millis(60)); + "zero" + }); + jobs.spawn(1, || "one"); + jobs.spawn(2, || "two"); + assert_eq!( + jobs.into_results(), + vec![Some("zero"), Some("one"), Some("two")] + ); + } + + #[test] + fn a_clip_without_a_job_keeps_its_empty_slot() { + // Les clips sans audio ne lancent rien ; leur case doit rester `None` pour que + // `assemble_concatenated_pcm` y mette du silence. + let mut jobs = ClipAudioJobs::new(3); + jobs.spawn(1, || 7u32); + assert_eq!(jobs.into_results(), vec![None, Some(7), None]); + } + + #[test] + fn never_more_than_the_cap_run_at_once() { + // Sans plafond, une timeline longue ouvrirait un contexte ffmpeg et un PCM complet + // par clip, tous en même temps. + let live = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let mut jobs = ClipAudioJobs::new(32); + for index in 0..32 { + let live = Arc::clone(&live); + let peak = Arc::clone(&peak); + jobs.spawn(index, move || { + let now = live.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(std::time::Duration::from_millis(5)); + live.fetch_sub(1, Ordering::SeqCst); + index + }); + } + let results = jobs.into_results(); + assert_eq!(results.len(), 32); + assert!(results.iter().enumerate().all(|(i, r)| *r == Some(i))); + assert!( + peak.load(Ordering::SeqCst) <= MAX_INFLIGHT_AUDIO_JOBS, + "jusqu'à {} jobs simultanés pour un plafond de {MAX_INFLIGHT_AUDIO_JOBS}", + peak.load(Ordering::SeqCst) + ); + } + + #[test] + fn dropping_the_collection_joins_its_jobs_instead_of_detaching_them() { + // Le chemin d'erreur : entre le premier `spawn` et `into_results` il y a des `?`. + // Sans le `Drop`, jusqu'à quatre décodages continuaient dans le vide après l'abandon + // de l'export, dans un addon que l'hôte peut décharger. + let finished = Arc::new(AtomicUsize::new(0)); + { + let mut jobs = ClipAudioJobs::new(4); + for index in 0..4 { + let finished = Arc::clone(&finished); + jobs.spawn(index, move || { + std::thread::sleep(std::time::Duration::from_millis(20)); + finished.fetch_add(1, Ordering::SeqCst); + }); + } + // Pas d'`into_results` : on abandonne, comme le ferait un `?`. + } + assert_eq!( + finished.load(Ordering::SeqCst), + 4, + "des jobs tournaient encore après la destruction de la collection" + ); + } + + #[test] + fn a_panicking_job_leaves_its_clip_silent_without_taking_the_export_down() { + let mut jobs = ClipAudioJobs::new(2); + jobs.spawn(0, || panic!("décodage impossible")); + jobs.spawn(1, || 42u32); + assert_eq!(jobs.into_results(), vec![None, Some(42)]); + } +} diff --git a/crates/compositor/src/lib.rs b/crates/compositor/src/lib.rs index 90dee8155..64a9c44af 100644 --- a/crates/compositor/src/lib.rs +++ b/crates/compositor/src/lib.rs @@ -28,6 +28,7 @@ //! — c'est précisément ce qui rend le port Metal possible (cf. PR #162). pub mod audio; +pub mod audio_jobs; pub mod config; pub mod cursor; pub mod ffi; diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 910738fc0..235a32917 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -21,9 +21,10 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::config::Cfg; use crate::d3d::Gpu; use crate::ffi::AVFrame; @@ -453,7 +454,7 @@ pub fn run_composited_multi( // Un PCM par clip, assemble apres la marche video (elle seule dit combien de // frames chaque clip a produit, donc combien d'audio lui revient). - let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let mut clip_frame_counts: Vec = vec![0; clips.len()]; let scene = comp.scene_snapshot(); @@ -497,18 +498,24 @@ pub fn run_composited_multi( clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = - Some(stretch_clip_pcm_by_speed(&pcm, speed_segments, out_fps as f64)); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{clip_index} declare audio mais sans flux decodable; silence", - ), - Err(error) => eprintln!( - "[pipeline] warning: decodage audio clip #{clip_index} echoue ({error:#}); silence", - ), - } + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa + // durée — rien n'appelle `progress()` entre deux clips. Le travail part + // sur un thread et se recouvre avec la composition du clip suivant ; les + // résultats sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( + clip_index, + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, @@ -532,6 +539,16 @@ pub fn run_composited_multi( drain_encoder(ectx, octx, ostream, opkt)?; // Audio : le plan part des frames REELLEMENT produites par clip (un clip // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre + // avant d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés + // par le plus lent, pas par leur somme ; les autres se sont recouverts avec + // l'encodage vidéo. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 10de3fac2..c7cf571bc 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,9 +30,10 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::compositor::Compositor; use crate::d3d::Gpu; use crate::timeline_walk::NextFrameTime; @@ -1064,7 +1065,7 @@ pub fn run_composited_multi( } // Un PCM par clip, assemblé après la marche vidéo : c'est elle qui dit combien de // frames chaque clip a réellement produit, donc combien d'audio lui revient. - let mut clip_pcm: Vec> = (0..clips.len()).map(|_| None).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let mut clip_frame_counts: Vec = vec![0; clips.len()]; let mut opkt = unsafe { crate::ffi::av_packet_alloc() }; @@ -1097,21 +1098,24 @@ pub fn run_composited_multi( clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( - &pcm, - speed_segments, - out_fps as f64, - )); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{clip_index} déclaré audio mais sans flux décodable; silence conservé", - ), - Err(error) => eprintln!( - "[pipeline] warning: décodage audio du clip #{clip_index} échoué ({error:#}); silence conservé", - ), - } + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa + // durée — rien n'appelle `progress()` entre deux clips. Le travail part + // sur un thread et se recouvre avec la composition du clip suivant ; les + // résultats sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( + clip_index, + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, @@ -1129,6 +1133,16 @@ pub fn run_composited_multi( // Le plan part des frames RÉELLEMENT produites par clip, pas des durées demandées : // un clip raccourci (source plus courte que sa borne) doit voir son audio raccourci // d'autant, sinon la piste dérive pour tous les suivants. + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre + // avant d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés + // par le plus lent, pas par leur somme ; les autres se sont recouverts avec + // l'encodage vidéo. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 11738bcd4..5fff82056 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,9 +3,10 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + assemble_concatenated_pcm, build_audio_concat_plan, finish_audio, + AacEncoder, PlanarPcm, }; +use crate::audio_jobs::{decode_and_stretch_clip_audio, ClipAudioJobs}; use crate::compositor::{Compositor, OUT_H, OUT_W}; use crate::config::Cfg; use crate::cpu_frames::CpuFrames; @@ -1390,8 +1391,7 @@ unsafe fn run_multi_inner( let opkt = av_packet_alloc(); let mut clip_frame_counts = vec![0u64; clips.len()]; - let mut clip_pcm: Vec> = - std::iter::repeat_with(|| None).take(clips.len()).collect(); + let mut audio_jobs: ClipAudioJobs> = ClipAudioJobs::new(clips.len()); let t0 = Instant::now(); let frames = walk_composited_timeline( @@ -1431,23 +1431,24 @@ unsafe fn run_multi_inner( clip_frame_counts[clip_index] = frames_in_clip; let clip = &clips[clip_index]; if clip.has_audio && frames_in_clip > 0 { - match decode_clip_audio(&clip.screen, clip.source_start_sec, source_end_sec) { - Ok(Some(pcm)) => { - clip_pcm[clip_index] = Some(stretch_clip_pcm_by_speed( - &pcm, - speed_segments, - out_fps as f64, - )); - } - Ok(None) => eprintln!( - "[pipeline] warning: clip #{} déclaré audio mais sans flux décodable; silence conservé", - clip_index, - ), - Err(error) => eprintln!( - "[pipeline] warning: décodage audio du clip #{} échoué ({error:#}); silence conservé", + // L'audio d'un clip ne dépend que de ce clip : le décoder et l'étirer ici, + // sur le thread de rendu, immobilisait la barre d'export pour toute sa durée + // — rien n'appelle `progress()` entre deux clips. Le travail part sur un + // thread et se recouvre avec la composition du clip suivant ; les résultats + // sont récupérés après le parcours, rangés par index de clip. + let path = clip.screen.clone(); + let source_start_sec = clip.source_start_sec; + let segments = speed_segments.to_vec(); + audio_jobs.spawn(clip_index, move || { + decode_and_stretch_clip_audio( clip_index, - ), - } + &path, + source_start_sec, + source_end_sec, + &segments, + out_fps as f64, + ) + }); } Ok(()) }, @@ -1460,6 +1461,15 @@ unsafe fn run_multi_inner( enc.send(ptr::null_mut())?; drain_encoder(ectx, octx, ostream, opkt)?; + // Récupération des jobs audio lancés pendant le parcours. `spawn` en admet quatre avant + // d'en collecter un, donc il en reste au plus quatre à attendre ici — bornés par le plus + // lent, pas par leur somme ; tous les autres se sont recouverts avec l'encodage. + let clip_pcm: Vec> = audio_jobs + .into_results() + .into_iter() + .map(|slot| slot.flatten()) + .collect(); + let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let audio_plan = build_audio_concat_plan( &clip_frame_counts, diff --git a/crates/compositor/src/regions.rs b/crates/compositor/src/regions.rs index 000939708..d60c3b14d 100644 --- a/crates/compositor/src/regions.rs +++ b/crates/compositor/src/regions.rs @@ -1021,3 +1021,63 @@ mod tilt_tests { } } } + +#[cfg(test)] +mod exporter_frame_totals { + use super::*; + use crate::scene::SceneSpeedRegion; + + fn region(start_sec: f64, end_sec: f64, speed: f64) -> SceneSpeedRegion { + SceneSpeedRegion { clip_index: None, start_sec, end_sec, speed } + } + + fn frames(start_sec: f64, end_sec: f64, regions: &[SceneSpeedRegion], fps: f64) -> u64 { + speed_segments_for_window(regions, start_sec, end_sec, fps) + .iter() + .map(|segment| segment.frame_count) + .sum() + } + + /// Le total que la barre d'export doit viser, mesuré sur ce que `walk_composited_timeline` + /// itère réellement. + /// + /// Le jumeau de ce test est `src/lib/exporter/outputFrameCount.test.ts`, avec la MÊME + /// table de chiffres. Le natif n'envoie qu'un compteur de frames brut ; le total et donc + /// le pourcentage sont calculés côté TS, et rien ne reliait les deux calculs. Résultat + /// livré : le total TS ignorait les speed regions, donc un clip entièrement en 1,25× + /// rendait 80 % des frames annoncées et la barre s'arrêtait à 80 % — le « figé à ~80 % » + /// d'OpenScreen#371, au chiffre près. Toucher un côté doit faire rougir l'autre. + #[test] + fn speed_segments_match_the_exporter_frame_totals() { + const FPS: f64 = 30.0; + assert_eq!(frames(0.0, 10.0, &[], FPS), 300, "sans région"); + assert_eq!( + frames(0.0, 10.0, &[region(0.0, 10.0, 1.25)], FPS), + 240, + "1,25× : 80 % de 300, exactement le symptôme" + ); + assert_eq!(frames(0.0, 10.0, &[region(0.0, 10.0, 0.5)], FPS), 600, "0,5×"); + assert_eq!( + frames(0.0, 10.0, &[region(2.0, 4.0, 2.0)], FPS), + 60 + 30 + 180, + "couverture partielle" + ); + assert_eq!( + frames(1.0, 5.0, &[region(0.0, 100.0, 2.0)], FPS), + 60, + "région débordant la fenêtre gardée" + ); + assert_eq!( + frames(0.0, 10.0, &[region(2.0, 6.0, 2.0), region(4.0, 8.0, 4.0)], FPS), + 60 + 60 + 15 + 60, + "recouvrement : la première région garde la portion déjà couverte" + ); + assert_eq!( + frames(0.0, 10.0, &[region(0.0, 10.0, 0.0)], FPS), + 300, + "vitesse non positive traitée comme 1×" + ); + assert_eq!(frames(4.0, 4.0, &[], FPS), 0, "fenêtre vide"); + assert_eq!(frames(0.0, 10.0, &[], 0.0), 0, "fps non positif"); + } +} diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 83cf31aaa..9a7f1b590 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -30,6 +30,7 @@ import { buildAutoZoomSuggestions } from "@/lib/ai-edition/timeline/zoom-suggest import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; +import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native"; import type { CompositorClipInput } from "@/native/contracts"; @@ -277,13 +278,9 @@ async function runExport(request: CliExportRequest): Promise { // Progress: native pushes raw encoded-frame counts; totals and pacing are // computed here, mirroring the ExportDialog. const outFps = format === "gif" ? gifFrameRate : MP4_EXPORT_FPS; - const totalFrames = Math.max( - 1, - Math.round( - clips.reduce((sum, clip) => sum + Math.max(0, clip.sourceEndSec - clip.sourceStartSec), 0) * - outFps, - ), - ); + // Speed-adjusted, not source seconds — see `outputFrameCount`. Counting raw duration + // is what made a 1.25x timeline stop the bar at 80% (OpenScreen#371). + const totalFrames = outputFrameCount(clips, sceneDesc.speedRegions, outFps); const exportStartedAt = Date.now(); const unsubscribeProgress = window.electronAPI.onNativeExportProgress?.((frames: number) => { const elapsedSec = (Date.now() - exportStartedAt) / 1000; diff --git a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx index 835d9c8fd..c5b45637c 100644 --- a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx +++ b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx @@ -17,7 +17,9 @@ vi.mock("@/native", () => ({ })); vi.mock("@/native/sceneDescription", () => ({ - buildSceneDescription: () => ({}), + // `speedRegions` is what the export dialog reads to size the progress total + // (`outputFrameCount`); an empty object here made it read `undefined`. + buildSceneDescription: () => ({ speedRegions: [] }), resolveVisibleClips: (doc: AxcutDocument) => doc.timeline.clips, })); diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index fbc4d9362..5dff50acf 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -33,6 +33,7 @@ import { type GifSizePreset, } from "@/lib/exporter"; import { calculateMp4ExportSettings, wouldUpscale } from "@/lib/exporter/mp4ExportSettings"; +import { outputFrameCount } from "@/lib/exporter/outputFrameCount"; import { exportGifNative, exportMultiNative, useIsCpuCompositor } from "@/native"; import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; @@ -287,15 +288,17 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { const clips = buildNativeClipList(document); // GIF runs at its own frame rate, so the progress total has to use it. const outFps = format === "gif" ? gifFrameRate : fps; - // Total frames the encoder will produce, known upfront from the timeline (sum of - // each clip's trimmed source duration) — the native side only reports frames - // AFTER encoding one (onNativeExportProgress), it doesn't know/send a total, so - // this is computed here to turn that raw count into a percentage. - const totalDurationSec = clips.reduce( - (sum, c) => sum + Math.max(0, c.sourceEndSec - c.sourceStartSec), - 0, - ); - const totalFrames = Math.max(1, Math.round(totalDurationSec * outFps)); + // Total frames the encoder will produce, known upfront from the timeline — the + // native side only reports frames AFTER composing one (onNativeExportProgress), + // it doesn't know or send a total, so this is computed here to turn that raw + // count into a percentage. + // + // It has to count SPEED-ADJUSTED frames, not source seconds: a clip under a 1.25x + // region emits 80% of `duration * fps`, which is where the "frozen at ~80%" of + // OpenScreen#371 came from — the bar climbed to 80% and the export finished + // there. `outputFrameCount` mirrors the compositor's own span arithmetic. + const sceneDesc = buildSceneDescription(document); + const totalFrames = outputFrameCount(clips, sceneDesc.speedRegions, outFps); const startedAt = Date.now(); const unsubscribeProgress = window.electronAPI?.onNativeExportProgress?.((frames) => { const elapsedS = (Date.now() - startedAt) / 1000; @@ -309,8 +312,6 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { }); }); try { - const sceneDesc = buildSceneDescription(document); - // The webcam background effect is applied by the compositor from the scene, // so the clip list needs no pre-rendering pass. const exportClips = clips; diff --git a/src/lib/exporter/outputFrameCount.test.ts b/src/lib/exporter/outputFrameCount.test.ts new file mode 100644 index 000000000..55a969ac6 --- /dev/null +++ b/src/lib/exporter/outputFrameCount.test.ts @@ -0,0 +1,110 @@ +// The numbers below are the contract with the Rust exporter, not a snapshot of this file's +// arithmetic. `speed_segments_match_the_exporter_frame_totals` in +// `crates/compositor/src/regions.rs` asserts the SAME table against +// `speed_segments_for_window`, which is what `walk_composited_timeline` iterates. Change one +// side and the other goes red — which is the point: a silent divergence here is a progress +// bar that lies, and it has already shipped once (OpenScreen#371, the "frozen at ~80%"). + +import { describe, expect, it } from "vitest"; +import { clipOutputFrameCount, outputFrameCount } from "./outputFrameCount"; + +const FPS = 30; + +describe("outputFrameCount", () => { + it("counts a clip with no speed region at its plain duration", () => { + expect(clipOutputFrameCount({ sourceStartSec: 0, sourceEndSec: 10 }, [], FPS)).toBe(300); + }); + + it("is the whole bug: a 1.25x clip emits 80% of the frames its duration suggests", () => { + // 10 s at 30 fps looks like 300 frames and renders 240. The old total was the former, + // so the bar stopped at exactly 80% and the export finished there. + const naive = Math.round(10 * FPS); + const real = clipOutputFrameCount( + { sourceStartSec: 0, sourceEndSec: 10 }, + [{ startSec: 0, endSec: 10, speed: 1.25 }], + FPS, + ); + expect(real).toBe(240); + expect(real / naive).toBeCloseTo(0.8, 5); + }); + + it("counts a slow-motion clip above its duration", () => { + // The other half of the same bug: a 0.5x region pins the bar at 100% for the second + // half of the export instead of stopping short. + expect( + clipOutputFrameCount( + { sourceStartSec: 0, sourceEndSec: 10 }, + [{ startSec: 0, endSec: 10, speed: 0.5 }], + FPS, + ), + ).toBe(600); + }); + + it("splits a partially covered clip into 1x and sped spans", () => { + expect( + clipOutputFrameCount( + { sourceStartSec: 0, sourceEndSec: 10 }, + [{ startSec: 2, endSec: 4, speed: 2 }], + FPS, + ), + ).toBe(60 + 30 + 180); + }); + + it("clamps a region that runs past the trimmed window", () => { + expect( + clipOutputFrameCount( + { sourceStartSec: 1, sourceEndSec: 5 }, + [{ startSec: 0, endSec: 100, speed: 2 }], + FPS, + ), + ).toBe(60); + }); + + it("never renders the same source time twice when two regions overlap", () => { + // A stale payload can overlap; the first region keeps the covered portion, matching + // `speed_segments_for_window`'s cursor. + expect( + clipOutputFrameCount( + { sourceStartSec: 0, sourceEndSec: 10 }, + [ + { startSec: 2, endSec: 6, speed: 2 }, + { startSec: 4, endSec: 8, speed: 4 }, + ], + FPS, + ), + ).toBe(60 + 60 + 15 + 60); + }); + + it("ignores a region belonging to another clip", () => { + const clips = [ + { sourceStartSec: 0, sourceEndSec: 10 }, + { sourceStartSec: 0, sourceEndSec: 10 }, + ]; + const regions = [{ startSec: 0, endSec: 10, speed: 2, clipIndex: 1 }]; + expect(outputFrameCount(clips, regions, FPS)).toBe(300 + 150); + }); + + it("applies a region with no clipIndex to every clip", () => { + const clips = [ + { sourceStartSec: 0, sourceEndSec: 10 }, + { sourceStartSec: 0, sourceEndSec: 10 }, + ]; + expect(outputFrameCount(clips, [{ startSec: 0, endSec: 10, speed: 2 }], FPS)).toBe(150 + 150); + }); + + it("treats a non-positive speed as 1x rather than dividing by it", () => { + expect( + clipOutputFrameCount( + { sourceStartSec: 0, sourceEndSec: 10 }, + [{ startSec: 0, endSec: 10, speed: 0 }], + FPS, + ), + ).toBe(300); + }); + + it("never returns a total the callers would divide by zero", () => { + expect(outputFrameCount([], [], FPS)).toBe(1); + expect(outputFrameCount([{ sourceStartSec: 4, sourceEndSec: 4 }], [], FPS)).toBe(1); + expect(outputFrameCount([{ sourceStartSec: 0, sourceEndSec: 10 }], [], 0)).toBe(1); + }); +}); diff --git a/src/lib/exporter/outputFrameCount.ts b/src/lib/exporter/outputFrameCount.ts new file mode 100644 index 000000000..71424ba08 --- /dev/null +++ b/src/lib/exporter/outputFrameCount.ts @@ -0,0 +1,115 @@ +// How many frames the native exporter will actually emit for a timeline. +// +// The progress bar needs a total, and the native side does not send one — it reports a raw +// running count of composed frames and nothing else (see `throttled_progress` in +// `crates/compositor-view-napi/src/lib.rs`). Both callers used to compute that total as +// `sum(sourceEndSec - sourceStartSec) * fps`, which ignores speed regions entirely. +// +// That is the "frozen at ~80%" in OpenScreen#371, quite literally. A clip covered by a 1.25x +// speed region emits `duration * fps / 1.25` frames — exactly 80% of what that formula +// predicts — so the bar climbed to 80%, stopped, and the export finished there. The audio +// phase stalling on the render thread made it read as a hang, but even with instant audio +// the bar would never have reached 100%. A 0.5x region overshoots the other way and the bar +// pins at 100% for the second half of the export. +// +// This mirrors `speed_segments_for_window` + `push_speed_segment` +// (`crates/compositor/src/regions.rs`), which is what `walk_composited_timeline` iterates to +// decide how many frames a clip produces. The two must agree; `outputFrameCount.test.ts` and +// the Rust `speed_segments_match_the_exporter_frame_totals` test share one fixture table so +// a change on either side shows up as a failure rather than as a drifting progress bar. +// +// One difference is unavoidable: the walk clamps each clip's end to the source's real +// duration, which only a decoder can know. A source shorter than its declared window makes +// this count slightly high — the same limitation the old formula had, and the same one that +// already makes `build_audio_concat_plan` work off produced frames rather than declared ones. + +/** `SPEED_FRAME_EPSILON_SEC` in `crates/compositor/src/regions.rs`. Absorbs the float error + * of a span boundary so a segment does not gain a frame it never renders. */ +const SPEED_FRAME_EPSILON_SEC = 0.001; + +/** `MIN_SPEED_SEGMENT_SEC` in the same file: a span thinner than this emits nothing. */ +const MIN_SPEED_SEGMENT_SEC = 0.0001; + +/** The trimmed source window of one clip, as `buildNativeClipList` produces it. */ +export interface OutputFrameClip { + sourceStartSec: number; + sourceEndSec: number; +} + +/** A speed region already projected onto source time — `SceneDescription.speedRegions`. */ +export interface OutputFrameSpeedRegion { + startSec: number; + endSec: number; + speed: number; + /** Absent means "every clip", matching `Scene::for_clip_window`. */ + clipIndex?: number; +} + +/** Frames one contiguous span renders. Port of `push_speed_segment`. */ +function segmentFrames(startSec: number, endSec: number, speed: number, fps: number): number { + const duration = endSec - startSec; + if (duration <= MIN_SPEED_SEGMENT_SEC) { + return 0; + } + return Math.max(0, Math.ceil(((duration - SPEED_FRAME_EPSILON_SEC) / speed) * fps)); +} + +/** Frames one clip renders across its speed spans. Port of `speed_segments_for_window`, + * including its overlap rule: regions are walked in start order and a later one never + * reclaims source time an earlier one already covered. */ +export function clipOutputFrameCount( + clip: OutputFrameClip, + regions: OutputFrameSpeedRegion[], + fps: number, +): number { + const { sourceStartSec, sourceEndSec } = clip; + if (!(sourceEndSec > sourceStartSec) || !Number.isFinite(fps) || fps <= 0) { + return 0; + } + const overlapping = regions + .filter((region) => region.startSec < sourceEndSec && region.endSec > sourceStartSec) + .sort((a, b) => a.startSec - b.startSec); + + let frames = 0; + let cursor = sourceStartSec; + for (const region of overlapping) { + const start = Math.max(region.startSec, sourceStartSec, cursor); + const end = Math.min(region.endSec, sourceEndSec); + if (start > cursor) { + frames += segmentFrames(cursor, start, 1, fps); + } + if (end > start) { + const speed = Number.isFinite(region.speed) && region.speed > 0 ? region.speed : 1; + frames += segmentFrames(start, end, speed, fps); + cursor = end; + } + } + if (cursor < sourceEndSec) { + frames += segmentFrames(cursor, sourceEndSec, 1, fps); + } + return frames; +} + +/** Frames the whole export will emit, for turning the native running count into a + * percentage. Never returns 0, so the callers' division stays safe, and tolerates a missing + * region list — this is progress-bar arithmetic and must not be able to abort an export. */ +export function outputFrameCount( + clips: OutputFrameClip[], + speedRegions: OutputFrameSpeedRegion[] | undefined, + fps: number, +): number { + const regions = speedRegions ?? []; + const total = clips.reduce( + (sum, clip, clipIndex) => + sum + + clipOutputFrameCount( + clip, + regions.filter( + (region) => region.clipIndex === undefined || region.clipIndex === clipIndex, + ), + fps, + ), + 0, + ); + return Math.max(1, total); +} diff --git a/technical-documentation/architecture/export-pipeline.md b/technical-documentation/architecture/export-pipeline.md index fbd223f9f..f5550a1eb 100644 --- a/technical-documentation/architecture/export-pipeline.md +++ b/technical-documentation/architecture/export-pipeline.md @@ -77,20 +77,38 @@ and **one** encoder + muxer pair: the target after the corrected pass (see [Audio](native-compositor.md#audio)). -- **The stretch is not overlapped with the encode.** Decode and stretch - run inside `walk_composited_timeline`'s `on_clip_end` callback - (`pipeline.rs`), which fires once per clip *after* that clip's frames - have been composed and submitted to the encoder, on the same thread — - so the stretch time is added to the export wall, not hidden behind it. - `progress()` counts composed frames as they are handed to the encoder, - and nothing calls it during the audio phase, so a long clip parks the - export at whatever percentage its last frame reported. (The encoder's - own flush comes later still, after the timeline walk.) That reporting - gap is why the `atempo` path matters so much here: WSOLA is - O(grain × radius) per rendered sample, which on a long clip meant - minutes of an apparently frozen export. Reporting progress across the - audio phase would fix the symptom rather than the cost, and is tracked - separately. +- **The stretch runs beside the encode, not inside it.** + `walk_composited_timeline`'s `on_clip_end` callback fires once per clip, + after that clip's frames have been composed and submitted to the + encoder. It used to decode and stretch that clip's audio right there, + on the render thread; since a clip's audio depends on nothing but that + clip, it now hands the work to `ClipAudioJobs` + ([`audio_jobs.rs`](../../crates/compositor/src/audio_jobs.rs), at most + four in flight) and the walk carries straight on to the next clip. The + results are collected after the walk, indexed by clip. `spawn` admits + four before it collects one, so what is left to wait for at the end is up + to four jobs — bounded by the slowest of them, not by their sum. Dropping + the collection joins them rather than detaching, so an export that fails + between the walk and the collection does not leave decoders running. + + This matters for reporting as much as for wall time. `progress()` + counts composed frames as they are handed to the encoder and nothing + calls it during the audio phase, so while that work sat on the render + thread the bar parked at whatever percentage the clip's last frame + reported — for minutes, back when WSOLA was O(grain × radius) per + rendered sample. That is the reporting half of "frozen at ~80%"; the + cost half was the move to `atempo`. + +- **The progress total is speed-adjusted.** The native side reports a raw + running count of composed frames and never a total, so the percentage + is computed in the renderer + ([`outputFrameCount`](../../src/lib/exporter/outputFrameCount.ts)). It + has to mirror `speed_segments_for_window`, because a clip under a 1.25× + region emits `duration × fps / 1.25` frames: counting source seconds + instead made the bar stop at exactly 80% and the export finish there — + the number in the title of the bug. The two sides share one fixture + table, asserted by `outputFrameCount.test.ts` and by + `speed_segments_match_the_exporter_frame_totals`. - **Output** honours the timeline's selected aspect ratio (`resolveAspectRatioValue` over `getEditorSettings(document).aspectRatio` —