diff --git a/CHANGELOG.md b/CHANGELOG.md index 0454a7f73..806a83dfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ the public-API contract. the live edge, so the rewind buffer discarded each video keyframe as older than its window, recovery seeked about once a second and the clock barely ran. `LiveAudioAxis` now moves audio more than 30 s from the video onto the video's axis, aligned on arrival. `LiveAudioAxisTests` covers the wrap, seams and a still image over sound. +- **A short hole in a live source's timestamps no longer freezes the picture on the software path's rewind route.** Only + jumps of 10 s or more were folded out of the timeline. A shorter hole, which a feed that pauses and skips what it + missed leaves behind, played out with a standing picture while the clock ran and the phase read playing. A forward + hole of 3 s or more that both streams have now folds (`LiveHoleFold`); a sparse picture over continuous sound is not + one. `LiveHoleFoldTests` covers both orders the streams can reach a hole in. ## [7.56.0] - 2026-11-03 diff --git a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift index b3393f95c..c0c37a428 100644 --- a/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift +++ b/Sources/AetherEngine/Native/SoftwarePlaybackHost.swift @@ -3171,7 +3171,8 @@ final class SoftwarePlaybackHost { var prevRawVideoPtsSec = Double.nan var frameIntervalSec = 0.0 var discontinuityOffsetSec = 0.0 - var loggedSWDiscontinuity = false + var loggedSWDiscontinuities = 0 + var holeFold = LiveHoleFold() var audioAxis = LiveAudioAxis() var lastSeenExtradata: Data? = nil let replayReader = LiveReplayPacketReader(videoIndex: videoStreamIndex, audioIndex: audioStreamIndex, @@ -3271,12 +3272,17 @@ final class SoftwarePlaybackHost { let rawPtsSec = Double(packet.pointee.pts) * videoTimeBaseSeconds if !prevRawVideoPtsSec.isNaN { let deltaSec = rawPtsSec - prevRawVideoPtsSec - if abs(deltaSec) >= discontinuityThresholdSeconds { + if abs(deltaSec) >= discontinuityThresholdSeconds + || holeFold.opensHole(videoSeconds: rawPtsSec) { let expectedContinuation = prevRawVideoPtsSec + (frameIntervalSec > 0 ? frameIntervalSec : 0) + if deltaSec > 0 { + holeFold.noteFold(fromVideoSeconds: prevRawVideoPtsSec, toVideoSeconds: rawPtsSec, + foldBefore: discontinuityOffsetSec) + } discontinuityOffsetSec += (rawPtsSec - expectedContinuation) - if !loggedSWDiscontinuity { - loggedSWDiscontinuity = true + if loggedSWDiscontinuities < 20 { + loggedSWDiscontinuities += 1 EngineLog.emit( "[SWHost] live PTS discontinuity (reader): prevPts=" + "\(String(format: "%.2f", prevRawVideoPtsSec))s " @@ -3292,14 +3298,27 @@ final class SoftwarePlaybackHost { } } prevRawVideoPtsSec = rawPtsSec + holeFold.noteVideo(seconds: rawPtsSec) + } + var packetFoldSeconds = discontinuityOffsetSec + if streamIdx == audioStreamIndex, audioTimeBaseSeconds > 0, packet.pointee.pts != Int64.min { + switch holeFold.audio(seconds: Double(packet.pointee.pts) * audioTimeBaseSeconds, + currentFoldSeconds: discontinuityOffsetSec) { + case .drop: + av_packet_unref(packet) + av_packet_free_safe(packet) + return true + case .play(let foldSeconds): + packetFoldSeconds = foldSeconds + } } - if discontinuityOffsetSec != 0 { + if packetFoldSeconds != 0 { let tbSec = (streamIdx == videoStreamIndex) ? videoTimeBaseSeconds : audioTimeBaseSeconds if tbSec > 0 { // Audit NAT-101: the offset spans twice the input range and converts per stream // base, so a packet on the far side of a seam can still leave Int64. - let offsetTicks = SourceTimestampBounds.roundedTicks(discontinuityOffsetSec / tbSec) + let offsetTicks = SourceTimestampBounds.roundedTicks(packetFoldSeconds / tbSec) packet.pointee.pts = SourceTimestampBounds.shifted(packet.pointee.pts, back: offsetTicks) packet.pointee.dts = SourceTimestampBounds.shifted(packet.pointee.dts, back: offsetTicks) } diff --git a/Sources/AetherEngine/Video/LiveHoleFold.swift b/Sources/AetherEngine/Video/LiveHoleFold.swift new file mode 100644 index 000000000..9becc82ec --- /dev/null +++ b/Sources/AetherEngine/Video/LiveHoleFold.swift @@ -0,0 +1,66 @@ +import Foundation + +/// A live source that lost a few seconds of its feed comes back with a hole in its timestamps. Played +/// out, the hole is a standing picture with the clock running through it, so a hole folds out of the +/// timeline like any larger jump. Audio decides what is a hole: under a sparse picture (a still image +/// over a radio channel) the video's stamps jump while the sound carries on. +struct LiveHoleFold { + enum AudioDecision: Equatable { + /// Plays, this many seconds earlier than stamped. + case play(foldSeconds: Double) + case drop + } + + static let forwardHoleSeconds = 3.0 + /// Audio past a hole the video stays short of (about 2 s of AAC) has a hole of its own. + static let audioPacketsHeldAtMost = 100 + + private var lastVideoSeconds = Double.nan + private var lastAudioSeconds = Double.nan + private var isHoldingAudio = false + private var audioPacketsHeld = 0 + private var seam: (middleSeconds: Double, foldBefore: Double)? + + /// Whether this video packet opened a hole both streams have in them. + func opensHole(videoSeconds: Double) -> Bool { + guard !lastVideoSeconds.isNaN, !lastAudioSeconds.isNaN else { return false } + return videoSeconds - lastVideoSeconds >= Self.forwardHoleSeconds + && videoSeconds - lastAudioSeconds >= Self.forwardHoleSeconds + } + + mutating func noteVideo(seconds: Double) { + lastVideoSeconds = seconds + } + + /// The video crossed a forward jump: audio still short of it keeps the fold it was stamped under. + /// The middle of the jump tells them apart, since audio past it may stamp just below the video. + mutating func noteFold(fromVideoSeconds: Double, toVideoSeconds: Double, foldBefore: Double) { + seam = ((fromVideoSeconds + toVideoSeconds) / 2, foldBefore) + } + + mutating func audio(seconds: Double, currentFoldSeconds: Double) -> AudioDecision { + if let seam { + if seconds < seam.middleSeconds { + lastAudioSeconds = seconds + return .play(foldSeconds: seam.foldBefore) + } + self.seam = nil + } + // Audio over a hole the video has yet to reach waits for the fold. + if isHoldingAudio { + if seconds - lastVideoSeconds >= Self.forwardHoleSeconds, audioPacketsHeld < Self.audioPacketsHeldAtMost { + audioPacketsHeld += 1 + return .drop + } + isHoldingAudio = false + } else if !lastAudioSeconds.isNaN, !lastVideoSeconds.isNaN, + seconds - lastAudioSeconds >= Self.forwardHoleSeconds, + seconds - lastVideoSeconds >= Self.forwardHoleSeconds { + isHoldingAudio = true + audioPacketsHeld = 1 + return .drop + } + lastAudioSeconds = seconds + return .play(foldSeconds: currentFoldSeconds) + } +} diff --git a/Tests/AetherEngineTests/LiveHoleFoldTests.swift b/Tests/AetherEngineTests/LiveHoleFoldTests.swift new file mode 100644 index 000000000..58b6a088a --- /dev/null +++ b/Tests/AetherEngineTests/LiveHoleFoldTests.swift @@ -0,0 +1,62 @@ +import Testing +@testable import AetherEngine + +/// A live feed that lost a few seconds keeps playing across the hole in its stamps, in whichever order +/// the two streams reach it; a sparse picture over continuous sound is no hole. +@Suite("Live hole fold") +struct LiveHoleFoldTests { + private func running(video: Double = 100, audio: Double = 100) -> LiveHoleFold { + var fold = LiveHoleFold() + fold.noteVideo(seconds: video) + _ = fold.audio(seconds: audio, currentFoldSeconds: 0) + return fold + } + + @Test("a hole in both streams opens at the video packet past it") + func holeInBoth() { + let fold = running() + #expect(fold.opensHole(videoSeconds: 107.76)) + #expect(!fold.opensHole(videoSeconds: 102.9), "below the threshold") + } + + @Test("video that jumps under continuous sound is no hole") + func sparsePicture() { + var fold = running() + for tick in 1...300 { + #expect(fold.audio(seconds: 100 + Double(tick) * 0.02, currentFoldSeconds: 0) == .play(foldSeconds: 0)) + } + #expect(!fold.opensHole(videoSeconds: 106)) + } + + @Test("audio over the hole first waits for the video, then plays folded") + func audioCrossesFirst() { + var fold = running() + #expect(fold.audio(seconds: 107.8, currentFoldSeconds: 0) == .drop) + #expect(fold.audio(seconds: 107.82, currentFoldSeconds: 0) == .drop) + #expect(fold.opensHole(videoSeconds: 107.76), "held audio does not count as having crossed") + fold.noteFold(fromVideoSeconds: 100, toVideoSeconds: 107.76, foldBefore: 0) + fold.noteVideo(seconds: 107.76) + #expect(fold.audio(seconds: 107.84, currentFoldSeconds: 7.72) == .play(foldSeconds: 7.72)) + } + + @Test("audio still short of the hole keeps the fold before it") + func videoCrossesFirst() { + var fold = running() + fold.noteFold(fromVideoSeconds: 100, toVideoSeconds: 107.76, foldBefore: 2) + fold.noteVideo(seconds: 107.76) + #expect(fold.audio(seconds: 100.02, currentFoldSeconds: 9.72) == .play(foldSeconds: 2)) + #expect(fold.audio(seconds: 107.7, currentFoldSeconds: 9.72) == .play(foldSeconds: 9.72), + "audio past the hole, just below the video, folds with it") + } + + @Test("audio with a hole of its own is held only briefly") + func audioOnlyHole() { + var fold = running() + var audio = 106.0 + for _ in 0..