fix(audio): keep APAC priming so TrueHD Atmos stays in sync with the video

CHANGELOG.md

@@ -25,6 +25,7 @@ the public-API contract.
 
 ### Fixed
 
+- TrueHD Atmos rendered to APAC (`LoadOptions.objectAudioRendering`) no longer plays 42.7 ms ahead of the video. The bridge dropped the encoder's 2048 frames of priming and stamped the first content packet on the source position, but AVFoundation presents an APAC packet's audio 2048 frames before its timestamp, so every session ran early, at load and after every seek. The priming packets now stay in the stream and take the source position's timestamp.
 - Authorized native HLS uses the engine relay from the initial load, without forwarding origin credentials to the loopback asset. Optional subtitle playlist preparation shares the authorizer and has a bounded deadline across redirects and refreshes.
 - Static-header HLS redirects apply the shared credential policy, including Emby and MediaBrowser token headers, before contacting another origin.
 - Session option corrections recognize `httpRequestAuthorization` and external subtitle provider replacements by identity.

Sources/AetherEngine/Audio/Spatial/APACEncoder.swift

@@ -13,7 +13,9 @@ import Foundation
 ///
 /// Facts about the encoder this relies on, all measured on macOS 27 and the tvOS 27 simulator:
 /// - packets are 1024 frames;
-/// - the encoder primes 2048 frames (`primeInfo.leadingFrames`), which the caller trims;
+/// - the encoder primes 2048 frames (`primeInfo.leadingFrames`). AVFoundation removes them on
+///   playback by presenting each packet's audio 2048 frames before its timestamp, so the caller
+///   keeps the priming packets and stamps them from the anchor;
 /// - with the converter's defaults every packet is independently decodable (an Audio Sync Packet),
 ///   which is what lets any fMP4 fragment, and so any HLS segment, start on one (HLS authoring 7.9);
 /// - the magic cookie IS the complete `dapa` box the sample entry carries;

Sources/AetherEngine/Audio/Spatial/SpatialAudioBridge.swift

@@ -59,7 +59,8 @@ final class SpatialAudioBridge: AudioTranscodingBridge, @unchecked Sendable {
     /// `originFrame`: the source PTS of the first packet pushed since the last reset.
     /// `encoderStartFrame`: where the first frame handed to the encoder since the reset sits.
     /// `encoderFramesIn`: frames handed to the encoder since the reset; `packetsOut`: packets it
-    /// has returned. Packet k covers frames [k·1024 − leadingFrames, +1024) of encoder input.
+    /// has returned. Packet k covers frames [k·1024 − leadingFrames, +1024) of encoder input and is
+    /// stamped `encoderStartFrame + k·1024`; see `emit`.
     private var originFrame: Int64?
     private var encoderStartFrame: Int64?
     private var encoderFramesIn: Int64 = 0
@@ -286,18 +287,21 @@ final class SpatialAudioBridge: AudioTranscodingBridge, @unchecked Sendable {
         emit(packets, into: &results)
     }
 
-    /// Stamp encoder packets onto the output timeline and wrap them for the muxer. The encoder's
-    /// priming packets precede the first input frame; they carry no content and would sit before
-    /// the anchor (below zero at the head of the file), so they are dropped. Every APAC packet the
-    /// encoder produces is independently decodable, so the first kept packet decodes on its own.
+    /// Stamp encoder packets onto the output timeline and wrap them for the muxer. Packet k is
+    /// stamped `encoderStartFrame + k·1024`, the encoder's priming packets included, because
+    /// AVFoundation presents an APAC packet's audio `leadingFrames` (2048) before its timestamp: it
+    /// counts the priming as part of the stream, the way AVAssetWriter writes it. So the priming
+    /// packets take the anchor's timestamp, their output falls before it and is never heard, and the
+    /// first content frame plays on the source position. Dropping them and stamping the first
+    /// content packet on the anchor played every session 2048 frames (42.7 ms) ahead of the video,
+    /// at load and after every seek (AVPlayer over HLS on macOS 27, with 5.1 APAC since macOS rejects
+    /// 7.1.4, against an ALAC reference; AVAssetReader shows the same 2048 frames for 7.1.4).
     private func emit(_ packets: [APACEncoder.Packet], into results: inout [UnsafeMutablePointer<AVPacket>]) {
         let frames = Int64(APACEncoder.framesPerPacket)
-        let leading = Int64(encoder.leadingFrames)
         for packet in packets {
             let k = packetsOut
             packetsOut += 1
-            let contentStart = k * frames - leading
-            guard contentStart + frames > 0, let start = encoderStartFrame else { continue }
+            guard let start = encoderStartFrame else { continue }
             guard let avpkt = trackedPacketAlloc() else { continue }
             guard av_new_packet(avpkt, Int32(packet.data.count)) >= 0 else {
                 var p: UnsafeMutablePointer<AVPacket>? = avpkt
@@ -307,7 +311,7 @@ final class SpatialAudioBridge: AudioTranscodingBridge, @unchecked Sendable {
             packet.data.withUnsafeBytes { raw in
                 avpkt.pointee.data.update(from: raw.bindMemory(to: UInt8.self).baseAddress!, count: packet.data.count)
             }
-            let pts = start + max(contentStart, 0)
+            let pts = start + k * frames
             avpkt.pointee.pts = pts
             avpkt.pointee.dts = pts
             avpkt.pointee.duration = frames

Tests/AetherEngineTests/SpatialAudioBridgeTests.swift

@@ -1,5 +1,6 @@
 import Testing
 import Foundation
+import AVFoundation
 import AetherLibavcodec
 import AetherLibavformat
 import AetherLibavutil
@@ -8,8 +9,10 @@ import AetherLibavutil
 /// The spatial bridge turns TrueHD Atmos into APAC for the fMP4 muxer. What it must get right
 /// beyond the audio itself is time: its packets land in segments by PTS, so the first packet after
 /// a load or a seek has to sit where the source says, however much the decoder skipped to find a
-/// major sync, and nothing after it may drift. These drive the bridge with a synthetic decoder
-/// that behaves like the real one on exactly those points.
+/// major sync, and nothing after it may drift. Packets are stamped the way AVFoundation reads APAC,
+/// with the encoder's 2048 frames of priming counted in, so the first packet takes the anchor and
+/// the first content frame plays there. These drive the bridge with a synthetic decoder that
+/// behaves like the real one on exactly those points.
 @Suite("TrueHD Atmos spatial bridge")
 struct SpatialAudioBridgeTests {
 
@@ -72,6 +75,41 @@ struct SpatialAudioBridgeTests {
         func reset() { skipped = 0; inputFrames = 0; blocks = []; pushes = 0 }
     }
 
+    /// Stand-in decoder with something to find: one pushed byte is one 40-frame access unit,
+    /// contiguous from the first push, and the left bed channel carries a 3 ms 1 kHz burst
+    /// starting at input frame `clickFrame`.
+    final class ClickDecoder: ObjectAudioDecoding {
+        let clickFrame: Int64
+        private var inputFrames: Int64 = 0
+        private var blocks: [(offset: Int64, frames: Int)] = []
+        private var stateSent = false
+        private let plane = UnsafeMutablePointer<Float>.allocate(capacity: 1 << 16)
+
+        init(clickFrame: Int64) { self.clickFrame = clickFrame }
+        deinit { plane.deallocate() }
+
+        func push(_ bytes: UnsafeRawBufferPointer) throws {
+            blocks.append((inputFrames, bytes.count * 40))
+            inputFrames += Int64(bytes.count * 40)
+        }
+
+        func nextBlock() throws -> ObjectAudioDecodedBlock? {
+            guard !blocks.isEmpty else { return nil }
+            let b = blocks.removeFirst()
+            for i in 0..<b.frames {
+                let d = b.offset + Int64(i) - clickFrame
+                plane[i] = (0..<144).contains(d) ? 0.8 * sinf(2 * .pi * 1000 * Float(d) / 48_000) : 0
+            }
+            defer { stateSent = true }
+            return ObjectAudioDecodedBlock(
+                sampleRate: 48_000, frameCount: b.frames, inputFrameOffset: b.offset,
+                configurationGeneration: 0, roles: [.bed(.left)], planes: [UnsafePointer(plane)],
+                updates: stateSent ? [] : [.init(frameOffset: 0, rampFrames: 0, states: [.init()])])
+        }
+
+        func reset() { inputFrames = 0; blocks = []; stateSent = false }
+    }
+
     /// Feed `count` 20 ms packets (24 access units) starting at `startMs` on a 1/1000 time base.
     private func feed(
         _ bridge: some AudioTranscodingBridge, startMs: Int64, count: Int
@@ -121,13 +159,15 @@ struct SpatialAudioBridgeTests {
             defer { var p: UnsafeMutablePointer<AVPacket>? = fp; trackedPacketFree(&p) }
             return (fp.pointee.pts, fp.pointee.flags & AV_PKT_FLAG_KEY != 0, fp.pointee.size)
         }
-        // 1000 ms = 48000 frames, plus five skipped 40-frame access units.
+        // 1000 ms = 48000 frames, plus five skipped 40-frame access units. The first packet is the
+        // encoder's priming, which AVFoundation presents before its timestamp, so the first content
+        // frame plays on the anchor.
         #expect(packets.first?.pts == Int64(48_200))
         #expect(zip(packets, packets.dropFirst()).allSatisfy { $1.pts - $0.pts == 1024 })
         #expect(packets.allSatisfy { $0.key && $0.size > 0 })
-        // 96000 frames in, 200 of them skipped; the encoder's priming packets are dropped and the
-        // flush pads the last partial packet.
-        #expect(packets.count == Int((Double(96_000 - 200) / 1024).rounded(.up)))
+        // 96000 frames in, 200 of them skipped, after two packets of priming; the flush pads the
+        // last partial packet.
+        #expect(packets.count == 2 + Int((Double(96_000 - 200) / 1024).rounded(.up)))
         #expect(bridge.feedStats.packetsEmitted == packets.count)
     }
 
@@ -159,19 +199,19 @@ struct SpatialAudioBridgeTests {
         }
         #expect(packets.first?.pts == Int64(0))
         #expect(zip(packets, packets.dropFirst()).allSatisfy { $1.pts - $0.pts == 1024 })
-        // 100 packets x 960 frames in, the dropped 480 included as silence: the tail reaches the end.
-        let end = packets.last!.pts + 1024
+        // 100 packets x 960 frames in, the dropped 480 included as silence: the tail, presented
+        // 2048 frames before its timestamp, reaches the end.
+        let end = packets.last!.pts + 1024 - 2048
         #expect(end >= 96_000 && end < 96_000 + 1024)
     }
 
-    @Test("movenc muxes the stand-in and the init segment carries the real apac entry")
-    func muxedInitCarriesAPAC() throws {
-        guard #available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) else { return }
-        let bridge = try SpatialAudioBridge(
-            srcTimeBase: AVRational(num: 1, den: 1000), layout: .l714,
-            decoder: SyntheticDecoder(skipAfterReset: 0))
-        defer { bridge.close() }
-
+    /// Mux `count` 20 ms source packets from `startMs` through the bridge, next to the probe video
+    /// movenc needs, and cut one segment. Returns the init segment, the segment file and the
+    /// session directory it sits in (the caller removes it).
+    @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *)
+    private func muxWithProbeVideo(
+        _ bridge: SpatialAudioBridge, startMs: Int64, count: Int
+    ) throws -> (initBytes: Data, segment: URL, sessionDir: URL)? {
         let videoDemuxer = Demuxer()
         defer { videoDemuxer.close() }
         try videoDemuxer.open(
@@ -181,7 +221,6 @@ struct SpatialAudioBridgeTests {
         let vStream = try #require(videoDemuxer.stream(at: videoDemuxer.videoStreamIndex))
         let sessionDir = FileManager.default.temporaryDirectory.appendingPathComponent("spatial-\(UUID().uuidString)")
         try FileManager.default.createDirectory(at: sessionDir, withIntermediateDirectories: true)
-        defer { try? FileManager.default.removeItem(at: sessionDir) }
 
         var initBytes: Data?
         let muxer = try MP4SegmentMuxer(
@@ -201,27 +240,40 @@ struct SpatialAudioBridgeTests {
             _ = muxer.writePacket(pkt)
         }
         var payload = [UInt8](repeating: 0, count: 24)
-        for i in 0..<50 {
+        for i in 0..<count {
             let pkt = av_packet_alloc()!
             defer { var p: UnsafeMutablePointer<AVPacket>? = pkt; av_packet_free(&p) }
             payload.withUnsafeMutableBytes { raw in
                 _ = av_new_packet(pkt, 24)
                 pkt.pointee.data.update(from: raw.bindMemory(to: UInt8.self).baseAddress!, count: 24)
             }
-            pkt.pointee.pts = Int64(i) * 20
-            for fp in try bridge.feed(packet: pkt) + (i == 49 ? bridge.flush() : []) {
+            pkt.pointee.pts = startMs + Int64(i) * 20
+            for fp in try bridge.feed(packet: pkt) + (i == count - 1 ? bridge.flush() : []) {
                 var p: UnsafeMutablePointer<AVPacket>? = fp
                 defer { trackedPacketFree(&p) }
                 fp.pointee.stream_index = muxer.audioOutputStreamIndex
                 av_packet_rescale_ts(fp, bridge.encoderTimeBase, muxer.muxerAudioTimeBase)
                 #expect(muxer.writePacket(fp).rc >= 0)
             }
         }
-        guard case .completed = muxer.cutFragmentForNextSegment(1) else {
+        guard case .completed(let segment, _) = muxer.cutFragmentForNextSegment(1) else {
             Issue.record("the stand-in needs no parsed packet, so the cut completes")
-            return
+            return nil
         }
-        let bytes = try #require(initBytes)
+        return (try #require(initBytes), segment, sessionDir)
+    }
+
+    @Test("movenc muxes the stand-in and the init segment carries the real apac entry")
+    func muxedInitCarriesAPAC() throws {
+        guard #available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) else { return }
+        let bridge = try SpatialAudioBridge(
+            srcTimeBase: AVRational(num: 1, den: 1000), layout: .l714,
+            decoder: SyntheticDecoder(skipAfterReset: 0))
+        defer { bridge.close() }
+        guard let muxed = try muxWithProbeVideo(bridge, startMs: 0, count: 50) else { return }
+        let sessionDir = muxed.sessionDir
+        defer { try? FileManager.default.removeItem(at: sessionDir) }
+        let bytes = muxed.initBytes
         let hex = bytes.map { String(format: "%02x", $0) }.joined()
         #expect(hex.contains("61706163"), "apac sample entry present")
         #expect(hex.contains("64617061"), "dapa configuration present")
@@ -236,4 +288,53 @@ struct SpatialAudioBridgeTests {
             for file in seg { try? FileManager.default.copyItem(at: file, to: dir.appendingPathComponent(file.lastPathComponent)) }
         }
     }
+
+    /// The end-to-end form of the timestamp contract: a click at a known source position is
+    /// rendered, encoded, muxed and decoded back by AVFoundation, which presents APAC the way
+    /// AVPlayer does, and has to come out where the source put it. With the encoder's priming
+    /// packets dropped it came out 2048 frames (42.7 ms) early, and so did every session's audio.
+    @Test("a click decodes back at its source position through AVFoundation")
+    func clickDecodesOnItsSourcePosition() async throws {
+        guard #available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) else { return }
+        // The source starts at 0 and the click is 12000 frames in, so it belongs at 0.25 s.
+        let bridge = try SpatialAudioBridge(
+            srcTimeBase: AVRational(num: 1, den: 1000), layout: .l714,
+            decoder: ClickDecoder(clickFrame: 12_000))
+        defer { bridge.close() }
+        guard let muxed = try muxWithProbeVideo(bridge, startMs: 0, count: 30) else { return }
+        let sessionDir = muxed.sessionDir
+        defer { try? FileManager.default.removeItem(at: sessionDir) }
+        let file = sessionDir.appendingPathComponent("joined.mp4")
+        try (muxed.initBytes + Data(contentsOf: muxed.segment)).write(to: file)
+
+        let asset = AVURLAsset(url: file)
+        let track = try #require(try await asset.loadTracks(withMediaType: .audio).first)
+        let reader = try AVAssetReader(asset: asset)
+        let output = AVAssetReaderTrackOutput(track: track, outputSettings: [
+            AVFormatIDKey: kAudioFormatLinearPCM, AVLinearPCMBitDepthKey: 32,
+            AVLinearPCMIsFloatKey: true, AVLinearPCMIsNonInterleaved: false,
+        ])
+        reader.add(output)
+        #expect(reader.startReading())
+        var onset: Double?
+        while onset == nil, let buffer = output.copyNextSampleBuffer() {
+            guard let format = CMSampleBufferGetFormatDescription(buffer),
+                  let asbd = CMAudioFormatDescriptionGetStreamBasicDescription(format)?.pointee else { continue }
+            let channels = Int(asbd.mChannelsPerFrame)
+            let frames = CMSampleBufferGetNumSamples(buffer)
+            let start = CMSampleBufferGetPresentationTimeStamp(buffer).seconds
+            var block: CMBlockBuffer?
+            var list = AudioBufferList()
+            _ = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
+                buffer, bufferListSizeNeededOut: nil, bufferListOut: &list,
+                bufferListSize: MemoryLayout<AudioBufferList>.size, blockBufferAllocator: nil,
+                blockBufferMemoryAllocator: nil, flags: 0, blockBufferOut: &block)
+            guard let samples = list.mBuffers.mData?.assumingMemoryBound(to: Float.self) else { continue }
+            if let i = (0..<frames).first(where: { abs(samples[$0 * channels]) > 0.2 }) {
+                onset = start + Double(i) / 48_000
+            }
+        }
+        let heard = try #require(onset, "the click reaches the left channel")
+        #expect(abs(heard - 0.25) < 0.001, "the click decodes at \(heard) s, not at its source position 0.25 s")
+    }
 }

Tests/AetherEngineTests/TrueHDAtmosSampleTests.swift

@@ -101,6 +101,7 @@ struct TrueHDAtmosSampleTests {
               + "\(bytes * 8 / Int(seconds) / 1000) kbps, \(String(format: "%.1f", seconds / elapsed))x realtime (debug build)")
         #expect(pts.first == Int64(0))
         #expect(zip(pts, pts.dropFirst()).allSatisfy { $1 - $0 == 1024 })
-        #expect(pts.count == Int((1_298_000.0 / 1024).rounded(.up)))
+        // Two packets of encoder priming, then the sample's 1,298,000 frames.
+        #expect(pts.count == 2 + Int((1_298_000.0 / 1024).rounded(.up)))
     }
 }