fix(mac): align native audio startup timestamps

electron/native/screencapturekit/Package.swift

@@ -25,6 +25,10 @@ let package = Package(
 		.executableTarget(
 			name: "OpenScreenMacOSCursorHelper",
 			path: "Sources/OpenScreenMacOSCursorHelper"
+		),
+		.testTarget(
+			name: "OpenScreenScreenCaptureKitHelperTests",
+			dependencies: ["OpenScreenScreenCaptureKitHelper"]
 		)
 	]
 )

electron/native/screencapturekit/Sources/OpenScreenScreenCaptureKitHelper/AudioTrackMixer.swift

@@ -2,6 +2,46 @@ import AVFoundation
 import CoreMedia
 import Foundation
 
+/// Rebases each ScreenCaptureKit audio output onto the writer's session start.
+///
+/// Screen, system-audio and microphone outputs become live asynchronously. Their PTS values
+/// share a clock, but the first buffer from an audio output can arrive well after the first
+/// screen frame because that capture branch is still warming up. Carrying that one-time
+/// startup offset into the file makes the entire track sound late. Once a source is live its
+/// own PTS deltas are authoritative, so remove only a bounded first-buffer offset and preserve
+/// every later gap, pause and drift correction. A source that takes longer than the normal
+/// warm-up window keeps its original offset so a device failure cannot masquerade as sync.
+@available(macOS 13.0, *)
+struct AudioStartAlignment {
+	private static let maximumCompensatedStartupDelay = CMTime(value: 1, timescale: 4)
+	private var firstPresentationTimes: [CMTime?]
+
+	init(sourceCount: Int) {
+		firstPresentationTimes = Array(repeating: nil, count: sourceCount)
+	}
+
+	mutating func align(
+		_ presentationTime: CMTime,
+		forSourceAt index: Int,
+		to sessionStart: CMTime
+	) -> CMTime? {
+		guard presentationTime.isValid,
+			sessionStart.isValid,
+			firstPresentationTimes.indices.contains(index)
+		else {
+			return nil
+		}
+
+		let firstPresentationTime = firstPresentationTimes[index] ?? presentationTime
+		firstPresentationTimes[index] = firstPresentationTime
+		let startupDelay = CMTimeSubtract(firstPresentationTime, sessionStart)
+		let isExpectedWarmup = CMTimeCompare(startupDelay, .zero) >= 0
+			&& CMTimeCompare(startupDelay, Self.maximumCompensatedStartupDelay) <= 0
+		let sourceOrigin = isExpectedWarmup ? firstPresentationTime : sessionStart
+		return CMTimeAdd(sessionStart, CMTimeSubtract(presentationTime, sourceOrigin))
+	}
+}
+
 /// Sums system audio and the microphone into the single AAC track the helper muxes.
 ///
 /// The helper used to give AVAssetWriter one input per source, so a recording with both
@@ -52,14 +92,16 @@ final class AudioTrackMixer {
 		static let finalFlushTimeout = 5.0
 	}
 
-	private let input: AVAssetWriterInput
+	private let isOutputReady: () -> Bool
+	private let appendOutput: (CMSampleBuffer) -> Void
 	private let includesSystemAudio: Bool
 	private let includesMicrophone: Bool
 	private let microphoneGain: Float
 	private let outputFormatDescription: CMAudioFormatDescription?
 
 	private var sources = [SourceTimeline](repeating: SourceTimeline(), count: Source.allCases.count)
 	private var sessionStart: CMTime?
+	private var startAlignment = AudioStartAlignment(sourceCount: Source.allCases.count)
 	/// Timeline origin: frame 0 of the mixed track, in the writer's time domain.
 	private var anchor: CMTime?
 	/// Absolute frame index of the next chunk to emit.
@@ -74,16 +116,42 @@ final class AudioTrackMixer {
 		includesMicrophone: Bool,
 		microphoneGain: Double
 	) {
-		self.input = input
+		self.isOutputReady = { input.isReadyForMoreMediaData }
+		self.appendOutput = { input.append($0) }
+		self.includesSystemAudio = includesSystemAudio
+		self.includesMicrophone = includesMicrophone
+		// The request carries MIC_GAIN_BOOST (1.4); Windows applies it unconditionally and so
+		// does this. A non-finite or negative value would poison every mixed sample.
+		self.microphoneGain = Self.sanitizeMicrophoneGain(microphoneGain)
+		self.outputFormatDescription = Self.makeOutputFormatDescription()
+	}
+
+	/// Test seam for observing mixed PCM without putting an AVAssetWriter into a writing
+	/// session. Production always uses the AVAssetWriterInput initializer above.
+	init(
+		includesSystemAudio: Bool,
+		includesMicrophone: Bool,
+		microphoneGain: Double,
+		isOutputReady: @escaping () -> Bool,
+		appendOutput: @escaping (CMSampleBuffer) -> Void
+	) {
+		self.isOutputReady = isOutputReady
+		self.appendOutput = appendOutput
 		self.includesSystemAudio = includesSystemAudio
 		self.includesMicrophone = includesMicrophone
 		// The request carries MIC_GAIN_BOOST (1.4); Windows applies it unconditionally and so
 		// does this. A non-finite or negative value would poison every mixed sample.
-		let sanitized = microphoneGain.isFinite ? max(0, microphoneGain) : 1
-		self.microphoneGain = Float(sanitized)
+		self.microphoneGain = Self.sanitizeMicrophoneGain(microphoneGain)
 		self.outputFormatDescription = Self.makeOutputFormatDescription()
 	}
 
+	private static func sanitizeMicrophoneGain(_ gain: Double) -> Float {
+		guard gain.isFinite else {
+			return 1
+		}
+		return Float(min(max(0, gain), Double(Float.greatestFiniteMagnitude)))
+	}
+
 	/// Anchors the mixer to the writer session. Audio delivered before this is dropped — the
 	/// writer would reject anything ahead of its session start anyway.
 	func beginTimeline(at sessionStart: CMTime) {
@@ -92,16 +160,18 @@ final class AudioTrackMixer {
 		}
 
 		self.sessionStart = sessionStart
+		anchor = CMTimeConvertScale(
+			sessionStart,
+			timescale: CMTimeScale(MixFormat.sampleRate),
+			method: .roundHalfAwayFromZero
+		)
 	}
 
 	func ingest(_ sampleBuffer: CMSampleBuffer, from source: Source) {
 		guard includes(source), let sessionStart else {
 			return
 		}
 		let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
-		guard presentationTime.isValid else {
-			return
-		}
 		guard let frames = decodeInterleavedStereo(sampleBuffer, gain: gain(for: source)),
 			!frames.isEmpty
 		else {
@@ -110,20 +180,20 @@ final class AudioTrackMixer {
 			warnAboutDecodeFailure(source, sampleBuffer)
 			return
 		}
-
-		if anchor == nil {
-			anchor = CMTimeConvertScale(
-				CMTimeMaximum(presentationTime, sessionStart),
-				timescale: CMTimeScale(MixFormat.sampleRate),
-				method: .roundHalfAwayFromZero
-			)
+		guard let alignedPresentationTime = startAlignment.align(
+			presentationTime,
+			forSourceAt: source.rawValue,
+			to: sessionStart
+		) else {
+			return
 		}
+
 		guard let anchor else {
 			return
 		}
 
 		let offset = CMTimeConvertScale(
-			CMTimeSubtract(presentationTime, anchor),
+			CMTimeSubtract(alignedPresentationTime, anchor),
 			timescale: CMTimeScale(MixFormat.sampleRate),
 			method: .roundHalfAwayFromZero
 		)
@@ -184,12 +254,30 @@ final class AudioTrackMixer {
 	/// system-audio device that stops delivering — from blocking the whole track.
 	private func drain(flushing: Bool) {
 		while true {
-			let delivered = sources.indices.filter { sources[$0].hasDelivered }
+			let enabled = Source.allCases.filter(includes).map(\.rawValue)
+			let delivered = enabled.filter { sources[$0].hasDelivered }
 			guard let furthest = delivered.map({ sources[$0].endFrame }).max(), furthest > cursor else {
 				break
 			}
 
 			let chunkEnd = cursor + Int64(MixFormat.chunkFrames)
+			// Alignment rebases each enabled source's ordinary startup warm-up onto frame zero.
+			// Do not advance past that frame until every source has had a chance to contribute:
+			// otherwise a source that arrives second has its newly aligned opening samples
+			// discarded by SourceTimeline.dropFrames. If a device never delivers, reuse the
+			// existing bounded stall tolerance and continue with silence for that source.
+			let awaitingFirstBuffer = enabled.filter {
+				!sources[$0].hasDelivered && !sources[$0].isStalled
+			}
+			if !awaitingFirstBuffer.isEmpty && !flushing {
+				if furthest < chunkEnd + Int64(MixFormat.stallToleranceFrames) {
+					break
+				}
+				for index in awaitingFirstBuffer {
+					sources[index].isStalled = true
+				}
+			}
+
 			let live = delivered.filter { !sources[$0].isStalled }
 			let complete = live.allSatisfy { sources[$0].endFrame >= chunkEnd }
 			if !complete && !flushing {
@@ -237,16 +325,16 @@ final class AudioTrackMixer {
 	/// `append` is not advisory backpressure — it raises an NSException when the input is not
 	/// ready — so every path here waits for readiness rather than pushing through it.
 	private func flushPending(force: Bool) {
-		while !pending.isEmpty && input.isReadyForMoreMediaData {
-			input.append(pending.removeFirst())
+		while !pending.isEmpty && isOutputReady() {
+			appendOutput(pending.removeFirst())
 		}
 		if force {
 			// Teardown: this is the tail's last chance, and the writer is still draining, so
 			// give it a bounded moment instead of dropping audio the recording just captured.
 			let deadline = Date().addingTimeInterval(MixFormat.finalFlushTimeout)
 			while !pending.isEmpty {
-				if input.isReadyForMoreMediaData {
-					input.append(pending.removeFirst())
+				if isOutputReady() {
+					appendOutput(pending.removeFirst())
 					continue
 				}
 				if Date() >= deadline {