fix(macos): keep microphone audio in sync by gap-filling dropped buffers

electron/native/ScreenCaptureKitRecorder.swift

@@ -59,6 +59,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 	private var lastVideoDuration: CMTime = .zero
 	private var lastInlineAudioPresentationTime: CMTime = .invalid
 	private var lastInlineAudioDuration: CMTime = .zero
+	private var lastSystemAudioDuration: CMTime = .zero
+	private var lastMicrophoneDuration: CMTime = .zero
+	private var droppedAudioBufferCount = 0
+	private var insertedSilenceFrames: Int64 = 0
+	/// Audio is delivered on its own queue so a slow video callback (5K crop +
+	/// encode) can never make ScreenCaptureKit discard microphone buffers.
+	private let audioQueue = DispatchQueue(label: "recordly.screencapturekit.audio")
 	private var isRecording = false
 	private var isPaused = false
 	private var pauseStartedHostTime: CMTime?
@@ -82,6 +89,9 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 
 	private let microphoneOutputTypeRawValue = 2
 
+	/// Configures the ScreenCaptureKit stream and asset writers from the JSON
+	/// config passed on the command line, then starts capturing. Screen frames
+	/// are delivered on `queue`; audio outputs are delivered on `audioQueue`.
 	func startCapture(configJSON: String) async throws {
 		guard !isRecording else {
 			throw NSError(domain: "RecordlyCapture", code: 1, userInfo: [NSLocalizedDescriptionKey: "Recording is already in progress"])
@@ -326,7 +336,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		self.stream = stream
 		try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: queue)
 		if capturesSystemAudio {
-			try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: queue)
+			try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: audioQueue)
 		}
 		if capturesMicrophone {
 			guard let microphoneOutputType = SCStreamOutputType(rawValue: microphoneOutputTypeRawValue) else {
@@ -336,7 +346,7 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 					userInfo: [NSLocalizedDescriptionKey: "Microphone stream output type is unavailable"]
 				)
 			}
-			try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: queue)
+			try stream.addStreamOutput(self, type: microphoneOutputType, sampleHandlerQueue: audioQueue)
 		}
 		try await stream.startCapture()
 
@@ -392,7 +402,24 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		}
 	}
 
+	/// ScreenCaptureKit delivery entry point. Screen frames arrive on `queue` and
+	/// are handled inline; audio arrives on `audioQueue` and is hopped onto
+	/// `queue` so recorder state is only ever touched from one queue and a slow
+	/// video callback can never block or drop audio delivery.
 	func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
+		if outputType != .screen {
+			queue.async { [weak self] in
+				self?.handleSampleBuffer(sampleBuffer, of: outputType)
+			}
+			return
+		}
+		handleSampleBuffer(sampleBuffer, of: outputType)
+	}
+
+	/// Retimes one screen, system-audio, or microphone sample onto the recording
+	/// timeline and appends it to the matching writer inputs. Always runs on
+	/// `queue`; drops everything once finalization has started.
+	private func handleSampleBuffer(_ sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
 		guard sessionStarted, sampleBuffer.isValid, isRecording else { return }
 		guard let presentationTime = adjustedPresentationTime(for: sampleBuffer, outputType: outputType) else { return }
 
@@ -445,21 +472,21 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 
 		if outputType == .audio {
 			guard let systemAudioInput else { return }
-			appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, of: systemAudioWriter, firstSampleTime: &firstSystemAudioSampleTime, lastPresentationTime: &lastSystemAudioPresentationTime, presentationTime: presentationTime)
+			appendAudioSampleBuffer(sampleBuffer, to: systemAudioInput, of: systemAudioWriter, firstSampleTime: &firstSystemAudioSampleTime, lastPresentationTime: &lastSystemAudioPresentationTime, lastDuration: &lastSystemAudioDuration, presentationTime: presentationTime)
 			// Also write system audio to the inline video track
-			if let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData {
-				appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime)
+			if let inlineAudioInput {
+				appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, lastDuration: &lastInlineAudioDuration, presentationTime: presentationTime)
 			}
 			return
 		}
 
 		if outputType.rawValue == microphoneOutputTypeRawValue {
 			if let microphoneOnlyInput {
-				appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime)
+				appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, lastDuration: &lastMicrophoneDuration, presentationTime: presentationTime)
 			}
 			// Write mic to inline video track only if there's no system audio (avoids double-writing)
-			if !capturesSystemAudio, let inlineAudioInput, inlineAudioInput.isReadyForMoreMediaData {
-				appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime)
+			if !capturesSystemAudio, let inlineAudioInput {
+				appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, lastDuration: &lastInlineAudioDuration, presentationTime: presentationTime)
 			}
 			return
 		}
@@ -518,7 +545,13 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 	/// the recorder queue have drained. Manual stop and automatic window-close
 	/// detection join the same operation instead of racing the asset writers.
 	private func finalizeCapture(interactive: Bool) async -> CaptureFinalizationResult {
-		await withCheckedContinuation { continuation in
+		// Audio arrives on audioQueue and hops onto the recorder queue. Drain that
+		// hop first so every buffer delivered before this stop request is already
+		// queued ahead of the finalization block instead of being dropped by the
+		// isRecording guard. finalizeCapture never runs on either queue (it is
+		// called from the command queue or a Task), so the barrier cannot deadlock.
+		audioQueue.sync {}
+		return await withCheckedContinuation { continuation in
 			queue.async {
 				if self.isFinalizing {
 					self.interactiveStopParticipated = self.interactiveStopParticipated || interactive
@@ -571,6 +604,10 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		}
 	}
 
+	/// Stops the stream, gives the last frame its full duration, closes every
+	/// writer, reports audio-gap diagnostics on stderr, and resets recorder state.
+	/// Throws if any writer failed so a half-written file is never reported as
+	/// a successful recording.
 	private func finishCapture() async throws -> String {
 
 		if let activeStream = stream {
@@ -633,6 +670,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 					: (writer.error ?? unfinalizedWriterError(status: writer.status))
 			}
 			.first
+		if droppedAudioBufferCount > 0 || insertedSilenceFrames > 0 {
+			fputs("AUDIO_GAPS: droppedBuffers=\(droppedAudioBufferCount) silenceFramesInserted=\(insertedSilenceFrames)\n", stderr)
+			fflush(stderr)
+		}
+		droppedAudioBufferCount = 0
+		insertedSilenceFrames = 0
+		lastSystemAudioDuration = .zero
+		lastMicrophoneDuration = .zero
+
 		let path = outputURL?.path ?? ""
 		assetWriter = nil
 		videoInput = nil
@@ -792,28 +838,133 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate {
 		return videoEndTime + CMTimeMinimum(tailExtension, maxInlineAudioTailExtension)
 	}
 
-	private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, of writer: AVAssetWriter?, firstSampleTime: inout CMTime?, lastPresentationTime: inout CMTime, presentationTime: CMTime) {
+	/// Appends one audio buffer to `input` at `presentationTime`, first filling
+	/// any gap since the previous buffer with silence so lost buffers never
+	/// compact the track. Buffers that arrive while the input is not ready are
+	/// counted as dropped; `lastPresentationTime`/`lastDuration` track the last
+	/// sample (real or silence) the writer accepted for this input.
+	private func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer, to input: AVAssetWriterInput, of writer: AVAssetWriter?, firstSampleTime: inout CMTime?, lastPresentationTime: inout CMTime, lastDuration: inout CMTime, presentationTime: CMTime) {
 		// A writer that failed mid-capture (a full disk, say) raises on every
 		// further append, which would abort the helper and lose the whole file.
-		guard writer?.status == .writing, input.isReadyForMoreMediaData else { return }
+		guard writer?.status == .writing else { return }
+		guard input.isReadyForMoreMediaData else {
+			// Back-pressure from the writer (typically the video encoder lagging on
+			// a high-resolution capture). The buffer is lost, but the hole is filled
+			// with silence on the next accepted buffer so the track keeps real time
+			// instead of compacting and drifting ahead of the video.
+			droppedAudioBufferCount += 1
+			return
+		}
 		guard !lastPresentationTime.isValid || CMTimeCompare(presentationTime, lastPresentationTime) > 0 else { return }
 
 		if firstSampleTime == nil {
 			firstSampleTime = presentationTime
 		}
 
+		if lastPresentationTime.isValid, lastDuration.isValid, lastDuration > .zero {
+			let expectedNext = lastPresentationTime + lastDuration
+			let gap = presentationTime - expectedNext
+			let tolerance = CMTimeMultiplyByFloat64(lastDuration, multiplier: 0.5)
+			if CMTimeCompare(gap, tolerance) > 0,
+			   !appendSilence(matching: sampleBuffer, from: expectedNext, to: presentationTime, into: input, lastPresentationTime: &lastPresentationTime, lastDuration: &lastDuration) {
+				// The writer stopped accepting data mid-fill. The silence already
+				// committed is recorded in lastPresentationTime/lastDuration, so drop
+				// this buffer and let the next one resume the fill from there; the
+				// track never carries an unfilled hole that the muxer could compact.
+				droppedAudioBufferCount += 1
+				return
+			}
+		}
+
+		// The fill may have just saturated the input, and appending to an input
+		// that is not ready raises an uncatchable Objective-C exception.
+		guard input.isReadyForMoreMediaData else {
+			droppedAudioBufferCount += 1
+			return
+		}
+
 		// presentationTime is already relative to the video's first frame
 		// (computed by adjustedPresentationTime), so use it directly.
 		let timing = CMSampleTimingInfo(duration: sampleBuffer.duration, presentationTimeStamp: presentationTime, decodeTimeStamp: sampleBuffer.decodeTimeStamp)
-		if let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]) {
-			let appended = input.append(retimedSampleBuffer)
-			if appended {
-				lastPresentationTime = presentationTime
-				if input === inlineAudioInput {
-					lastInlineAudioDuration = sampleBuffer.duration
-				}
+		guard let retimedSampleBuffer = try? CMSampleBuffer(copying: sampleBuffer, withNewTiming: [timing]),
+			  input.append(retimedSampleBuffer) else {
+			// A failed retime or a rejected append loses this buffer as well; count
+			// it so AUDIO_GAPS reflects every buffer missing from the track. The
+			// hole is filled by the next accepted buffer like any other drop.
+			droppedAudioBufferCount += 1
+			return
+		}
+		lastPresentationTime = presentationTime
+		lastDuration = sampleBuffer.duration
+	}
+
+	/// Appends zeroed LPCM covering [start, end) in the same format as `sampleBuffer`,
+	/// so buffers lost to back-pressure or late delivery leave a silent hole instead
+	/// of shifting every later sample earlier.
+	/// Returns false only when the writer stopped accepting data before the
+	/// (capped) fill completed; an unfillable format is treated as complete.
+	/// `lastPresentationTime`/`lastDuration` advance to every silence chunk that
+	/// was actually accepted, so a retry resumes where the fill stopped instead
+	/// of re-appending earlier timestamps (which would fail the writer).
+	@discardableResult
+	private func appendSilence(matching sampleBuffer: CMSampleBuffer, from start: CMTime, to end: CMTime, into input: AVAssetWriterInput, lastPresentationTime: inout CMTime, lastDuration: inout CMTime) -> Bool {
+		guard let formatDescription = sampleBuffer.formatDescription,
+			  let asbd = formatDescription.audioStreamBasicDescription else { return true }
+		let sampleRate = asbd.mSampleRate
+		guard sampleRate > 0, asbd.mBytesPerFrame > 0 else { return true }
+		let gapSeconds = CMTimeGetSeconds(end - start)
+		guard gapSeconds.isFinite, gapSeconds > 0 else { return true }
+
+		// Cap a single hole so a stalled device cannot balloon the file.
+		let totalFrames = Int(min(gapSeconds, 10.0) * sampleRate)
+		let isNonInterleaved = (asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved) != 0
+		let bytesPerFrameAllChannels = Int(asbd.mBytesPerFrame) * (isNonInterleaved ? Int(max(1, asbd.mChannelsPerFrame)) : 1)
+		let chunkFrames = 4096
+		var framesWritten = 0
+
+		while framesWritten < totalFrames, input.isReadyForMoreMediaData {
+			let frames = min(chunkFrames, totalFrames - framesWritten)
+			let byteCount = frames * bytesPerFrameAllChannels
+			var blockBuffer: CMBlockBuffer?
+			guard CMBlockBufferCreateWithMemoryBlock(
+					allocator: kCFAllocatorDefault,
+					memoryBlock: nil,
+					blockLength: byteCount,
+					blockAllocator: kCFAllocatorDefault,
+					customBlockSource: nil,
+					offsetToData: 0,
+					dataLength: byteCount,
+					flags: 0,
+					blockBufferOut: &blockBuffer) == kCMBlockBufferNoErr,
+				  let blockBuffer,
+				  CMBlockBufferFillDataBytes(with: 0, blockBuffer: blockBuffer, offsetIntoDestination: 0, dataLength: byteCount) == kCMBlockBufferNoErr else {
+				return true
+			}
+
+			let pts = start + CMTime(value: CMTimeValue(framesWritten), timescale: CMTimeScale(sampleRate))
+			var silence: CMSampleBuffer?
+			guard CMAudioSampleBufferCreateReadyWithPacketDescriptions(
+					allocator: kCFAllocatorDefault,
+					dataBuffer: blockBuffer,
+					formatDescription: formatDescription,
+					sampleCount: frames,
+					presentationTimeStamp: pts,
+					packetDescriptions: nil,
+					sampleBufferOut: &silence) == noErr,
+				  let silence else {
+				return true
 			}
+			guard input.append(silence) else {
+				insertedSilenceFrames += Int64(framesWritten)
+				return false
+			}
+			lastPresentationTime = pts
+			lastDuration = CMTime(value: CMTimeValue(frames), timescale: CMTimeScale(sampleRate))
+			framesWritten += frames
 		}
+
+		insertedSilenceFrames += Int64(framesWritten)
+		return framesWritten >= totalFrames
 	}
 
 	private static func audioOutputSettings(bitRate: Int) -> [String: Any] {

electron/native/ScreenCaptureKitRecorder.test.ts

@@ -84,3 +84,31 @@ describe("ScreenCaptureKitRecorder window capture", () => {
 		expect(recorderSource).toContain("self.windowCropRect = cropRect");
 	});
 });
+
+describe("ScreenCaptureKitRecorder audio continuity", () => {
+	it("delivers audio on a dedicated queue and hops onto the recorder queue", () => {
+		expect(recorderSource).toContain('DispatchQueue(label: "recordly.screencapturekit.audio")');
+		expect(recorderSource).toContain("type: .audio, sampleHandlerQueue: audioQueue");
+		expect(recorderSource).toContain(
+			"type: microphoneOutputType, sampleHandlerQueue: audioQueue",
+		);
+		expect(recorderSource).toMatch(/if outputType != \.screen \{\s*queue\.async/);
+		expect(recorderSource).toContain("audioQueue.sync {}");
+	});
+
+	it("fills audio timestamp gaps with silence instead of compacting the track", () => {
+		expect(recorderSource).toContain(
+			"appendSilence(matching: sampleBuffer, from: expectedNext, to: presentationTime, into: input, lastPresentationTime: &lastPresentationTime, lastDuration: &lastDuration)",
+		);
+		expect(recorderSource).toContain("CMAudioSampleBufferCreateReadyWithPacketDescriptions(");
+		expect(recorderSource).toContain("lastDuration = sampleBuffer.duration");
+		expect(recorderSource).toContain(
+			"lastDuration = CMTime(value: CMTimeValue(frames), timescale: CMTimeScale(sampleRate))",
+		);
+	});
+
+	it("counts dropped audio buffers and reports gaps at finalization", () => {
+		expect(recorderSource).toContain("droppedAudioBufferCount += 1");
+		expect(recorderSource).toContain("AUDIO_GAPS: droppedBuffers=");
+	});
+});