diff --git a/Sources/IO/MediaLink.swift b/Sources/IO/MediaLink.swift index 5af4bcabd..01a018a24 100644 --- a/Sources/IO/MediaLink.swift +++ b/Sources/IO/MediaLink.swift @@ -54,6 +54,7 @@ final class MediaLink { private var scheduledAudioBuffers: Atomic = .init(0) private var presentationTimeStampOrigin: CMTime = .invalid private var audioTime = IOAudioTime() + private var startedAt: TimeInterval = 0 func enqueue(_ buffer: CMSampleBuffer) { guard buffer.presentationTimeStamp != .invalid else { @@ -102,14 +103,17 @@ final class MediaLink { }) } - private func duration(_ duraiton: Double) -> Double { + private func duration(_ timestamp: TimeInterval) -> Double { if playerNode.isPlaying { guard let nodeTime = playerNode.lastRenderTime, let playerTime = playerNode.playerTime(forNodeTime: nodeTime) else { return 0.0 } return TimeInterval(playerTime.sampleTime) / playerTime.sampleRate } - return duraiton + if startedAt == 0 { + startedAt = timestamp + } + return timestamp - startedAt } private func makeBufferkQueue() { @@ -123,11 +127,11 @@ final class MediaLink { extension MediaLink: ChoreographerDelegate { // MARK: ChoreographerDelegate - func choreographer(_ choreographer: some Choreographer, didFrame duration: Double) { + func choreographer(_ choreographer: some Choreographer, didFrame timestamp: TimeInterval) { guard let bufferQueue else { return } - let duration = self.duration(duration) + let duration = self.duration(timestamp) var frameCount = 0 while !bufferQueue.isEmpty { guard let first = bufferQueue.head else { @@ -158,6 +162,7 @@ extension MediaLink: Running { hasVideo = false bufferingTime = kMediaLink_bufferingTime isBuffering = true + startedAt = 0 choreographer.startRunning() makeBufferkQueue() isRunning.mutate { $0 = true } diff --git a/Sources/Screen/Choreographer.swift b/Sources/Screen/Choreographer.swift index 5e23a6471..7abc4768d 100644 --- a/Sources/Screen/Choreographer.swift +++ b/Sources/Screen/Choreographer.swift @@ -97,18 +97,15 @@ typealias DisplayLink = CADisplayLink #endif protocol ChoreographerDelegate: AnyObject { - func choreographer(_ choreographer: some Choreographer, didFrame duration: Double) + func choreographer(_ choreographer: some Choreographer, didFrame timestamp: TimeInterval) } protocol Choreographer: Running { var isPaused: Bool { get set } var delegate: (any ChoreographerDelegate)? { get set } - - func clear() } final class DisplayLinkChoreographer: NSObject, Choreographer { - private static let duration = 0.0 private static let preferredFramesPerSecond = 0 var isPaused: Bool { @@ -122,7 +119,6 @@ final class DisplayLinkChoreographer: NSObject, Choreographer { weak var delegate: (any ChoreographerDelegate)? var isRunning: Atomic = .init(false) var preferredFramesPerSecond = DisplayLinkChoreographer.preferredFramesPerSecond - private var duration: Double = DisplayLinkChoreographer.duration private var displayLink: DisplayLink? { didSet { oldValue?.invalidate() @@ -135,14 +131,9 @@ final class DisplayLinkChoreographer: NSObject, Choreographer { } } - func clear() { - duration = Self.duration - } - @objc private func update(displayLink: DisplayLink) { - delegate?.choreographer(self, didFrame: duration) - duration += displayLink.duration + delegate?.choreographer(self, didFrame: displayLink.timestamp) } } @@ -154,7 +145,6 @@ extension DisplayLinkChoreographer: Running { func stopRunning() { displayLink = nil - duration = DisplayLinkChoreographer.duration isRunning.mutate { $0 = false } } } diff --git a/Sources/Screen/Screen.swift b/Sources/Screen/Screen.swift index 0788c6acf..e41398082 100644 --- a/Sources/Screen/Screen.swift +++ b/Sources/Screen/Screen.swift @@ -58,6 +58,8 @@ public final class Screen: ScreenObjectContainerConvertible { return choreographer.isRunning } + public var delayTime: TimeInterval = (1 / 33) + #if os(macOS) /// Specifies the background color. public var backgroundColor: CGColor = NSColor.black.cgColor { @@ -122,6 +124,7 @@ public final class Screen: ScreenObjectContainerConvertible { defer { sampleBuffer.imageBuffer?.unlockBaseAddress(Self.lockFrags) } + renderer.presentationTimeStamp = sampleBuffer.presentationTimeStamp renderer.setTarget(sampleBuffer.imageBuffer) if let dimensions = sampleBuffer.formatDescription?.dimensions { root.size = dimensions.size @@ -155,7 +158,7 @@ extension Screen: Running { extension Screen: ChoreographerDelegate { // MARK: ChoreographerDelegate - func choreographer(_ choreographer: some Choreographer, didFrame duration: Double) { + func choreographer(_ choreographer: some Choreographer, didFrame timestamp: TimeInterval) { var pixelBuffer: CVPixelBuffer? pixelBufferPool?.createPixelBuffer(&pixelBuffer) guard let pixelBuffer else { @@ -174,7 +177,7 @@ extension Screen: ChoreographerDelegate { if let dictionary = CVBufferGetAttachments(pixelBuffer, .shouldNotPropagate) { CVBufferSetAttachments(pixelBuffer, dictionary, .shouldPropagate) } - let now = CMClock.hostTimeClock.time + let now = CMTime(seconds: timestamp - delayTime, preferredTimescale: 1000000000) var timingInfo = CMSampleTimingInfo( duration: timeStamp == .invalid ? .zero : now - timeStamp, presentationTimeStamp: now, diff --git a/Sources/Screen/ScreenObject.swift b/Sources/Screen/ScreenObject.swift index 75e2bb1d2..467f47712 100644 --- a/Sources/Screen/ScreenObject.swift +++ b/Sources/Screen/ScreenObject.swift @@ -207,6 +207,7 @@ public final class ImageScreenObject: ScreenObject { /// An object that manages offscreen rendering a video track source. public final class VideoTrackScreenObject: ScreenObject, ChromaKeyProcessable { + static let capacity: Int = 3 public var chromaKeyColor: CGColor? /// Specifies the track number how the displays the visual content. @@ -244,7 +245,7 @@ public final class VideoTrackScreenObject: ScreenObject, ChromaKeyProcessable { super.init() horizontalAlignment = .center do { - queue = try TypedBlockQueue(capacity: 1, handlers: .outputPTSSortedSampleBuffers) + queue = try TypedBlockQueue(capacity: Self.capacity, handlers: .outputPTSSortedSampleBuffers) } catch { logger.error(error) } @@ -269,7 +270,8 @@ public final class VideoTrackScreenObject: ScreenObject, ChromaKeyProcessable { } override public func makeImage(_ renderer: some ScreenRenderer) -> CGImage? { - guard let sampleBuffer = queue?.dequeue(), let pixelBuffer = sampleBuffer.imageBuffer else { + guard let sampleBuffer = queue?.query(renderer.presentationTimeStamp), + let pixelBuffer = sampleBuffer.imageBuffer else { return nil } // Resizing before applying the filter for performance optimization. diff --git a/Sources/Screen/ScreenRenderer.swift b/Sources/Screen/ScreenRenderer.swift index 1c83046f6..88ba40d23 100644 --- a/Sources/Screen/ScreenRenderer.swift +++ b/Sources/Screen/ScreenRenderer.swift @@ -11,6 +11,8 @@ public protocol ScreenRenderer: AnyObject { var backgroundColor: CGColor { get set } /// The current screen bounds. var bounds: CGRect { get } + /// The current presentationTimeStamp. + var presentationTimeStamp: CMTime { get } /// Layouts a screen object. func layout(_ screenObject: ScreenObject) /// Draws a sceen object. @@ -24,6 +26,7 @@ final class ScreenRendererByCPU: ScreenRenderer { static let doNotTile = vImage_Flags(kvImageDoNotTile) var bounds: CGRect = .init(origin: .zero, size: Screen.size) + var presentationTimeStamp: CMTime = .zero lazy var context = { guard let deive = MTLCreateSystemDefaultDevice() else { @@ -64,6 +67,7 @@ final class ScreenRendererByCPU: ScreenRenderer { } } } + private var format = vImage_CGImageFormat( bitsPerComponent: 8, bitsPerPixel: 32, diff --git a/Sources/Util/TypedBlockQueue.swift b/Sources/Util/TypedBlockQueue.swift index e5b914e8b..5ef661d5d 100644 --- a/Sources/Util/TypedBlockQueue.swift +++ b/Sources/Util/TypedBlockQueue.swift @@ -46,3 +46,20 @@ final class TypedBlockQueue { try queue.reset() } } + +extension TypedBlockQueue where T == CMSampleBuffer { + func query(_ presentationTimeStamp: CMTime) -> CMSampleBuffer? { + var result: CMSampleBuffer? + while !queue.isEmpty { + guard let head else { + break + } + if head.presentationTimeStamp <= presentationTimeStamp { + result = dequeue() + } else { + return result + } + } + return result + } +}