Added a mechanism to obtain the CMSampleBuffer of the video through a…

… delegate.
shogo4405 · Jan 10, 2024 · 73fd81a · 73fd81a
1 parent 6eb40db
commit 73fd81a
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 53 deletions.
diff --git a/Sources/Extension/CMVideoFormatDescription+Extension.swift b/Sources/Extension/CMVideoFormatDescription+Extension.swift
@@ -8,4 +8,69 @@ extension CMVideoFormatDescription {
     var dimensions: CMVideoDimensions {
         CMVideoFormatDescriptionGetDimensions(self)
     }
+
+    var isCompressed: Bool {
+        switch CMFormatDescriptionGetMediaSubType(self) {
+        case kCVPixelFormatType_1Monochrome,
+             kCVPixelFormatType_2Indexed,
+             kCVPixelFormatType_8Indexed,
+             kCVPixelFormatType_1IndexedGray_WhiteIsZero,
+             kCVPixelFormatType_2IndexedGray_WhiteIsZero,
+             kCVPixelFormatType_4IndexedGray_WhiteIsZero,
+             kCVPixelFormatType_8IndexedGray_WhiteIsZero,
+             kCVPixelFormatType_16BE555,
+             kCVPixelFormatType_16LE555,
+             kCVPixelFormatType_16LE5551,
+             kCVPixelFormatType_16BE565,
+             kCVPixelFormatType_16LE565,
+             kCVPixelFormatType_24RGB,
+             kCVPixelFormatType_24BGR,
+             kCVPixelFormatType_32ARGB,
+             kCVPixelFormatType_32BGRA,
+             kCVPixelFormatType_32ABGR,
+             kCVPixelFormatType_32RGBA,
+             kCVPixelFormatType_64ARGB,
+             kCVPixelFormatType_48RGB,
+             kCVPixelFormatType_32AlphaGray,
+             kCVPixelFormatType_16Gray,
+             kCVPixelFormatType_30RGB,
+             kCVPixelFormatType_422YpCbCr8,
+             kCVPixelFormatType_4444YpCbCrA8,
+             kCVPixelFormatType_4444YpCbCrA8R,
+             kCVPixelFormatType_4444AYpCbCr8,
+             kCVPixelFormatType_4444AYpCbCr16,
+             kCVPixelFormatType_444YpCbCr8,
+             kCVPixelFormatType_422YpCbCr16,
+             kCVPixelFormatType_422YpCbCr10,
+             kCVPixelFormatType_444YpCbCr10,
+             kCVPixelFormatType_420YpCbCr8Planar,
+             kCVPixelFormatType_420YpCbCr8PlanarFullRange,
+             kCVPixelFormatType_422YpCbCr_4A_8BiPlanar,
+             kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
+             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
+             kCVPixelFormatType_422YpCbCr8_yuvs,
+             kCVPixelFormatType_422YpCbCr8FullRange,
+             kCVPixelFormatType_OneComponent8,
+             kCVPixelFormatType_TwoComponent8,
+             kCVPixelFormatType_OneComponent16Half,
+             kCVPixelFormatType_OneComponent32Float,
+             kCVPixelFormatType_TwoComponent16Half,
+             kCVPixelFormatType_TwoComponent32Float,
+             kCVPixelFormatType_64RGBAHalf,
+             kCVPixelFormatType_128RGBAFloat,
+             kCVPixelFormatType_Lossy_32BGRA,
+             kCVPixelFormatType_Lossless_32BGRA,
+             kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange,
+             kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange,
+             kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange,
+             kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange,
+             kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange,
+             kCVPixelFormatType_Lossy_422YpCbCr10PackedBiPlanarVideoRange,
+             kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange,
+             kCVPixelFormatType_Lossless_422YpCbCr10PackedBiPlanarVideoRange:
+            return false
+        default:
+            return true
+        }
+    }
 }
diff --git a/Sources/IO/IOMixer.swift b/Sources/IO/IOMixer.swift
@@ -102,6 +102,9 @@ extension IOMixer: VideoCodecDelegate {
     }
 
     func videoCodec(_ codec: VideoCodec<IOMixer>, didOutput sampleBuffer: CMSampleBuffer) {
+        if sampleBuffer.formatDescription?.isCompressed == false {
+            delegate?.mixer(self, didOutput: sampleBuffer)
+        }
         muxer?.append(sampleBuffer)
     }
 
@@ -189,3 +192,10 @@ extension IOMixer: IOAudioUnitDelegate {
         recorder.append(audioBuffer, when: when)
     }
 }
+
+extension IOMixer: IOVideoUnitDelegate {
+    // MARK: IOVideoUnitDelegate
+    func videoUnit(_ videoUnit: IOVideoUnit, didOutput sampleBuffer: CMSampleBuffer) {
+        delegate?.mixer(self, didOutput: sampleBuffer)
+    }
+}
diff --git a/Sources/IO/IOStream.swift b/Sources/IO/IOStream.swift
@@ -16,9 +16,9 @@ public typealias NetStreamDelegate = IOStreamDelegate
 
 /// The interface an IOStream uses to inform its delegate.
 public protocol IOStreamDelegate: AnyObject {
-    /// Tells the receiver an audio packet incoming.
+    /// Tells the receiver to an audio packet incoming.
     func stream(_ stream: IOStream, didOutput audio: AVAudioBuffer, when: AVAudioTime)
-    /// Tells the receiver to playback a video incoming.
+    /// Tells the receiver to a video incoming.
     func stream(_ stream: IOStream, didOutput video: CMSampleBuffer)
     #if os(iOS) || os(tvOS)
     /// Tells the receiver to session was interrupted.

diff --git a/Sources/IO/IOVideoUnit.swift b/Sources/IO/IOVideoUnit.swift
@@ -15,6 +15,10 @@ public enum IOVideoUnitError: Error {
     case failedToSetOption(status: OSStatus, option: VTSessionOption)
 }
 
+protocol IOVideoUnitDelegate: AnyObject {
+    func videoUnit(_ videoUnit: IOVideoUnit, didOutput sampleBuffer: CMSampleBuffer)
+}
+
 final class IOVideoUnit: NSObject, IOUnit {
     typealias FormatDescription = CMVideoFormatDescription
 
@@ -163,58 +167,11 @@ final class IOVideoUnit: NSObject, IOUnit {
     }
 
     func append(_ sampleBuffer: CMSampleBuffer, channel: UInt8 = 0) {
-        switch sampleBuffer.formatDescription?._mediaSubType {
-        case kCVPixelFormatType_1Monochrome,
-             kCVPixelFormatType_2Indexed,
-             kCVPixelFormatType_8Indexed,
-             kCVPixelFormatType_1IndexedGray_WhiteIsZero,
-             kCVPixelFormatType_2IndexedGray_WhiteIsZero,
-             kCVPixelFormatType_4IndexedGray_WhiteIsZero,
-             kCVPixelFormatType_8IndexedGray_WhiteIsZero,
-             kCVPixelFormatType_16BE555,
-             kCVPixelFormatType_16LE555,
-             kCVPixelFormatType_16LE5551,
-             kCVPixelFormatType_16BE565,
-             kCVPixelFormatType_16LE565,
-             kCVPixelFormatType_24RGB,
-             kCVPixelFormatType_24BGR,
-             kCVPixelFormatType_32ARGB,
-             kCVPixelFormatType_32BGRA,
-             kCVPixelFormatType_32ABGR,
-             kCVPixelFormatType_32RGBA,
-             kCVPixelFormatType_64ARGB,
-             kCVPixelFormatType_48RGB,
-             kCVPixelFormatType_32AlphaGray,
-             kCVPixelFormatType_16Gray,
-             kCVPixelFormatType_30RGB,
-             kCVPixelFormatType_422YpCbCr8,
-             kCVPixelFormatType_4444YpCbCrA8,
-             kCVPixelFormatType_4444YpCbCrA8R,
-             kCVPixelFormatType_4444AYpCbCr8,
-             kCVPixelFormatType_4444AYpCbCr16,
-             kCVPixelFormatType_444YpCbCr8,
-             kCVPixelFormatType_422YpCbCr16,
-             kCVPixelFormatType_422YpCbCr10,
-             kCVPixelFormatType_444YpCbCr10,
-             kCVPixelFormatType_420YpCbCr8Planar,
-             kCVPixelFormatType_420YpCbCr8PlanarFullRange,
-             kCVPixelFormatType_422YpCbCr_4A_8BiPlanar,
-             kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
-             kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
-             kCVPixelFormatType_422YpCbCr8_yuvs,
-             kCVPixelFormatType_422YpCbCr8FullRange,
-             kCVPixelFormatType_OneComponent8,
-             kCVPixelFormatType_TwoComponent8,
-             kCVPixelFormatType_OneComponent16Half,
-             kCVPixelFormatType_OneComponent32Float,
-             kCVPixelFormatType_TwoComponent16Half,
-             kCVPixelFormatType_TwoComponent32Float,
-             kCVPixelFormatType_64RGBAHalf,
-             kCVPixelFormatType_128RGBAFloat:
-            videoMixer.append(sampleBuffer, channel: channel, isVideoMirrored: false)
-        default:
+        if sampleBuffer.formatDescription?.isCompressed == true {
             inputFormat = sampleBuffer.formatDescription
             codec.append(sampleBuffer)
+        } else {
+            videoMixer.append(sampleBuffer, channel: channel, isVideoMirrored: false)
         }
     }
 
@@ -322,8 +279,8 @@ extension IOVideoUnit {
 extension IOVideoUnit: IOVideoMixerDelegate {
     // MARK: IOVideoMixerDelegate
     func videoMixer(_ videoMixer: IOVideoMixer<IOVideoUnit>, didOutput sampleBuffer: CMSampleBuffer) {
-        inputFormat = sampleBuffer.formatDescription
         drawable?.enqueue(sampleBuffer)
+        mixer?.videoUnit(self, didOutput: sampleBuffer)
     }
 
     func videoMixer(_ videoMixer: IOVideoMixer<IOVideoUnit>, didOutput imageBuffer: CVImageBuffer, presentationTimeStamp: CMTime) {