From 73fd81a9dfbd482067a97aa370802c1242285174 Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Thu, 11 Jan 2024 02:21:26 +0900 Subject: [PATCH] Added a mechanism to obtain the CMSampleBuffer of the video through a delegate. --- .../CMVideoFormatDescription+Extension.swift | 65 +++++++++++++++++++ Sources/IO/IOMixer.swift | 10 +++ Sources/IO/IOStream.swift | 4 +- Sources/IO/IOVideoUnit.swift | 59 +++-------------- 4 files changed, 85 insertions(+), 53 deletions(-) diff --git a/Sources/Extension/CMVideoFormatDescription+Extension.swift b/Sources/Extension/CMVideoFormatDescription+Extension.swift index 2970d84ab..60d224fcd 100644 --- a/Sources/Extension/CMVideoFormatDescription+Extension.swift +++ b/Sources/Extension/CMVideoFormatDescription+Extension.swift @@ -8,4 +8,69 @@ extension CMVideoFormatDescription { var dimensions: CMVideoDimensions { CMVideoFormatDescriptionGetDimensions(self) } + + var isCompressed: Bool { + switch CMFormatDescriptionGetMediaSubType(self) { + case kCVPixelFormatType_1Monochrome, + kCVPixelFormatType_2Indexed, + kCVPixelFormatType_8Indexed, + kCVPixelFormatType_1IndexedGray_WhiteIsZero, + kCVPixelFormatType_2IndexedGray_WhiteIsZero, + kCVPixelFormatType_4IndexedGray_WhiteIsZero, + kCVPixelFormatType_8IndexedGray_WhiteIsZero, + kCVPixelFormatType_16BE555, + kCVPixelFormatType_16LE555, + kCVPixelFormatType_16LE5551, + kCVPixelFormatType_16BE565, + kCVPixelFormatType_16LE565, + kCVPixelFormatType_24RGB, + kCVPixelFormatType_24BGR, + kCVPixelFormatType_32ARGB, + kCVPixelFormatType_32BGRA, + kCVPixelFormatType_32ABGR, + kCVPixelFormatType_32RGBA, + kCVPixelFormatType_64ARGB, + kCVPixelFormatType_48RGB, + kCVPixelFormatType_32AlphaGray, + kCVPixelFormatType_16Gray, + kCVPixelFormatType_30RGB, + kCVPixelFormatType_422YpCbCr8, + kCVPixelFormatType_4444YpCbCrA8, + kCVPixelFormatType_4444YpCbCrA8R, + kCVPixelFormatType_4444AYpCbCr8, + kCVPixelFormatType_4444AYpCbCr16, + kCVPixelFormatType_444YpCbCr8, + kCVPixelFormatType_422YpCbCr16, + kCVPixelFormatType_422YpCbCr10, + kCVPixelFormatType_444YpCbCr10, + kCVPixelFormatType_420YpCbCr8Planar, + kCVPixelFormatType_420YpCbCr8PlanarFullRange, + kCVPixelFormatType_422YpCbCr_4A_8BiPlanar, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_422YpCbCr8_yuvs, + kCVPixelFormatType_422YpCbCr8FullRange, + kCVPixelFormatType_OneComponent8, + kCVPixelFormatType_TwoComponent8, + kCVPixelFormatType_OneComponent16Half, + kCVPixelFormatType_OneComponent32Float, + kCVPixelFormatType_TwoComponent16Half, + kCVPixelFormatType_TwoComponent32Float, + kCVPixelFormatType_64RGBAHalf, + kCVPixelFormatType_128RGBAFloat, + kCVPixelFormatType_Lossy_32BGRA, + kCVPixelFormatType_Lossless_32BGRA, + kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange, + kCVPixelFormatType_Lossy_422YpCbCr10PackedBiPlanarVideoRange, + kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange, + kCVPixelFormatType_Lossless_422YpCbCr10PackedBiPlanarVideoRange: + return false + default: + return true + } + } } diff --git a/Sources/IO/IOMixer.swift b/Sources/IO/IOMixer.swift index e71918a40..e861a61ba 100644 --- a/Sources/IO/IOMixer.swift +++ b/Sources/IO/IOMixer.swift @@ -102,6 +102,9 @@ extension IOMixer: VideoCodecDelegate { } func videoCodec(_ codec: VideoCodec, didOutput sampleBuffer: CMSampleBuffer) { + if sampleBuffer.formatDescription?.isCompressed == false { + delegate?.mixer(self, didOutput: sampleBuffer) + } muxer?.append(sampleBuffer) } @@ -189,3 +192,10 @@ extension IOMixer: IOAudioUnitDelegate { recorder.append(audioBuffer, when: when) } } + +extension IOMixer: IOVideoUnitDelegate { + // MARK: IOVideoUnitDelegate + func videoUnit(_ videoUnit: IOVideoUnit, didOutput sampleBuffer: CMSampleBuffer) { + delegate?.mixer(self, didOutput: sampleBuffer) + } +} diff --git a/Sources/IO/IOStream.swift b/Sources/IO/IOStream.swift index c7a5ba3ac..1c529d59f 100644 --- a/Sources/IO/IOStream.swift +++ b/Sources/IO/IOStream.swift @@ -16,9 +16,9 @@ public typealias NetStreamDelegate = IOStreamDelegate /// The interface an IOStream uses to inform its delegate. public protocol IOStreamDelegate: AnyObject { - /// Tells the receiver an audio packet incoming. + /// Tells the receiver to an audio packet incoming. func stream(_ stream: IOStream, didOutput audio: AVAudioBuffer, when: AVAudioTime) - /// Tells the receiver to playback a video incoming. + /// Tells the receiver to a video incoming. func stream(_ stream: IOStream, didOutput video: CMSampleBuffer) #if os(iOS) || os(tvOS) /// Tells the receiver to session was interrupted. diff --git a/Sources/IO/IOVideoUnit.swift b/Sources/IO/IOVideoUnit.swift index dcbd1ac01..c0be99713 100644 --- a/Sources/IO/IOVideoUnit.swift +++ b/Sources/IO/IOVideoUnit.swift @@ -15,6 +15,10 @@ public enum IOVideoUnitError: Error { case failedToSetOption(status: OSStatus, option: VTSessionOption) } +protocol IOVideoUnitDelegate: AnyObject { + func videoUnit(_ videoUnit: IOVideoUnit, didOutput sampleBuffer: CMSampleBuffer) +} + final class IOVideoUnit: NSObject, IOUnit { typealias FormatDescription = CMVideoFormatDescription @@ -163,58 +167,11 @@ final class IOVideoUnit: NSObject, IOUnit { } func append(_ sampleBuffer: CMSampleBuffer, channel: UInt8 = 0) { - switch sampleBuffer.formatDescription?._mediaSubType { - case kCVPixelFormatType_1Monochrome, - kCVPixelFormatType_2Indexed, - kCVPixelFormatType_8Indexed, - kCVPixelFormatType_1IndexedGray_WhiteIsZero, - kCVPixelFormatType_2IndexedGray_WhiteIsZero, - kCVPixelFormatType_4IndexedGray_WhiteIsZero, - kCVPixelFormatType_8IndexedGray_WhiteIsZero, - kCVPixelFormatType_16BE555, - kCVPixelFormatType_16LE555, - kCVPixelFormatType_16LE5551, - kCVPixelFormatType_16BE565, - kCVPixelFormatType_16LE565, - kCVPixelFormatType_24RGB, - kCVPixelFormatType_24BGR, - kCVPixelFormatType_32ARGB, - kCVPixelFormatType_32BGRA, - kCVPixelFormatType_32ABGR, - kCVPixelFormatType_32RGBA, - kCVPixelFormatType_64ARGB, - kCVPixelFormatType_48RGB, - kCVPixelFormatType_32AlphaGray, - kCVPixelFormatType_16Gray, - kCVPixelFormatType_30RGB, - kCVPixelFormatType_422YpCbCr8, - kCVPixelFormatType_4444YpCbCrA8, - kCVPixelFormatType_4444YpCbCrA8R, - kCVPixelFormatType_4444AYpCbCr8, - kCVPixelFormatType_4444AYpCbCr16, - kCVPixelFormatType_444YpCbCr8, - kCVPixelFormatType_422YpCbCr16, - kCVPixelFormatType_422YpCbCr10, - kCVPixelFormatType_444YpCbCr10, - kCVPixelFormatType_420YpCbCr8Planar, - kCVPixelFormatType_420YpCbCr8PlanarFullRange, - kCVPixelFormatType_422YpCbCr_4A_8BiPlanar, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, - kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, - kCVPixelFormatType_422YpCbCr8_yuvs, - kCVPixelFormatType_422YpCbCr8FullRange, - kCVPixelFormatType_OneComponent8, - kCVPixelFormatType_TwoComponent8, - kCVPixelFormatType_OneComponent16Half, - kCVPixelFormatType_OneComponent32Float, - kCVPixelFormatType_TwoComponent16Half, - kCVPixelFormatType_TwoComponent32Float, - kCVPixelFormatType_64RGBAHalf, - kCVPixelFormatType_128RGBAFloat: - videoMixer.append(sampleBuffer, channel: channel, isVideoMirrored: false) - default: + if sampleBuffer.formatDescription?.isCompressed == true { inputFormat = sampleBuffer.formatDescription codec.append(sampleBuffer) + } else { + videoMixer.append(sampleBuffer, channel: channel, isVideoMirrored: false) } } @@ -322,8 +279,8 @@ extension IOVideoUnit { extension IOVideoUnit: IOVideoMixerDelegate { // MARK: IOVideoMixerDelegate func videoMixer(_ videoMixer: IOVideoMixer, didOutput sampleBuffer: CMSampleBuffer) { - inputFormat = sampleBuffer.formatDescription drawable?.enqueue(sampleBuffer) + mixer?.videoUnit(self, didOutput: sampleBuffer) } func videoMixer(_ videoMixer: IOVideoMixer, didOutput imageBuffer: CVImageBuffer, presentationTimeStamp: CMTime) {