From 6b24a7be88fe104ee139ff3a48b641b81599e09c Mon Sep 17 00:00:00 2001 From: shogo4405 Date: Wed, 27 Mar 2024 20:02:30 +0900 Subject: [PATCH] Support attachVideo() for visionOS. --- Examples/visionOS/ContentView.swift | 2 - .../Extension/AVCaptureDevice+Extension.swift | 4 -- .../AVCaptureDevice.Format+Extension.swift | 8 +++- .../AVCaptureSession+Extension.swift | 4 +- .../AVFrameRateRange+Extension.swift | 2 - Sources/IO/IOCaptureSession.swift | 43 +++++++++++-------- Sources/IO/IOCaptureUnit.swift | 2 - Sources/IO/IOMixer.swift | 12 +++--- Sources/IO/IOStream.swift | 15 +++---- Sources/IO/IOVideoCaptureUnit.swift | 13 ++++-- Sources/IO/IOVideoUnit.swift | 14 +++--- 11 files changed, 60 insertions(+), 59 deletions(-) diff --git a/Examples/visionOS/ContentView.swift b/Examples/visionOS/ContentView.swift index c712d6884..f1bf235cb 100644 --- a/Examples/visionOS/ContentView.swift +++ b/Examples/visionOS/ContentView.swift @@ -1,5 +1,3 @@ -import RealityKit -import RealityKitContent import SwiftUI struct ContentView: View { diff --git a/Sources/Extension/AVCaptureDevice+Extension.swift b/Sources/Extension/AVCaptureDevice+Extension.swift index fea7a6ecc..827c020f2 100644 --- a/Sources/Extension/AVCaptureDevice+Extension.swift +++ b/Sources/Extension/AVCaptureDevice+Extension.swift @@ -1,5 +1,3 @@ -#if os(iOS) || os(tvOS) || os(macOS) - import AVFoundation import Foundation @@ -21,5 +19,3 @@ extension AVCaptureDevice { } } } - -#endif diff --git a/Sources/Extension/AVCaptureDevice.Format+Extension.swift b/Sources/Extension/AVCaptureDevice.Format+Extension.swift index 9e3064efe..72528f3f2 100644 --- a/Sources/Extension/AVCaptureDevice.Format+Extension.swift +++ b/Sources/Extension/AVCaptureDevice.Format+Extension.swift @@ -14,9 +14,14 @@ extension AVCaptureDevice.Format { return true } } +#elseif os(visionOS) +extension AVCaptureDevice.Format { + var isMultiCamSupported: Bool { + return false + } +} #endif -#if os(iOS) || os(tvOS) || os(macOS) @available(tvOS 17.0, *) extension AVCaptureDevice.Format { func isFrameRateSupported(_ frameRate: Float64) -> Bool { @@ -42,4 +47,3 @@ extension AVCaptureDevice.Format { return false } } -#endif diff --git a/Sources/Extension/AVCaptureSession+Extension.swift b/Sources/Extension/AVCaptureSession+Extension.swift index c21383276..8df04b1eb 100644 --- a/Sources/Extension/AVCaptureSession+Extension.swift +++ b/Sources/Extension/AVCaptureSession+Extension.swift @@ -17,7 +17,8 @@ extension AVCaptureSession { } } } -#elseif os(iOS) || os(tvOS) || os(macOS) +#endif + @available(tvOS 17.0, *) extension AVCaptureSession { @available(iOS, obsoleted: 16.0) @@ -35,5 +36,4 @@ extension AVCaptureSession { } } } -#endif // swiftlint:enable unused_setter_value diff --git a/Sources/Extension/AVFrameRateRange+Extension.swift b/Sources/Extension/AVFrameRateRange+Extension.swift index 76518f5e9..ae961a8dd 100644 --- a/Sources/Extension/AVFrameRateRange+Extension.swift +++ b/Sources/Extension/AVFrameRateRange+Extension.swift @@ -1,7 +1,6 @@ import AVFoundation import Foundation -#if os(iOS) || os(tvOS) || os(macOS) @available(tvOS 17.0, *) extension AVFrameRateRange { func clamp(rate: Float64) -> Float64 { @@ -12,4 +11,3 @@ extension AVFrameRateRange { (minFrameRate...maxFrameRate) ~= frameRate } } -#endif diff --git a/Sources/IO/IOCaptureSession.swift b/Sources/IO/IOCaptureSession.swift index ea47c8a62..ee03e8b3b 100644 --- a/Sources/IO/IOCaptureSession.swift +++ b/Sources/IO/IOCaptureSession.swift @@ -1,10 +1,9 @@ -#if os(iOS) || os(tvOS) || os(macOS) import AVFoundation protocol IOCaptureSessionDelegate: AnyObject { @available(tvOS 17.0, *) func captureSession(_ session: IOCaptureSession, sessionRuntimeError session: AVCaptureSession, error: AVError) - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @available(tvOS 17.0, *) func captureSession(_ session: IOCaptureSession, sessionWasInterrupted session: AVCaptureSession, reason: AVCaptureSession.InterruptionReason?) @available(tvOS 17.0, *) @@ -21,8 +20,10 @@ final class IOCaptureSession { return false } } - #else + #elseif os(macOS) static let isMultiCamSupported = true + #elseif os(visionOS) + static let isMultiCamSupported = false #endif #if os(iOS) || os(tvOS) @@ -39,9 +40,12 @@ final class IOCaptureSession { var isMultitaskingCameraAccessEnabled: Bool { return session.isMultitaskingCameraAccessEnabled } - #else + #elseif os(macOS) let isMultiCamSessionEnabled = true let isMultitaskingCameraAccessEnabled = true + #elseif os(visionOS) + let isMultiCamSessionEnabled = false + let isMultitaskingCameraAccessEnabled = false #endif weak var delegate: (any IOCaptureSessionDelegate)? @@ -76,7 +80,10 @@ final class IOCaptureSession { session.commitConfiguration() } } - #elseif os(iOS) || os(macOS) + #elseif os(visionOS) + /// The capture session instance. + private(set) lazy var session = AVCaptureSession() + #else var sessionPreset: AVCaptureSession.Preset = .default { didSet { guard sessionPreset != oldValue, session.canSetSessionPreset(sessionPreset) else { @@ -125,6 +132,7 @@ final class IOCaptureSession { @available(tvOS 17.0, *) func attachCapture(_ capture: any IOCaptureUnit) { + #if !os(visionOS) if let connection = capture.connection { if let input = capture.input, session.canAddInput(input) { session.addInputWithNoConnections(input) @@ -135,23 +143,26 @@ final class IOCaptureSession { if session.canAddConnection(connection) { session.addConnection(connection) } - } else { - if let input = capture.input, session.canAddInput(input) { - session.addInput(input) - } - if let output = capture.output, session.canAddOutput(output) { - session.addOutput(output) - } + return + } + #endif + if let input = capture.input, session.canAddInput(input) { + session.addInput(input) + } + if let output = capture.output, session.canAddOutput(output) { + session.addOutput(output) } } @available(tvOS 17.0, *) func detachCapture(_ capture: any IOCaptureUnit) { + #if !os(visionOS) if let connection = capture.connection { if capture.output?.connections.contains(connection) == true { session.removeConnection(connection) } } + #endif if let input = capture.input, session.inputs.contains(input) { session.removeInput(input) } @@ -199,7 +210,7 @@ final class IOCaptureSession { @available(tvOS 17.0, *) private func addSessionObservers(_ session: AVCaptureSession) { NotificationCenter.default.addObserver(self, selector: #selector(sessionRuntimeError(_:)), name: .AVCaptureSessionRuntimeError, object: session) - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) NotificationCenter.default.addObserver(self, selector: #selector(sessionInterruptionEnded(_:)), name: .AVCaptureSessionInterruptionEnded, object: session) NotificationCenter.default.addObserver(self, selector: #selector(sessionWasInterrupted(_:)), name: .AVCaptureSessionWasInterrupted, object: session) #endif @@ -207,7 +218,7 @@ final class IOCaptureSession { @available(tvOS 17.0, *) private func removeSessionObservers(_ session: AVCaptureSession) { - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) NotificationCenter.default.removeObserver(self, name: .AVCaptureSessionWasInterrupted, object: session) NotificationCenter.default.removeObserver(self, name: .AVCaptureSessionInterruptionEnded, object: session) #endif @@ -234,7 +245,7 @@ final class IOCaptureSession { delegate?.captureSession(self, sessionRuntimeError: session, error: error) } - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @available(tvOS 17.0, *) @objc private func sessionWasInterrupted(_ notification: Notification) { @@ -286,5 +297,3 @@ extension IOCaptureSession: Running { } } } - -#endif diff --git a/Sources/IO/IOCaptureUnit.swift b/Sources/IO/IOCaptureUnit.swift index c3fdb66ad..9a503780f 100644 --- a/Sources/IO/IOCaptureUnit.swift +++ b/Sources/IO/IOCaptureUnit.swift @@ -1,4 +1,3 @@ -#if os(iOS) || os(tvOS) || os(macOS) import AVFoundation import Foundation @@ -14,4 +13,3 @@ protocol IOCaptureUnit { var output: Output? { get set } var connection: AVCaptureConnection? { get set } } -#endif diff --git a/Sources/IO/IOMixer.swift b/Sources/IO/IOMixer.swift index e861a61ba..f7a53e0f5 100644 --- a/Sources/IO/IOMixer.swift +++ b/Sources/IO/IOMixer.swift @@ -9,7 +9,7 @@ protocol IOMixerDelegate: AnyObject { func mixer(_ mixer: IOMixer, didOutput video: CMSampleBuffer) func mixer(_ mixer: IOMixer, videoErrorOccurred error: IOVideoUnitError) func mixer(_ mixer: IOMixer, audioErrorOccurred error: IOAudioUnitError) - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @available(tvOS 17.0, *) func mixer(_ mixer: IOMixer, sessionWasInterrupted session: AVCaptureSession, reason: AVCaptureSession.InterruptionReason?) @available(tvOS 17.0, *) @@ -41,13 +41,11 @@ final class IOMixer { return videoIO }() - #if os(iOS) || os(tvOS) || os(macOS) private(set) lazy var session = { var session = IOCaptureSession() session.delegate = self return session }() - #endif private(set) lazy var audioEngine: AVAudioEngine? = { return IOStream.audioEngineHolder.retain() @@ -57,7 +55,7 @@ final class IOMixer { IOStream.audioEngineHolder.release(audioEngine) } - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) func setBackgroundMode(_ background: Bool) { guard #available(tvOS 17.0, *) else { return @@ -135,11 +133,11 @@ extension IOMixer: AudioCodecDelegate { } } -#if os(iOS) || os(tvOS) || os(macOS) extension IOMixer: IOCaptureSessionDelegate { // MARK: IOCaptureSessionDelegate @available(tvOS 17.0, *) func captureSession(_ capture: IOCaptureSession, sessionRuntimeError session: AVCaptureSession, error: AVError) { + #if os(iOS) || os(tvOS) || os(macOS) switch error.code { case .unsupportedDeviceActiveFormat: guard let device = error.device, let format = device.videoFormat( @@ -165,9 +163,10 @@ extension IOMixer: IOCaptureSessionDelegate { default: break } + #endif } - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @available(tvOS 17.0, *) func captureSession(_ _: IOCaptureSession, sessionWasInterrupted session: AVCaptureSession, reason: AVCaptureSession.InterruptionReason?) { delegate?.mixer(self, sessionWasInterrupted: session, reason: reason) @@ -179,7 +178,6 @@ extension IOMixer: IOCaptureSessionDelegate { } #endif } -#endif extension IOMixer: IOAudioUnitDelegate { // MARK: IOAudioUnitDelegate diff --git a/Sources/IO/IOStream.swift b/Sources/IO/IOStream.swift index 2cdbd3571..ffe57ec10 100644 --- a/Sources/IO/IOStream.swift +++ b/Sources/IO/IOStream.swift @@ -20,7 +20,7 @@ public protocol IOStreamDelegate: AnyObject { func stream(_ stream: IOStream, didOutput audio: AVAudioBuffer, when: AVAudioTime) /// Tells the receiver to a video incoming. func stream(_ stream: IOStream, didOutput video: CMSampleBuffer) - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) /// Tells the receiver to session was interrupted. @available(tvOS 17.0, *) func stream(_ stream: IOStream, sessionWasInterrupted session: AVCaptureSession, reason: AVCaptureSession.InterruptionReason?) @@ -290,11 +290,9 @@ open class IOStream: NSObject { guard #available(tvOS 17.0, *) else { return } - #if os(iOS) || os(tvOS) || os(macOS) if newValue != nil && self.mixer.videoIO.hasDevice { self.mixer.session.startRunning() } - #endif } } } @@ -332,13 +330,12 @@ open class IOStream: NSObject { /// Creates a NetStream object. override public init() { super.init() - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) NotificationCenter.default.addObserver(self, selector: #selector(didEnterBackground(_:)), name: UIApplication.didEnterBackgroundNotification, object: nil) NotificationCenter.default.addObserver(self, selector: #selector(willEnterForeground(_:)), name: UIApplication.willEnterForegroundNotification, object: nil) #endif } - #if os(iOS) || os(macOS) || os(tvOS) /// Attaches the primary camera object. /// - Warning: This method can't use appendSampleBuffer at the same time. @available(tvOS 17.0, *) @@ -387,6 +384,7 @@ open class IOStream: NSObject { } } + #if os(iOS) || os(macOS) || os(tvOS) /// Attaches the audio capture object. /// - Warning: This method can't use appendSampleBuffer at the same time. @available(tvOS 17.0, *) @@ -483,10 +481,7 @@ open class IOStream: NSObject { mixer.muxer = telly mixer.startRunning() case .publish: - #if os(iOS) || os(tvOS) || os(macOS) - // Start capture audio and video data. mixer.session.startRunning() - #endif case .publishing(let muxer): mixer.muxer = muxer mixer.startRunning() @@ -495,7 +490,7 @@ open class IOStream: NSObject { } } - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @objc private func didEnterBackground(_ notification: Notification) { // Require main thread. Otherwise the microphone cannot be used in the background. @@ -529,7 +524,7 @@ extension IOStream: IOMixerDelegate { delegate?.stream(self, videoErrorOccurred: error) } - #if os(iOS) || os(tvOS) + #if os(iOS) || os(tvOS) || os(visionOS) @available(tvOS 17.0, *) func mixer(_ mixer: IOMixer, sessionWasInterrupted session: AVCaptureSession, reason: AVCaptureSession.InterruptionReason?) { delegate?.stream(self, sessionWasInterrupted: session, reason: reason) diff --git a/Sources/IO/IOVideoCaptureUnit.swift b/Sources/IO/IOVideoCaptureUnit.swift index 3a17dcfde..a1b58b5b1 100644 --- a/Sources/IO/IOVideoCaptureUnit.swift +++ b/Sources/IO/IOVideoCaptureUnit.swift @@ -1,4 +1,3 @@ -#if os(iOS) || os(tvOS) || os(macOS) import AVFoundation import Foundation @@ -37,6 +36,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } #endif + #if os(iOS) || os(macOS) || os(tvOS) /// Spcifies the video mirroed indicates whether the video flowing through the connection should be mirrored about its vertical axis. public var isVideoMirrored = false { didSet { @@ -45,6 +45,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } } } + #endif #if os(iOS) /// Specifies the preferredVideoStabilizationMode most appropriate for use with the connection. @@ -65,11 +66,13 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { return } output.alwaysDiscardsLateVideoFrames = true + #if os(iOS) || os(macOS) || os(tvOS) if output.availableVideoPixelFormatTypes.contains(colorFormat) { output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: NSNumber(value: colorFormat)] } else { logger.warn("device doesn't support this color format ", colorFormat, ".") } + #endif } } var connection: AVCaptureConnection? @@ -98,7 +101,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } else { connection = nil } - #else + #elseif os(tvOS) || os(macOS) if let output, let port = input?.ports.first(where: { $0.mediaType == .video }) { connection = AVCaptureConnection(inputPorts: [port], output: output) } else { @@ -106,6 +109,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } #endif videoUnit.mixer?.session.attachCapture(self) + #if os(iOS) || os(tvOS) || os(macOS) output?.connections.forEach { if $0.isVideoMirroringSupported { $0.isVideoMirrored = isVideoMirrored @@ -121,6 +125,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } #endif } + #endif setSampleBufferDelegate(videoUnit) } @@ -164,6 +169,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { } } + #if os(iOS) || os(tvOS) || os(macOS) func setTorchMode(_ torchMode: AVCaptureDevice.TorchMode) { guard let device, device.isTorchModeSupported(torchMode) else { return @@ -176,6 +182,7 @@ public final class IOVideoCaptureUnit: IOCaptureUnit { logger.error("while setting torch:", error) } } + #endif func setSampleBufferDelegate(_ videoUnit: IOVideoUnit?) { if let videoUnit { @@ -204,5 +211,3 @@ final class IOVideoCaptureUnitVideoDataOutputSampleBuffer: NSObject, AVCaptureVi videoMixer.append(sampleBuffer, channel: channel, isVideoMirrored: connection.isVideoMirrored) } } - -#endif diff --git a/Sources/IO/IOVideoUnit.swift b/Sources/IO/IOVideoUnit.swift index 5385c63b9..f21edba9a 100644 --- a/Sources/IO/IOVideoUnit.swift +++ b/Sources/IO/IOVideoUnit.swift @@ -63,7 +63,7 @@ final class IOVideoUnit: NSObject, IOUnit { var outputFormat: FormatDescription? { codec.outputFormat } - #if os(iOS) || os(macOS) || os(tvOS) + var frameRate = IOMixer.defaultFrameRate { didSet { guard #available(tvOS 17.0, *) else { @@ -75,6 +75,7 @@ final class IOVideoUnit: NSObject, IOUnit { } } + #if os(iOS) || os(tvOS) || os(macOS) var torch = false { didSet { guard #available(tvOS 17.0, *), torch != oldValue else { @@ -83,11 +84,12 @@ final class IOVideoUnit: NSObject, IOUnit { setTorchMode(torch ? .on : .off) } } + #endif + @available(tvOS 17.0, *) var hasDevice: Bool { !captures.lazy.filter { $0.value.device != nil }.isEmpty } - #endif var context: CIContext { get { @@ -132,7 +134,7 @@ final class IOVideoUnit: NSObject, IOUnit { private var captures: [UInt8: IOVideoCaptureUnit] { return _captures as! [UInt8: IOVideoCaptureUnit] } - #elseif os(iOS) || os(macOS) + #elseif os(iOS) || os(macOS) || os(visionOS) private var captures: [UInt8: IOVideoCaptureUnit] = [:] #endif @@ -175,7 +177,6 @@ final class IOVideoUnit: NSObject, IOUnit { } } - #if os(iOS) || os(tvOS) || os(macOS) @available(tvOS 17.0, *) func attachCamera(_ device: AVCaptureDevice?, channel: UInt8, configuration: IOVideoCaptureConfigurationBlock?) throws { guard captures[channel]?.device != device else { @@ -201,12 +202,14 @@ final class IOVideoUnit: NSObject, IOUnit { } } + #if os(iOS) || os(tvOS) || os(macOS) @available(tvOS 17.0, *) func setTorchMode(_ torchMode: AVCaptureDevice.TorchMode) { for capture in captures.values { capture.setTorchMode(torchMode) } } + #endif @available(tvOS 17.0, *) func capture(for channel: UInt8) -> IOVideoCaptureUnit? { @@ -238,7 +241,6 @@ final class IOVideoUnit: NSObject, IOUnit { } } } - #endif #if os(macOS) func attachScreen(_ input: AVCaptureScreenInput?, channel: UInt8) { @@ -267,14 +269,12 @@ extension IOVideoUnit: Running { } } -#if os(iOS) || os(tvOS) || os(macOS) extension IOVideoUnit { @available(tvOS 17.0, *) func makeVideoDataOutputSampleBuffer(_ channel: UInt8) -> IOVideoCaptureUnitVideoDataOutputSampleBuffer { return .init(channel: channel, videoMixer: videoMixer) } } -#endif extension IOVideoUnit: IOVideoMixerDelegate { // MARK: IOVideoMixerDelegate