Skip to content

Commit

Permalink
Support downmix feature and refactor.
Browse files Browse the repository at this point in the history
  • Loading branch information
@shogo4405
shogo4405 committed Sep 22, 2023
1 parent 5e43171 commit a6725f4
Show file tree
Hide file tree
Showing 8 changed files with 115 additions and 105 deletions.
4 changes: 0 additions & 4 deletions HaishinKit.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,6 @@
BCFB355524FA27EA00DC5108 /* PlaybackViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCFB355324FA275600DC5108 /* PlaybackViewController.swift */; };
BCFB355A24FA40DD00DC5108 /* PlaybackContainerViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCFB355924FA40DD00DC5108 /* PlaybackContainerViewController.swift */; };
BCFC51FE2AAB420700014428 /* IOAudioResampler.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCFC51FD2AAB420700014428 /* IOAudioResampler.swift */; };
BCFC9BE02AB43A3A00378E56 /* AudioCodecSettingsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BCFC9BDF2AB43A3A00378E56 /* AudioCodecSettingsTests.swift */; };
BCFF640B29C0C44B004EFF2F /* SampleVideo_360x240_5mb_2ch.ts in Resources */ = {isa = PBXBuildFile; fileRef = BCFF640A29C0C44B004EFF2F /* SampleVideo_360x240_5mb_2ch.ts */; };
/* End PBXBuildFile section */

Expand Down Expand Up @@ -645,7 +644,6 @@
BCFB355324FA275600DC5108 /* PlaybackViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlaybackViewController.swift; sourceTree = "<group>"; };
BCFB355924FA40DD00DC5108 /* PlaybackContainerViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlaybackContainerViewController.swift; sourceTree = "<group>"; };
BCFC51FD2AAB420700014428 /* IOAudioResampler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOAudioResampler.swift; sourceTree = "<group>"; };
BCFC9BDF2AB43A3A00378E56 /* AudioCodecSettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCodecSettingsTests.swift; sourceTree = "<group>"; };
BCFF640A29C0C44B004EFF2F /* SampleVideo_360x240_5mb_2ch.ts */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.typescript; path = SampleVideo_360x240_5mb_2ch.ts; sourceTree = "<group>"; };
/* End PBXFileReference section */

Expand Down Expand Up @@ -896,7 +894,6 @@
isa = PBXGroup;
children = (
2950181F1FFA1BD700358E10 /* AudioCodecTests.swift */,
BCFC9BDF2AB43A3A00378E56 /* AudioCodecSettingsTests.swift */,
);
path = Codec;
sourceTree = "<group>";
Expand Down Expand Up @@ -1844,7 +1841,6 @@
290EA8AA1DFB61E700053022 /* CRC32Tests.swift in Sources */,
035AFA042263868E009DD0BB /* RTMPStreamTests.swift in Sources */,
290686031DFDB7A7008EB7ED /* RTMPConnectionTests.swift in Sources */,
BCFC9BE02AB43A3A00378E56 /* AudioCodecSettingsTests.swift in Sources */,
BCC9E9092636FF7400948774 /* DataBufferTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down
1 change: 1 addition & 0 deletions Sources/Codec/AudioCodec.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public class AudioCodec {
guard var inSourceFormat, inSourceFormat != oldValue else {
return
}
cursor = 0
inputBuffers.removeAll()
outputBuffers.removeAll()
audioConverter = makeAudioConverter(&inSourceFormat)
Expand Down
67 changes: 21 additions & 46 deletions Sources/Codec/AudioCodecSettings.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ import Foundation
public struct AudioCodecSettings: Codable {
/// The default value.
public static let `default` = AudioCodecSettings()

/// Maximum number of channels supported by the system
public static let maximumNumberOfChannels: UInt32 = 2
/// Maximum sampleRate supported by the system
public static let mamimumSampleRate: Float64 = 48000
public static let mamimumSampleRate: Float64 = 48000.0

/// The type of the AudioCodec supports format.
enum Format: Codable {
Expand Down Expand Up @@ -133,33 +132,34 @@ public struct AudioCodecSettings: Codable {
/// Specifies the sampleRate of audio output.
public var sampleRate: Float64
/// Specifies the channels of audio output.
public var channels: Int
/// Map of the output to input channels.
public var channelMap: [Int: Int]
public var channels: UInt32
/// Specifies the mixes the channels or not. Currently, it supports input sources with 4, 5, 6, and 8 channels.
public var downmix: Bool
/// Specifies the map of the output to input channels.
/// ## Example code:
/// ```
/// // If you want to use the 3rd and 4th channels from a 4-channel input source for a 2-channel output, you would specify it like this.
/// channelMap = [2, 3]
/// ```
public var channelMap: [Int]?
/// Specifies the output format.
var format: AudioCodecSettings.Format = .aac

/// Create an new AudioCodecSettings instance.
/// Create an new AudioCodecSettings instance. A value of 0 will use the same value as the input source.
public init(
bitRate: Int = 64 * 1000,
sampleRate: Float64 = 0,
channels: Int = 0,
channelMap: [Int: Int] = [0: 0, 1: 1]
channels: UInt32 = 0,
downmix: Bool = false,
channelMap: [Int]? = nil
) {
self.bitRate = bitRate
self.sampleRate = sampleRate
self.channels = channels
self.downmix = downmix
self.channelMap = channelMap
}

func invalidateConverter(_ oldValue: AudioCodecSettings) -> Bool {
return !(
sampleRate == oldValue.sampleRate &&
channels == oldValue.channels &&
channelMap == oldValue.channelMap
)
}

func apply(_ converter: AVAudioConverter?, oldValue: AudioCodecSettings?) {
guard let converter else {
return
Expand All @@ -175,37 +175,12 @@ public struct AudioCodecSettings: Codable {
}
}

func makeOutputChannels(_ inChannels: Int) -> Int {
return min(channels == 0 ? inChannels : channels, Int(Self.maximumNumberOfChannels))
}

func makeChannelMap(_ inChannels: Int) -> [NSNumber] {
let outChannels = makeOutputChannels(inChannels)
var result = Array(repeating: -1, count: outChannels)
for inputIndex in 0..<min(inChannels, outChannels) {
result[inputIndex] = inputIndex
}
for currentIndex in 0..<outChannels {
if let inputIndex = channelMap[currentIndex], inputIndex < inChannels {
result[currentIndex] = inputIndex
}
}
return result.map { NSNumber(value: $0) }
}

func makeOutputFormat(_ inputFormat: AVAudioFormat?) -> AVAudioFormat? {
guard let inputFormat else {
return nil
}
let numberOfChannels = makeOutputChannels(Int(inputFormat.channelCount))
guard let channelLayout = AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_DiscreteInOrder | UInt32(numberOfChannels)) else {
return nil
}
func makeAudioResamplerSettings() -> IOAudioResamplerSettings {
return .init(
commonFormat: inputFormat.commonFormat,
sampleRate: min(sampleRate == 0 ? inputFormat.sampleRate : sampleRate, Self.mamimumSampleRate),
interleaved: inputFormat.isInterleaved,
channelLayout: channelLayout
sampleRate: sampleRate,
channels: channels,
downmix: downmix,
channelMap: channelMap?.map { NSNumber(value: $0) }
)
}
}
65 changes: 60 additions & 5 deletions Sources/Media/IOAudioResampler.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,68 @@ protocol IOAudioResamplerDelegate: AnyObject {
func resampler(_ resampler: IOAudioResampler<Self>, errorOccurred error: AudioCodec.Error)
}

struct IOAudioResamplerSettings {
let sampleRate: Float64
let channels: UInt32
let downmix: Bool
let channelMap: [NSNumber]?

init(sampleRate: Float64 = 0, channels: UInt32 = 0, downmix: Bool = false, channelMap: [NSNumber]? = nil) {
self.sampleRate = sampleRate
self.channels = channels
self.downmix = downmix
self.channelMap = channelMap
}

func invalidate(_ oldValue: IOAudioResamplerSettings!) -> Bool {
return (sampleRate != oldValue.sampleRate &&
channels != oldValue.channels)
}

func apply(_ converter: AVAudioConverter?, oldValue: IOAudioResamplerSettings?) {
guard let converter else {
return
}
if converter.downmix != downmix {
converter.downmix = downmix
}
if let channelMap {
converter.channelMap = channelMap
} else {
switch converter.outputFormat.channelCount {
case 1:
converter.channelMap = [0]
case 2:
converter.channelMap = (converter.inputFormat.channelCount == 1) ? [0, 0] : [0, 1]
default:
logger.error("channelCount must be 2 or less.")
}
}
}

func makeOutputFormat(_ inputFormat: AVAudioFormat?) -> AVAudioFormat? {
guard let inputFormat else {
return nil
}
return .init(
commonFormat: inputFormat.commonFormat,
sampleRate: min(sampleRate == 0 ? inputFormat.sampleRate : sampleRate, AudioCodecSettings.mamimumSampleRate),
channels: min(channels == 0 ? inputFormat.channelCount : channels, AudioCodecSettings.maximumNumberOfChannels),
interleaved: inputFormat.isInterleaved
)
}
}

final class IOAudioResampler<T: IOAudioResamplerDelegate> {
var settings: AudioCodecSettings = .default {
var settings: IOAudioResamplerSettings = .init() {
didSet {
guard var inSourceFormat, settings.invalidateConverter(oldValue) else {
return
if settings.invalidate(oldValue) {
if var inSourceFormat {
setUp(&inSourceFormat)
}
} else {
settings.apply(audioConverter, oldValue: oldValue)
}
setUp(&inSourceFormat)
}
}
weak var delegate: T?
Expand All @@ -42,7 +97,7 @@ final class IOAudioResampler<T: IOAudioResamplerDelegate> {
guard let audioConverter else {
return
}
audioConverter.channelMap = settings.makeChannelMap(Int(audioConverter.inputFormat.channelCount))
settings.apply(audioConverter, oldValue: nil)
audioConverter.primeMethod = .normal
delegate?.resampler(self, didOutput: audioConverter.outputFormat)
}
Expand Down
2 changes: 1 addition & 1 deletion Sources/Media/IOAudioUnit.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ final class IOAudioUnit: NSObject, IOUnit {
var settings: AudioCodecSettings = .default {
didSet {
codec.settings = settings
resampler.settings = settings
resampler.settings = settings.makeAudioResamplerSettings()
}
}

Expand Down
30 changes: 26 additions & 4 deletions Sources/Util/AVAudioFormatFactory.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,21 @@ enum AVAudioFormatFactory {
guard inSourceFormat.mBitsPerChannel == 16 else {
return nil
}
if let layout = Self.makeChannelLayout(inSourceFormat.mChannelsPerFrame) {
return .init(commonFormat: .pcmFormatInt16, sampleRate: inSourceFormat.mSampleRate, interleaved: true, channelLayout: layout)
let interleaved = !((inSourceFormat.mFormatFlags & kLinearPCMFormatFlagIsNonInterleaved) == kLinearPCMFormatFlagIsNonInterleaved)
if let channelLayout = Self.makeChannelLayout(inSourceFormat.mChannelsPerFrame) {
return .init(
commonFormat: .pcmFormatInt16,
sampleRate: inSourceFormat.mSampleRate,
interleaved: interleaved,
channelLayout: channelLayout
)
}
return AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: inSourceFormat.mSampleRate, channels: inSourceFormat.mChannelsPerFrame, interleaved: true)
return .init(
commonFormat: .pcmFormatInt16,
sampleRate: inSourceFormat.mSampleRate,
channels: inSourceFormat.mChannelsPerFrame,
interleaved: interleaved
)
}
if let layout = Self.makeChannelLayout(inSourceFormat.mChannelsPerFrame) {
return .init(streamDescription: &inSourceFormat, channelLayout: layout)
Expand All @@ -23,6 +34,17 @@ enum AVAudioFormatFactory {
guard 2 < numberOfChannels else {
return nil
}
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_DiscreteInOrder | numberOfChannels)
switch numberOfChannels {
case 4:
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_AudioUnit_4)
case 5:
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_AudioUnit_5)
case 6:
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_AudioUnit_6)
case 8:
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_AudioUnit_8)
default:
return AVAudioChannelLayout(layoutTag: kAudioChannelLayoutTag_DiscreteInOrder | numberOfChannels)
}
}
}
39 changes: 0 additions & 39 deletions Tests/Codec/AudioCodecSettingsTests.swift
View file

This file was deleted.

12 changes: 6 additions & 6 deletions Tests/Media/IOAudioResamplerTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpKeep16000() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 16000, channels: 1)
resampler.settings = .init(sampleRate: 16000, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(48000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 16000)
Expand All @@ -30,7 +30,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpKeep44100() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 44100, channels: 1)
resampler.settings = .init(sampleRate: 44100, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(48000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 44100)
Expand All @@ -44,7 +44,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpKeep48000() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 48000, channels: 1)
resampler.settings = .init(sampleRate: 48000, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(48000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 48000)
Expand All @@ -54,7 +54,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpPassthrough48000_44100() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 0, channels: 1)
resampler.settings = .init(sampleRate: 0, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(44000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 44000)
Expand All @@ -64,7 +64,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpPassthrough44100_48000() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 0, channels: 1)
resampler.settings = .init(sampleRate: 0, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(48000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 48000)
Expand All @@ -74,7 +74,7 @@ final class IOAudioResamplerTests: XCTestCase {

func testpPassthrough16000_48000() {
let resampler = IOAudioResampler<NullIOAudioResamplerDelegate>()
resampler.settings = .init(bitRate: 0, sampleRate: 0, channels: 1)
resampler.settings = .init(sampleRate: 0, channels: 1)
resampler.delegate = nullIOAudioResamplerDelegate
resampler.appendSampleBuffer(CMAudioSampleBufferFactory.makeSinWave(16000, numSamples: 1024, channels: 1)!)
XCTAssertEqual(resampler.outputFormat?.sampleRate, 16000)
Expand Down

0 comments on commit a6725f4

Please sign in to comment.