From fedeb648fbb51bf1543e2656755b1550b9e8d8e3 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sun, 10 Mar 2024 19:11:11 +0000 Subject: [PATCH 1/2] support downmixing 4, 6 and 8 channel audio --- alvr/audio/src/lib.rs | 124 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 14 deletions(-) diff --git a/alvr/audio/src/lib.rs b/alvr/audio/src/lib.rs index b15c26ffb4..f44b13a73c 100644 --- a/alvr/audio/src/lib.rs +++ b/alvr/audio/src/lib.rs @@ -210,6 +210,108 @@ pub enum AudioRecordState { Err(Option), } +pub enum AudioChannel { + FrontLeft, + FrontRight, + Center, + SurroundLeft, + SurroundRight, + BackLeft, + BackRight, + Top, + HighFrontLeft, + HighFrontRight, + HighFrontCenter, + HighBackLeft, + HighBackRight, + LowFrequency, +} + +macro_rules! channel_mix { + ( $x:expr ) => { + match $x { + AudioChannel::FrontLeft => [1.0, 0.0], + AudioChannel::FrontRight => [0.0, 1.0], + AudioChannel::Center => [0.707, 0.707], + AudioChannel::SurroundLeft => [0.707, 0.0], + AudioChannel::SurroundRight => [0.0, 0.707], + AudioChannel::BackLeft => [0.707, 0.0], + AudioChannel::BackRight => [0.0, 0.707], + AudioChannel::Top => [0.577, 0.577], + AudioChannel::HighFrontLeft => [0.707, 0.0], + AudioChannel::HighFrontRight => [0.0, 0.707], + AudioChannel::HighFrontCenter => [0.5, 0.5], + AudioChannel::HighBackLeft => [0.5, 0.0], + AudioChannel::HighBackRight => [0.0, 0.5], + _ => [0.0, 0.0], + } + }; +} + +fn downmix_channels(channels: &[AudioChannel], data: &[u8], out_channels: u16) -> Vec { + let mut left = 0.0; + let mut right = 0.0; + + for i in 0..channels.len() { + let chan = &channels[i]; + let [l, r] = channel_mix!(chan); + let val = i16::from_ne_bytes([data[i * 2], data[i * 2 + 1]]).to_sample::(); + left += val * l; + right += val * r; + } + + if out_channels == 1 { + let bytes = ((left + right) / 2.0).to_sample::().to_ne_bytes(); + vec![bytes[0], bytes[1]] + } else { + let left_bytes = left.to_sample::().to_ne_bytes(); + let right_bytes = right.to_sample::().to_ne_bytes(); + vec![left_bytes[0], left_bytes[1], right_bytes[0], right_bytes[1]] + } +} + +fn downmix_audio(data: Vec, in_channels: u16, out_channels: u16) -> Vec { + if in_channels == out_channels { + data + } else if in_channels == 1 && out_channels == 2 { + data.chunks_exact(2) + .flat_map(|c| vec![c[0], c[1], c[0], c[1]]) + .collect() + } else { + let channels = match in_channels { + 2 => vec![AudioChannel::FrontLeft, AudioChannel::FrontRight], + 4 => vec![ + AudioChannel::FrontLeft, + AudioChannel::FrontRight, + AudioChannel::BackLeft, + AudioChannel::BackRight, + ], + 6 => vec![ + AudioChannel::FrontRight, + AudioChannel::Center, + AudioChannel::LowFrequency, + AudioChannel::SurroundLeft, // Sometimes actually BackLeft, has same level so it's okay + AudioChannel::SurroundRight, // Sometimes actually BackRight, has same level so it's okay + ], + 8 => vec![ + AudioChannel::FrontLeft, + AudioChannel::FrontRight, + AudioChannel::Center, + AudioChannel::LowFrequency, + AudioChannel::BackLeft, + AudioChannel::BackRight, + AudioChannel::SurroundLeft, + AudioChannel::SurroundRight, + ], + _ => unreachable!("Invalid input channel count"), + }; + + data.chunks_exact(in_channels as usize * 2) + .flat_map(|c| downmix_channels(&channels, c, out_channels)) + .collect() + } +} + #[allow(unused_variables)] pub fn record_audio_blocking( is_running: Arc bool + Send + Sync>, @@ -224,12 +326,16 @@ pub fn record_audio_blocking( // On Windows, loopback devices are not recognized as input devices. Use output config. .or_else(|_| device.inner.default_output_config())?; - if config.channels() > 2 { - // todo: handle more than 2 channels + if config.channels() > 8 { bail!( - "Audio devices with more than 2 channels are not supported. {}", + "Audio devices with more than 8 channels are not supported. {}", "Please turn off surround audio." ); + } else if config.channels() == 3 || config.channels() == 5 || config.channels() == 7 { + bail!( + "Audio devices with {} channels are not supported.", + config.channels() + ); } let stream_config = StreamConfig { @@ -261,17 +367,7 @@ pub fn record_audio_blocking( data.bytes().to_vec() }; - let data = if config.channels() == 1 && channels_count == 2 { - data.chunks_exact(2) - .flat_map(|c| vec![c[0], c[1], c[0], c[1]]) - .collect() - } else if config.channels() == 2 && channels_count == 1 { - data.chunks_exact(4) - .flat_map(|c| vec![c[0], c[1]]) - .collect() - } else { - data - }; + let data = downmix_audio(data, config.channels(), channels_count); if is_running() { let mut buffer = sender.get_buffer(&()).unwrap(); From 87f3b841cb09414c60c0adc8f7334ff6762a7b84 Mon Sep 17 00:00:00 2001 From: Barnaby <22575741+barnabwhy@users.noreply.github.com> Date: Sun, 10 Mar 2024 20:53:22 +0000 Subject: [PATCH 2/2] add 2.1 support (FL, FR SW) --- alvr/audio/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/alvr/audio/src/lib.rs b/alvr/audio/src/lib.rs index f44b13a73c..9e677f61d7 100644 --- a/alvr/audio/src/lib.rs +++ b/alvr/audio/src/lib.rs @@ -280,6 +280,11 @@ fn downmix_audio(data: Vec, in_channels: u16, out_channels: u16) -> Vec } else { let channels = match in_channels { 2 => vec![AudioChannel::FrontLeft, AudioChannel::FrontRight], + 3 => vec![ + AudioChannel::FrontLeft, + AudioChannel::FrontRight, + AudioChannel::LowFrequency, + ], 4 => vec![ AudioChannel::FrontLeft, AudioChannel::FrontRight, @@ -331,7 +336,7 @@ pub fn record_audio_blocking( "Audio devices with more than 8 channels are not supported. {}", "Please turn off surround audio." ); - } else if config.channels() == 3 || config.channels() == 5 || config.channels() == 7 { + } else if config.channels() == 5 || config.channels() == 7 { bail!( "Audio devices with {} channels are not supported.", config.channels()