Skip to content

Commit

Permalink
Merge PR #6511: REFAC(client): Introduce and use AudioPreprocessor
Browse files Browse the repository at this point in the history
  • Loading branch information
davidebeatrici authored Jan 21, 2025
2 parents 5452eef + a4b69de commit 104bab8
Show file tree
Hide file tree
Showing 7 changed files with 438 additions and 75 deletions.
2 changes: 1 addition & 1 deletion src/mumble/AudioConfigDialog.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,7 @@ void AudioInputDialog::showSpeexNoiseSuppressionSlider(bool show) {
void AudioInputDialog::on_Tick_timeout() {
AudioInputPtr ai = Global::get().ai;

if (!ai.get() || !ai->sppPreprocess)
if (!ai.get() || !ai->m_preprocessor)
return;

abSpeech->iBelow = qsTransmitMin->value();
Expand Down
67 changes: 24 additions & 43 deletions src/mumble/AudioInput.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,7 @@ AudioInput::AudioInput()

bEchoMulti = false;

sppPreprocess = nullptr;
sesEcho = nullptr;
sesEcho = nullptr;
srsMic = srsEcho = nullptr;

iEchoChannels = iMicChannels = 0;
Expand Down Expand Up @@ -298,8 +297,6 @@ AudioInput::~AudioInput() {
}
#endif

if (sppPreprocess)
speex_preprocess_state_destroy(sppPreprocess);
if (sesEcho)
speex_echo_state_destroy(sesEcho);

Expand Down Expand Up @@ -740,44 +737,34 @@ void AudioInput::resetAudioProcessor() {
if (!bResetProcessor)
return;

int iArg;

if (sppPreprocess)
speex_preprocess_state_destroy(sppPreprocess);
if (sesEcho)
speex_echo_state_destroy(sesEcho);

sppPreprocess = speex_preprocess_state_init(iFrameSize, iSampleRate);
m_preprocessor.init(iSampleRate, iFrameSize);
resync.reset();
selectNoiseCancel();

iArg = 1;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_VAD, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DEREVERB, &iArg);

iArg = 30000;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_TARGET, &iArg);
m_preprocessor.setVAD(true);
m_preprocessor.setAGC(true);
m_preprocessor.setDereverb(true);

float v = 30000.0f / static_cast< float >(Global::get().s.iMinLoudness);
iArg = static_cast< int >(floorf(20.0f * log10f(v)));
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_MAX_GAIN, &iArg);
m_preprocessor.setAGCTarget(30000);

iArg = -60;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_DECREMENT, &iArg);
const float v = 30000.0f / static_cast< float >(Global::get().s.iMinLoudness);
m_preprocessor.setAGCMaxGain(static_cast< std::int32_t >(floorf(20.0f * log10f(v))));
m_preprocessor.setAGCDecrement(-60);

if (noiseCancel == Settings::NoiseCancelSpeex) {
iArg = Global::get().s.iSpeexNoiseCancelStrength;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
m_preprocessor.setNoiseSuppress(Global::get().s.iSpeexNoiseCancelStrength);
}

if (iEchoChannels > 0) {
int filterSize = iFrameSize * (10 + resync.getNominalLag());
sesEcho =
speex_echo_state_init_mc(iFrameSize, filterSize, 1, bEchoMulti ? static_cast< int >(iEchoChannels) : 1);
iArg = iSampleRate;
int iArg = iSampleRate;
speex_echo_ctl(sesEcho, SPEEX_ECHO_SET_SAMPLING_RATE, &iArg);
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_ECHO_STATE, sesEcho);
m_preprocessor.setEchoState(sesEcho);

qWarning("AudioInput: ECHO CANCELLER ACTIVE");
} else {
Expand Down Expand Up @@ -821,24 +808,24 @@ void AudioInput::selectNoiseCancel() {
#endif
}

int iArg = 0;
bool preprocessorDenoise = false;
switch (noiseCancel) {
case Settings::NoiseCancelOff:
qWarning("AudioInput: Noise canceller disabled");
break;
case Settings::NoiseCancelSpeex:
qWarning("AudioInput: Using Speex as noise canceller");
iArg = 1;
preprocessorDenoise = true;
break;
case Settings::NoiseCancelRNN:
qWarning("AudioInput: Using ReNameNoise as noise canceller");
break;
case Settings::NoiseCancelBoth:
iArg = 1;
preprocessorDenoise = true;
qWarning("AudioInput: Using ReNameNoise and Speex as noise canceller");
break;
}
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_DENOISE, &iArg);
m_preprocessor.setDenoise(preprocessorDenoise);
}

int AudioInput::encodeOpusFrame(short *source, int size, EncodingOutputBuffer &buffer) {
Expand All @@ -857,7 +844,6 @@ int AudioInput::encodeOpusFrame(short *source, int size, EncodingOutputBuffer &b
}

void AudioInput::encodeAudioFrame(AudioChunk chunk) {
int iArg;
float sum;
short max;

Expand Down Expand Up @@ -897,11 +883,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
QMutexLocker l(&qmSpeex);
resetAudioProcessor();

speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_AGC_GAIN, &iArg);
float gainValue = static_cast< float >(iArg);
const std::int32_t gainValue = m_preprocessor.getAGCGain();

if (noiseCancel == Settings::NoiseCancelSpeex || noiseCancel == Settings::NoiseCancelBoth) {
iArg = Global::get().s.iSpeexNoiseCancelStrength - iArg;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, &iArg);
m_preprocessor.setNoiseSuppress(Global::get().s.iSpeexNoiseCancelStrength - gainValue);
}

short psClean[iFrameSize];
Expand All @@ -924,7 +909,7 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
}
#endif

speex_preprocess_run(sppPreprocess, psSource);
m_preprocessor.run(*psSource);

sum = 1.0f;
for (unsigned int i = 0; i < iFrameSize; i++)
Expand All @@ -942,12 +927,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
static_cast< std::streamsize >(iFrameSize * sizeof(short)));
}

spx_int32_t prob = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_GET_PROB, &prob);
fSpeechProb = static_cast< float >(prob) / 100.0f;
fSpeechProb = static_cast< float >(m_preprocessor.getSpeechProb()) / 100.0f;

// clean microphone level: peak of filtered signal attenuated by AGC gain
dPeakCleanMic = qMax(dPeakSignal - gainValue, -96.0f);
dPeakCleanMic = qMax(dPeakSignal - static_cast< float >(gainValue), -96.0f);
float level = (Global::get().s.vsVAD == Settings::SignalToNoise) ? fSpeechProb : (1.0f + dPeakCleanMic / 96.0f);

bool bIsSpeech = false;
Expand Down Expand Up @@ -1075,12 +1058,10 @@ void AudioInput::encodeAudioFrame(AudioChunk chunk) {
}
}

spx_int32_t increment = 0;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
m_preprocessor.setAGCIncrement(0);
return;
} else {
spx_int32_t increment = 12;
speex_preprocess_ctl(sppPreprocess, SPEEX_PREPROCESS_SET_AGC_INCREMENT, &increment);
m_preprocessor.setAGCIncrement(12);
}

if (bIsSpeech && !bPreviousVoice) {
Expand Down
4 changes: 2 additions & 2 deletions src/mumble/AudioInput.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@
#include <vector>

#include <speex/speex_echo.h>
#include <speex/speex_preprocess.h>
#include <speex/speex_resampler.h>

#include "Audio.h"
#include "AudioOutputToken.h"
#include "AudioPreprocessor.h"
#include "EchoCancelOption.h"
#include "MumbleProtocol.h"
#include "Settings.h"
Expand Down Expand Up @@ -224,7 +224,7 @@ class AudioInput : public QThread {
static const int iFrameSize = SAMPLE_RATE / 100;

QMutex qmSpeex;
SpeexPreprocessState *sppPreprocess;
AudioPreprocessor m_preprocessor;
SpeexEchoState *sesEcho;

/// bResetEncoder is a flag that notifies
Expand Down
179 changes: 179 additions & 0 deletions src/mumble/AudioPreprocessor.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
// Copyright The Mumble Developers. All rights reserved.
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file at the root of the
// Mumble source tree or at <https://www.mumble.info/LICENSE>.

#include "AudioPreprocessor.h"

#include <utility>

#include <speex/speex_preprocess.h>

AudioPreprocessor::AudioPreprocessor(AudioPreprocessor &&other) : m_handle(std::exchange(other.m_handle, nullptr)) {
}

AudioPreprocessor::~AudioPreprocessor() {
deinit();
}

AudioPreprocessor &AudioPreprocessor::operator=(AudioPreprocessor &&other) {
m_handle = std::exchange(other.m_handle, nullptr);
return *this;
}

bool AudioPreprocessor::init(const std::uint32_t sampleRate, const std::uint32_t quantum) {
deinit();

m_handle = speex_preprocess_state_init(static_cast< int >(quantum), static_cast< int >(sampleRate));
return m_handle != nullptr;
}

void AudioPreprocessor::deinit() {
if (m_handle) {
speex_preprocess_state_destroy(m_handle);
m_handle = nullptr;
}
}

bool AudioPreprocessor::run(std::int16_t &samples) {
return speex_preprocess_run(m_handle, &samples);
}

SpeexEchoState_ *AudioPreprocessor::getEchoState() {
SpeexEchoState_ *handle;
return speex_preprocess_ctl(m_handle, SPEEX_PREPROCESS_GET_ECHO_STATE, &handle) == 0 ? handle : nullptr;
}

bool AudioPreprocessor::setEchoState(SpeexEchoState_ *handle) {
return speex_preprocess_ctl(m_handle, SPEEX_PREPROCESS_SET_ECHO_STATE, handle) == 0;
}

bool AudioPreprocessor::usesAGC() const {
return getBool(SPEEX_PREPROCESS_GET_AGC);
}

bool AudioPreprocessor::setAGC(const bool enable) {
return setBool(SPEEX_PREPROCESS_SET_AGC, enable);
}

std::int32_t AudioPreprocessor::getAGCDecrement() const {
return getInt32(SPEEX_PREPROCESS_GET_AGC_DECREMENT);
}

bool AudioPreprocessor::setAGCDecrement(const std::int32_t value) {
return setInt32(SPEEX_PREPROCESS_SET_AGC_DECREMENT, value);
}

std::int32_t AudioPreprocessor::getAGCGain() const {
return getInt32(SPEEX_PREPROCESS_GET_AGC_GAIN);
}

std::int32_t AudioPreprocessor::getAGCIncrement() const {
return getInt32(SPEEX_PREPROCESS_GET_AGC_INCREMENT);
}

bool AudioPreprocessor::setAGCIncrement(const std::int32_t value) {
return setInt32(SPEEX_PREPROCESS_SET_AGC_INCREMENT, value);
}

std::int32_t AudioPreprocessor::getAGCMaxGain() const {
return getInt32(SPEEX_PREPROCESS_GET_AGC_MAX_GAIN);
}

bool AudioPreprocessor::setAGCMaxGain(const std::int32_t value) {
return setInt32(SPEEX_PREPROCESS_SET_AGC_MAX_GAIN, value);
}

std::int32_t AudioPreprocessor::getAGCTarget() const {
return getInt32(SPEEX_PREPROCESS_GET_AGC_TARGET);
}

bool AudioPreprocessor::setAGCTarget(const std::int32_t value) {
return setInt32(SPEEX_PREPROCESS_SET_AGC_TARGET, value);
}

bool AudioPreprocessor::usesDenoise() const {
return getBool(SPEEX_PREPROCESS_GET_DENOISE);
}

bool AudioPreprocessor::setDenoise(const bool enable) {
return setBool(SPEEX_PREPROCESS_SET_DENOISE, enable);
}

bool AudioPreprocessor::usesDereverb() const {
return getBool(SPEEX_PREPROCESS_GET_DEREVERB);
}

bool AudioPreprocessor::setDereverb(const bool enable) {
return setBool(SPEEX_PREPROCESS_SET_DEREVERB, enable);
}

std::int32_t AudioPreprocessor::getNoiseSuppress() const {
return getInt32(SPEEX_PREPROCESS_GET_NOISE_SUPPRESS);
}

bool AudioPreprocessor::setNoiseSuppress(const std::int32_t value) {
return setInt32(SPEEX_PREPROCESS_SET_NOISE_SUPPRESS, value);
}

AudioPreprocessor::psd_t AudioPreprocessor::getPSD() const {
const auto size = getInt32(SPEEX_PREPROCESS_GET_PSD_SIZE);
if (!size) {
return {};
}

psd_t ret(static_cast< size_t >(size));
if (speex_preprocess_ctl(m_handle, SPEEX_PREPROCESS_GET_PSD, ret.data()) != 0) {
return {};
}

return ret;
}

AudioPreprocessor::psd_t AudioPreprocessor::getNoisePSD() const {
const auto size = getInt32(SPEEX_PREPROCESS_GET_PSD_SIZE);
if (!size) {
return {};
}

psd_t ret(static_cast< size_t >(size));
if (speex_preprocess_ctl(m_handle, SPEEX_PREPROCESS_GET_NOISE_PSD, ret.data()) != 0) {
return {};
}

return ret;
}

std::int32_t AudioPreprocessor::getSpeechProb() const {
return getInt32(SPEEX_PREPROCESS_GET_PROB);
}

bool AudioPreprocessor::usesVAD() const {
return getBool(SPEEX_PREPROCESS_GET_VAD);
}

bool AudioPreprocessor::setVAD(const bool enable) {
return setBool(SPEEX_PREPROCESS_SET_VAD, enable);
}

bool AudioPreprocessor::getBool(const int op) const {
const auto val = getInt32(op);
return static_cast< bool >(val);
}

bool AudioPreprocessor::setBool(const int op, const bool value) {
return setInt32(op, value);
}

std::int32_t AudioPreprocessor::getInt32(const int op) const {
spx_int32_t value;
if (speex_preprocess_ctl(m_handle, op, &value) != 0) {
return 0;
}

return value;
}

bool AudioPreprocessor::setInt32(const int op, std::int32_t value) {
return speex_preprocess_ctl(m_handle, op, &value) == 0;
}
Loading

0 comments on commit 104bab8

Please sign in to comment.