From c61ca0d1169e4ba69a07ecab27400290cfc1e5a4 Mon Sep 17 00:00:00 2001 From: Kanat Kiialbaev Date: Tue, 17 Oct 2023 12:37:36 -0700 Subject: [PATCH] Add a way to intercept the audio samples before processing (#22) * Add a way to intercept the audio samples before processing * fix BUILD.gn (cherry picked from commit b33e7bdf77a950dd5e94a442bc9ddb518aec0cda) --- sdk/android/BUILD.gn | 1 + .../org/webrtc/audio/JavaAudioDeviceModule.java | 13 ++++++++++++- .../webrtc/audio/AudioRecordDataCallback.java | 16 ++++++++++++++++ .../java/org/webrtc/audio/WebRtcAudioRecord.java | 13 ++++++++++++- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 4cd8b682cb..9d82264d79 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -429,6 +429,7 @@ if (is_android) { "src/java/org/webrtc/audio/WebRtcAudioRecord.java", "src/java/org/webrtc/audio/WebRtcAudioTrack.java", "src/java/org/webrtc/audio/WebRtcAudioUtils.java", + "src/java/org/webrtc/audio/AudioRecordDataCallback.java", ] deps = [ diff --git a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java index b118843ea0..8515f48e3a 100644 --- a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -51,6 +51,7 @@ public static class Builder { private AudioAttributes audioAttributes; private boolean useLowLatency; private boolean enableVolumeLogger; + private AudioRecordDataCallback audioRecordDataCallback; private Builder(Context context) { this.context = context; @@ -221,6 +222,16 @@ public Builder setEnableVolumeLogger(boolean enableVolumeLogger) { return this; } + /** + * Can be used to gain access to the raw ByteBuffer from the recording device before it's + * fed into WebRTC. You can use this to manipulate the ByteBuffer (e.g. audio filters). + * Make sure that the operation is fast. + */ + public Builder setAudioRecordDataCallback(AudioRecordDataCallback audioRecordDataCallback) { + this.audioRecordDataCallback = audioRecordDataCallback; + return this; + } + /** * Construct an AudioDeviceModule based on the supplied arguments. The caller takes ownership * and is responsible for calling release(). @@ -255,7 +266,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() { } final WebRtcAudioRecord audioInput = new WebRtcAudioRecord(context, executor, audioManager, audioSource, audioFormat, audioRecordErrorCallback, audioRecordStateCallback, - samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); + samplesReadyCallback, audioRecordDataCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); final WebRtcAudioTrack audioOutput = new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, audioTrackStateCallback, useLowLatency, enableVolumeLogger); diff --git a/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java b/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java new file mode 100644 index 0000000000..421e559e2d --- /dev/null +++ b/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java @@ -0,0 +1,16 @@ +package org.webrtc.audio; + +import androidx.annotation.NonNull; + +import java.nio.ByteBuffer; + +public interface AudioRecordDataCallback { + /** + * Invoked after an audio sample is recorded. Can be used to manipulate + * the ByteBuffer before it's fed into WebRTC. Currently the audio in the + * ByteBuffer is always PCM 16bit and the buffer sample size is ~10ms. + * + * @param audioFormat format in android.media.AudioFormat + */ + void onAudioDataRecorded(int audioFormat, int channelCount, int sampleRate, @NonNull ByteBuffer audioBuffer); +} \ No newline at end of file diff --git a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java index cfb651f6cd..451d93f908 100644 --- a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java +++ b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -104,6 +104,7 @@ class WebRtcAudioRecord { private final @Nullable AudioRecordErrorCallback errorCallback; private final @Nullable AudioRecordStateCallback stateCallback; + private final @Nullable AudioRecordDataCallback audioRecordDataCallback; private final @Nullable SamplesReadyCallback audioSamplesReadyCallback; private final boolean isAcousticEchoCancelerSupported; private final boolean isNoiseSuppressorSupported; @@ -153,6 +154,13 @@ public void run() { captureTimeNs = audioTimestamp.nanoTime; } } + + // Allow the client to intercept the ByteBuffer (to modify it) + if (audioRecordDataCallback != null) { + audioRecordDataCallback.onAudioDataRecorded(audioRecord.getAudioFormat(), + audioRecord.getChannelCount(), audioRecord.getSampleRate(), byteBuffer); + } + nativeDataIsRecorded(nativeAudioRecord, bytesRead, captureTimeNs); } if (audioSamplesReadyCallback != null) { @@ -196,7 +204,8 @@ public void stopThread() { WebRtcAudioRecord(Context context, AudioManager audioManager) { this(context, newDefaultScheduler() /* scheduler */, audioManager, DEFAULT_AUDIO_SOURCE, DEFAULT_AUDIO_FORMAT, null /* errorCallback */, null /* stateCallback */, - null /* audioSamplesReadyCallback */, WebRtcAudioEffects.isAcousticEchoCancelerSupported(), + null /* audioSamplesReadyCallback */, null /* audioRecordCallback */, + WebRtcAudioEffects.isAcousticEchoCancelerSupported(), WebRtcAudioEffects.isNoiseSuppressorSupported()); } @@ -205,6 +214,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler, @Nullable AudioRecordErrorCallback errorCallback, @Nullable AudioRecordStateCallback stateCallback, @Nullable SamplesReadyCallback audioSamplesReadyCallback, + @Nullable AudioRecordDataCallback audioRecordDataCallback, boolean isAcousticEchoCancelerSupported, boolean isNoiseSuppressorSupported) { if (isAcousticEchoCancelerSupported && !WebRtcAudioEffects.isAcousticEchoCancelerSupported()) { throw new IllegalArgumentException("HW AEC not supported"); @@ -220,6 +230,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler, this.errorCallback = errorCallback; this.stateCallback = stateCallback; this.audioSamplesReadyCallback = audioSamplesReadyCallback; + this.audioRecordDataCallback = audioRecordDataCallback; this.isAcousticEchoCancelerSupported = isAcousticEchoCancelerSupported; this.isNoiseSuppressorSupported = isNoiseSuppressorSupported; Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo());