Patch/android/external audio processing

extensions/android-external-audio-processing/BUILD.gn

@@ -0,0 +1,40 @@
+import("../../webrtc.gni")
+import("//build/config/android/config.gni")
+import("//build/config/android/rules.gni")
+
+rtc_library("external_processing") {
+    visibility = [ "*" ]
+    configs += [ "../../modules/audio_processing:apm_debug_dump", ]
+    configs += [ "//build/config/android:hide_all_but_jni" ]
+    defines = ["WEBRTC_ANDROID", "WEBRTC_POSIX",]
+    sources = [
+        "include/external_processor.hpp",
+        "external_processing.hpp",
+        "external_processing.cc",
+        "external_processing_factory_jni.cc",
+    ]
+    lib_dirs = []
+    deps = [
+        ":generated_external_jni",
+        ":external_java",
+        "../../rtc_base:logging",
+        "../../sdk/android:native_api_jni",
+        "../../modules/audio_processing:audio_processing",
+    ]
+}
+
+rtc_android_library("external_java") {
+    visibility = [ "*" ]
+    sources = [
+        "java/src/org/webrtc/ExternalAudioProcessingFactory.java",
+    ]
+    deps = ["//sdk/android:peerconnection_java",]
+}
+
+generate_jni("generated_external_jni") {
+    sources = [
+        "java/src/org/webrtc/ExternalAudioProcessingFactory.java",
+    ]
+    namespace = "external"
+    jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
+}

extensions/android-external-audio-processing/external_processing.cc

@@ -0,0 +1,177 @@
+#include "external_processing.hpp"
+
+#include <dlfcn.h>
+#include <syslog.h>
+
+#include "rtc_base/time_utils.h"
+
+namespace external {
+
+ExternalProcessing* ExternalProcessing::m_instance = nullptr;
+
+ExternalProcessing::ExternalProcessing() {
+  ::syslog(LOG_INFO, "ExternalProcessing: #Constructor;");
+}
+
+ExternalProcessing::~ExternalProcessing() {
+  ::syslog(LOG_INFO, "ExternalProcessing: #Destructor;");
+
+  Destroy();
+}
+
+bool ExternalProcessing::Create(const char* libname) {
+  ::syslog(LOG_INFO, "ExternalProcessing: #Create; libname: %s", libname);
+
+  // Load the shared library
+  dlerror();  // Clear any existing errors
+  void* handle = dlopen(libname, RTLD_LAZY);
+  if (!handle) {
+    ::syslog(LOG_ERR, "ExternalProcessing: #Create; Failed to load library: %s",
+             dlerror());
+    return false;
+  }
+
+  // Load external processor functions
+  for (size_t functionId = 0; functionId < kFunctionCount; ++functionId) {
+    const char* functionName = kFunctionNames[functionId];
+    syslog(LOG_INFO, "ExternalProcessing: #Create; Loading function: %s",
+           functionName);
+    void* functionPtr = dlsym(handle, functionName);
+    const char* dlsym_error = dlerror();
+    if (dlsym_error) {
+      syslog(LOG_ERR,
+             "ExternalProcessing: #Create; Failed to load the function: %s",
+             dlsym_error);
+      return false;
+    }
+    m_functionPointers[functionId] = functionPtr;
+  }
+
+  void* createPtr = m_functionPointers[FunctionId::ExternalProcessorCreate];
+  if (!createPtr) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Create; Failed to access "
+             "ExternalProcessorCreate function");
+    dlclose(handle);
+    return false;
+  }
+
+  auto createFunc =
+      reinterpret_cast<ExternalProcessorCreateFuncType>(createPtr);
+  if (!createFunc()) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Create; Failed to invoke "
+             "ExternalProcessorCreate function");
+    dlclose(handle);
+    return false;
+  }
+
+  m_handle = handle;
+
+  ::syslog(LOG_INFO, "ExternalProcessing: #Create; completed successfully");
+
+  return true;
+}
+
+bool ExternalProcessing::Destroy() {
+  ::syslog(LOG_INFO, "ExternalProcessing: #Destroy;");
+
+  void* destroyPtr = m_functionPointers[FunctionId::ExternalProcessorDestroy];
+  if (destroyPtr) {
+    ::syslog(LOG_INFO,
+             "ExternalProcessing: #Destroy; Invoke ExternalProcessorDestroy "
+             "function");
+
+    auto destroyFunc =
+        reinterpret_cast<ExternalProcessorDestroyFuncType>(destroyPtr);
+    if (destroyFunc()) {
+      ::syslog(LOG_INFO,
+               "ExternalProcessing: #Destroy; Invoked ExternalProcessorDestroy "
+               "successfully");
+    }
+  }
+  for (auto& functionPtr : m_functionPointers) {
+    functionPtr = nullptr;
+  }
+  if (m_handle) {
+    dlclose(m_handle);
+    m_handle = nullptr;
+  }
+
+  return true;
+}
+
+void ExternalProcessing::Initialize(int sample_rate_hz, int num_channels) {
+  if (m_functionPointers.size() <=
+      static_cast<size_t>(FunctionId::ExternalProcessorInitialize)) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Initialize; m_functionPointers is not large "
+             "enough");
+    return;
+  }
+  void* initializePtr =
+      m_functionPointers[FunctionId::ExternalProcessorInitialize];
+  if (!initializePtr) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Initialize; Failed to access "
+             "ExternalProcessorInitialize function");
+    return;
+  }
+
+  auto initializeFunc =
+      reinterpret_cast<ExternalProcessorInitializeFuncType>(initializePtr);
+  if (!initializeFunc(sample_rate_hz, num_channels)) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Initialize; Failed to invoke "
+             "ExternalProcessorInitialize function");
+    return;
+  }
+  ::syslog(LOG_INFO,
+           "ExternalProcessing: #Initialize; Invoked "
+           "ExternalProcessorInitialize; sample_rate_hz: %i, "
+           "num_channels: %i",
+           sample_rate_hz, num_channels);
+}
+
+void ExternalProcessing::Process(webrtc::AudioBuffer* audio) {
+  float* const* channels = audio->channels();
+  size_t num_frames = audio->num_frames();
+  size_t num_bands = audio->num_bands();
+  size_t num_channels = audio->num_channels();
+
+  if (m_functionPointers.size() <=
+      static_cast<size_t>(FunctionId::ExternalProcessorProcessFrame)) {
+    ::syslog(
+        LOG_ERR,
+        "ExternalProcessing: #Process; m_functionPointers is not large enough");
+    return;
+  }
+  void* processPtr =
+      m_functionPointers[FunctionId::ExternalProcessorProcessFrame];
+  if (!processPtr) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Process; Failed to access "
+             "ExternalProcessorProcessFrame function");
+    return;
+  }
+
+  auto processFunc =
+      reinterpret_cast<ExternalProcessorProcessFrameFuncType>(processPtr);
+  if (!processFunc(channels, num_frames, num_bands, num_channels)) {
+    ::syslog(LOG_ERR,
+             "ExternalProcessing: #Process; Failed to invoke "
+             "ExternalProcessorProcessFrame function");
+    return;
+  }
+}
+
+std::string ExternalProcessing::ToString() const {
+  return "ExternalProcessing";
+}
+
+void ExternalProcessing::SetRuntimeSetting(
+    webrtc::AudioProcessing::RuntimeSetting setting) {
+  ::syslog(LOG_INFO, "ExternalProcessing: #SetRuntimeSetting;");
+}
+
+}  // end of namespace external

extensions/android-external-audio-processing/external_processing.hpp

@@ -0,0 +1,49 @@
+#ifndef EXTERNAL_PROCESSING_HPP
+#define EXTERNAL_PROCESSING_HPP
+
+#include <syslog.h>
+
+#include "include/external_processor.hpp"
+
+#include "modules/audio_processing/audio_buffer.h"
+#include "modules/audio_processing/audio_processing_impl.h"
+#include "modules/audio_processing/include/audio_processing.h"
+
+namespace external {
+
+class ExternalProcessing : public webrtc::CustomProcessing {
+ public:
+  ExternalProcessing(const ExternalProcessing&) = delete;
+  ExternalProcessing(ExternalProcessing&&) = delete;
+  ExternalProcessing& operator=(const ExternalProcessing&) = delete;
+  ExternalProcessing& operator=(ExternalProcessing&&) = delete;
+  ~ExternalProcessing();
+
+  static ExternalProcessing* getInstance() {
+    if (m_instance == nullptr) {
+      m_instance = new ExternalProcessing();
+    }
+    return m_instance;
+  }
+
+  bool Create(const char* libname);
+
+  bool Destroy();
+
+  void Initialize(int sample_rate_hz, int num_channels) override;
+  void Process(webrtc::AudioBuffer* audio) override;
+  std::string ToString() const override;
+  void SetRuntimeSetting(
+      webrtc::AudioProcessing::RuntimeSetting setting) override;
+
+ private:
+  ExternalProcessing();
+
+  void* m_handle = nullptr;
+  std::array<void *, kFunctionCount> m_functionPointers = {};
+
+  static ExternalProcessing* m_instance;
+};
+}  // namespace external
+
+#endif  // EXTERNAL_PROCESSING_HPP

extensions/android-external-audio-processing/external_processing_factory_jni.cc

@@ -0,0 +1,60 @@
+#include <syslog.h>
+
+#include <cstring>
+
+#include "extensions/android-external-audio-processing/generated_external_jni/ExternalAudioProcessingFactory_jni.h"
+#include "external_processing.hpp"
+#include "rtc_base/checks.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/thread.h"
+#include "rtc_base/time_utils.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+namespace external {
+
+webrtc::AudioProcessing* apm_ptr = nullptr;
+
+static jlong JNI_ExternalAudioProcessingFactory_CreateAudioProcessingModule(
+    JNIEnv* env,
+    const webrtc::JavaParamRef<jstring>& libnameRef
+) {
+
+  if (libnameRef.is_null()) {
+    ::syslog(LOG_ERR, "EXTERNAL-JNI: #GetApm; libname is null");
+    return 0;
+  }
+
+  const char* libname = env->GetStringUTFChars(libnameRef.obj(), nullptr);
+
+  ::syslog(LOG_INFO, "EXTERNAL-JNI: #GetApm; libname: %s", libname);
+
+  auto instance = ExternalProcessing::getInstance();
+  if (!instance->Create(libname)) {
+    ::syslog(LOG_ERR, "EXTERNAL-JNI: #GetApm; Failed to load external processor");
+    env->ReleaseStringUTFChars(libnameRef.obj(), libname);
+    return 0;
+  }
+
+  env->ReleaseStringUTFChars(libnameRef.obj(), libname);
+
+  std::unique_ptr<webrtc::CustomProcessing> external_processing(instance);
+  auto apm = webrtc::AudioProcessingBuilder()
+                 .SetCapturePostProcessing(std::move(external_processing))
+                 .Create();
+  webrtc::AudioProcessing::Config config;
+  config.echo_canceller.enabled = false;
+  config.echo_canceller.mobile_mode = true;
+  apm->ApplyConfig(config);
+  apm_ptr = apm.release();
+  return webrtc::jni::jlongFromPointer(apm_ptr);
+}
+
+static void JNI_ExternalAudioProcessingFactory_DestroyAudioProcessingModule(
+    JNIEnv* env
+) {
+  ExternalProcessing::getInstance()->Destroy();
+  delete apm_ptr;
+  apm_ptr = nullptr;
+}
+
+}  // end of namespace external

extensions/android-external-audio-processing/include/external_processor.hpp

@@ -0,0 +1,50 @@
+#ifndef EXTERNAL_PROCESSOR_HPP
+#define EXTERNAL_PROCESSOR_HPP
+
+#include <cstdint>
+#include <array>
+
+namespace external {
+
+static constexpr unsigned int kFunctionCount = 4;
+
+// Function ID struct
+struct FunctionId {
+  static constexpr unsigned int ExternalProcessorCreate = 0;
+  static constexpr unsigned int ExternalProcessorInitialize = 1;
+  static constexpr unsigned int ExternalProcessorProcessFrame = 2;
+  static constexpr unsigned int ExternalProcessorDestroy = 3;
+};
+
+static constexpr std::array<const char *, kFunctionCount> kFunctionNames =
+    {
+        "ExternalProcessorCreate",
+        "ExternalProcessorInitialize",
+        "ExternalProcessorProcessFrame",
+        "ExternalProcessorDestroy"
+};
+
+// Function type definitions
+using ExternalProcessorCreateFuncType = bool(*)();
+using ExternalProcessorInitializeFuncType = bool(*)(int sample_rate_hz, int num_channels);
+using ExternalProcessorProcessFrameFuncType = bool(*)(float* const* channels,
+                                                       size_t num_frames,
+                                                       size_t num_bands,
+                                                       size_t num_channels);
+using ExternalProcessorDestroyFuncType = bool(*)();
+
+extern "C" bool ExternalProcessorCreate();
+
+extern "C" bool ExternalProcessorInitialize(int sample_rate_hz,
+                                            int num_channels);
+
+extern "C" bool ExternalProcessorProcessFrame(float* const* channels,
+                                              size_t num_frames,
+                                              size_t num_bands,
+                                              size_t num_channels);
+
+extern "C" bool ExternalProcessorDestroy();
+
+}  // namespace external
+
+#endif  // EXTERNAL_PROCESSOR_HPP