fix: e2ee and upgrade webrtc

livekit-webrtc/src/native/frame_cryptor.rs

@@ -3,6 +3,7 @@ use parking_lot::Mutex;
 use std::sync::Arc;
 use webrtc_sys::frame_cryptor::{self as sys_fc};
 
+use crate::peer_connection_factory::PeerConnectionFactory;
 use crate::{rtp_receiver::RtpReceiver, rtp_sender::RtpSender};
 
 pub type OnStateChange = Box<dyn FnMut(String, EncryptionState) + Send + Sync>;
@@ -77,13 +78,15 @@ pub struct FrameCryptor {
 
 impl FrameCryptor {
     pub fn new_for_rtp_sender(
+        peer_factory: &PeerConnectionFactory,
         participant_id: String,
         algorithm: EncryptionAlgorithm,
         key_provider: KeyProvider,
         sender: RtpSender,
     ) -> Self {
         let observer = Arc::new(RtcFrameCryptorObserver::default());
         let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_sender(
+            peer_factory.handle.sys_handle.clone(),
             participant_id,
             algorithm.into(),
             key_provider.sys_handle,
@@ -101,13 +104,15 @@ impl FrameCryptor {
     }
 
     pub fn new_for_rtp_receiver(
+        peer_factory: &PeerConnectionFactory,
         participant_id: String,
         algorithm: EncryptionAlgorithm,
         key_provider: KeyProvider,
         receiver: RtpReceiver,
     ) -> Self {
         let observer = Arc::new(RtcFrameCryptorObserver::default());
         let sys_handle = sys_fc::ffi::new_frame_cryptor_for_rtp_receiver(
+            peer_factory.handle.sys_handle.clone(),
             participant_id,
             algorithm.into(),
             key_provider.sys_handle,

livekit-webrtc/src/native/peer_connection_factory.rs

@@ -40,7 +40,7 @@ lazy_static! {
 
 #[derive(Clone)]
 pub struct PeerConnectionFactory {
-    sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
+    pub(crate) sys_handle: SharedPtr<sys_pcf::ffi::PeerConnectionFactory>,
 }
 
 impl Default for PeerConnectionFactory {

livekit/src/room/e2ee/manager.rs

@@ -18,6 +18,7 @@ use crate::e2ee::E2eeOptions;
 use crate::id::{ParticipantIdentity, TrackSid};
 use crate::participant::{LocalParticipant, RemoteParticipant};
 use crate::prelude::{LocalTrack, LocalTrackPublication, RemoteTrack, RemoteTrackPublication};
+use crate::rtc_engine::lk_runtime::LkRuntime;
 use livekit_webrtc::native::frame_cryptor::{EncryptionAlgorithm, EncryptionState, FrameCryptor};
 use livekit_webrtc::{rtp_receiver::RtpReceiver, rtp_sender::RtpSender};
 use parking_lot::Mutex;
@@ -194,6 +195,7 @@ impl E2eeManager {
         let options = inner.options.as_ref().unwrap();
 
         let frame_cryptor = FrameCryptor::new_for_rtp_sender(
+            LkRuntime::instance().pc_factory(),
             participant_identity.to_string(),
             EncryptionAlgorithm::AesGcm,
             options.key_provider.handle.clone(),
@@ -212,6 +214,7 @@ impl E2eeManager {
         let options = inner.options.as_ref().unwrap();
 
         let frame_cryptor = FrameCryptor::new_for_rtp_receiver(
+            LkRuntime::instance().pc_factory(),
             participant_identity.to_string(),
             EncryptionAlgorithm::AesGcm,
             options.key_provider.handle.clone(),

webrtc-sys/build/src/lib.rs

@@ -26,7 +26,7 @@ use regex::Regex;
 use reqwest::StatusCode;
 
 pub const SCRATH_PATH: &str = "livekit_webrtc";
-pub const WEBRTC_TAG: &str = "webrtc-fcdef0d";
+pub const WEBRTC_TAG: &str = "webrtc-6af5bfd";
 pub const IGNORE_DEFINES: [&str; 2] = ["CR_CLANG_REVISION", "CR_XCODE_VERSION"];
 
 pub fn target_os() -> String {

webrtc-sys/include/livekit/frame_cryptor.h

@@ -23,8 +23,11 @@
 #include <vector>
 
 #include "api/crypto/frame_crypto_transformer.h"
+#include "api/scoped_refptr.h"
+#include "livekit/peer_connection.h"
 #include "livekit/rtp_receiver.h"
 #include "livekit/rtp_sender.h"
+#include "livekit/webrtc.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rust/cxx.h"
 
@@ -114,12 +117,14 @@ class KeyProvider {
 
 class FrameCryptor {
  public:
-  FrameCryptor(const std::string participant_id,
+  FrameCryptor(std::shared_ptr<RtcRuntime> rtc_runtime,
+               const std::string participant_id,
                webrtc::FrameCryptorTransformer::Algorithm algorithm,
                rtc::scoped_refptr<webrtc::KeyProvider> key_provider,
                rtc::scoped_refptr<webrtc::RtpSenderInterface> sender);
 
-  FrameCryptor(const std::string participant_id,
+  FrameCryptor(std::shared_ptr<RtcRuntime> rtc_runtime,
+               const std::string participant_id,
                webrtc::FrameCryptorTransformer::Algorithm algorithm,
                rtc::scoped_refptr<webrtc::KeyProvider> key_provider,
                rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver);
@@ -146,13 +151,14 @@ class FrameCryptor {
   void unregister_observer() const;
 
  private:
+  std::shared_ptr<RtcRuntime> rtc_runtime_;
   const rust::String participant_id_;
   mutable webrtc::Mutex mutex_;
   rtc::scoped_refptr<webrtc::FrameCryptorTransformer> e2ee_transformer_;
   rtc::scoped_refptr<webrtc::KeyProvider> key_provider_;
   rtc::scoped_refptr<webrtc::RtpSenderInterface> sender_;
   rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver_;
-  mutable std::unique_ptr<NativeFrameCryptorObserver> observer_;
+  mutable rtc::scoped_refptr<NativeFrameCryptorObserver> observer_;
 };
 
 class NativeFrameCryptorObserver
@@ -171,12 +177,14 @@ class NativeFrameCryptorObserver
 };
 
 std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_sender(
+    std::shared_ptr<PeerConnectionFactory> peer_factory,
     const ::rust::String participant_id,
     Algorithm algorithm,
     std::shared_ptr<KeyProvider> key_provider,
     std::shared_ptr<RtpSender> sender);
 
 std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_receiver(
+    std::shared_ptr<PeerConnectionFactory> peer_factory,
     const ::rust::String participant_id,
     Algorithm algorithm,
     std::shared_ptr<KeyProvider> key_provider,

webrtc-sys/include/livekit/peer_connection_factory.h

@@ -56,6 +56,8 @@ class PeerConnectionFactory {
 
   RtpCapabilities rtp_receiver_capabilities(MediaType type) const;
 
+  std::shared_ptr<RtcRuntime> rtc_runtime() const { return rtc_runtime_; }
+
  private:
   std::shared_ptr<RtcRuntime> rtc_runtime_;
   rtc::scoped_refptr<AudioDevice> audio_device_;

webrtc-sys/src/frame_cryptor.cpp

@@ -19,6 +19,11 @@
 #include <memory>
 
 #include "absl/types/optional.h"
+#include "api/make_ref_counted.h"
+#include "livekit/peer_connection.h"
+#include "livekit/peer_connection_factory.h"
+#include "livekit/webrtc.h"
+#include "rtc_base/thread.h"
 #include "webrtc-sys/src/frame_cryptor.rs.h"
 
 namespace livekit {
@@ -57,38 +62,44 @@ KeyProvider::KeyProvider(KeyProviderOptions options) {
 }
 
 FrameCryptor::FrameCryptor(
+    std::shared_ptr<RtcRuntime> rtc_runtime,
     const std::string participant_id,
     webrtc::FrameCryptorTransformer::Algorithm algorithm,
     rtc::scoped_refptr<webrtc::KeyProvider> key_provider,
     rtc::scoped_refptr<webrtc::RtpSenderInterface> sender)
-    : participant_id_(participant_id),
+    : rtc_runtime_(rtc_runtime),
+      participant_id_(participant_id),
       key_provider_(key_provider),
       sender_(sender) {
   auto mediaType =
       sender->track()->kind() == "audio"
           ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame
           : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame;
   e2ee_transformer_ = rtc::scoped_refptr<webrtc::FrameCryptorTransformer>(
-      new webrtc::FrameCryptorTransformer(participant_id, mediaType, algorithm,
+      new webrtc::FrameCryptorTransformer(rtc_runtime->signaling_thread(),
+                                          participant_id, mediaType, algorithm,
                                           key_provider_));
   sender->SetEncoderToPacketizerFrameTransformer(e2ee_transformer_);
   e2ee_transformer_->SetEnabled(false);
 }
 
 FrameCryptor::FrameCryptor(
+    std::shared_ptr<RtcRuntime> rtc_runtime,
     const std::string participant_id,
     webrtc::FrameCryptorTransformer::Algorithm algorithm,
     rtc::scoped_refptr<webrtc::KeyProvider> key_provider,
     rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver)
-    : participant_id_(participant_id),
+    : rtc_runtime_(rtc_runtime),
+      participant_id_(participant_id),
       key_provider_(key_provider),
       receiver_(receiver) {
   auto mediaType =
       receiver->track()->kind() == "audio"
           ? webrtc::FrameCryptorTransformer::MediaType::kAudioFrame
           : webrtc::FrameCryptorTransformer::MediaType::kVideoFrame;
   e2ee_transformer_ = rtc::scoped_refptr<webrtc::FrameCryptorTransformer>(
-      new webrtc::FrameCryptorTransformer(participant_id, mediaType, algorithm,
+      new webrtc::FrameCryptorTransformer(rtc_runtime->signaling_thread(),
+                                          participant_id, mediaType, algorithm,
                                           key_provider_));
   receiver->SetDepacketizerToDecoderFrameTransformer(e2ee_transformer_);
   e2ee_transformer_->SetEnabled(false);
@@ -99,15 +110,15 @@ FrameCryptor::~FrameCryptor() {}
 void FrameCryptor::register_observer(
     rust::Box<RtcFrameCryptorObserverWrapper> observer) const {
   webrtc::MutexLock lock(&mutex_);
-  observer_ =
-      std::make_unique<NativeFrameCryptorObserver>(std::move(observer), this);
-  e2ee_transformer_->SetFrameCryptorTransformerObserver(observer_.get());
+  observer_ = rtc::make_ref_counted<NativeFrameCryptorObserver>(
+      std::move(observer), this);
+  e2ee_transformer_->RegisterFrameCryptorTransformerObserver(observer_);
 }
 
 void FrameCryptor::unregister_observer() const {
   webrtc::MutexLock lock(&mutex_);
   observer_ = nullptr;
-  e2ee_transformer_->SetFrameCryptorTransformerObserver(nullptr);
+  e2ee_transformer_->UnRegisterFrameCryptorTransformerObserver();
 }
 
 NativeFrameCryptorObserver::NativeFrameCryptorObserver(
@@ -151,22 +162,26 @@ std::shared_ptr<KeyProvider> new_key_provider(KeyProviderOptions options) {
 }
 
 std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_sender(
+    std::shared_ptr<PeerConnectionFactory> peer_factory,
     const ::rust::String participant_id,
     Algorithm algorithm,
     std::shared_ptr<KeyProvider> key_provider,
     std::shared_ptr<RtpSender> sender) {
   return std::make_shared<FrameCryptor>(
+      peer_factory->rtc_runtime(),
       std::string(participant_id.data(), participant_id.size()),
       AlgorithmToFrameCryptorAlgorithm(algorithm),
       key_provider->rtc_key_provider(), sender->rtc_sender());
 }
 
 std::shared_ptr<FrameCryptor> new_frame_cryptor_for_rtp_receiver(
+    std::shared_ptr<PeerConnectionFactory> peer_factory,
     const ::rust::String participant_id,
     Algorithm algorithm,
     std::shared_ptr<KeyProvider> key_provider,
     std::shared_ptr<RtpReceiver> receiver) {
   return std::make_shared<FrameCryptor>(
+      peer_factory->rtc_runtime(),
       std::string(participant_id.data(), participant_id.size()),
       AlgorithmToFrameCryptorAlgorithm(algorithm),
       key_provider->rtc_key_provider(), receiver->rtc_receiver());

webrtc-sys/src/frame_cryptor.rs

@@ -83,20 +83,24 @@ pub mod ffi {
         include!("livekit/frame_cryptor.h");
         include!("livekit/rtp_sender.h");
         include!("livekit/rtp_receiver.h");
+        include!("livekit/peer_connection_factory.h");
 
         type RtpSender = crate::rtp_sender::ffi::RtpSender;
         type RtpReceiver = crate::rtp_receiver::ffi::RtpReceiver;
+        type PeerConnectionFactory = crate::peer_connection_factory::ffi::PeerConnectionFactory;
 
         pub type FrameCryptor;
 
         pub fn new_frame_cryptor_for_rtp_sender(
+            peer_factory: SharedPtr<PeerConnectionFactory>,
             participant_id: String,
             algorithm: Algorithm,
             key_provider: SharedPtr<KeyProvider>,
             sender: SharedPtr<RtpSender>,
         ) -> SharedPtr<FrameCryptor>;
 
         pub fn new_frame_cryptor_for_rtp_receiver(
+            peer_factory: SharedPtr<PeerConnectionFactory>,
             participant_id: String,
             algorithm: Algorithm,
             key_provider: SharedPtr<KeyProvider>,