From a1c7145c8702f05050e0971dad18e17293774e56 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 31 Aug 2023 15:21:47 +0800 Subject: [PATCH 01/28] Improve e2ee, add setSharedKey to KeyProvider. --- api/crypto/frame_crypto_transformer.cc | 10 +- api/crypto/frame_crypto_transformer.h | 94 ++++++++++++------- .../org/webrtc/FrameCryptorKeyProvider.java | 8 +- .../src/jni/pc/frame_cryptor_key_provider.cc | 10 ++ .../RTCFrameCryptorKeyProvider.h | 2 + .../RTCFrameCryptorKeyProvider.mm | 6 ++ 6 files changed, 89 insertions(+), 41 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index ed5a17c957..0453e14db4 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -565,15 +565,15 @@ void FrameCryptorTransformer::decryptFrame( RTC_LOG(LS_ERROR) << "FrameCryptorTransformer::decryptFrame() failed"; std::shared_ptr ratcheted_key_set; auto currentKeyMaterial = key_set->material; - if (key_handler->options().ratchet_window_size > 0) { - while (ratchet_count < key_handler->options().ratchet_window_size) { + if (key_provider_->options().ratchet_window_size > 0) { + while (ratchet_count < key_provider_->options().ratchet_window_size) { ratchet_count++; RTC_LOG(LS_INFO) << "ratcheting key attempt " << ratchet_count << " of " - << key_handler->options().ratchet_window_size; + << key_provider_->options().ratchet_window_size; auto new_material = key_handler->RatchetKeyMaterial(currentKeyMaterial); - ratcheted_key_set = key_handler->DeriveKeys(new_material, key_handler->options().ratchet_salt, 128); + ratcheted_key_set = key_handler->DeriveKeys(new_material, key_provider_->options().ratchet_salt, 128); if (AesEncryptDecrypt(EncryptOrDecrypt::kDecrypt, algorithm_, ratcheted_key_set->encryption_key, iv, frame_header, @@ -603,7 +603,7 @@ void FrameCryptorTransformer::decryptFrame( times, we come back to the initial key. */ if (!decryption_success || - ratchet_count >= key_handler->options().ratchet_window_size) { + ratchet_count >= key_provider_->options().ratchet_window_size) { key_handler->SetKeyFromMaterial(initialKeyMaterial, key_index); } } diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 7f98deb11d..1f7fc3bc21 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -34,6 +34,8 @@ namespace webrtc { const size_t KEYRING_SIZE = 16; +class ParticipantKeyHandler; + struct KeyProviderOptions { bool shared_key; std::vector ratchet_salt; @@ -47,6 +49,33 @@ struct KeyProviderOptions { ratchet_window_size(copy.ratchet_window_size) {} }; +class KeyProvider : public rtc::RefCountInterface { + public: + enum { kRawKeySize = 32 }; + + public: + virtual const std::shared_ptr GetKey( + const std::string participant_id) const = 0; + + virtual bool SetSharedKey(int key_index, std::vector key) = 0; + + virtual bool SetKey(const std::string participant_id, + int key_index, + std::vector key) = 0; + + virtual const std::vector RatchetKey( + const std::string participant_id, + int key_index) = 0; + + virtual const std::vector ExportKey(const std::string participant_id, + int key_index) const = 0; + + virtual KeyProviderOptions& options() = 0; + + protected: + virtual ~KeyProvider() {} +}; + class ParticipantKeyHandler { friend class FrameCryptorTransformer; public: @@ -56,18 +85,18 @@ class ParticipantKeyHandler { KeySet(std::vector material, std::vector encryptionKey) : material(material), encryption_key(encryptionKey) {} }; - public: - ParticipantKeyHandler(KeyProviderOptions options) : options_(options) { + ParticipantKeyHandler(KeyProvider* key_provider) : key_provider_(key_provider) { crypto_key_ring_.resize(KEYRING_SIZE); } virtual ~ParticipantKeyHandler() = default; virtual std::vector RatchetKey(int key_index) { + webrtc::MutexLock lock(&mutex_); auto current_material = GetKeySet(key_index)->material; std::vector new_material; - if (DerivePBKDF2KeyFromRawKey(current_material, options_.ratchet_salt, 256, + if (DerivePBKDF2KeyFromRawKey(current_material, key_provider_->options().ratchet_salt, 256, &new_material) != 0) { return std::vector(); } @@ -77,24 +106,25 @@ class ParticipantKeyHandler { } virtual std::shared_ptr GetKeySet(int key_index) { + webrtc::MutexLock lock(&mutex_); return crypto_key_ring_[key_index != -1 ? key_index : current_key_index_]; } virtual void SetKey(std::vector password, int key_index) { + webrtc::MutexLock lock(&mutex_); SetKeyFromMaterial(password, key_index); have_valid_key = true; } - virtual void SetKeyFromMaterial(std::vector password, int key_index) { +private: + void SetKeyFromMaterial(std::vector password, int key_index) { if (key_index >= 0) { current_key_index_ = key_index % crypto_key_ring_.size(); } crypto_key_ring_[current_key_index_] = - DeriveKeys(password, options_.ratchet_salt, 128); + DeriveKeys(password, key_provider_->options().ratchet_salt, 128); } - virtual KeyProviderOptions& options() { return options_; } - std::shared_ptr DeriveKeys(std::vector password, std::vector ratchet_salt, unsigned int optional_length_bits) { @@ -108,8 +138,9 @@ class ParticipantKeyHandler { std::vector RatchetKeyMaterial( std::vector current_material) { + webrtc::MutexLock lock(&mutex_); std::vector new_material; - if (DerivePBKDF2KeyFromRawKey(current_material, options_.ratchet_salt, 256, + if (DerivePBKDF2KeyFromRawKey(current_material, key_provider_->options().ratchet_salt, 256, &new_material) != 0) { return std::vector(); } @@ -118,41 +149,29 @@ class ParticipantKeyHandler { protected: bool have_valid_key = false; private: + mutable webrtc::Mutex mutex_; int current_key_index_ = 0; - KeyProviderOptions options_; + KeyProvider* key_provider_; std::vector> crypto_key_ring_; }; -class KeyProvider : public rtc::RefCountInterface { - public: - enum { kRawKeySize = 32 }; - - public: - virtual const std::shared_ptr GetKey( - const std::string participant_id) const = 0; - - virtual bool SetKey(const std::string participant_id, - int index, - std::vector key) = 0; - - virtual const std::vector RatchetKey( - const std::string participant_id, - int key_index) = 0; - - virtual const std::vector ExportKey(const std::string participant_id, - int key_index) const = 0; - - virtual KeyProviderOptions& options() = 0; - - protected: - virtual ~KeyProvider() {} -}; - class DefaultKeyProviderImpl : public KeyProvider { public: DefaultKeyProviderImpl(KeyProviderOptions options) : options_(options) {} ~DefaultKeyProviderImpl() override = default; + /// Set the shared key. + bool SetSharedKey(int key_index, std::vector key) override { + webrtc::MutexLock lock(&mutex_); + if (keys_.find("shared") == keys_.end()) { + keys_["shared"] = std::make_shared(this); + } + + auto key_handler = keys_["shared"]; + key_handler->SetKey(key, key_index); + return true; + } + /// Set the key at the given index. bool SetKey(const std::string participant_id, int index, @@ -160,7 +179,7 @@ class DefaultKeyProviderImpl : public KeyProvider { webrtc::MutexLock lock(&mutex_); if (keys_.find(participant_id) == keys_.end()) { - keys_[participant_id] = std::make_shared(options_); + keys_[participant_id] = std::make_shared(this); } auto key_handler = keys_[participant_id]; @@ -171,6 +190,11 @@ class DefaultKeyProviderImpl : public KeyProvider { const std::shared_ptr GetKey( const std::string participant_id) const override { webrtc::MutexLock lock(&mutex_); + + if(options_.shared_key && keys_.find(participant_id) != keys_.end()) { + return keys_.find("shared")->second; + } + if (keys_.find(participant_id) == keys_.end()) { return nullptr; } diff --git a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java index 1c89eac55b..6e4a73a2bf 100644 --- a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java +++ b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java @@ -30,6 +30,11 @@ public long getNativeKeyProvider() { return nativeKeyProvider; } + public boolean setSharedKey(int index, byte[] key) { + checkKeyProviderExists(); + return nativeSetSharedKey(nativeKeyProvider,index, key); + } + public boolean setKey(String participantId, int index, byte[] key) { checkKeyProviderExists(); return nativeSetKey(nativeKeyProvider, participantId, index, key); @@ -56,7 +61,8 @@ private void checkKeyProviderExists() { throw new IllegalStateException("FrameCryptorKeyProvider has been disposed."); } } - + private static native boolean nativeSetSharedKey( + long keyProviderPointer, int index, byte[] key); private static native boolean nativeSetKey( long keyProviderPointer, String participantId, int index, byte[] key); private static native byte[] nativeRatchetKey( diff --git a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc index 2732693c0f..81ceb458d6 100644 --- a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc +++ b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc @@ -33,6 +33,16 @@ ScopedJavaLocalRef NativeToJavaFrameCryptorKeyProvider( env, jlongFromPointer(key_provider.release())); } +static jboolean JNI_FrameCryptorKeyProvider_SetSharedKey( + JNIEnv* jni, + jlong j_key_provider, + jint j_index, + const base::android::JavaParamRef& j_key) { + auto key = JavaToNativeByteArray(jni, j_key); + return reinterpret_cast(j_key_provider) + ->SetSharedKey(j_index,std::vector(key.begin(), key.end())); +} + static jboolean JNI_FrameCryptorKeyProvider_SetKey( JNIEnv* jni, jlong j_key_provider, diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h index 276b2e730c..ab879b7b9e 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h @@ -25,6 +25,8 @@ RTC_OBJC_EXPORT - (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId; +- (void)setSharedKey:(NSData *)key withIndex:(int)index; + - (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index; - (NSData *)exportKey:(NSString *)participantId withIndex:(int)index; diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm index c27e18975b..c0683c6a32 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm @@ -56,6 +56,12 @@ - (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)par std::vector((const uint8_t *)key.bytes, ((const uint8_t *)key.bytes) + key.length)); } +- (void)setSharedKey:(NSData *)key withIndex:(int)index { + _nativeKeyProvider->SetSharedKey( + index, + std::vector((const uint8_t *)key.bytes, ((const uint8_t *)key.bytes) + key.length)); +} + - (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index { std::vector nativeKey = _nativeKeyProvider->RatchetKey([participantId stdString], index); return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; From 677bec53bb420ede909f7d9fbc5a79a789166655 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 31 Aug 2023 15:24:36 +0800 Subject: [PATCH 02/28] update. --- api/crypto/frame_crypto_transformer.h | 29 +++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 1f7fc3bc21..8f358f8d48 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -116,13 +116,14 @@ class ParticipantKeyHandler { have_valid_key = true; } -private: - void SetKeyFromMaterial(std::vector password, int key_index) { - if (key_index >= 0) { - current_key_index_ = key_index % crypto_key_ring_.size(); + std::vector RatchetKeyMaterial( + std::vector current_material) { + std::vector new_material; + if (DerivePBKDF2KeyFromRawKey(current_material, key_provider_->options().ratchet_salt, 256, + &new_material) != 0) { + return std::vector(); } - crypto_key_ring_[current_key_index_] = - DeriveKeys(password, key_provider_->options().ratchet_salt, 128); + return new_material; } std::shared_ptr DeriveKeys(std::vector password, @@ -136,15 +137,13 @@ class ParticipantKeyHandler { return nullptr; } - std::vector RatchetKeyMaterial( - std::vector current_material) { - webrtc::MutexLock lock(&mutex_); - std::vector new_material; - if (DerivePBKDF2KeyFromRawKey(current_material, key_provider_->options().ratchet_salt, 256, - &new_material) != 0) { - return std::vector(); +private: + void SetKeyFromMaterial(std::vector password, int key_index) { + if (key_index >= 0) { + current_key_index_ = key_index % crypto_key_ring_.size(); } - return new_material; + crypto_key_ring_[current_key_index_] = + DeriveKeys(password, key_provider_->options().ratchet_salt, 128); } protected: bool have_valid_key = false; @@ -191,7 +190,7 @@ class DefaultKeyProviderImpl : public KeyProvider { const std::string participant_id) const override { webrtc::MutexLock lock(&mutex_); - if(options_.shared_key && keys_.find(participant_id) != keys_.end()) { + if(options_.shared_key && keys_.find("shared") != keys_.end()) { return keys_.find("shared")->second; } From bb20946d75ae83464b8192b12e33f9af4428cc69 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 31 Aug 2023 16:07:07 +0800 Subject: [PATCH 03/28] reset has_valid_key after RatchetKey. --- api/crypto/frame_crypto_transformer.cc | 5 +++-- api/crypto/frame_crypto_transformer.h | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 0453e14db4..4f92d42b64 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -534,7 +534,7 @@ void FrameCryptorTransformer::decryptFrame( return; } - if(last_dec_error_ == kDecryptionFailed && !key_handler->have_valid_key) { + if(last_dec_error_ == kDecryptionFailed && !key_handler->has_valid_key) { // if decryption failed and we have an invalid key, // please try to decrypt with the next new key return; @@ -584,6 +584,7 @@ void FrameCryptorTransformer::decryptFrame( decryption_success = true; // success, so we set the new key key_handler->SetKeyFromMaterial(new_material, key_index); + key_handler->has_valid_key = true; if (last_dec_error_ != FrameCryptionState::kKeyRatcheted) { last_dec_error_ = FrameCryptionState::kKeyRatcheted; if (observer_) @@ -612,7 +613,7 @@ void FrameCryptorTransformer::decryptFrame( if (!decryption_success) { if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { last_dec_error_ = FrameCryptionState::kDecryptionFailed; - key_handler->have_valid_key = false; + key_handler->has_valid_key = false; if (observer_) observer_->OnFrameCryptionStateChanged(participant_id_, last_dec_error_); diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 8f358f8d48..06e11a35f8 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -102,6 +102,7 @@ class ParticipantKeyHandler { } SetKeyFromMaterial(new_material, key_index != -1 ? key_index : current_key_index_); + has_valid_key = true; return new_material; } @@ -113,7 +114,7 @@ class ParticipantKeyHandler { virtual void SetKey(std::vector password, int key_index) { webrtc::MutexLock lock(&mutex_); SetKeyFromMaterial(password, key_index); - have_valid_key = true; + has_valid_key = true; } std::vector RatchetKeyMaterial( @@ -145,8 +146,8 @@ class ParticipantKeyHandler { crypto_key_ring_[current_key_index_] = DeriveKeys(password, key_provider_->options().ratchet_salt, 128); } - protected: - bool have_valid_key = false; +protected: + bool has_valid_key = false; private: mutable webrtc::Mutex mutex_; int current_key_index_ = 0; From a4dba5c759f6fb956a05a83bed0c94e7331c76f3 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 31 Aug 2023 22:05:30 +0800 Subject: [PATCH 04/28] update. --- api/crypto/frame_crypto_transformer.cc | 6 +-- api/crypto/frame_crypto_transformer.h | 58 ++++++++++++++------------ 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 4f92d42b64..554f3f5acf 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -534,7 +534,7 @@ void FrameCryptorTransformer::decryptFrame( return; } - if(last_dec_error_ == kDecryptionFailed && !key_handler->has_valid_key) { + if(last_dec_error_ == kDecryptionFailed && !key_handler->HasValidKey()) { // if decryption failed and we have an invalid key, // please try to decrypt with the next new key return; @@ -584,7 +584,7 @@ void FrameCryptorTransformer::decryptFrame( decryption_success = true; // success, so we set the new key key_handler->SetKeyFromMaterial(new_material, key_index); - key_handler->has_valid_key = true; + key_handler->SetHasValidKey(true); if (last_dec_error_ != FrameCryptionState::kKeyRatcheted) { last_dec_error_ = FrameCryptionState::kKeyRatcheted; if (observer_) @@ -613,7 +613,7 @@ void FrameCryptorTransformer::decryptFrame( if (!decryption_success) { if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { last_dec_error_ = FrameCryptionState::kDecryptionFailed; - key_handler->has_valid_key = false; + key_handler->SetHasValidKey(false); if (observer_) observer_->OnFrameCryptionStateChanged(participant_id_, last_dec_error_); diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 06e11a35f8..b670a62473 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -77,7 +77,6 @@ class KeyProvider : public rtc::RefCountInterface { }; class ParticipantKeyHandler { - friend class FrameCryptorTransformer; public: struct KeySet { std::vector material; @@ -93,8 +92,11 @@ class ParticipantKeyHandler { virtual ~ParticipantKeyHandler() = default; virtual std::vector RatchetKey(int key_index) { - webrtc::MutexLock lock(&mutex_); - auto current_material = GetKeySet(key_index)->material; + auto key_set = GetKeySet(key_index); + if (!key_set) { + return std::vector(); + } + auto current_material = key_set->material; std::vector new_material; if (DerivePBKDF2KeyFromRawKey(current_material, key_provider_->options().ratchet_salt, 256, &new_material) != 0) { @@ -102,7 +104,7 @@ class ParticipantKeyHandler { } SetKeyFromMaterial(new_material, key_index != -1 ? key_index : current_key_index_); - has_valid_key = true; + SetHasValidKey(true); return new_material; } @@ -112,9 +114,8 @@ class ParticipantKeyHandler { } virtual void SetKey(std::vector password, int key_index) { - webrtc::MutexLock lock(&mutex_); SetKeyFromMaterial(password, key_index); - has_valid_key = true; + SetHasValidKey(true); } std::vector RatchetKeyMaterial( @@ -138,17 +139,27 @@ class ParticipantKeyHandler { return nullptr; } -private: - void SetKeyFromMaterial(std::vector password, int key_index) { + bool HasValidKey() { + webrtc::MutexLock lock(&mutex_); + return has_valid_key_; + } + + void SetHasValidKey(bool has_valid_key) { + webrtc::MutexLock lock(&mutex_); + has_valid_key_ = has_valid_key; + } + + void SetKeyFromMaterial(std::vector password, int key_index) { + webrtc::MutexLock lock(&mutex_); if (key_index >= 0) { current_key_index_ = key_index % crypto_key_ring_.size(); } crypto_key_ring_[current_key_index_] = DeriveKeys(password, key_provider_->options().ratchet_salt, 128); } -protected: - bool has_valid_key = false; + private: + bool has_valid_key_ = false; mutable webrtc::Mutex mutex_; int current_key_index_ = 0; KeyProvider* key_provider_; @@ -204,28 +215,23 @@ class DefaultKeyProviderImpl : public KeyProvider { const std::vector RatchetKey(const std::string participant_id, int key_index) override { - webrtc::MutexLock lock(&mutex_); - if (keys_.find(participant_id) == keys_.end()) { - return std::vector(); + auto key_handler = GetKey(participant_id); + if (key_handler) { + return key_handler->RatchetKey(key_index); } - - return keys_[participant_id]->RatchetKey(key_index); + return std::vector(); } const std::vector ExportKey(const std::string participant_id, int key_index) const override { - webrtc::MutexLock lock(&mutex_); - if (keys_.find(participant_id) == keys_.end()) { - return std::vector(); + auto key_handler = GetKey(participant_id); + if (key_handler) { + auto key_set = key_handler->GetKeySet(key_index); + if (key_set) { + return key_set->material; + } } - - auto key_set = GetKey(participant_id); - - if (!key_set) { - return std::vector(); - } - - return key_set->GetKeySet(key_index)->material; + return std::vector(); } KeyProviderOptions& options() override { return options_; } From c2d3c7dfcce827f1ca5b9e5d72a6edd0891c04de Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 31 Aug 2023 22:59:56 +0800 Subject: [PATCH 05/28] clone key_handler from share_key for each participant. --- api/crypto/frame_crypto_transformer.cc | 56 +++++++++++++++----------- api/crypto/frame_crypto_transformer.h | 42 ++++++++++++++++--- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 554f3f5acf..9d8ee44c9e 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -354,7 +354,10 @@ void FrameCryptorTransformer::encryptFrame( return; } - auto key_handler = key_provider_->GetKey(participant_id_); + auto key_handler = key_provider_->options().shared_key + ? key_provider_->GetSharedKey(participant_id_) + : key_provider_->GetKey(participant_id_); + if (key_handler == nullptr || key_handler->GetKeySet(key_index_) == nullptr) { RTC_LOG(LS_INFO) << "FrameCryptorTransformer::encryptFrame() no keys, or " "key_index[" @@ -462,30 +465,31 @@ void FrameCryptorTransformer::decryptFrame( sink_callback->OnTransformedFrame(std::move(frame)); return; } - + auto uncrypted_magic_bytes = key_provider_->options().uncrypted_magic_bytes; if (uncrypted_magic_bytes.size() > 0 && date_in.size() >= uncrypted_magic_bytes.size() + 1) { - auto tmp = date_in.subview(date_in.size() - (uncrypted_magic_bytes.size() + 1), - uncrypted_magic_bytes.size()); + auto tmp = + date_in.subview(date_in.size() - (uncrypted_magic_bytes.size() + 1), + uncrypted_magic_bytes.size()); if (uncrypted_magic_bytes == std::vector(tmp.begin(), tmp.end())) { - RTC_CHECK_EQ(tmp.size(), uncrypted_magic_bytes.size()); auto frame_type = date_in.subview(date_in.size() - 1, 1); RTC_CHECK_EQ(frame_type.size(), 1); - RTC_LOG(LS_INFO) << "FrameCryptorTransformer::uncrypted_magic_bytes( type " - << frame_type[0] << ", tmp " - << to_hex(tmp.data(), tmp.size()) << ", magic bytes " - << to_hex(uncrypted_magic_bytes.data(), - uncrypted_magic_bytes.size()) - << ")"; + RTC_LOG(LS_INFO) + << "FrameCryptorTransformer::uncrypted_magic_bytes( type " + << frame_type[0] << ", tmp " << to_hex(tmp.data(), tmp.size()) + << ", magic bytes " + << to_hex(uncrypted_magic_bytes.data(), uncrypted_magic_bytes.size()) + << ")"; - // magic bytes detected, this is a non-encrypted frame, skip frame decryption. + // magic bytes detected, this is a non-encrypted frame, skip frame + // decryption. rtc::Buffer data_out; - data_out.AppendData( - date_in.subview(0, date_in.size() - uncrypted_magic_bytes.size() - 1)); + data_out.AppendData(date_in.subview( + 0, date_in.size() - uncrypted_magic_bytes.size() - 1)); frame->SetData(data_out); sink_callback->OnTransformedFrame(std::move(frame)); return; @@ -518,7 +522,10 @@ void FrameCryptorTransformer::decryptFrame( return; } - auto key_handler = key_provider_->GetKey(participant_id_); + auto key_handler = key_provider_->options().shared_key + ? key_provider_->GetSharedKey(participant_id_) + : key_provider_->GetKey(participant_id_); + if (key_index >= KEYRING_SIZE || key_handler == nullptr || key_handler->GetKeySet(key_index) == nullptr) { RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() no keys, or " @@ -534,10 +541,10 @@ void FrameCryptorTransformer::decryptFrame( return; } - if(last_dec_error_ == kDecryptionFailed && !key_handler->HasValidKey()) { - // if decryption failed and we have an invalid key, - // please try to decrypt with the next new key - return; + if (last_dec_error_ == kDecryptionFailed && !key_handler->HasValidKey()) { + // if decryption failed and we have an invalid key, + // please try to decrypt with the next new key + return; } auto key_set = key_handler->GetKeySet(key_index); @@ -573,11 +580,13 @@ void FrameCryptorTransformer::decryptFrame( << key_provider_->options().ratchet_window_size; auto new_material = key_handler->RatchetKeyMaterial(currentKeyMaterial); - ratcheted_key_set = key_handler->DeriveKeys(new_material, key_provider_->options().ratchet_salt, 128); + ratcheted_key_set = key_handler->DeriveKeys( + new_material, key_provider_->options().ratchet_salt, 128); if (AesEncryptDecrypt(EncryptOrDecrypt::kDecrypt, algorithm_, - ratcheted_key_set->encryption_key, iv, frame_header, - encrypted_payload, &buffer) == Success) { + ratcheted_key_set->encryption_key, iv, + frame_header, encrypted_payload, + &buffer) == Success) { RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() " "ratcheted to key_index=" << static_cast(key_index); @@ -630,7 +639,8 @@ void FrameCryptorTransformer::decryptFrame( RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() ivLength=" << static_cast(ivLength) << " unencrypted_bytes=" << static_cast(unencrypted_bytes) - << " key_index=" << static_cast(key_index_) << " aesKey=" + << " key_index=" << static_cast(key_index_) + << " aesKey=" << to_hex(key_set->encryption_key.data(), key_set->encryption_key.size()) << " iv=" << to_hex(iv.data(), iv.size()); diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index b670a62473..02239e6e49 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -54,15 +54,18 @@ class KeyProvider : public rtc::RefCountInterface { enum { kRawKeySize = 32 }; public: - virtual const std::shared_ptr GetKey( - const std::string participant_id) const = 0; virtual bool SetSharedKey(int key_index, std::vector key) = 0; + virtual const std::shared_ptr GetSharedKey(const std::string participant_id) = 0; + virtual bool SetKey(const std::string participant_id, int key_index, std::vector key) = 0; + virtual const std::shared_ptr GetKey( + const std::string participant_id) const = 0; + virtual const std::vector RatchetKey( const std::string participant_id, int key_index) = 0; @@ -91,6 +94,14 @@ class ParticipantKeyHandler { virtual ~ParticipantKeyHandler() = default; + std::shared_ptr Clone() { + auto clone = std::make_shared(key_provider_); + clone->crypto_key_ring_ = crypto_key_ring_; + clone->current_key_index_ = current_key_index_; + clone->has_valid_key_ = has_valid_key_; + return clone; + } + virtual std::vector RatchetKey(int key_index) { auto key_set = GetKeySet(key_index); if (!key_set) { @@ -180,9 +191,32 @@ class DefaultKeyProviderImpl : public KeyProvider { auto key_handler = keys_["shared"]; key_handler->SetKey(key, key_index); + + if(options_.shared_key) { + for(auto& key_pair : keys_) { + if(key_pair.first != "shared") { + key_pair.second->SetKey(key, key_index); + } + } + } return true; } + const std::shared_ptr GetSharedKey(const std::string participant_id) override { + webrtc::MutexLock lock(&mutex_); + if(options_.shared_key && keys_.find("shared") != keys_.end()) { + auto shared_key_handler = keys_["shared"]; + if (keys_.find(participant_id) != keys_.end()) { + return keys_[participant_id]; + } else { + auto key_handler_clone = shared_key_handler->Clone(); + keys_[participant_id] = key_handler_clone; + return key_handler_clone; + } + } + return nullptr; + } + /// Set the key at the given index. bool SetKey(const std::string participant_id, int index, @@ -202,10 +236,6 @@ class DefaultKeyProviderImpl : public KeyProvider { const std::string participant_id) const override { webrtc::MutexLock lock(&mutex_); - if(options_.shared_key && keys_.find("shared") != keys_.end()) { - return keys_.find("shared")->second; - } - if (keys_.find(participant_id) == keys_.end()) { return nullptr; } From 5a549637ab37d0d0db956940caf673373b1b157f Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 1 Sep 2023 08:47:56 +0800 Subject: [PATCH 06/28] add RatchetSharedKey and ExportSharedKey. --- api/crypto/frame_crypto_transformer.h | 38 ++++++++++++++++++- .../org/webrtc/FrameCryptorKeyProvider.java | 14 +++++++ .../src/jni/pc/frame_cryptor_key_provider.cc | 25 ++++++++++++ .../RTCFrameCryptorKeyProvider.h | 8 +++- .../RTCFrameCryptorKeyProvider.mm | 10 +++++ 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 02239e6e49..f1ffcee62e 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -59,6 +59,10 @@ class KeyProvider : public rtc::RefCountInterface { virtual const std::shared_ptr GetSharedKey(const std::string participant_id) = 0; + virtual const std::vector RatchetSharedKey(int key_index) = 0; + + virtual const std::vector ExportSharedKey(int key_index) const = 0; + virtual bool SetKey(const std::string participant_id, int key_index, std::vector key) = 0; @@ -185,6 +189,7 @@ class DefaultKeyProviderImpl : public KeyProvider { /// Set the shared key. bool SetSharedKey(int key_index, std::vector key) override { webrtc::MutexLock lock(&mutex_); + if (keys_.find("shared") == keys_.end()) { keys_["shared"] = std::make_shared(this); } @@ -198,8 +203,39 @@ class DefaultKeyProviderImpl : public KeyProvider { key_pair.second->SetKey(key, key_index); } } + return true; } - return true; + return false; + } + + const std::vector RatchetSharedKey(int key_index) override { + webrtc::MutexLock lock(&mutex_); + auto it = keys_.find("shared"); + if(it == keys_.end()) { + return std::vector(); + } + auto new_key = it->second->RatchetKey(key_index); + if(options_.shared_key) { + for(auto& key_pair : keys_) { + if(key_pair.first != "shared") { + key_pair.second->SetKey(new_key, key_index); + } + } + } + return new_key; + } + + const std::vector ExportSharedKey(int key_index) const override { + webrtc::MutexLock lock(&mutex_); + auto it = keys_.find("shared"); + if(it == keys_.end()) { + return std::vector(); + } + auto key_set = it->second->GetKeySet(key_index); + if(key_set) { + return key_set->material; + } + return std::vector(); } const std::shared_ptr GetSharedKey(const std::string participant_id) override { diff --git a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java index 6e4a73a2bf..af2e3ff651 100644 --- a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java +++ b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java @@ -35,6 +35,16 @@ public boolean setSharedKey(int index, byte[] key) { return nativeSetSharedKey(nativeKeyProvider,index, key); } + public byte[] ratchetSharedKey(int index) { + checkKeyProviderExists(); + return nativeRatchetSharedKey(nativeKeyProvider, index); + } + + public byte[] exportSharedKey(int index) { + checkKeyProviderExists(); + return nativeExportSharedKey(nativeKeyProvider, index); + } + public boolean setKey(String participantId, int index, byte[] key) { checkKeyProviderExists(); return nativeSetKey(nativeKeyProvider, participantId, index, key); @@ -63,6 +73,10 @@ private void checkKeyProviderExists() { } private static native boolean nativeSetSharedKey( long keyProviderPointer, int index, byte[] key); + private static native byte[] nativeRatchetSharedKey( + long keyProviderPointer, int index); + private static native byte[] nativeExportSharedKey( + long keyProviderPointer, int index); private static native boolean nativeSetKey( long keyProviderPointer, String participantId, int index, byte[] key); private static native byte[] nativeRatchetKey( diff --git a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc index 81ceb458d6..2665bdc9ab 100644 --- a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc +++ b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc @@ -43,6 +43,31 @@ static jboolean JNI_FrameCryptorKeyProvider_SetSharedKey( ->SetSharedKey(j_index,std::vector(key.begin(), key.end())); } +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_RatchetSharedKey( + JNIEnv* env, + jlong keyProviderPointer, + jint j_index) { + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto newKey = key_provider->RatchetSharedKey(j_index); + std::vector int8tKey = + std::vector(newKey.begin(), newKey.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + +static base::android::ScopedJavaLocalRef +JNI_FrameCryptorKeyProvider_ExportSharedKey( + JNIEnv* env, + jlong keyProviderPointer, + jint j_index) { + auto key_provider = + reinterpret_cast(keyProviderPointer); + auto key = key_provider->ExportSharedKey(j_index); + std::vector int8tKey = std::vector(key.begin(), key.end()); + return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); +} + static jboolean JNI_FrameCryptorKeyProvider_SetKey( JNIEnv* jni, jlong j_key_provider, diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h index ab879b7b9e..0122a31444 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h @@ -23,10 +23,14 @@ NS_ASSUME_NONNULL_BEGIN RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCFrameCryptorKeyProvider) : NSObject -- (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId; - - (void)setSharedKey:(NSData *)key withIndex:(int)index; +- (NSData *)ratchetSharedKey:(int)index; + +- (NSData *)exportSharedKey:(int)index; + +- (void)setKey:(NSData *)key withIndex:(int)index forParticipant:(NSString *)participantId; + - (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index; - (NSData *)exportKey:(NSString *)participantId withIndex:(int)index; diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm index c0683c6a32..218a3fafb0 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm @@ -62,6 +62,16 @@ - (void)setSharedKey:(NSData *)key withIndex:(int)index { std::vector((const uint8_t *)key.bytes, ((const uint8_t *)key.bytes) + key.length)); } +- (NSData *)ratchetSharedKey:(int)index { + std::vector nativeKey = _nativeKeyProvider->RatchetSharedKey(index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + +- (NSData *)exportSharedKey:(int)index { + std::vector nativeKey = _nativeKeyProvider->ExportSharedKey(index); + return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; +} + - (NSData *)ratchetKey:(NSString *)participantId withIndex:(int)index { std::vector nativeKey = _nativeKeyProvider->RatchetKey([participantId stdString], index); return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; From 38894feb0d9d3ee4b0949a38ea13222154506ca6 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 1 Sep 2023 08:57:19 +0800 Subject: [PATCH 07/28] make KeyProvider::SetSharedKey only valid when KeyProviderOptions::shared_key == true. --- api/crypto/frame_crypto_transformer.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index f1ffcee62e..c919feefcb 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -189,15 +189,14 @@ class DefaultKeyProviderImpl : public KeyProvider { /// Set the shared key. bool SetSharedKey(int key_index, std::vector key) override { webrtc::MutexLock lock(&mutex_); - - if (keys_.find("shared") == keys_.end()) { - keys_["shared"] = std::make_shared(this); - } + if(options_.shared_key) { + if (keys_.find("shared") == keys_.end()) { + keys_["shared"] = std::make_shared(this); + } - auto key_handler = keys_["shared"]; - key_handler->SetKey(key, key_index); + auto key_handler = keys_["shared"]; + key_handler->SetKey(key, key_index); - if(options_.shared_key) { for(auto& key_pair : keys_) { if(key_pair.first != "shared") { key_pair.second->SetKey(key, key_index); From 2f8bbd8cb364ebae8b90d5592d5082da4883ea4b Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 1 Sep 2023 09:14:41 +0800 Subject: [PATCH 08/28] remove unused enum. --- api/crypto/frame_crypto_transformer.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index c919feefcb..bd245729cf 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -50,9 +50,6 @@ struct KeyProviderOptions { }; class KeyProvider : public rtc::RefCountInterface { - public: - enum { kRawKeySize = 32 }; - public: virtual bool SetSharedKey(int key_index, std::vector key) = 0; From 31774d304b5967c1df09706754e9bf43c18cb61d Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Fri, 1 Sep 2023 23:31:16 +0900 Subject: [PATCH 09/28] Fix memory leak when creating audio CMSampleBuffer #86 --- sdk/objc/api/peerconnection/RTCAudioTrack.mm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/objc/api/peerconnection/RTCAudioTrack.mm b/sdk/objc/api/peerconnection/RTCAudioTrack.mm index 3d3af34893..87deaf8aa8 100644 --- a/sdk/objc/api/peerconnection/RTCAudioTrack.mm +++ b/sdk/objc/api/peerconnection/RTCAudioTrack.mm @@ -151,6 +151,9 @@ void OnData(const void *audio_data, 0, NULL, &buffer); + // format is no longer required + CFRelease(format); + if (status != 0) { NSLog(@"RTCAudioTrack: Failed to allocate sample buffer"); return; From 703978160745012d2c1c1115819962bd7dc807ff Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 8 Sep 2023 11:35:51 +0800 Subject: [PATCH 10/28] add scalabilityMode for AV1. --- .../peerconnection/RTCVideoCodecInfo+Private.mm | 17 +++++++++++++++-- sdk/objc/api/video_codec/RTCVideoEncoderAV1.h | 2 ++ sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm | 16 ++++++++++++++++ sdk/objc/base/RTCVideoCodecInfo.h | 6 ++++++ sdk/objc/base/RTCVideoCodecInfo.m | 17 ++++++++++++++++- .../video_codec/RTCDefaultVideoEncoderFactory.m | 4 +++- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm b/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm index 2eb8d366d2..88c2add1df 100644 --- a/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm +++ b/sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.mm @@ -12,6 +12,11 @@ #import "helpers/NSString+StdString.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/svc/create_scalability_structure.h" + @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) (Private) @@ -31,8 +36,16 @@ - (instancetype)initWithNativeSdpVideoFormat : (webrtc::SdpVideoFormat)format { std::string value = [NSString stdStringForString:self.parameters[paramKey]]; parameters[key] = value; } - - return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters); + + absl::InlinedVector + scalability_modes; + for (NSString *scalabilityMode in self.scalabilityModes) { + auto scalability_mode = webrtc::ScalabilityModeFromString([NSString stdStringForString:scalabilityMode]); + if (scalability_mode != absl::nullopt) { + scalability_modes.push_back(*scalability_mode); + } + } + return webrtc::SdpVideoFormat([NSString stdStringForString:self.name], parameters, scalability_modes); } @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h index 8aa55e4bfa..a900728049 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h +++ b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.h @@ -24,4 +24,6 @@ RTC_OBJC_EXPORT + (bool)isSupported; ++ (NSArray *)scalabilityModes; + @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm index d2fe65293b..b1141ed8b0 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm @@ -14,6 +14,12 @@ #import "RTCMacros.h" #import "RTCVideoEncoderAV1.h" #import "RTCWrappedNativeVideoEncoder.h" + +#import "helpers/NSString+StdString.h" + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" @implementation RTC_OBJC_TYPE (RTCVideoEncoderAV1) @@ -28,4 +34,14 @@ + (bool)isSupported { return true; } ++ (NSArray *)scalabilityModes { + absl::InlinedVector + scalability_modes = webrtc::LibaomAv1EncoderSupportedScalabilityModes(); + NSMutableArray *scalabilityModes = [NSMutableArray array]; + for (const webrtc::ScalabilityMode &scalability_mode : scalability_modes) { + [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + } + return scalabilityModes; +} + @end diff --git a/sdk/objc/base/RTCVideoCodecInfo.h b/sdk/objc/base/RTCVideoCodecInfo.h index fa28958f25..9da0c7aa81 100644 --- a/sdk/objc/base/RTCVideoCodecInfo.h +++ b/sdk/objc/base/RTCVideoCodecInfo.h @@ -26,10 +26,16 @@ RTC_OBJC_EXPORT parameters:(nullable NSDictionary *)parameters NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary *)parameters + scalabilityModes:(nullable NSArray *)scalabilityModes + NS_DESIGNATED_INITIALIZER; + - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info; @property(nonatomic, readonly) NSString *name; @property(nonatomic, readonly) NSDictionary *parameters; +@property(nonatomic, readonly) NSArray *scalabilityModes; @end diff --git a/sdk/objc/base/RTCVideoCodecInfo.m b/sdk/objc/base/RTCVideoCodecInfo.m index ce26ae1de3..441ecde7c5 100644 --- a/sdk/objc/base/RTCVideoCodecInfo.m +++ b/sdk/objc/base/RTCVideoCodecInfo.m @@ -14,6 +14,7 @@ @implementation RTC_OBJC_TYPE (RTCVideoCodecInfo) @synthesize name = _name; @synthesize parameters = _parameters; +@synthesize scalabilityModes = _scalabilityModes; - (instancetype)initWithName:(NSString *)name { return [self initWithName:name parameters:nil]; @@ -24,15 +25,29 @@ - (instancetype)initWithName:(NSString *)name if (self = [super init]) { _name = name; _parameters = (parameters ? parameters : @{}); + _scalabilityModes = @[]; } return self; } +- (instancetype)initWithName:(NSString *)name + parameters:(nullable NSDictionary *)parameters + scalabilityModes:(nullable NSArray *)scalabilityModes { + if (self = [super init]) { + _name = name; + _parameters = (parameters ? parameters : @{}); + _scalabilityModes = (scalabilityModes ? scalabilityModes : @[]); + } + + return self; + } + - (BOOL)isEqualToCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { if (!info || ![self.name isEqualToString:info.name] || - ![self.parameters isEqualToDictionary:info.parameters]) { + ![self.parameters isEqualToDictionary:info.parameters] || + ![self.scalabilityModes isEqualToArray:info.scalabilityModes]) { return NO; } return YES; diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m index 8de55bde4a..3b1d4e49b1 100644 --- a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m +++ b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m @@ -59,7 +59,9 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) } #if defined(RTC_USE_LIBAOM_AV1_ENCODER) - [result addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name]]; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *av1Info = + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecAv1Name parameters:nil scalabilityModes:[RTCVideoEncoderAV1 scalabilityModes]]; + [result addObject:av1Info]; #endif return result; From de5d25f4ca0d290d0e754aba31fa978d32eface0 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 8 Sep 2023 12:36:39 +0800 Subject: [PATCH 11/28] fix bug for scalability-mode. --- api/video_codecs/video_encoder_factory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h index d28a2a4035..da9204bd38 100644 --- a/api/video_codecs/video_encoder_factory.h +++ b/api/video_codecs/video_encoder_factory.h @@ -89,7 +89,7 @@ class VideoEncoderFactory { // specified format is supported. Returns false if scalability_mode is // specified. CodecSupport codec_support; - if (!scalability_mode) { + if (scalability_mode) { codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); } return codec_support; From 0e4c1716ae50e558795917c83381a6b9d15d6591 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Fri, 8 Sep 2023 12:52:39 +0800 Subject: [PATCH 12/28] add scalability-mode support for VP9. --- sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm | 10 +++++----- sdk/objc/api/video_codec/RTCVideoEncoderVP9.h | 2 ++ sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm | 15 +++++++++++++++ .../video_codec/RTCDefaultVideoEncoderFactory.m | 2 +- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm index b1141ed8b0..f9256387dc 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderAV1.mm @@ -19,8 +19,8 @@ #include "absl/container/inlined_vector.h" #include "api/video_codecs/sdp_video_format.h" -#include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" +#include "modules/video_coding/svc/create_scalability_structure.h" @implementation RTC_OBJC_TYPE (RTCVideoEncoderAV1) @@ -35,11 +35,11 @@ + (bool)isSupported { } + (NSArray *)scalabilityModes { - absl::InlinedVector - scalability_modes = webrtc::LibaomAv1EncoderSupportedScalabilityModes(); NSMutableArray *scalabilityModes = [NSMutableArray array]; - for (const webrtc::ScalabilityMode &scalability_mode : scalability_modes) { - [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + } } return scalabilityModes; } diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h index f7dac6117d..adfca0f9a4 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h +++ b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.h @@ -24,4 +24,6 @@ RTC_OBJC_EXPORT + (bool)isSupported; ++ (NSArray *)scalabilityModes; + @end diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm index 18a9353f7e..3db0629074 100644 --- a/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm +++ b/sdk/objc/api/video_codec/RTCVideoEncoderVP9.mm @@ -15,7 +15,12 @@ #import "RTCVideoEncoderVP9.h" #import "RTCWrappedNativeVideoEncoder.h" +#import "helpers/NSString+StdString.h" + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/svc/create_scalability_structure.h" @implementation RTC_OBJC_TYPE (RTCVideoEncoderVP9) @@ -36,4 +41,14 @@ + (bool)isSupported { #endif } ++ (NSArray *)scalabilityModes { + NSMutableArray *scalabilityModes = [NSMutableArray array]; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + [scalabilityModes addObject:[NSString stringForAbslStringView:webrtc::ScalabilityModeToString(scalability_mode)]]; + } + } + return scalabilityModes; +} + @end diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m index 3b1d4e49b1..61da5f4514 100644 --- a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m +++ b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m @@ -55,7 +55,7 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) if ([RTC_OBJC_TYPE(RTCVideoEncoderVP9) isSupported]) { [result - addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name]]; + addObject:[[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecVp9Name parameters:nil scalabilityModes:[RTCVideoEncoderVP9 scalabilityModes]]]; } #if defined(RTC_USE_LIBAOM_AV1_ENCODER) From 6afb1551c55f3f025e5d92f92e8f779972e88f70 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Sat, 9 Sep 2023 08:56:07 +0800 Subject: [PATCH 13/28] add failure tolerance for framecryptor. --- api/crypto/frame_crypto_transformer.cc | 6 +++--- api/crypto/frame_crypto_transformer.h | 28 ++++++++++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 9d8ee44c9e..023c9ca661 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -219,7 +219,7 @@ int AesGcmEncryptDecrypt(EncryptOrDecrypt mode, } if (!ok) { - RTC_LOG(LS_ERROR) << "Failed to perform AES-GCM operation."; + RTC_LOG(LS_WARNING) << "Failed to perform AES-GCM operation."; return OperationError; } @@ -593,7 +593,7 @@ void FrameCryptorTransformer::decryptFrame( decryption_success = true; // success, so we set the new key key_handler->SetKeyFromMaterial(new_material, key_index); - key_handler->SetHasValidKey(true); + key_handler->SetHasValidKey(); if (last_dec_error_ != FrameCryptionState::kKeyRatcheted) { last_dec_error_ = FrameCryptionState::kKeyRatcheted; if (observer_) @@ -622,7 +622,7 @@ void FrameCryptorTransformer::decryptFrame( if (!decryption_success) { if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { last_dec_error_ = FrameCryptionState::kDecryptionFailed; - key_handler->SetHasValidKey(false); + key_handler->DecryptionFailure(); if (observer_) observer_->OnFrameCryptionStateChanged(participant_id_, last_dec_error_); diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index bd245729cf..96143b2637 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -41,12 +41,14 @@ struct KeyProviderOptions { std::vector ratchet_salt; std::vector uncrypted_magic_bytes; int ratchet_window_size; - KeyProviderOptions() : shared_key(false), ratchet_window_size(0) {} + int failure_tolerance; + KeyProviderOptions() : shared_key(false), ratchet_window_size(0), failure_tolerance(-1) {} KeyProviderOptions(KeyProviderOptions& copy) : shared_key(copy.shared_key), ratchet_salt(copy.ratchet_salt), uncrypted_magic_bytes(copy.uncrypted_magic_bytes), - ratchet_window_size(copy.ratchet_window_size) {} + ratchet_window_size(copy.ratchet_window_size), + failure_tolerance(copy.failure_tolerance) {} }; class KeyProvider : public rtc::RefCountInterface { @@ -116,7 +118,7 @@ class ParticipantKeyHandler { } SetKeyFromMaterial(new_material, key_index != -1 ? key_index : current_key_index_); - SetHasValidKey(true); + SetHasValidKey(); return new_material; } @@ -127,7 +129,7 @@ class ParticipantKeyHandler { virtual void SetKey(std::vector password, int key_index) { SetKeyFromMaterial(password, key_index); - SetHasValidKey(true); + SetHasValidKey(); } std::vector RatchetKeyMaterial( @@ -156,9 +158,10 @@ class ParticipantKeyHandler { return has_valid_key_; } - void SetHasValidKey(bool has_valid_key) { + void SetHasValidKey() { webrtc::MutexLock lock(&mutex_); - has_valid_key_ = has_valid_key; + decryption_failure_count_ = 0; + has_valid_key_ = true; } void SetKeyFromMaterial(std::vector password, int key_index) { @@ -170,8 +173,21 @@ class ParticipantKeyHandler { DeriveKeys(password, key_provider_->options().ratchet_salt, 128); } + void DecryptionFailure() { + webrtc::MutexLock lock(&mutex_); + if (key_provider_->options().failure_tolerance < 0) { + return; + } + decryption_failure_count_ += 1; + + if (decryption_failure_count_ > key_provider_->options().failure_tolerance) { + has_valid_key_ = false; + } + } + private: bool has_valid_key_ = false; + int decryption_failure_count_ = 0; mutable webrtc::Mutex mutex_; int current_key_index_ = 0; KeyProvider* key_provider_; From 4911ef869c224572d38b2c99eaec8e3e17d3667c Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Sat, 9 Sep 2023 12:43:33 +0800 Subject: [PATCH 14/28] add failureTolerance for android/objc. --- sdk/android/api/org/webrtc/FrameCryptorFactory.java | 6 +++--- sdk/android/src/jni/pc/frame_cryptor.cc | 4 +++- .../api/peerconnection/RTCFrameCryptorKeyProvider.h | 6 ++++++ .../peerconnection/RTCFrameCryptorKeyProvider.mm | 13 +++++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/sdk/android/api/org/webrtc/FrameCryptorFactory.java b/sdk/android/api/org/webrtc/FrameCryptorFactory.java index 74df6a5b29..b70f61cc6b 100644 --- a/sdk/android/api/org/webrtc/FrameCryptorFactory.java +++ b/sdk/android/api/org/webrtc/FrameCryptorFactory.java @@ -18,8 +18,8 @@ public class FrameCryptorFactory { public static FrameCryptorKeyProvider createFrameCryptorKeyProvider( - boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes) { - return nativeCreateFrameCryptorKeyProvider(sharedKey, ratchetSalt, ratchetWindowSize, uncryptedMagicBytes); + boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes, int failureTolerance) { + return nativeCreateFrameCryptorKeyProvider(sharedKey, ratchetSalt, ratchetWindowSize, uncryptedMagicBytes, failureTolerance); } public static FrameCryptor createFrameCryptorForRtpSender(RtpSender rtpSender, @@ -40,5 +40,5 @@ private static native FrameCryptor nativeCreateFrameCryptorForRtpReceiver( long rtpReceiver, String participantId, int algorithm, long nativeFrameCryptorKeyProvider); private static native FrameCryptorKeyProvider nativeCreateFrameCryptorKeyProvider( - boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes); + boolean sharedKey, byte[] ratchetSalt, int ratchetWindowSize, byte[] uncryptedMagicBytes, int failureTolerance); } diff --git a/sdk/android/src/jni/pc/frame_cryptor.cc b/sdk/android/src/jni/pc/frame_cryptor.cc index d02f0c62da..6c36ece9ee 100644 --- a/sdk/android/src/jni/pc/frame_cryptor.cc +++ b/sdk/android/src/jni/pc/frame_cryptor.cc @@ -171,7 +171,8 @@ JNI_FrameCryptorFactory_CreateFrameCryptorKeyProvider( jboolean j_shared, const base::android::JavaParamRef& j_ratchetSalt, jint j_ratchetWindowSize, - const base::android::JavaParamRef& j_uncryptedMagicBytes) { + const base::android::JavaParamRef& j_uncryptedMagicBytes, + jint j_failureTolerance) { auto ratchetSalt = JavaToNativeByteArray(env, j_ratchetSalt); KeyProviderOptions options; options.ratchet_salt = @@ -182,6 +183,7 @@ JNI_FrameCryptorFactory_CreateFrameCryptorKeyProvider( options.uncrypted_magic_bytes = std::vector(uncryptedMagicBytes.begin(), uncryptedMagicBytes.end()); options.shared_key = j_shared; + options.failure_tolerance = j_failureTolerance; return NativeToJavaFrameCryptorKeyProvider( env, rtc::make_ref_counted(options)); } diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h index 0122a31444..1b207e59a8 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h @@ -40,6 +40,12 @@ RTC_OBJC_EXPORT sharedKeyMode:(BOOL)sharedKey uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes; +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance; + @end NS_ASSUME_NONNULL_END diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm index 218a3fafb0..8f961b002d 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm @@ -34,12 +34,25 @@ - (instancetype)initWithRatchetSalt:(NSData *)salt ratchetWindowSize:(int)windowSize sharedKeyMode:(BOOL)sharedKey uncryptedMagicBytes:(NSData *)uncryptedMagicBytes { + return [self initWithRatchetSalt:salt + ratchetWindowSize:windowSize + sharedKeyMode:sharedKey + uncryptedMagicBytes:uncryptedMagicBytes + failureTolerance:-1]; +} + +- (instancetype)initWithRatchetSalt:(NSData *)salt + ratchetWindowSize:(int)windowSize + sharedKeyMode:(BOOL)sharedKey + uncryptedMagicBytes:(nullable NSData *)uncryptedMagicBytes + failureTolerance:(int)failureTolerance { if (self = [super init]) { webrtc::KeyProviderOptions options; options.ratchet_salt = std::vector((const uint8_t *)salt.bytes, ((const uint8_t *)salt.bytes) + salt.length); options.ratchet_window_size = windowSize; options.shared_key = sharedKey; + options.failure_tolerance = failureTolerance; if(uncryptedMagicBytes != nil) { options.uncrypted_magic_bytes = std::vector((const uint8_t *)uncryptedMagicBytes.bytes, ((const uint8_t *)uncryptedMagicBytes.bytes) + uncryptedMagicBytes.length); From bd21d2c0d957954b9d88394f831fec0eecff0242 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Mon, 11 Sep 2023 16:24:18 +0800 Subject: [PATCH 15/28] fix: make H264's unencrypted_bytes consistent with js-sdk. --- api/crypto/frame_crypto_transformer.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 023c9ca661..9ef356ce0f 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -136,7 +136,7 @@ uint8_t get_unencrypted_bytes(webrtc::TransformableFrameInterface* frame, << "NonParameterSetNalu::payload_size: " << index.payload_size << ", nalu_type " << nalu_type << ", NaluIndex [" << idx++ << "] offset: " << index.payload_start_offset; - break; + return unencrypted_bytes; default: break; } @@ -510,7 +510,7 @@ void FrameCryptorTransformer::decryptFrame( uint8_t key_index = frame_trailer[1]; if (ivLength != getIvSize()) { - RTC_LOG(LS_ERROR) << "FrameCryptorTransformer::decryptFrame() ivLength[" + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::decryptFrame() ivLength[" << static_cast(ivLength) << "] != getIvSize()[" << static_cast(getIvSize()) << "]"; if (last_dec_error_ != FrameCryptionState::kDecryptionFailed) { @@ -569,7 +569,7 @@ void FrameCryptorTransformer::decryptFrame( encrypted_payload, &buffer) == Success) { decryption_success = true; } else { - RTC_LOG(LS_ERROR) << "FrameCryptorTransformer::decryptFrame() failed"; + RTC_LOG(LS_WARNING) << "FrameCryptorTransformer::decryptFrame() failed"; std::shared_ptr ratcheted_key_set; auto currentKeyMaterial = key_set->material; if (key_provider_->options().ratchet_window_size > 0) { From 3c26cf7578806f78d1d9049f7bd17c1a30e48c14 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 08:34:58 +0800 Subject: [PATCH 16/28] wip: ScalabilityModes for android. --- sdk/android/api/org/webrtc/LibaomAv1Encoder.java | 6 ++++++ sdk/android/api/org/webrtc/LibvpxVp9Encoder.java | 6 ++++++ sdk/android/api/org/webrtc/VideoCodecInfo.java | 16 ++++++++++++++++ sdk/android/src/jni/libaom_av1_encoder.cc | 10 ++++++++++ sdk/android/src/jni/video_codec_info.cc | 13 ++++++++++++- sdk/android/src/jni/vp9_codec.cc | 13 +++++++++++++ 6 files changed, 63 insertions(+), 1 deletion(-) diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java index 569a719f44..83ac1c8a4e 100644 --- a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java +++ b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -22,4 +22,10 @@ public long createNativeVideoEncoder() { public boolean isHardwareEncoder() { return false; } + + List scalabilityModes() { + return nativeGetSupportedScalabilityModes(); + } + + static native List nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java index 1211ae93fb..2319512632 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -24,4 +24,10 @@ public boolean isHardwareEncoder() { } static native boolean nativeIsSupported(); + + List scalabilityModes() { + return nativeGetSupportedScalabilityModes(); + } + + static native List nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index 4f97cf74cf..efff2746dd 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -34,6 +34,8 @@ public class VideoCodecInfo { public final String name; public final Map params; + public final List scalabilityModes; + @Deprecated public final int payload; @CalledByNative @@ -41,6 +43,7 @@ public VideoCodecInfo(String name, Map params) { this.payload = 0; this.name = name; this.params = params; + this.scalabilityModes = new ArrayList<>(); } @Deprecated @@ -48,6 +51,14 @@ public VideoCodecInfo(int payload, String name, Map params) { this.payload = payload; this.name = name; this.params = params; + this.scalabilityModes = new ArrayList<>(); + } + + public VideoCodecInfo(String name, Map params, List scalabilityModes) { + this.payload = 0; + this.name = name; + this.params = params; + this.scalabilityModes = scalabilityModes; } @Override @@ -83,4 +94,9 @@ String getName() { Map getParams() { return params; } + + @CalledByNative + List getScalabilityModes() { + return scalabilityModes; + } } diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 96e0dda169..68a7d55a93 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -22,5 +22,15 @@ static jlong JNI_LibaomAv1Encoder_CreateEncoder(JNIEnv* jni) { return jlongFromPointer(webrtc::CreateLibaomAv1Encoder().release()); } +static webrtc::ScopedJavaLocalRef JNI_LibaomAv1Encoder_SGetSupportedScalabilityModes(JNIEnv* jni) { + std::vector modes; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + modes.push_back(webrtc::ScalabilityModeToString(scalability_mode)); + } + } + return NativeToJavaList(jni, modes, &JavaStringFromStdString); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index a218a1d23f..5d6a0833cf 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -19,9 +19,20 @@ namespace jni { SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, const JavaRef& j_info) { + std::vector params = + JavaToStdVectorStrings(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); + absl::InlinedVector + scalability_modes; + for (auto mode : params) { + auto scalability_mode = webrtc::ScalabilityModeFromString(mode); + if (scalability_mode != absl::nullopt) { + scalability_modes.push_back(*scalability_mode); + } + } return SdpVideoFormat( JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)), - JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info))); + JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)), + scalability_modes); } ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc index ad9ca793ce..0f2a0026d7 100644 --- a/sdk/android/src/jni/vp9_codec.cc +++ b/sdk/android/src/jni/vp9_codec.cc @@ -10,7 +10,11 @@ #include +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/svc/create_scalability_structure.h" + #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Decoder_jni.h" #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" @@ -34,5 +38,14 @@ static jboolean JNI_LibvpxVp9Decoder_IsSupported(JNIEnv* jni) { return !SupportedVP9Codecs().empty(); } +static webrtc::ScopedJavaLocalRef JNI_LibvpxVp9Decoder_SGetSupportedScalabilityModes(JNIEnv* jni) { + std::vector modes; + for (const auto scalability_mode : webrtc::kAllScalabilityModes) { + if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { + modes.push_back(webrtc::ScalabilityModeToString(scalability_mode)); + } + } + return NativeToJavaList(jni, modes, &JavaStringFromStdString); +} } // namespace jni } // namespace webrtc From 717260201db4ed40537d3144438afc3c308ac345 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 08:40:03 +0800 Subject: [PATCH 17/28] update. --- sdk/android/api/org/webrtc/LibaomAv1Encoder.java | 1 + sdk/android/api/org/webrtc/LibvpxVp9Encoder.java | 1 + sdk/android/api/org/webrtc/VideoCodecInfo.java | 2 ++ sdk/android/src/jni/libaom_av1_encoder.cc | 3 +++ sdk/android/src/jni/vp9_codec.cc | 3 +++ 5 files changed, 10 insertions(+) diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java index 83ac1c8a4e..9cffd9051d 100644 --- a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java +++ b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -9,6 +9,7 @@ */ package org.webrtc; +import java.util.List; public class LibaomAv1Encoder extends WrappedNativeVideoEncoder { @Override diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java index 2319512632..6c4bc6a5ad 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -9,6 +9,7 @@ */ package org.webrtc; +import java.util.List; public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder { @Override diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index efff2746dd..eed06aa513 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -14,6 +14,8 @@ import java.util.Arrays; import java.util.Locale; import java.util.Map; +import java.util.List; +import java.util.ArrayList; /** * Represent a video codec as encoded in SDP. diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 68a7d55a93..027acbe435 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -15,6 +15,9 @@ #include "sdk/android/generated_libaom_av1_encoder_jni/LibaomAv1Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" +#include +#include + namespace webrtc { namespace jni { diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc index 0f2a0026d7..d4bfa7191a 100644 --- a/sdk/android/src/jni/vp9_codec.cc +++ b/sdk/android/src/jni/vp9_codec.cc @@ -19,6 +19,9 @@ #include "sdk/android/generated_libvpx_vp9_jni/LibvpxVp9Encoder_jni.h" #include "sdk/android/src/jni/jni_helpers.h" +#include +#include + namespace webrtc { namespace jni { From 52fce22d38bf2d5ca4009257e025ae763c0c6471 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 08:55:42 +0800 Subject: [PATCH 18/28] wip. --- sdk/android/src/jni/libaom_av1_encoder.cc | 5 ++--- sdk/android/src/jni/video_codec_info.cc | 6 ++++-- sdk/android/src/jni/vp9_codec.cc | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 027acbe435..72651eb627 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -29,11 +29,10 @@ static webrtc::ScopedJavaLocalRef JNI_LibaomAv1Encoder_SGetSupportedSc std::vector modes; for (const auto scalability_mode : webrtc::kAllScalabilityModes) { if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { - modes.push_back(webrtc::ScalabilityModeToString(scalability_mode)); + modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode))); } } - return NativeToJavaList(jni, modes, &JavaStringFromStdString); + return NativeToJavaStringArray(jni, modes); } - } // namespace jni } // namespace webrtc diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index 5d6a0833cf..a71ed38568 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -13,6 +13,8 @@ #include "sdk/android/generated_video_jni/VideoCodecInfo_jni.h" #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" +#include "api/video_codecs/scalability_mode.h" +#include "modules/video_coding/svc/scalability_mode_util.h" namespace webrtc { namespace jni { @@ -21,10 +23,10 @@ SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, const JavaRef& j_info) { std::vector params = JavaToStdVectorStrings(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); - absl::InlinedVector + absl::InlinedVector scalability_modes; for (auto mode : params) { - auto scalability_mode = webrtc::ScalabilityModeFromString(mode); + auto scalability_mode = ScalabilityModeFromString(mode); if (scalability_mode != absl::nullopt) { scalability_modes.push_back(*scalability_mode); } diff --git a/sdk/android/src/jni/vp9_codec.cc b/sdk/android/src/jni/vp9_codec.cc index d4bfa7191a..e5929ad2f4 100644 --- a/sdk/android/src/jni/vp9_codec.cc +++ b/sdk/android/src/jni/vp9_codec.cc @@ -41,14 +41,14 @@ static jboolean JNI_LibvpxVp9Decoder_IsSupported(JNIEnv* jni) { return !SupportedVP9Codecs().empty(); } -static webrtc::ScopedJavaLocalRef JNI_LibvpxVp9Decoder_SGetSupportedScalabilityModes(JNIEnv* jni) { +static webrtc::ScopedJavaLocalRef JNI_LibvpxVp9Encoder_GetSupportedScalabilityModes(JNIEnv* jni) { std::vector modes; for (const auto scalability_mode : webrtc::kAllScalabilityModes) { if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { - modes.push_back(webrtc::ScalabilityModeToString(scalability_mode)); + modes.push_back(std::string(webrtc::ScalabilityModeToString(scalability_mode))); } } - return NativeToJavaList(jni, modes, &JavaStringFromStdString); + return NativeToJavaStringArray(jni, modes); } } // namespace jni } // namespace webrtc From 871b74fac88ed4476fb53b69522dd51dcbcd5086 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 09:00:59 +0800 Subject: [PATCH 19/28] wip. --- sdk/android/src/jni/libaom_av1_encoder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/android/src/jni/libaom_av1_encoder.cc b/sdk/android/src/jni/libaom_av1_encoder.cc index 72651eb627..32c1fc7993 100644 --- a/sdk/android/src/jni/libaom_av1_encoder.cc +++ b/sdk/android/src/jni/libaom_av1_encoder.cc @@ -25,7 +25,7 @@ static jlong JNI_LibaomAv1Encoder_CreateEncoder(JNIEnv* jni) { return jlongFromPointer(webrtc::CreateLibaomAv1Encoder().release()); } -static webrtc::ScopedJavaLocalRef JNI_LibaomAv1Encoder_SGetSupportedScalabilityModes(JNIEnv* jni) { +static webrtc::ScopedJavaLocalRef JNI_LibaomAv1Encoder_GetSupportedScalabilityModes(JNIEnv* jni) { std::vector modes; for (const auto scalability_mode : webrtc::kAllScalabilityModes) { if (webrtc::ScalabilityStructureConfig(scalability_mode).has_value()) { From 994cc2cddc7d2cf1550fd13229826cb27f879026 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 09:30:26 +0800 Subject: [PATCH 20/28] wip. --- sdk/android/api/org/webrtc/LibaomAv1Encoder.java | 2 +- sdk/android/api/org/webrtc/LibvpxVp9Encoder.java | 2 +- sdk/android/src/jni/video_codec_info.cc | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java index 9cffd9051d..1bb5ff9c5e 100644 --- a/sdk/android/api/org/webrtc/LibaomAv1Encoder.java +++ b/sdk/android/api/org/webrtc/LibaomAv1Encoder.java @@ -24,7 +24,7 @@ public boolean isHardwareEncoder() { return false; } - List scalabilityModes() { + static List scalabilityModes() { return nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java index 6c4bc6a5ad..c1cc86a2f2 100644 --- a/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java +++ b/sdk/android/api/org/webrtc/LibvpxVp9Encoder.java @@ -26,7 +26,7 @@ public boolean isHardwareEncoder() { static native boolean nativeIsSupported(); - List scalabilityModes() { + static List scalabilityModes() { return nativeGetSupportedScalabilityModes(); } diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index a71ed38568..1412b3af59 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -42,8 +42,12 @@ ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( const SdpVideoFormat& format) { ScopedJavaLocalRef j_params = NativeToJavaStringMap(jni, format.parameters); + std::vector scalability_modes; + for (auto mode : format.scalability_modes) { + scalability_modes.push_back(std::string(ScalabilityModeToString(mode))); + } return Java_VideoCodecInfo_Constructor( - jni, NativeToJavaString(jni, format.name), j_params); + jni, NativeToJavaString(jni, format.name), j_params, NativeToJavaStringArray(jni, scalability_modes)); } } // namespace jni From 275277ecb7c0d770c440480dc69acd8f3556b3b2 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 09:39:25 +0800 Subject: [PATCH 21/28] wip. --- .../api/org/webrtc/HardwareVideoEncoderFactory.java | 4 ++-- sdk/android/api/org/webrtc/VideoCodecInfo.java | 11 ++--------- .../AndroidVideoDecoderInstrumentationTest.java | 2 +- .../org/webrtc/DefaultVideoEncoderFactoryTest.java | 4 ++-- .../org/webrtc/SoftwareVideoDecoderFactoryTest.java | 3 ++- .../org/webrtc/SoftwareVideoEncoderFactoryTest.java | 3 ++- .../org/webrtc/CodecsWrapperTestHelper.java | 3 ++- sdk/android/src/java/org/webrtc/H264Utils.java | 5 +++-- .../org/webrtc/MediaCodecVideoDecoderFactory.java | 4 ++-- 9 files changed, 18 insertions(+), 21 deletions(-) diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java index d43fc27fa0..05c39a4b38 100644 --- a/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java @@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() { // supported by the decoder. if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) { supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>())); } supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>())); } } diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index eed06aa513..e0f5153d47 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -41,11 +41,11 @@ public class VideoCodecInfo { @Deprecated public final int payload; @CalledByNative - public VideoCodecInfo(String name, Map params) { + public VideoCodecInfo(String name, Map params, List scalabilityModes) { this.payload = 0; this.name = name; this.params = params; - this.scalabilityModes = new ArrayList<>(); + this.scalabilityModes = scalabilityModes; } @Deprecated @@ -56,13 +56,6 @@ public VideoCodecInfo(int payload, String name, Map params) { this.scalabilityModes = new ArrayList<>(); } - public VideoCodecInfo(String name, Map params, List scalabilityModes) { - this.payload = 0; - this.name = name; - this.params = params; - this.scalabilityModes = scalabilityModes; - } - @Override public boolean equals(@Nullable Object obj) { if (obj == null) diff --git a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java index 6f448124e8..d9fadabfd9 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java @@ -48,7 +48,7 @@ public AndroidVideoDecoderInstrumentationTest(String codecName, boolean useEglCo if (codecName.equals("H264")) { this.codecType = H264Utils.DEFAULT_H264_BASELINE_PROFILE_CODEC; } else { - this.codecType = new VideoCodecInfo(codecName, new HashMap<>()); + this.codecType = new VideoCodecInfo(codecName, new HashMap<>(), new ArrayList<>()); } this.useEglContext = useEglContext; } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java index fe608c794e..5910a72f93 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java @@ -47,7 +47,7 @@ public void setUp() { @SmallTest @Test public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() { - VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>()); + VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>(), new ArrayList<>()); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs(); @@ -62,7 +62,7 @@ public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() { public void getSupportedCodecs_hwVp8WithDifferentParams_twoVp8() { VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap() { { put("param", "value"); } - }); + }, new ArrayList<>()); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs(); diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java index 8a5d9788ee..8be15624da 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoDecoderFactoryTest.java @@ -14,6 +14,7 @@ import androidx.annotation.Nullable; import androidx.test.filters.SmallTest; +import java.util.ArrayList; import java.util.HashMap; import org.junit.Before; import org.junit.Test; @@ -55,7 +56,7 @@ public void createDecoder_supportedCodec_returnsNotNull() { @Test public void createDecoder_unsupportedCodec_returnsNull() { VideoDecoderFactory factory = new SoftwareVideoDecoderFactory(); - VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap()); + VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap(), new ArrayList<>()); VideoDecoder decoder = factory.createDecoder(codec); assertThat(decoder).isNull(); } diff --git a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java index 696b423cde..0fa4c4cc17 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/SoftwareVideoEncoderFactoryTest.java @@ -14,6 +14,7 @@ import androidx.annotation.Nullable; import androidx.test.filters.SmallTest; +import java.util.ArrayList; import java.util.HashMap; import org.junit.Before; import org.junit.Test; @@ -52,7 +53,7 @@ public void createEncoder_supportedCodec_returnsNotNull() { @Test public void createEncoder_unsupportedCodec_returnsNull() { VideoEncoderFactory factory = new SoftwareVideoEncoderFactory(); - VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap()); + VideoCodecInfo codec = new VideoCodecInfo("unsupported", new HashMap(), new ArrayList<>()); VideoEncoder encoder = factory.createEncoder(codec); assertThat(encoder).isNull(); } diff --git a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java index 70151d3b78..72c5c64191 100644 --- a/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java +++ b/sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; public class CodecsWrapperTestHelper { @CalledByNative @@ -20,7 +21,7 @@ public static VideoCodecInfo createTestVideoCodecInfo() { params.put( VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID, VideoCodecInfo.H264_CONSTRAINED_BASELINE_3_1); - VideoCodecInfo codec_info = new VideoCodecInfo("H264", params); + VideoCodecInfo codec_info = new VideoCodecInfo("H264", params, new ArrayList<>()); return codec_info; } diff --git a/sdk/android/src/java/org/webrtc/H264Utils.java b/sdk/android/src/java/org/webrtc/H264Utils.java index abb79c6582..4bf292ee12 100644 --- a/sdk/android/src/java/org/webrtc/H264Utils.java +++ b/sdk/android/src/java/org/webrtc/H264Utils.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.HashMap; +import java.util.ArrayList; /** Container for static helper functions related to dealing with H264 codecs. */ class H264Utils { @@ -38,9 +39,9 @@ public static Map getDefaultH264Params(boolean isHighProfile) { } public static VideoCodecInfo DEFAULT_H264_BASELINE_PROFILE_CODEC = - new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false)); + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ false), new ArrayList<>()); public static VideoCodecInfo DEFAULT_H264_HIGH_PROFILE_CODEC = - new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true)); + new VideoCodecInfo("H264", getDefaultH264Params(/* isHighProfile= */ true), new ArrayList<>()); public static boolean isSameH264Profile( Map params1, Map params2) { diff --git a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java index 9a73bc49ff..d5b892279c 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java @@ -72,11 +72,11 @@ public VideoCodecInfo[] getSupportedCodecs() { String name = type.name(); if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) { supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>())); } supportedCodecInfos.add(new VideoCodecInfo( - name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false))); + name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>())); } } From 5fdfb7698bf37a5285c71d042cc4d14744ba20a4 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 13 Sep 2023 12:25:06 +0900 Subject: [PATCH 22/28] Fix camera rotation (#92) Use UIInterfaceOrientation instead of UIDeviceOrientation for RTCVideoFrame's rotation --- .../capturer/RTCCameraVideoCapturer.m | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m index f7ae04e814..018c1d41cf 100644 --- a/sdk/objc/components/capturer/RTCCameraVideoCapturer.m +++ b/sdk/objc/components/capturer/RTCCameraVideoCapturer.m @@ -42,7 +42,7 @@ @implementation RTC_OBJC_TYPE (RTCCameraVideoCapturer) { FourCharCode _outputPixelFormat; RTCVideoRotation _rotation; #if TARGET_OS_IPHONE - UIDeviceOrientation _orientation; + UIInterfaceOrientation _orientation; BOOL _generatingOrientationNotifications; #endif } @@ -75,7 +75,7 @@ - (instancetype)initWithDelegate:(__weak id_generatingOrientationNotifications = YES; } + // Must be called on main + [self updateOrientation]; }); #endif @@ -183,7 +185,6 @@ - (void)startCaptureWithDevice:(AVCaptureDevice *)device return; } [self reconfigureCaptureSessionInput]; - [self updateOrientation]; [self updateDeviceCaptureFormat:format fps:fps]; [self updateVideoDataOutputPixelFormat:format]; [self.captureSession startRunning]; @@ -226,10 +227,7 @@ - (void)stopCaptureWithCompletionHandler:(nullable void (^)(void))completionHand #if TARGET_OS_IPHONE - (void)deviceOrientationDidChange:(NSNotification *)notification { - [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession - block:^{ - [self updateOrientation]; - }]; + [self updateOrientation]; } #endif @@ -265,22 +263,20 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position; } switch (_orientation) { - case UIDeviceOrientationPortrait: + case UIInterfaceOrientationPortrait: _rotation = RTCVideoRotation_90; break; - case UIDeviceOrientationPortraitUpsideDown: + case UIInterfaceOrientationPortraitUpsideDown: _rotation = RTCVideoRotation_270; break; - case UIDeviceOrientationLandscapeLeft: - _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; - break; - case UIDeviceOrientationLandscapeRight: + case UIInterfaceOrientationLandscapeLeft: _rotation = usingFrontCamera ? RTCVideoRotation_0 : RTCVideoRotation_180; break; - case UIDeviceOrientationFaceUp: - case UIDeviceOrientationFaceDown: - case UIDeviceOrientationUnknown: - // Ignore. + case UIInterfaceOrientationLandscapeRight: + _rotation = usingFrontCamera ? RTCVideoRotation_180 : RTCVideoRotation_0; + break; + case UIInterfaceOrientationUnknown: + _rotation = RTCVideoRotation_0; break; } #else @@ -495,8 +491,8 @@ - (void)updateDeviceCaptureFormat:(AVCaptureDeviceFormat *)format fps:(NSInteger @"updateDeviceCaptureFormat must be called on the capture queue."); @try { _currentDevice.activeFormat = format; - if(![NSStringFromClass([_currentDevice class]) isEqualToString:@"AVCaptureDALDevice"]) { - _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); + if (![NSStringFromClass([_currentDevice class]) isEqualToString:@"AVCaptureDALDevice"]) { + _currentDevice.activeVideoMinFrameDuration = CMTimeMake(1, fps); } } @catch (NSException *exception) { RTCLogError(@"Failed to set active format!\n User info:%@", exception.userInfo); @@ -508,8 +504,8 @@ - (void)reconfigureCaptureSessionInput { NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], @"reconfigureCaptureSessionInput must be called on the capture queue."); NSError *error = nil; - AVCaptureDeviceInput *input = - [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice error:&error]; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:_currentDevice + error:&error]; if (!input) { RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); return; @@ -526,12 +522,19 @@ - (void)reconfigureCaptureSessionInput { [_captureSession commitConfiguration]; } -- (void)updateOrientation { - NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeCaptureSession], - @"updateOrientation must be called on the capture queue."); #if TARGET_OS_IPHONE - _orientation = [UIDevice currentDevice].orientation; -#endif +- (void)updateOrientation { + NSAssert([RTC_OBJC_TYPE(RTCDispatcher) isOnQueueForType:RTCDispatcherTypeMain], + @"statusBarOrientation must be called on the main queue."); + // statusBarOrientation must be called on the main queue + UIInterfaceOrientation newOrientation = [UIApplication sharedApplication].statusBarOrientation; + + [RTC_OBJC_TYPE(RTCDispatcher) dispatchAsyncOnType:RTCDispatcherTypeCaptureSession + block:^{ + // Must be called on the capture queue + self->_orientation = newOrientation; + }]; } +#endif @end From 8b60bcfa6dca2a0974cf8e01d1afba5267ce6018 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 12:52:35 +0800 Subject: [PATCH 23/28] done. --- sdk/android/api/org/webrtc/RtpParameters.java | 13 +++++++++++-- sdk/android/src/jni/pc/rtp_parameters.cc | 6 ++++++ sdk/android/src/jni/video_codec_info.cc | 11 ++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java index 9ca8311610..4e3f106785 100644 --- a/sdk/android/api/org/webrtc/RtpParameters.java +++ b/sdk/android/api/org/webrtc/RtpParameters.java @@ -76,6 +76,8 @@ public static class Encoding { // If non-null, scale the width and height down by this factor for video. If null, // implementation default scaling factor will be used. @Nullable public Double scaleResolutionDownBy; + // Scalability modes are used to represent simulcast and SVC layers. + @Nullable public String scalabilityMode; // SSRC to be used by this encoding. // Can't be changed between getParameters/setParameters. public Long ssrc; @@ -93,8 +95,8 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { @CalledByNative("Encoding") Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority, Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate, - Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc, - boolean adaptiveAudioPacketTime) { + Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, + Long ssrc, boolean adaptiveAudioPacketTime) { this.rid = rid; this.active = active; this.bitratePriority = bitratePriority; @@ -104,6 +106,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { this.maxFramerate = maxFramerate; this.numTemporalLayers = numTemporalLayers; this.scaleResolutionDownBy = scaleResolutionDownBy; + this.scalabilityMode = scalabilityMode; this.ssrc = ssrc; this.adaptiveAudioPacketTime = adaptiveAudioPacketTime; } @@ -160,6 +163,12 @@ Double getScaleResolutionDownBy() { return scaleResolutionDownBy; } + @Nullable + @CalledByNative("Encoding") + String getScalabilityMode() { + return scalabilityMode; + } + @CalledByNative("Encoding") Long getSsrc() { return ssrc; diff --git a/sdk/android/src/jni/pc/rtp_parameters.cc b/sdk/android/src/jni/pc/rtp_parameters.cc index 4bd9ee0e1d..6feb6a631b 100644 --- a/sdk/android/src/jni/pc/rtp_parameters.cc +++ b/sdk/android/src/jni/pc/rtp_parameters.cc @@ -53,6 +53,7 @@ ScopedJavaLocalRef NativeToJavaRtpEncodingParameter( NativeToJavaInteger(env, encoding.max_framerate), NativeToJavaInteger(env, encoding.num_temporal_layers), NativeToJavaDouble(env, encoding.scale_resolution_down_by), + NativeToJavaString(env, encoding.scalability_mode), encoding.ssrc ? NativeToJavaLong(env, *encoding.ssrc) : nullptr, encoding.adaptive_ptime); } @@ -116,6 +117,11 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters( Java_Encoding_getScaleResolutionDownBy(jni, j_encoding_parameters); encoding.scale_resolution_down_by = JavaToNativeOptionalDouble(jni, j_scale_resolution_down_by); + ScopedJavaLocalRef j_scalability_mode = + Java_Encoding_getScalabilityMode(jni, j_encoding_parameters); + if (!IsNull(jni, j_scalability_mode)) { + encoding.scalability_mode = JavaToNativeString(jni,j_scalability_mode); + } encoding.adaptive_ptime = Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters); ScopedJavaLocalRef j_ssrc = diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index 1412b3af59..a85dde67dc 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -42,12 +42,17 @@ ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( const SdpVideoFormat& format) { ScopedJavaLocalRef j_params = NativeToJavaStringMap(jni, format.parameters); - std::vector scalability_modes; + webrtc::ScopedJavaLocalRef j_scalability_modes; + if (!format.scalability_modes.empty()) { + JavaListBuilder builder(jni); for (auto mode : format.scalability_modes) { - scalability_modes.push_back(std::string(ScalabilityModeToString(mode))); + std::string scalability_mode(ScalabilityModeToString(mode)); + builder.add(NativeToJavaString(jni, scalability_mode)); } + j_scalability_modes = builder.java_list(); + } return Java_VideoCodecInfo_Constructor( - jni, NativeToJavaString(jni, format.name), j_params, NativeToJavaStringArray(jni, scalability_modes)); + jni, NativeToJavaString(jni, format.name), j_params, j_scalability_modes); } } // namespace jni From e4a4012bbb704bea8a14327aac1a9a9dd48ef86a Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 12:58:48 +0800 Subject: [PATCH 24/28] fix --- .../src/org/webrtc/DefaultVideoEncoderFactoryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java index 5910a72f93..1a9dd5fc38 100644 --- a/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java +++ b/sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java @@ -47,7 +47,7 @@ public void setUp() { @SmallTest @Test public void getSupportedCodecs_hwVp8SameParamsAsSwVp8_oneVp8() { - VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>(), new ArrayList<>()); + VideoCodecInfo hwVp8Encoder = new VideoCodecInfo("VP8", new HashMap<>(), new ArrayList<>()); VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(hwVp8Encoder); DefaultVideoEncoderFactory defFactory = new DefaultVideoEncoderFactory(hwFactory); VideoCodecInfo[] supportedCodecs = defFactory.getSupportedCodecs(); From 6e96936514c1940ff2f235f46360c6ad2cc7ed13 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 15:12:02 +0800 Subject: [PATCH 25/28] update. --- api/video_codecs/video_encoder_factory.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/api/video_codecs/video_encoder_factory.h b/api/video_codecs/video_encoder_factory.h index da9204bd38..bf19540914 100644 --- a/api/video_codecs/video_encoder_factory.h +++ b/api/video_codecs/video_encoder_factory.h @@ -85,13 +85,8 @@ class VideoEncoderFactory { virtual CodecSupport QueryCodecSupport( const SdpVideoFormat& format, absl::optional scalability_mode) const { - // Default implementation, query for supported formats and check if the - // specified format is supported. Returns false if scalability_mode is - // specified. CodecSupport codec_support; - if (scalability_mode) { - codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); - } + codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); return codec_support; } From 25372dfa219e05fed9b1edad91fe640ffbed3d7e Mon Sep 17 00:00:00 2001 From: davidliu Date: Wed, 13 Sep 2023 16:41:03 +0900 Subject: [PATCH 26/28] Expose audio sample buffers for Android (#89) * Initial draft * Working impl * doc and cleanup * doc update --- sdk/android/BUILD.gn | 4 ++ sdk/android/api/org/webrtc/AudioTrack.java | 48 +++++++++++++++++++ .../api/org/webrtc/AudioTrackSink.java | 27 +++++++++++ .../webrtc/audio/JavaAudioDeviceModule.java | 16 ++++++- .../org/webrtc/audio/WebRtcAudioTrack.java | 21 ++++++-- sdk/android/src/jni/pc/audio_sink.cc | 39 +++++++++++++++ sdk/android/src/jni/pc/audio_sink.h | 41 ++++++++++++++++ sdk/android/src/jni/pc/audio_track.cc | 26 ++++++++++ 8 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 sdk/android/api/org/webrtc/AudioTrackSink.java create mode 100644 sdk/android/src/jni/pc/audio_sink.cc create mode 100644 sdk/android/src/jni/pc/audio_sink.h diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 596dbc9693..494b43f6d3 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -255,6 +255,7 @@ if (is_android) { "api/org/webrtc/AudioProcessingFactory.java", "api/org/webrtc/AudioSource.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", @@ -716,6 +717,8 @@ if (current_os == "linux" || is_android) { "src/jni/pc/add_ice_candidate_observer.cc", "src/jni/pc/add_ice_candidate_observer.h", "src/jni/pc/android_network_monitor.h", + "src/jni/pc/audio_sink.cc", + "src/jni/pc/audio_sink.h", "src/jni/pc/audio_track.cc", "src/jni/pc/call_session_file_rotating_log_sink.cc", "src/jni/pc/crypto_options.cc", @@ -1405,6 +1408,7 @@ if (current_os == "linux" || is_android) { sources = [ "api/org/webrtc/AddIceObserver.java", "api/org/webrtc/AudioTrack.java", + "api/org/webrtc/AudioTrackSink.java", "api/org/webrtc/CallSessionFileRotatingLogSink.java", "api/org/webrtc/CandidatePairChangeEvent.java", "api/org/webrtc/CryptoOptions.java", diff --git a/sdk/android/api/org/webrtc/AudioTrack.java b/sdk/android/api/org/webrtc/AudioTrack.java index ca745db634..b30e46cebc 100644 --- a/sdk/android/api/org/webrtc/AudioTrack.java +++ b/sdk/android/api/org/webrtc/AudioTrack.java @@ -10,8 +10,12 @@ package org.webrtc; +import java.util.IdentityHashMap; + /** Java wrapper for a C++ AudioTrackInterface */ public class AudioTrack extends MediaStreamTrack { + private final IdentityHashMap sinks = new IdentityHashMap(); + public AudioTrack(long nativeTrack) { super(nativeTrack); } @@ -23,10 +27,54 @@ public void setVolume(double volume) { nativeSetVolume(getNativeAudioTrack(), volume); } + /** + * Adds an AudioTrackSink to the track. This callback is only + * called for remote audio tracks. + * + * Repeated addSink calls will not add the sink multiple times. + */ + public void addSink(AudioTrackSink sink) { + if (sink == null) { + throw new IllegalArgumentException("The AudioTrackSink is not allowed to be null"); + } + if (!sinks.containsKey(sink)) { + final long nativeSink = nativeWrapSink(sink); + sinks.put(sink, nativeSink); + nativeAddSink(getNativeMediaStreamTrack(), nativeSink); + } + } + + /** + * Removes an AudioTrackSink from the track. + * + * If the AudioTrackSink was not attached to the track, this is a no-op. + */ + public void removeSink(AudioTrackSink sink) { + final Long nativeSink = sinks.remove(sink); + if (nativeSink != null) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + } + + @Override + public void dispose() { + for (long nativeSink : sinks.values()) { + nativeRemoveSink(getNativeMediaStreamTrack(), nativeSink); + nativeFreeSink(nativeSink); + } + sinks.clear(); + super.dispose(); + } + /** Returns a pointer to webrtc::AudioTrackInterface. */ long getNativeAudioTrack() { return getNativeMediaStreamTrack(); } private static native void nativeSetVolume(long track, double volume); + private static native void nativeAddSink(long track, long nativeSink); + private static native void nativeRemoveSink(long track, long nativeSink); + private static native long nativeWrapSink(AudioTrackSink sink); + private static native void nativeFreeSink(long sink); } diff --git a/sdk/android/api/org/webrtc/AudioTrackSink.java b/sdk/android/api/org/webrtc/AudioTrackSink.java new file mode 100644 index 0000000000..eca390f82c --- /dev/null +++ b/sdk/android/api/org/webrtc/AudioTrackSink.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import java.nio.ByteBuffer; + +/** + * Java version of rtc::AudioTrackSinkInterface. + */ +public interface AudioTrackSink { + /** + * Implementations should copy the audio data into a local copy if they wish + * to use the data after this function returns. + */ + @CalledByNative + void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate, + int numberOfChannels, int numberOfFrames, + long absoluteCaptureTimestampMs); +} diff --git a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java index d3d57602a8..fc3dd6836c 100644 --- a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -42,6 +42,7 @@ public static class Builder { private AudioTrackErrorCallback audioTrackErrorCallback; private AudioRecordErrorCallback audioRecordErrorCallback; private SamplesReadyCallback samplesReadyCallback; + private PlaybackSamplesReadyCallback playbackSamplesReadyCallback; private AudioTrackStateCallback audioTrackStateCallback; private AudioRecordStateCallback audioRecordStateCallback; private boolean useHardwareAcousticEchoCanceler = isBuiltInAcousticEchoCancelerSupported(); @@ -140,6 +141,14 @@ public Builder setSamplesReadyCallback(SamplesReadyCallback samplesReadyCallback return this; } + /** + * Set a callback to listen to the audio output passed to the AudioTrack. + */ + public Builder setPlaybackSamplesReadyCallback(PlaybackSamplesReadyCallback playbackSamplesReadyCallback) { + this.playbackSamplesReadyCallback = playbackSamplesReadyCallback; + return this; + } + /** * Set a callback to retrieve information from the AudioTrack on when audio starts and stop. */ @@ -258,7 +267,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() { samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); final WebRtcAudioTrack audioOutput = new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, - audioTrackStateCallback, useLowLatency, enableVolumeLogger); + audioTrackStateCallback, playbackSamplesReadyCallback, useLowLatency, enableVolumeLogger); return new JavaAudioDeviceModule(context, audioManager, audioInput, audioOutput, inputSampleRate, outputSampleRate, useStereoInput, useStereoOutput); } @@ -325,6 +334,11 @@ public static interface SamplesReadyCallback { void onWebRtcAudioRecordSamplesReady(AudioSamples samples); } + /** Called when new audio samples are ready. This should only be set for debug purposes */ + public static interface PlaybackSamplesReadyCallback { + void onWebRtcAudioTrackSamplesReady(AudioSamples samples); + } + /* AudioTrack */ // Audio playout/track error handler functions. public enum AudioTrackStartErrorCode { diff --git a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java index 2b34e34013..2f745cc62e 100644 --- a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java +++ b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -20,12 +20,14 @@ import android.os.Process; import androidx.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Arrays; import org.webrtc.CalledByNative; import org.webrtc.Logging; import org.webrtc.ThreadUtils; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback; +import org.webrtc.audio.JavaAudioDeviceModule.PlaybackSamplesReadyCallback; import org.webrtc.audio.LowLatencyAudioBufferManager; class WebRtcAudioTrack { @@ -76,6 +78,7 @@ class WebRtcAudioTrack { private final @Nullable AudioTrackErrorCallback errorCallback; private final @Nullable AudioTrackStateCallback stateCallback; + private final @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback; /** * Audio thread which keeps calling AudioTrack.write() to stream audio. @@ -129,6 +132,17 @@ public void run() { reportWebRtcAudioTrackError("AudioTrack.write failed: " + bytesWritten); } } + + if (audioSamplesReadyCallback != null && keepAlive) { + // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily + // at index 0. + byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), + sizeInBytes + byteBuffer.arrayOffset()); + audioSamplesReadyCallback.onWebRtcAudioTrackSamplesReady( + new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), + audioTrack.getChannelCount(), audioTrack.getSampleRate(), data)); + } + if (useLowLatency) { bufferManager.maybeAdjustBufferSize(audioTrack); } @@ -154,13 +168,13 @@ public void stopThread() { @CalledByNative WebRtcAudioTrack(Context context, AudioManager audioManager) { this(context, audioManager, null /* audioAttributes */, null /* errorCallback */, - null /* stateCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); + null /* stateCallback */, null /* audioSamplesReadyCallback */, false /* useLowLatency */, true /* enableVolumeLogger */); } WebRtcAudioTrack(Context context, AudioManager audioManager, @Nullable AudioAttributes audioAttributes, @Nullable AudioTrackErrorCallback errorCallback, - @Nullable AudioTrackStateCallback stateCallback, boolean useLowLatency, - boolean enableVolumeLogger) { + @Nullable AudioTrackStateCallback stateCallback, @Nullable PlaybackSamplesReadyCallback audioSamplesReadyCallback, + boolean useLowLatency, boolean enableVolumeLogger) { threadChecker.detachThread(); this.context = context; this.audioManager = audioManager; @@ -168,6 +182,7 @@ public void stopThread() { this.errorCallback = errorCallback; this.stateCallback = stateCallback; this.volumeLogger = enableVolumeLogger ? new VolumeLogger(audioManager) : null; + this.audioSamplesReadyCallback = audioSamplesReadyCallback; this.useLowLatency = useLowLatency; Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); } diff --git a/sdk/android/src/jni/pc/audio_sink.cc b/sdk/android/src/jni/pc/audio_sink.cc new file mode 100644 index 0000000000..5bd88c75f6 --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "sdk/android/src/jni/pc/audio_sink.h" + +#include "sdk/android/generated_peerconnection_jni/AudioTrackSink_jni.h" + +namespace webrtc { +namespace jni { + +AudioTrackSinkWrapper::AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink) + : j_sink_(jni, j_sink) {} + +AudioTrackSinkWrapper::~AudioTrackSinkWrapper() {} + +void AudioTrackSinkWrapper::OnData( + const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + int length = (bits_per_sample / 8) * number_of_channels * number_of_frames; + ScopedJavaLocalRef audio_buffer = + NewDirectByteBuffer(jni, (void *) audio_data, length); + Java_AudioTrackSink_onData(jni, j_sink_, + audio_buffer, bits_per_sample, sample_rate, (int) number_of_channels, (int) number_of_frames, (absolute_capture_timestamp_ms ? absolute_capture_timestamp_ms.value() : 0)); +} + +} // namespace jni +} // namespace webrtc diff --git a/sdk/android/src/jni/pc/audio_sink.h b/sdk/android/src/jni/pc/audio_sink.h new file mode 100644 index 0000000000..809f460e0c --- /dev/null +++ b/sdk/android/src/jni/pc/audio_sink.h @@ -0,0 +1,41 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ +#define SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ + +#include + +#include "api/media_stream_interface.h" +#include "sdk/android/src/jni/jni_helpers.h" + +namespace webrtc { +namespace jni { + +class AudioTrackSinkWrapper : public webrtc::AudioTrackSinkInterface { + public: + AudioTrackSinkWrapper(JNIEnv* jni, const JavaRef& j_sink); + ~AudioTrackSinkWrapper() override; + + private: + void OnData(const void* audio_data, + int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + absl::optional absolute_capture_timestamp_ms) override; + + const ScopedJavaGlobalRef j_sink_; +}; + +} // namespace jni +} // namespace webrtc + +#endif // SDK_ANDROID_SRC_JNI_AUDIO_TRACK_SINK_H_ diff --git a/sdk/android/src/jni/pc/audio_track.cc b/sdk/android/src/jni/pc/audio_track.cc index 36ed43f1d4..df2d605893 100644 --- a/sdk/android/src/jni/pc/audio_track.cc +++ b/sdk/android/src/jni/pc/audio_track.cc @@ -9,6 +9,8 @@ */ #include "api/media_stream_interface.h" +#include "sdk/android/src/jni/pc/audio_sink.h" + #include "sdk/android/generated_peerconnection_jni/AudioTrack_jni.h" namespace webrtc { @@ -20,5 +22,29 @@ static void JNI_AudioTrack_SetVolume(JNIEnv*, jlong j_p, jdouble volume) { source->SetVolume(volume); } +static void JNI_AudioTrack_AddSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->AddSink(reinterpret_cast(j_native_sink)); +} + +static void JNI_AudioTrack_RemoveSink(JNIEnv* jni, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) + ->RemoveSink(reinterpret_cast(j_native_sink)); +} + +static jlong JNI_AudioTrack_WrapSink(JNIEnv* jni, + const JavaParamRef& sink) { + return jlongFromPointer(new AudioTrackSinkWrapper(jni, sink)); +} + +static void JNI_AudioTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { + delete reinterpret_cast(j_native_sink); +} + + } // namespace jni } // namespace webrtc From aba25fb287d2b070ee47c87fe26b2811da8dfc41 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Wed, 13 Sep 2023 23:00:34 +0800 Subject: [PATCH 27/28] add SetSifTrailer. --- api/crypto/frame_crypto_transformer.h | 7 +++++++ sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java | 7 +++++++ sdk/android/src/jni/pc/frame_cryptor_key_provider.cc | 9 +++++++++ sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h | 2 ++ .../api/peerconnection/RTCFrameCryptorKeyProvider.mm | 6 ++++++ 5 files changed, 31 insertions(+) diff --git a/api/crypto/frame_crypto_transformer.h b/api/crypto/frame_crypto_transformer.h index 96143b2637..c3042b2021 100644 --- a/api/crypto/frame_crypto_transformer.h +++ b/api/crypto/frame_crypto_transformer.h @@ -76,6 +76,8 @@ class KeyProvider : public rtc::RefCountInterface { virtual const std::vector ExportKey(const std::string participant_id, int key_index) const = 0; + virtual void SetSifTrailer(const std::vector trailer) = 0; + virtual KeyProviderOptions& options() = 0; protected: @@ -312,6 +314,11 @@ class DefaultKeyProviderImpl : public KeyProvider { return std::vector(); } + void SetSifTrailer(const std::vector trailer) override { + webrtc::MutexLock lock(&mutex_); + options_.uncrypted_magic_bytes = trailer; + } + KeyProviderOptions& options() override { return options_; } private: diff --git a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java index af2e3ff651..6ab0cdddf5 100644 --- a/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java +++ b/sdk/android/api/org/webrtc/FrameCryptorKeyProvider.java @@ -60,6 +60,11 @@ public byte[] exportKey(String participantId, int index) { return nativeExportKey(nativeKeyProvider, participantId, index); } + public void setSifTrailer(byte[] sifTrailer) { + checkKeyProviderExists(); + nativeSetSifTrailer(nativeKeyProvider, sifTrailer); + } + public void dispose() { checkKeyProviderExists(); JniCommon.nativeReleaseRef(nativeKeyProvider); @@ -83,4 +88,6 @@ private static native byte[] nativeRatchetKey( long keyProviderPointer, String participantId, int index); private static native byte[] nativeExportKey( long keyProviderPointer, String participantId, int index); + private static native void nativeSetSifTrailer( + long keyProviderPointer, byte[] sifTrailer); } \ No newline at end of file diff --git a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc index 2665bdc9ab..e41d16ed91 100644 --- a/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc +++ b/sdk/android/src/jni/pc/frame_cryptor_key_provider.cc @@ -110,5 +110,14 @@ JNI_FrameCryptorKeyProvider_ExportKey( return NativeToJavaByteArray(env, rtc::ArrayView(int8tKey)); } +static void JNI_FrameCryptorKeyProvider_SetSifTrailer( + JNIEnv* jni, + jlong j_key_provider, + const base::android::JavaParamRef& j_trailer) { + auto trailer = JavaToNativeByteArray(jni, j_trailer); + reinterpret_cast(j_key_provider) + ->SetSifTrailer(std::vector(trailer.begin(), trailer.end())); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h index 1b207e59a8..b10cf8c4b2 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.h @@ -35,6 +35,8 @@ RTC_OBJC_EXPORT - (NSData *)exportKey:(NSString *)participantId withIndex:(int)index; +- (void)setSifTrailer:(NSData *)trailer; + - (instancetype)initWithRatchetSalt:(NSData *)salt ratchetWindowSize:(int)windowSize sharedKeyMode:(BOOL)sharedKey diff --git a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm index 8f961b002d..1596b98016 100644 --- a/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm +++ b/sdk/objc/api/peerconnection/RTCFrameCryptorKeyProvider.mm @@ -95,4 +95,10 @@ - (NSData *)exportKey:(NSString *)participantId withIndex:(int)index { return [NSData dataWithBytes:nativeKey.data() length:nativeKey.size()]; } +- (void)setSifTrailer:(NSData *)trailer { + _nativeKeyProvider->SetSifTrailer( + std::vector((const uint8_t *)trailer.bytes, + ((const uint8_t *)trailer.bytes) + trailer.length)); +} + @end From 78c7b1f6ece029e9ae1334bc1e0f49c72fd026e5 Mon Sep 17 00:00:00 2001 From: cloudwebrtc Date: Thu, 14 Sep 2023 10:19:54 +0800 Subject: [PATCH 28/28] fix h264 freeze. --- api/crypto/frame_crypto_transformer.cc | 93 ++++++++++++++++++++------ 1 file changed, 72 insertions(+), 21 deletions(-) diff --git a/api/crypto/frame_crypto_transformer.cc b/api/crypto/frame_crypto_transformer.cc index 9ef356ce0f..c957220757 100644 --- a/api/crypto/frame_crypto_transformer.cc +++ b/api/crypto/frame_crypto_transformer.cc @@ -82,6 +82,28 @@ const EVP_CIPHER* GetAesCbcAlgorithmFromKeySize(size_t key_size_bytes) { } } +inline bool FrameIsH264(webrtc::TransformableFrameInterface* frame, + webrtc::FrameCryptorTransformer::MediaType type) { + switch (type) { + case webrtc::FrameCryptorTransformer::MediaType::kVideoFrame: { + auto videoFrame = + static_cast(frame); + return videoFrame->header().codec == + webrtc::VideoCodecType::kVideoCodecH264; + } + default: + return false; + } +} + +inline bool NeedsRbspUnescaping(const uint8_t* frameData, size_t frameSize) { + for (size_t i = 0; i < frameSize - 3; ++i) { + if (frameData[i] == 0 && frameData[i + 1] == 0 && frameData[i + 2] == 3) + return true; + } + return false; +} + std::string to_uint8_list(const uint8_t* data, int len) { std::stringstream ss; ss << "["; @@ -395,26 +417,41 @@ void FrameCryptorTransformer::encryptFrame( key_set->encryption_key, iv, frame_header, payload, &buffer) == Success) { rtc::Buffer encrypted_payload(buffer.data(), buffer.size()); + rtc::Buffer tag(encrypted_payload.data() + encrypted_payload.size() - 16, + 16); + rtc::Buffer data_without_header; + data_without_header.AppendData(encrypted_payload); + data_without_header.AppendData(iv); + data_without_header.AppendData(frame_trailer); + rtc::Buffer data_out; data_out.AppendData(frame_header); - data_out.AppendData(encrypted_payload); - data_out.AppendData(iv); - data_out.AppendData(frame_trailer); - RTC_CHECK_EQ(data_out.size(), frame_header.size() + - encrypted_payload.size() + iv.size() + - frame_trailer.size()); + if (FrameIsH264(frame.get(), type_)) { + H264::WriteRbsp(data_without_header.data(),data_without_header.size(), &data_out); + } else { + data_out.AppendData(data_without_header); + RTC_CHECK_EQ(data_out.size(), frame_header.size() + + encrypted_payload.size() + iv.size() + + frame_trailer.size()); + } frame->SetData(data_out); - RTC_LOG(LS_INFO) << "FrameCryptorTransformer::encryptFrame() ivLength=" - << static_cast(iv.size()) << " unencrypted_bytes=" + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::encryptFrame() " + << "frame length = " << static_cast(date_in.size()) + << " encrypted_length = " + << static_cast(data_out.size()) + << " ivLength=" << static_cast(iv.size()) + << " unencrypted_bytes=" << static_cast(unencrypted_bytes) + << " tag=" << to_hex(tag.data(), tag.size()) << " key_index=" << static_cast(key_index_) << " aesKey=" << to_hex(key_set->encryption_key.data(), key_set->encryption_key.size()) << " iv=" << to_hex(iv.data(), iv.size()); + if (last_enc_error_ != FrameCryptionState::kOk) { last_enc_error_ = FrameCryptionState::kOk; if (observer_) @@ -554,11 +591,34 @@ void FrameCryptorTransformer::decryptFrame( iv[i] = date_in[date_in.size() - 2 - ivLength + i]; } - rtc::Buffer encrypted_payload(date_in.size() - unencrypted_bytes - ivLength - - 2); - for (size_t i = unencrypted_bytes; i < date_in.size() - ivLength - 2; i++) { - encrypted_payload[i - unencrypted_bytes] = date_in[i]; + rtc::Buffer encrypted_buffer(date_in.size() - unencrypted_bytes); + for (size_t i = unencrypted_bytes; i < date_in.size(); i++) { + encrypted_buffer[i - unencrypted_bytes] = date_in[i]; + } + + if (FrameIsH264(frame.get(), type_) && + NeedsRbspUnescaping(encrypted_buffer.data(), encrypted_buffer.size())) { + encrypted_buffer.SetData(H264::ParseRbsp(encrypted_buffer.data(), encrypted_buffer.size())); + } + + rtc::Buffer encrypted_payload(encrypted_buffer.size() - ivLength - 2); + for (size_t i = 0; i < encrypted_payload.size(); i++) { + encrypted_payload[i] = encrypted_buffer[i]; } + + rtc::Buffer tag(encrypted_payload.data() + encrypted_payload.size() - 16, 16); + RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() " + << " frame length = " << static_cast(date_in.size()) + << " ivLength=" << static_cast(iv.size()) + << " unencrypted_bytes=" + << static_cast(unencrypted_bytes) + << " tag=" << to_hex(tag.data(), tag.size()) + << " key_index=" << static_cast(key_index_) + << " aesKey=" + << to_hex(key_set->encryption_key.data(), + key_set->encryption_key.size()) + << " iv=" << to_hex(iv.data(), iv.size()); + std::vector buffer; int ratchet_count = 0; @@ -636,15 +696,6 @@ void FrameCryptorTransformer::decryptFrame( data_out.AppendData(payload); frame->SetData(data_out); - RTC_LOG(LS_INFO) << "FrameCryptorTransformer::decryptFrame() ivLength=" - << static_cast(ivLength) << " unencrypted_bytes=" - << static_cast(unencrypted_bytes) - << " key_index=" << static_cast(key_index_) - << " aesKey=" - << to_hex(key_set->encryption_key.data(), - key_set->encryption_key.size()) - << " iv=" << to_hex(iv.data(), iv.size()); - if (last_dec_error_ != FrameCryptionState::kOk) { last_dec_error_ = FrameCryptionState::kOk; if (observer_)