diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index 6b0a6a9297..d8a79896f3 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -19,6 +19,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index 9d336739e4..6d9b4aa6ca 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 2282b57d7e..15da82e457 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -1001,6 +1001,8 @@ class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface { webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) = 0; virtual bool AddDefaultRecvStreamForTesting(const StreamParams& sp) = 0; + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; } // namespace cricket diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index e8bffc2d05..3e62e7cdbe 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2579,6 +2579,24 @@ WebRtcVideoReceiveChannel::~WebRtcVideoReceiveChannel() { delete kv.second; } +void WebRtcVideoReceiveChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoReceiveChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + void WebRtcVideoReceiveChannel::SetReceiverFeedbackParameters( bool lntf_enabled, bool nack_enabled, @@ -3478,6 +3496,17 @@ void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::SetReceiverParameters( } } +void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StartStream(){ + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream::StopStream(){ + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoReceiveChannel::WebRtcVideoReceiveStream:: RecreateReceiveStream() { RTC_DCHECK_RUN_ON(&thread_checker_); diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 11f1b99ac2..599b8063a6 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -294,7 +294,6 @@ class WebRtcVideoSendChannel : public MediaChannelUtil, } return send_codec()->rtx_time; } - private: struct ChangedSenderParameters { // These optionals are unset if not changed. @@ -643,6 +642,8 @@ class WebRtcVideoReceiveChannel : public MediaChannelUtil, webrtc::RtcpMode rtcp_mode, absl::optional rtx_time) override; + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; struct ChangedReceiverParameters { @@ -744,6 +745,9 @@ class WebRtcVideoReceiveChannel : public MediaChannelUtil, rtc::scoped_refptr frame_transformer); + void StartStream(); + void StopStream(); + void SetLocalSsrc(uint32_t local_ssrc); void UpdateRtxSsrc(uint32_t ssrc); void StartReceiveStream(); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index 2af3aedb22..fab23d17ec 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -55,6 +55,8 @@ PROXY_SECONDARY_METHOD2(void, PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) PROXY_SECONDARY_METHOD0(void, RequestRefreshFrame) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 4432982027..1ed13b9088 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -41,15 +41,20 @@ VideoRtpReceiver::VideoRtpReceiver( rtc::Thread::Current(), worker_thread, VideoTrack::Create(receiver_id, source_, worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_should_receive_(track_->should_receive()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kInitializing); } VideoRtpReceiver::~VideoRtpReceiver() { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); RTC_DCHECK(!media_channel_); + + track_->UnregisterObserver(this); } std::vector VideoRtpReceiver::stream_ids() const { @@ -114,6 +119,39 @@ void VideoRtpReceiver::Stop() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); + worker_thread_->PostTask( + [this, receive = cached_track_should_receive_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(receive) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + }); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(signaled_ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} + +void VideoRtpReceiver::StopMediaChannel() { + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(signaled_ssrc_.value_or(0)); +} + void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); MediaSourceInterface::SourceState state = source_->state(); @@ -209,6 +247,7 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index ef88016052..491efe2931 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -42,7 +42,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -60,6 +61,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -115,6 +119,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { cricket::MediaReceiveChannelInterface* media_channel); private: + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc) RTC_RUN_ON(&signaling_thread_checker_); void RestartMediaChannel_w(absl::optional ssrc, @@ -162,6 +168,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -177,6 +185,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index 0bf8687af3..8922cdaf1f 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -76,6 +76,19 @@ VideoTrackSourceInterface* VideoTrack::GetSourceInternal() const { return video_source_->internal(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; diff --git a/pc/video_track.h b/pc/video_track.h index 13a51c454b..b56c64ef20 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -48,6 +48,9 @@ class VideoTrack : public MediaStreamTrack, void RequestRefreshFrame() override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -81,6 +84,7 @@ class VideoTrack : public MediaStreamTrack, // be queried without blocking on the worker thread by callers that don't // use an api proxy to call the `enabled()` method. bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 4f5ceb5ed3..7c306645a6 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -722,6 +722,7 @@ if (is_ios || is_mac) { ] deps = [ + ":simulcast", ":base_objc", ":native_video", ":videocodec_objc", @@ -807,6 +808,22 @@ if (is_ios || is_mac) { ] } + rtc_library("simulcast") { + sources = [ + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h", + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm", + "objc/api/video_codec/RTCVideoEncoderSimulcast.h", + "objc/api/video_codec/RTCVideoEncoderSimulcast.mm", + ] + + deps = [ + ":base_objc", + ":wrapped_native_codec_objc", + "../media:rtc_media_base", + "../media:rtc_simulcast_encoder_adapter", + ] + } + rtc_library("libaom_av1_encoder") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] @@ -1334,6 +1351,9 @@ if (is_ios || is_mac) { "objc/api/video_codec/RTCVideoEncoderAV1.h", "objc/api/video_frame_buffer/RTCNativeI420Buffer.h", "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h", + # Added for Simulcast support + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h", + "objc/api/video_codec/RTCVideoEncoderSimulcast.h", ] if (!build_with_chromium) { @@ -1478,6 +1498,9 @@ if (is_ios || is_mac) { "objc/components/video_codec/RTCVideoEncoderH264.h", "objc/components/video_frame_buffer/RTCCVPixelBuffer.h", "objc/helpers/RTCDispatcher.h", + # Added for Simulcast support + "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h", + "objc/api/video_codec/RTCVideoEncoderSimulcast.h", ] if (!build_with_chromium) { sources += [ diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn index 5f494052e2..2e29916261 100644 --- a/sdk/android/BUILD.gn +++ b/sdk/android/BUILD.gn @@ -232,6 +232,7 @@ if (is_android) { "api/org/webrtc/WrappedNativeVideoEncoder.java", "api/org/webrtc/YuvConverter.java", "api/org/webrtc/YuvHelper.java", + "api/org/webrtc/ResolutionAdjustment.java", "src/java/org/webrtc/EglBase10Impl.java", "src/java/org/webrtc/EglBase14Impl.java", "src/java/org/webrtc/GlGenericDrawer.java", @@ -362,6 +363,9 @@ if (is_android) { sources = [ "api/org/webrtc/DefaultVideoDecoderFactory.java", "api/org/webrtc/DefaultVideoEncoderFactory.java", + "api/org/webrtc/WrappedVideoDecoderFactory.java", + "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java", + "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java", ] deps = [ @@ -393,6 +397,8 @@ if (is_android) { sources = [ "api/org/webrtc/HardwareVideoDecoderFactory.java", "api/org/webrtc/HardwareVideoEncoderFactory.java", + "api/org/webrtc/HardwareVideoEncoderWrapper.java", + "api/org/webrtc/HardwareVideoEncoderWrapperFactory.java", "api/org/webrtc/PlatformSoftwareVideoDecoderFactory.java", "src/java/org/webrtc/AndroidVideoDecoder.java", "src/java/org/webrtc/BaseBitrateAdjuster.java", @@ -428,6 +434,7 @@ if (is_android) { "src/java/org/webrtc/audio/WebRtcAudioRecord.java", "src/java/org/webrtc/audio/WebRtcAudioTrack.java", "src/java/org/webrtc/audio/WebRtcAudioUtils.java", + "src/java/org/webrtc/audio/AudioRecordDataCallback.java", ] deps = [ @@ -525,6 +532,8 @@ if (is_android) { sources = [ "api/org/webrtc/SoftwareVideoDecoderFactory.java", "api/org/webrtc/SoftwareVideoEncoderFactory.java", + "api/org/webrtc/SimulcastVideoEncoder.java", + "api/org/webrtc/SimulcastVideoEncoderFactory.java", ] deps = [ @@ -864,6 +873,23 @@ if (current_os == "linux" || is_android) { ] } + rtc_library("simulcast_jni") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "src/jni/simulcast_video_encoder.cc", + "src/jni/simulcast_video_encoder.h", + "src/jni/simulcast_video_encoder_factory.cc", + "src/jni/simulcast_video_encoder_factory.h" + ] + deps = [ + ":base_jni", + ":video_jni", + ":native_api_codecs", + "../../media:rtc_simulcast_encoder_adapter" + ] + } + rtc_library("libaom_av1_encoder_jni") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] @@ -900,6 +926,7 @@ if (current_os == "linux" || is_android) { ":generated_swcodecs_jni", ":libvpx_vp8_jni", ":libvpx_vp9_jni", + ":simulcast_jni", ":native_api_jni", ":video_jni", "../../api/video_codecs:builtin_video_decoder_factory", diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..65720ef6a0 --- /dev/null +++ b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.Arrays; +import java.util.LinkedHashSet; + +/** + * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes + * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco + * derFactoryWrapper.kt#L18 + */ +public class DefaultAlignedVideoEncoderFactory implements VideoEncoderFactory { + private final VideoEncoderFactory hardwareVideoEncoderFactory; + private final VideoEncoderFactory softwareVideoEncoderFactory; + + public DefaultAlignedVideoEncoderFactory( + EglBase.Context eglContext, + boolean enableIntelVp8Encoder, + boolean enableH264HighProfile, + ResolutionAdjustment resolutionAdjustment + ) { + HardwareVideoEncoderFactory defaultFactory = + new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile); + hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ? + defaultFactory : + new HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.getValue()); + softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory(); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info); + VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info); + if (hardwareEncoder != null && softwareEncoder != null) { + return new VideoEncoderFallback(softwareEncoder, hardwareEncoder); + } + return hardwareEncoder != null ? hardwareEncoder : softwareEncoder; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet<>(); + supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs())); + return supportedCodecInfos.toArray(new VideoCodecInfo[0]); + } +} + diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java new file mode 100644 index 0000000000..ac3b0c4684 --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapper implements VideoEncoder { + + private static final String TAG = "HardwareVideoEncoderWrapper"; + + private final VideoEncoder internalEncoder; + private final int alignment; + + public HardwareVideoEncoderWrapper(VideoEncoder internalEncoder, int alignment) { + this.internalEncoder = internalEncoder; + this.alignment = alignment; + } + + private static class CropSizeCalculator { + + private static final String TAG = "CropSizeCalculator"; + + private final int alignment; + private final int originalWidth; + private final int originalHeight; + private final int cropX; + private final int cropY; + + public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) { + this.alignment = alignment; + this.originalWidth = originalWidth; + this.originalHeight = originalHeight; + this.cropX = originalWidth % alignment; + this.cropY = originalHeight % alignment; + if (originalWidth != 0 && originalHeight != 0) { + Logging.v(TAG, "init(): alignment=" + alignment + + " size=" + originalWidth + "x" + originalHeight + " => " + getCroppedWidth() + "x" + getCroppedHeight()); + } + } + + public int getCroppedWidth() { + return originalWidth - cropX; + } + + public int getCroppedHeight() { + return originalHeight - cropY; + } + + public boolean isCropRequired() { + return cropX != 0 || cropY != 0; + } + + public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) { + if (originalWidth == nextWidth && originalHeight == nextHeight) { + return false; + } else { + Logging.v(TAG, "frame size has changed: " + + originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight); + return true; + } + } + } + + private CropSizeCalculator calculator = new CropSizeCalculator(1, 0, 0); + + private VideoCodecStatus retryWithoutCropping(int width, int height, Runnable retryFunc) { + Logging.v(TAG, "retrying without resolution adjustment"); + calculator = new CropSizeCalculator(1, width, height); + retryFunc.run(); + return VideoCodecStatus.OK; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings originalSettings, VideoEncoder.Callback callback) { + calculator = new CropSizeCalculator(alignment, originalSettings.width, originalSettings.height); + if (!calculator.isCropRequired()) { + return internalEncoder.initEncode(originalSettings, callback); + } else { + VideoEncoder.Settings croppedSettings = new VideoEncoder.Settings( + originalSettings.numberOfCores, + calculator.getCroppedWidth(), + calculator.getCroppedHeight(), + originalSettings.startBitrate, + originalSettings.maxFramerate, + originalSettings.numberOfSimulcastStreams, + originalSettings.automaticResizeOn, + originalSettings.capabilities + ); + try { + VideoCodecStatus result = internalEncoder.initEncode(croppedSettings, callback); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " + + "croppedSettings " + croppedSettings); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.initEncode() failed", e); + return retryWithoutCropping( + originalSettings.width, + originalSettings.height, + () -> internalEncoder.initEncode(originalSettings, callback) + ); + } + } + } + + @Override + public VideoCodecStatus release() { + return internalEncoder.release(); + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + if (calculator.hasFrameSizeChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight())) { + calculator = new CropSizeCalculator(alignment, frame.getBuffer().getWidth(), frame.getBuffer().getHeight()); + } + if (!calculator.isCropRequired()) { + return internalEncoder.encode(frame, encodeInfo); + } else { + int croppedWidth = calculator.getCroppedWidth(); + int croppedHeight = calculator.getCroppedHeight(); + VideoFrame.Buffer croppedBuffer = frame.getBuffer().cropAndScale( + calculator.cropX / 2, + calculator.cropY / 2, + croppedWidth, + croppedHeight, + croppedWidth, + croppedHeight + ); + VideoFrame croppedFrame = new VideoFrame(croppedBuffer, frame.getRotation(), frame.getTimestampNs()); + try { + VideoCodecStatus result = internalEncoder.encode(croppedFrame, encodeInfo); + if (result == VideoCodecStatus.FALLBACK_SOFTWARE) { + Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE"); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } else { + return result; + } + } catch (Exception e) { + Logging.e(TAG, "internalEncoder.encode() failed", e); + return retryWithoutCropping( + frame.getBuffer().getWidth(), + frame.getBuffer().getHeight(), + () -> internalEncoder.encode(frame, encodeInfo) + ); + } finally { + croppedBuffer.release(); + } + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + return internalEncoder.setRateAllocation(allocation, frameRate); + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + return internalEncoder.getScalingSettings(); + } + + @Override + public String getImplementationName() { + return internalEncoder.getImplementationName(); + } + + @Override + public long createNativeVideoEncoder() { + return internalEncoder.createNativeVideoEncoder(); + } + + @Override + public boolean isHardwareEncoder() { + return internalEncoder.isHardwareEncoder(); + } + + @Override + public VideoCodecStatus setRates(VideoEncoder.RateControlParameters rcParameters) { + return internalEncoder.setRates(rcParameters); + } + + @Override + public VideoEncoder.ResolutionBitrateLimits[] getResolutionBitrateLimits() { + return internalEncoder.getResolutionBitrateLimits(); + } + + @Override + public VideoEncoder.EncoderInfo getEncoderInfo() { + return internalEncoder.getEncoderInfo(); + } +} + + diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java new file mode 100644 index 0000000000..2b5b1c8f7f --- /dev/null +++ b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapperFactory.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207 + * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco + * derWrapperFactory.kt + */ +class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory { + + private static final String TAG = "HardwareVideoEncoderWrapperFactory"; + + private final HardwareVideoEncoderFactory factory; + private final int resolutionPixelAlignment; + + public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) { + this.factory = factory; + this.resolutionPixelAlignment = resolutionPixelAlignment; + if (resolutionPixelAlignment == 0) { + throw new IllegalArgumentException("resolutionPixelAlignment should not be 0"); + } + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + try { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment); + } catch (Exception e) { + Logging.e(TAG, "createEncoder failed", e); + return null; + } + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } +} \ No newline at end of file diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.java b/sdk/android/api/org/webrtc/ResolutionAdjustment.java new file mode 100644 index 0000000000..0ed741db74 --- /dev/null +++ b/sdk/android/api/org/webrtc/ResolutionAdjustment.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +/** + * Resolution alignment values. Generally the MULTIPLE_OF_16 is recommended + * for both VP8 and H264 + */ +public enum ResolutionAdjustment { + NONE(1), + MULTIPLE_OF_2(2), + MULTIPLE_OF_4(4), + MULTIPLE_OF_8(8), + MULTIPLE_OF_16(16); + + private final int value; + + private ResolutionAdjustment(int value) { + this.value = value; + } + + public int getValue() { + return value; + } +} + diff --git a/sdk/android/api/org/webrtc/RtpParameters.java b/sdk/android/api/org/webrtc/RtpParameters.java index 9ca8311610..01f8082c57 100644 --- a/sdk/android/api/org/webrtc/RtpParameters.java +++ b/sdk/android/api/org/webrtc/RtpParameters.java @@ -76,6 +76,7 @@ public static class Encoding { // If non-null, scale the width and height down by this factor for video. If null, // implementation default scaling factor will be used. @Nullable public Double scaleResolutionDownBy; + @Nullable public String scalabilityMode; // SSRC to be used by this encoding. // Can't be changed between getParameters/setParameters. public Long ssrc; @@ -93,7 +94,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { @CalledByNative("Encoding") Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority, Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate, - Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc, + Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode, Long ssrc, boolean adaptiveAudioPacketTime) { this.rid = rid; this.active = active; @@ -104,6 +105,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) { this.maxFramerate = maxFramerate; this.numTemporalLayers = numTemporalLayers; this.scaleResolutionDownBy = scaleResolutionDownBy; + this.scalabilityMode = scalabilityMode; this.ssrc = ssrc; this.adaptiveAudioPacketTime = adaptiveAudioPacketTime; } @@ -160,6 +162,12 @@ Double getScaleResolutionDownBy() { return scaleResolutionDownBy; } + @Nullable + @CalledByNative("Encoding") + String getScalabilityMode() { + return scalabilityMode; + } + @CalledByNative("Encoding") Long getSsrc() { return ssrc; diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java new file mode 100644 index 0000000000..ab9faaa0c4 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.webrtc; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * The main difference with the standard SimulcastVideoEncoderFactory is that this fixes issues + * with simulcasting resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can + * set the alignment by setting resolutionAdjustment. Internally the resolutions during simulcast + * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the + * standard SimulcastVideoEncoderFactory and it will use the SW encoder if HW fails + * or is not available. + * + * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072 + * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnc + * oderFactoryWrapper.kt#L18 + */ +public class SimulcastAlignedVideoEncoderFactory implements VideoEncoderFactory { + private static class StreamEncoderWrapper implements VideoEncoder { + private static final String TAG = "StreamEncoderWrapper"; + private final VideoEncoder encoder; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private VideoEncoder.Settings streamSettings; + + public StreamEncoderWrapper(VideoEncoder encoder) { + this.encoder = encoder; + } + + @Override + public VideoCodecStatus initEncode(VideoEncoder.Settings settings, VideoEncoder.Callback callback) { + streamSettings = settings; + Callable callable = () -> { + Logging.v(TAG, "initEncode() thread=" + Thread.currentThread().getName() + " [" + Thread.currentThread().getId() + "]"); + Logging.v(TAG, " encoder=" + encoder.getImplementationName()); + Logging.v(TAG, " streamSettings:"); + Logging.v(TAG, " numberOfCores=" + settings.numberOfCores); + Logging.v(TAG, " width=" + settings.width); + Logging.v(TAG, " height=" + settings.height); + Logging.v(TAG, " startBitrate=" + settings.startBitrate); + Logging.v(TAG, " maxFramerate=" + settings.maxFramerate); + Logging.v(TAG, " automaticResizeOn=" + settings.automaticResizeOn); + Logging.v(TAG, " numberOfSimulcastStreams=" + settings.numberOfSimulcastStreams); + Logging.v(TAG, " lossNotification=" + settings.capabilities.lossNotification); + return encoder.initEncode(settings, callback); + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus release() { + Callable callable = () -> encoder.release(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) { + Callable callable = () -> { + if (streamSettings != null) { + if (frame.getBuffer().getWidth() == streamSettings.width) { + return encoder.encode(frame, encodeInfo); + } else { + int originalWidth = frame.getBuffer().getWidth(); + int originalHeight = frame.getBuffer().getHeight(); + VideoFrame.Buffer scaledBuffer = frame.getBuffer().cropAndScale( + 0, 0, originalWidth, originalHeight, + streamSettings.width, streamSettings.height + ); + VideoFrame scaledFrame = new VideoFrame(scaledBuffer, frame.getRotation(), frame.getTimestampNs()); + VideoCodecStatus result = encoder.encode(scaledFrame, encodeInfo); + scaledBuffer.release(); + return result; + } + } else { + return VideoCodecStatus.ERROR; + } + }; + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) { + Callable callable = () -> encoder.setRateAllocation(allocation, frameRate); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return VideoCodecStatus.ERROR; + } + } + + @Override + public VideoEncoder.ScalingSettings getScalingSettings() { + Callable callable = () -> encoder.getScalingSettings(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + + @Override + public String getImplementationName() { + Callable callable = () -> encoder.getImplementationName(); + try { + return executor.submit(callable).get(); + } catch (Exception e) { + return null; + } + } + } + + private static class StreamEncoderWrapperFactory implements VideoEncoderFactory { + private final VideoEncoderFactory factory; + + public StreamEncoderWrapperFactory(VideoEncoderFactory factory) { + this.factory = factory; + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) { + VideoEncoder encoder = factory.createEncoder(videoCodecInfo); + if (encoder == null) { + return null; + } + return new StreamEncoderWrapper(encoder); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return factory.getSupportedCodecs(); + } + } + + private final VideoEncoderFactory primary; + private final VideoEncoderFactory fallback; + private final SimulcastVideoEncoderFactory delegate; + + public SimulcastAlignedVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile, ResolutionAdjustment resolutionAdjustment) { + HardwareVideoEncoderFactory hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory(sharedContext, enableIntelVp8Encoder, enableH264HighProfile); + VideoEncoderFactory encoderFactory; + if (resolutionAdjustment == ResolutionAdjustment.NONE) { + encoderFactory = hardwareVideoEncoderFactory; + } else { + encoderFactory = new HardwareVideoEncoderWrapperFactory(hardwareVideoEncoderFactory, resolutionAdjustment.getValue()); + } + primary = new StreamEncoderWrapperFactory(encoderFactory); + fallback = new SoftwareVideoEncoderFactory(); + delegate = new SimulcastVideoEncoderFactory(primary, fallback); + } + + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return delegate.createEncoder(info); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + return delegate.getSupportedCodecs(); + } +} + + diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java new file mode 100644 index 0000000000..da39a1b6a7 --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastVideoEncoder.java @@ -0,0 +1,27 @@ +package org.webrtc; + +public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder { + + static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info); + + VideoEncoderFactory primary; + VideoEncoderFactory fallback; + VideoCodecInfo info; + + public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) { + this.primary = primary; + this.fallback = fallback; + this.info = info; + } + + @Override + public long createNativeVideoEncoder() { + return nativeCreateEncoder(primary, fallback, info); + } + + @Override + public boolean isHardwareEncoder() { + return false; + } + +} diff --git a/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java new file mode 100644 index 0000000000..de1d8086fe --- /dev/null +++ b/sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +import androidx.annotation.Nullable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Arrays; + +public class SimulcastVideoEncoderFactory implements VideoEncoderFactory { + + static native List nativeVP9Codecs(); + static native VideoCodecInfo nativeAV1Codec(); + + VideoEncoderFactory primary; + VideoEncoderFactory fallback; + + public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) { + this.primary = primary; + this.fallback = fallback; + } + + @Nullable + @Override + public VideoEncoder createEncoder(VideoCodecInfo info) { + return new SimulcastVideoEncoder(primary, fallback, info); + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + List codecs = new ArrayList(); + codecs.addAll(Arrays.asList(primary.getSupportedCodecs())); + if (fallback != null) { + codecs.addAll(Arrays.asList(fallback.getSupportedCodecs())); + } + codecs.addAll(nativeVP9Codecs()); + codecs.add(nativeAV1Codec()); + return codecs.toArray(new VideoCodecInfo[codecs.size()]); + } + +} diff --git a/sdk/android/api/org/webrtc/VideoCodecInfo.java b/sdk/android/api/org/webrtc/VideoCodecInfo.java index 4f97cf74cf..86d67d6d5c 100644 --- a/sdk/android/api/org/webrtc/VideoCodecInfo.java +++ b/sdk/android/api/org/webrtc/VideoCodecInfo.java @@ -34,6 +34,7 @@ public class VideoCodecInfo { public final String name; public final Map params; + public int[] scalabilityModes; @Deprecated public final int payload; @CalledByNative @@ -41,6 +42,7 @@ public VideoCodecInfo(String name, Map params) { this.payload = 0; this.name = name; this.params = params; + this.scalabilityModes = new int[0]; } @Deprecated @@ -48,6 +50,7 @@ public VideoCodecInfo(int payload, String name, Map params) { this.payload = payload; this.name = name; this.params = params; + this.scalabilityModes = new int[0]; } @Override @@ -83,4 +86,16 @@ String getName() { Map getParams() { return params; } + + @CalledByNative + int[] getScalabilityModes() { + return scalabilityModes; + } + + @CalledByNative + void setScalabilityModes(int[] values) { + scalabilityModes = values; + } + + } diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 512e46c26e..1791592b56 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts/stops receiving the video stream. + * + * If this is a local video track, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * The current receive status for a remote video track. + * + * This has no meaning for a local video track. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ public long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java b/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java new file mode 100644 index 0000000000..a7acd37289 --- /dev/null +++ b/sdk/android/api/org/webrtc/WrappedVideoDecoderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.webrtc; + +import android.media.MediaCodecInfo; +import androidx.annotation.Nullable; + +import java.util.Arrays; +import java.util.LinkedHashSet; + +public class WrappedVideoDecoderFactory implements VideoDecoderFactory { + public WrappedVideoDecoderFactory(@Nullable EglBase.Context eglContext) { + this.hardwareVideoDecoderFactory = new HardwareVideoDecoderFactory(eglContext); + this.platformSoftwareVideoDecoderFactory = new PlatformSoftwareVideoDecoderFactory(eglContext); + } + + private final VideoDecoderFactory hardwareVideoDecoderFactory; + private final VideoDecoderFactory hardwareVideoDecoderFactoryWithoutEglContext = new HardwareVideoDecoderFactory(null) ; + private final VideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory(); + @Nullable + private final VideoDecoderFactory platformSoftwareVideoDecoderFactory; + + @Override + public VideoDecoder createDecoder(VideoCodecInfo codecType) { + VideoDecoder softwareDecoder = this.softwareVideoDecoderFactory.createDecoder(codecType); + VideoDecoder hardwareDecoder = this.hardwareVideoDecoderFactory.createDecoder(codecType); + if (softwareDecoder == null && this.platformSoftwareVideoDecoderFactory != null) { + softwareDecoder = this.platformSoftwareVideoDecoderFactory.createDecoder(codecType); + } + + if(hardwareDecoder != null && disableSurfaceTextureFrame(hardwareDecoder.getImplementationName())) { + hardwareDecoder.release(); + hardwareDecoder = this.hardwareVideoDecoderFactoryWithoutEglContext.createDecoder(codecType); + } + + if (hardwareDecoder != null && softwareDecoder != null) { + return new VideoDecoderFallback(softwareDecoder, hardwareDecoder); + } else { + return hardwareDecoder != null ? hardwareDecoder : softwareDecoder; + } + } + + private boolean disableSurfaceTextureFrame(String name) { + if (name.startsWith("OMX.qcom.") || name.startsWith("OMX.hisi.")) { + return true; + } + return false; + } + + @Override + public VideoCodecInfo[] getSupportedCodecs() { + LinkedHashSet supportedCodecInfos = new LinkedHashSet(); + supportedCodecInfos.addAll(Arrays.asList(this.softwareVideoDecoderFactory.getSupportedCodecs())); + supportedCodecInfos.addAll(Arrays.asList(this.hardwareVideoDecoderFactory.getSupportedCodecs())); + if (this.platformSoftwareVideoDecoderFactory != null) { + supportedCodecInfos.addAll(Arrays.asList(this.platformSoftwareVideoDecoderFactory.getSupportedCodecs())); + } + + return (VideoCodecInfo[])supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]); + } +} diff --git a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java index b118843ea0..8515f48e3a 100644 --- a/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/sdk/android/api/org/webrtc/audio/JavaAudioDeviceModule.java @@ -51,6 +51,7 @@ public static class Builder { private AudioAttributes audioAttributes; private boolean useLowLatency; private boolean enableVolumeLogger; + private AudioRecordDataCallback audioRecordDataCallback; private Builder(Context context) { this.context = context; @@ -221,6 +222,16 @@ public Builder setEnableVolumeLogger(boolean enableVolumeLogger) { return this; } + /** + * Can be used to gain access to the raw ByteBuffer from the recording device before it's + * fed into WebRTC. You can use this to manipulate the ByteBuffer (e.g. audio filters). + * Make sure that the operation is fast. + */ + public Builder setAudioRecordDataCallback(AudioRecordDataCallback audioRecordDataCallback) { + this.audioRecordDataCallback = audioRecordDataCallback; + return this; + } + /** * Construct an AudioDeviceModule based on the supplied arguments. The caller takes ownership * and is responsible for calling release(). @@ -255,7 +266,7 @@ public JavaAudioDeviceModule createAudioDeviceModule() { } final WebRtcAudioRecord audioInput = new WebRtcAudioRecord(context, executor, audioManager, audioSource, audioFormat, audioRecordErrorCallback, audioRecordStateCallback, - samplesReadyCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); + samplesReadyCallback, audioRecordDataCallback, useHardwareAcousticEchoCanceler, useHardwareNoiseSuppressor); final WebRtcAudioTrack audioOutput = new WebRtcAudioTrack(context, audioManager, audioAttributes, audioTrackErrorCallback, audioTrackStateCallback, useLowLatency, enableVolumeLogger); diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 94dfdf0728..6d0edcf972 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -45,8 +45,8 @@ class HardwareVideoEncoder implements VideoEncoder { private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; - // Size of the input frames should be multiple of 16 for the H/W encoder. - private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16; + // Size of the input frames should be multiple of 2 for the H/W encoder. + private static final int REQUIRED_RESOLUTION_ALIGNMENT = 2; /** * Keeps track of the number of output buffers that have been passed down the pipeline and not yet @@ -210,6 +210,11 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) { this.callback = callback; automaticResizeOn = settings.automaticResizeOn; + if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec requires 2x2 alignment."); + return VideoCodecStatus.ERR_SIZE; + } this.width = settings.width; this.height = settings.height; useSurfaceMode = canUseSurface(); @@ -533,6 +538,12 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS if (status != VideoCodecStatus.OK) { return status; } + + if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec requires 2x2 alignment."); + return VideoCodecStatus.ERR_SIZE; + } width = newWidth; height = newHeight; useSurfaceMode = newUseSurfaceMode; diff --git a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java index 5417fec4d4..fdcdfb3cf2 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecUtils.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecUtils.java @@ -27,6 +27,7 @@ class MediaCodecUtils { // Prefixes for supported hardware encoder/decoder component names. static final String EXYNOS_PREFIX = "OMX.Exynos."; static final String INTEL_PREFIX = "OMX.Intel."; + static final String MARVELL_PREFIX = "OMX.Marvell."; static final String NVIDIA_PREFIX = "OMX.Nvidia."; static final String QCOM_PREFIX = "OMX.qcom."; static final String[] SOFTWARE_IMPLEMENTATION_PREFIXES = { @@ -46,7 +47,8 @@ class MediaCodecUtils { MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka, MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka, MediaCodecUtils.COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka, - MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m}; + MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m, + MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}; // Color formats supported by hardware encoder - in order of preference. static final int[] ENCODER_COLOR_FORMATS = { diff --git a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java index 9a73bc49ff..9bf6805094 100644 --- a/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java +++ b/sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java @@ -11,6 +11,7 @@ package org.webrtc; import static org.webrtc.MediaCodecUtils.EXYNOS_PREFIX; +import static org.webrtc.MediaCodecUtils.MARVELL_PREFIX; import static org.webrtc.MediaCodecUtils.QCOM_PREFIX; import android.media.MediaCodecInfo; @@ -127,8 +128,8 @@ private boolean isCodecAllowed(MediaCodecInfo info) { private boolean isH264HighProfileSupported(MediaCodecInfo info) { String name = info.getName(); - // Support H.264 HP decoding on QCOM chips. - if (name.startsWith(QCOM_PREFIX)) { + // Support H.264 HP decoding on QCOM and Marvell chips + if (name.startsWith(QCOM_PREFIX) || name.startsWith(MARVELL_PREFIX)) { return true; } // Support H.264 HP decoding on Exynos chips for Android M and above. diff --git a/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java b/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java new file mode 100644 index 0000000000..421e559e2d --- /dev/null +++ b/sdk/android/src/java/org/webrtc/audio/AudioRecordDataCallback.java @@ -0,0 +1,16 @@ +package org.webrtc.audio; + +import androidx.annotation.NonNull; + +import java.nio.ByteBuffer; + +public interface AudioRecordDataCallback { + /** + * Invoked after an audio sample is recorded. Can be used to manipulate + * the ByteBuffer before it's fed into WebRTC. Currently the audio in the + * ByteBuffer is always PCM 16bit and the buffer sample size is ~10ms. + * + * @param audioFormat format in android.media.AudioFormat + */ + void onAudioDataRecorded(int audioFormat, int channelCount, int sampleRate, @NonNull ByteBuffer audioBuffer); +} \ No newline at end of file diff --git a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java index cfb651f6cd..451d93f908 100644 --- a/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java +++ b/sdk/android/src/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -104,6 +104,7 @@ class WebRtcAudioRecord { private final @Nullable AudioRecordErrorCallback errorCallback; private final @Nullable AudioRecordStateCallback stateCallback; + private final @Nullable AudioRecordDataCallback audioRecordDataCallback; private final @Nullable SamplesReadyCallback audioSamplesReadyCallback; private final boolean isAcousticEchoCancelerSupported; private final boolean isNoiseSuppressorSupported; @@ -153,6 +154,13 @@ public void run() { captureTimeNs = audioTimestamp.nanoTime; } } + + // Allow the client to intercept the ByteBuffer (to modify it) + if (audioRecordDataCallback != null) { + audioRecordDataCallback.onAudioDataRecorded(audioRecord.getAudioFormat(), + audioRecord.getChannelCount(), audioRecord.getSampleRate(), byteBuffer); + } + nativeDataIsRecorded(nativeAudioRecord, bytesRead, captureTimeNs); } if (audioSamplesReadyCallback != null) { @@ -196,7 +204,8 @@ public void stopThread() { WebRtcAudioRecord(Context context, AudioManager audioManager) { this(context, newDefaultScheduler() /* scheduler */, audioManager, DEFAULT_AUDIO_SOURCE, DEFAULT_AUDIO_FORMAT, null /* errorCallback */, null /* stateCallback */, - null /* audioSamplesReadyCallback */, WebRtcAudioEffects.isAcousticEchoCancelerSupported(), + null /* audioSamplesReadyCallback */, null /* audioRecordCallback */, + WebRtcAudioEffects.isAcousticEchoCancelerSupported(), WebRtcAudioEffects.isNoiseSuppressorSupported()); } @@ -205,6 +214,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler, @Nullable AudioRecordErrorCallback errorCallback, @Nullable AudioRecordStateCallback stateCallback, @Nullable SamplesReadyCallback audioSamplesReadyCallback, + @Nullable AudioRecordDataCallback audioRecordDataCallback, boolean isAcousticEchoCancelerSupported, boolean isNoiseSuppressorSupported) { if (isAcousticEchoCancelerSupported && !WebRtcAudioEffects.isAcousticEchoCancelerSupported()) { throw new IllegalArgumentException("HW AEC not supported"); @@ -220,6 +230,7 @@ public WebRtcAudioRecord(Context context, ScheduledExecutorService scheduler, this.errorCallback = errorCallback; this.stateCallback = stateCallback; this.audioSamplesReadyCallback = audioSamplesReadyCallback; + this.audioRecordDataCallback = audioRecordDataCallback; this.isAcousticEchoCancelerSupported = isAcousticEchoCancelerSupported; this.isNoiseSuppressorSupported = isNoiseSuppressorSupported; Logging.d(TAG, "ctor" + WebRtcAudioUtils.getThreadInfo()); diff --git a/sdk/android/src/jni/pc/rtp_parameters.cc b/sdk/android/src/jni/pc/rtp_parameters.cc index 4bd9ee0e1d..dfc7f17f6d 100644 --- a/sdk/android/src/jni/pc/rtp_parameters.cc +++ b/sdk/android/src/jni/pc/rtp_parameters.cc @@ -53,6 +53,7 @@ ScopedJavaLocalRef NativeToJavaRtpEncodingParameter( NativeToJavaInteger(env, encoding.max_framerate), NativeToJavaInteger(env, encoding.num_temporal_layers), NativeToJavaDouble(env, encoding.scale_resolution_down_by), + NativeToJavaString(env, encoding.scalability_mode), encoding.ssrc ? NativeToJavaLong(env, *encoding.ssrc) : nullptr, encoding.adaptive_ptime); } @@ -116,6 +117,11 @@ RtpEncodingParameters JavaToNativeRtpEncodingParameters( Java_Encoding_getScaleResolutionDownBy(jni, j_encoding_parameters); encoding.scale_resolution_down_by = JavaToNativeOptionalDouble(jni, j_scale_resolution_down_by); + ScopedJavaLocalRef j_scalability_mode = + Java_Encoding_getScalabilityMode(jni, j_encoding_parameters); + if (!IsNull(jni, j_scalability_mode)) { + encoding.scalability_mode = JavaToNativeString(jni, j_scalability_mode); + } encoding.adaptive_ptime = Java_Encoding_getAdaptivePTime(jni, j_encoding_parameters); ScopedJavaLocalRef j_ssrc = diff --git a/sdk/android/src/jni/simulcast_video_encoder.cc b/sdk/android/src/jni/simulcast_video_encoder.cc new file mode 100644 index 0000000000..18daf6f062 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder.cc @@ -0,0 +1,34 @@ +#include + +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/video_encoder_factory_wrapper.h" +#include "sdk/android/src/jni/video_codec_info.h" +#include "sdk/android/native_api/codecs/wrapper.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "rtc_base/logging.h" + +using namespace webrtc; +using namespace webrtc::jni; + +#ifdef __cplusplus +extern "C" { +#endif + +// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) +JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) { + RTC_LOG(LS_INFO) << "Create simulcast video encoder"; + JavaParamRef info_ref(info); + SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref); + + // TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい + // https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795 + return NativeToJavaPointer(std::make_unique( + JavaToNativeVideoEncoderFactory(env, primary).release(), + fallback != nullptr ? JavaToNativeVideoEncoderFactory(env, fallback).release() : nullptr, + format).release()); +} + + +#ifdef __cplusplus +} +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder.h b/sdk/android/src/jni/simulcast_video_encoder.h new file mode 100644 index 0000000000..519be778e8 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder.h @@ -0,0 +1,22 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_webrtc_SimulcastVideoEncoder */ + +#ifndef _Included_org_webrtc_SimulcastVideoEncoder +#define _Included_org_webrtc_SimulcastVideoEncoder +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_webrtc_SimulcastVideoEncoder + * Method: nativeCreateEncoder + * Signature: (Lorg/webrtc/VideoEncoderFactory;Lorg/webrtc/VideoEncoderFactory;Lorg/webrtc/VideoCodecInfo;)J + */ + +JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder + (JNIEnv *, jclass, jobject, jobject, jobject); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder_factory.cc b/sdk/android/src/jni/simulcast_video_encoder_factory.cc new file mode 100644 index 0000000000..bc6ea5be6a --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder_factory.cc @@ -0,0 +1,40 @@ +#include + +#include "sdk/android/src/jni/jni_helpers.h" +#include "sdk/android/src/jni/video_encoder_factory_wrapper.h" +#include "sdk/android/src/jni/video_codec_info.h" +#include "sdk/android/native_api/codecs/wrapper.h" +#include "sdk/android/native_api/jni/class_loader.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "media/base/media_constants.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" + +using namespace webrtc; +using namespace webrtc::jni; + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeVP9Codecs + (JNIEnv *env, jclass klass) { + std::vector formats = SupportedVP9Codecs(true); + return NativeToJavaList(env, formats, &SdpVideoFormatToVideoCodecInfo).Release(); +} + +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeAV1Codec + (JNIEnv *env, jclass klass) { + SdpVideoFormat format = SdpVideoFormat( + cricket::kAv1CodecName, SdpVideoFormat::Parameters(), + LibaomAv1EncoderSupportedScalabilityModes()); + return SdpVideoFormatToVideoCodecInfo(env, format).Release(); +} + +#ifdef __cplusplus +} +#endif diff --git a/sdk/android/src/jni/simulcast_video_encoder_factory.h b/sdk/android/src/jni/simulcast_video_encoder_factory.h new file mode 100644 index 0000000000..d7426953a0 --- /dev/null +++ b/sdk/android/src/jni/simulcast_video_encoder_factory.h @@ -0,0 +1,29 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class org_webrtc_SimulcastVideoEncoderFactory */ + +#ifndef _Included_org_webrtc_SimulcastVideoEncoderFactory +#define _Included_org_webrtc_SimulcastVideoEncoderFactory +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: org_webrtc_SimulcastVideoEncoderFactory + * Method: nativeVP9Codecs + * Signature: ()Ljava/util/List; + */ +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeVP9Codecs + (JNIEnv *, jclass); + +/* + * Class: org_webrtc_SimulcastVideoEncoderFactory + * Method: nativeAV1Codec + * Signature: ()Lorg/webrtc/VideoCodecInfo; + */ +JNIEXPORT jobject JNICALL Java_org_webrtc_SimulcastVideoEncoderFactory_nativeAV1Codec + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/sdk/android/src/jni/video_codec_info.cc b/sdk/android/src/jni/video_codec_info.cc index a218a1d23f..42e7b5d897 100644 --- a/sdk/android/src/jni/video_codec_info.cc +++ b/sdk/android/src/jni/video_codec_info.cc @@ -14,14 +14,22 @@ #include "sdk/android/native_api/jni/java_types.h" #include "sdk/android/src/jni/jni_helpers.h" +#include "absl/container/inlined_vector.h" + namespace webrtc { namespace jni { SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni, const JavaRef& j_info) { + absl::InlinedVector scalabilityModes; + std::vector javaScalabilityModes = JavaToNativeIntArray(jni, Java_VideoCodecInfo_getScalabilityModes(jni, j_info)); + for (const auto& scalabilityMode : javaScalabilityModes) { + scalabilityModes.push_back(static_cast(scalabilityMode)); + } return SdpVideoFormat( JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)), - JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info))); + JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)), + scalabilityModes); } ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( @@ -29,8 +37,18 @@ ScopedJavaLocalRef SdpVideoFormatToVideoCodecInfo( const SdpVideoFormat& format) { ScopedJavaLocalRef j_params = NativeToJavaStringMap(jni, format.parameters); - return Java_VideoCodecInfo_Constructor( + + ScopedJavaLocalRef codec = Java_VideoCodecInfo_Constructor( jni, NativeToJavaString(jni, format.name), j_params); + + size_t size = format.scalability_modes.size(); + std::vector temp(size); + for (size_t i = 0; i < size; i++) { + temp[i] = static_cast(format.scalability_modes[i]); + } + Java_VideoCodecInfo_setScalabilityModes(jni, codec, NativeToJavaIntArray(jni, temp)); + + return codec; } } // namespace jni diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index eb343ebdb3..2078359cbc 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -44,5 +44,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, jlong j_native_sink) { delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index d3296f6279..df294d2f3e 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -70,6 +70,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return self.nativeVideoTrack->should_receive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + self.nativeVideoTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { if (!_workerThread->IsCurrent()) { _workerThread->BlockingCall([renderer, self] { [self addRenderer:renderer]; }); diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h new file mode 100644 index 0000000000..c3e3d4538f --- /dev/null +++ b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h @@ -0,0 +1,13 @@ +#import "RTCMacros.h" +#import "RTCVideoEncoder.h" +#import "RTCVideoEncoderFactory.h" +#import "RTCVideoCodecInfo.h" + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject + ++ (id)simulcastEncoderWithPrimary:(id)primary + fallback:(id)fallback + videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo; + +@end \ No newline at end of file diff --git a/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm new file mode 100644 index 0000000000..568e1bd517 --- /dev/null +++ b/sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm @@ -0,0 +1,26 @@ +#import + +#import "RTCMacros.h" +#import "RTCVideoEncoderSimulcast.h" +#import "RTCWrappedNativeVideoEncoder.h" +#import "api/peerconnection/RTCVideoCodecInfo+Private.h" + +#include "native/api/video_encoder_factory.h" +#include "media/engine/simulcast_encoder_adapter.h" + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) + ++ (id)simulcastEncoderWithPrimary:(id)primary + fallback:(id)fallback + videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo { + auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary); + auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback); + auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat]; + return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc] + initWithNativeEncoder: std::make_unique( + nativePrimary.release(), + nativeFallback.release(), + std::move(nativeFormat))]; +} + +@end \ No newline at end of file diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h new file mode 100644 index 0000000000..8ba7584c8e --- /dev/null +++ b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h @@ -0,0 +1,16 @@ +#import + +#import "RTCMacros.h" +#import "RTCVideoEncoderFactory.h" + +NS_ASSUME_NONNULL_BEGIN + +RTC_OBJC_EXPORT +@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject + +- (instancetype)initWithPrimary:(id)primary + fallback:(id)fallback; + +@end + +NS_ASSUME_NONNULL_END \ No newline at end of file diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm new file mode 100644 index 0000000000..6ba2074768 --- /dev/null +++ b/sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm @@ -0,0 +1,39 @@ +#import + +#import "RTCMacros.h" +#import "RTCVideoCodecInfo.h" +#import "RTCVideoEncoderFactorySimulcast.h" +#import "api/video_codec/RTCVideoEncoderSimulcast.h" + +@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) () + +@property id primary; +@property id fallback; + +@end + + +@implementation RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) + +@synthesize primary = _primary; +@synthesize fallback = _fallback; + +- (instancetype)initWithPrimary:(id)primary + fallback:(id)fallback { + if (self = [super init]) { + _primary = primary; + _fallback = fallback; + } + return self; +} + +- (nullable id)createEncoder: (RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { + return [RTCVideoEncoderSimulcast simulcastEncoderWithPrimary: _primary fallback: _fallback videoCodecInfo: info]; +} + +- (NSArray *)supportedCodecs { + return [[_primary supportedCodecs] arrayByAddingObjectsFromArray: [_fallback supportedCodecs]]; +} + + +@end \ No newline at end of file