From 5f37410268873868263d44635134c5ba95c55e1d Mon Sep 17 00:00:00 2001 From: David Liu Date: Fri, 6 May 2022 23:10:04 +0900 Subject: [PATCH 1/8] Initial implementation of start/stop receive --- media/base/media_channel.h | 3 ++ media/engine/webrtc_video_engine.cc | 33 +++++++++++++++++ media/engine/webrtc_video_engine.h | 6 +++- pc/media_stream_track_proxy.h | 8 ++--- pc/video_rtp_receiver.cc | 55 +++++++++++++++++++++++++++-- pc/video_rtp_receiver.h | 12 ++++++- pc/video_track.cc | 35 ++++++++++++------ pc/video_track.h | 7 +++- video/video_receive_stream2.cc | 2 ++ 9 files changed, 141 insertions(+), 20 deletions(-) diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 2607a7a4aa..95e7e961e0 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -897,6 +897,9 @@ class VideoMediaChannel : public MediaChannel, public Delayable { virtual void GenerateKeyFrame(uint32_t ssrc) = 0; virtual std::vector GetSources(uint32_t ssrc) const = 0; + + virtual void StartReceive(uint32_t ssrc) {} + virtual void StopReceive(uint32_t ssrc) {} }; // Info about data received in DataMediaChannel. For use in diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index f497d9046e..2baee37d67 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -949,6 +949,26 @@ void WebRtcVideoChannel::RequestEncoderSwitch( << format.ToString() << " not negotiated."; } +void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_WARNING) << "WebRtcVideoChannel::StartReceive"; + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StartStream(); +} + +void WebRtcVideoChannel::StopReceive(uint32_t ssrc) { + RTC_DCHECK_RUN_ON(&thread_checker_); + RTC_LOG(LS_WARNING) << "WebRtcVideoChannel::StopReceive"; + WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); + if(!stream) { + return; + } + stream->StopStream(); +} + bool WebRtcVideoChannel::ApplyChangedParams( const ChangedSendParameters& changed_params) { RTC_DCHECK_RUN_ON(&thread_checker_); @@ -3017,6 +3037,19 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){ + RTC_LOG(LS_ERROR) << "WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream"; + if (stream_) { + stream_->Start(); + } +} +void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){ + RTC_LOG(LS_ERROR) << "WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream"; + if (stream_) { + stream_->Stop(); + } +} + void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { absl::optional base_minimum_playout_delay_ms; absl::optional recording_state; diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index 90d824a55b..16858dac0a 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -245,7 +245,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, uint32_t ssrc, rtc::scoped_refptr frame_transformer) override; - + void StartReceive(uint32_t ssrc) override; + void StopReceive(uint32_t ssrc) override; private: class WebRtcVideoReceiveStream; @@ -479,6 +480,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, void SetDepacketizerToDecoderFrameTransformer( rtc::scoped_refptr frame_transformer); + + void StartStream(); + void StopStream(); private: void RecreateWebRtcVideoStream(); diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index f563137c77..e2841154e9 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -44,10 +44,10 @@ PROXY_PRIMARY_THREAD_DESTRUCTOR() BYPASS_PROXY_CONSTMETHOD0(std::string, kind) BYPASS_PROXY_CONSTMETHOD0(std::string, id) PROXY_SECONDARY_CONSTMETHOD0(TrackState, state) -PROXY_SECONDARY_CONSTMETHOD0(bool, enabled) -PROXY_SECONDARY_METHOD1(bool, set_enabled, bool) -PROXY_SECONDARY_CONSTMETHOD0(ContentHint, content_hint) -PROXY_SECONDARY_METHOD1(void, set_content_hint, ContentHint) +PROXY_CONSTMETHOD0(bool, enabled) +PROXY_METHOD1(bool, set_enabled, bool) +PROXY_CONSTMETHOD0(ContentHint, content_hint) +PROXY_METHOD1(void, set_content_hint, ContentHint) PROXY_SECONDARY_METHOD2(void, AddOrUpdateSink, rtc::VideoSinkInterface*, diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 8db4d9f02f..91d348cdf0 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -21,6 +21,7 @@ #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" +#include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { @@ -46,9 +47,13 @@ VideoRtpReceiver::VideoRtpReceiver( worker_thread, source_), worker_thread))), - attachment_id_(GenerateUniqueId()) { + cached_track_enabled_(track_->enabled()), + attachment_id_(GenerateUniqueId()), + worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); SetStreams(streams); + track_->RegisterObserver(this); + RTC_LOG(LS_ERROR) << "VideoRtpReceiver::VideoRtpReceiver called, id = " << receiver_id; RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kLive); } @@ -137,6 +142,42 @@ void VideoRtpReceiver::StopAndEndTrack() { track_->internal()->set_ended(); } +void VideoRtpReceiver::OnChanged() { + RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + if (cached_track_enabled_ != track_->enabled()) { + cached_track_enabled_ = track_->enabled(); + worker_thread_->PostTask(ToQueuedTask( + worker_thread_safety_, + [this, enabled = cached_track_enabled_]() { + RTC_DCHECK_RUN_ON(worker_thread_); + if(enabled) { + StartMediaChannel(); + } else { + StopMediaChannel(); + } + })); + } +} + +void VideoRtpReceiver::StartMediaChannel() { + RTC_LOG(LS_ERROR) << "VideoRtpReceiver::StartMediaChannel id = " << id(); + + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StartReceive(ssrc_.value_or(0)); + OnGenerateKeyFrame(); +} +void VideoRtpReceiver::StopMediaChannel() { + RTC_LOG(LS_ERROR) << "VideoRtpReceiver::StopMediaChannelid = " << id(); + RTC_DCHECK_RUN_ON(worker_thread_); + if (!media_channel_) { + return; + } + media_channel_->StopReceive(ssrc_.value_or(0)); +} + void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); @@ -235,6 +276,11 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); + + for (size_t i = 0; i < streams.size(); ++i) { + RTC_LOG(LS_ERROR) << id() << ": stream_id: " << streams[i]->id(); + } + // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { bool removed = true; @@ -301,13 +347,18 @@ void VideoRtpReceiver::SetMediaChannel(cricket::MediaChannel* media_channel) { void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) { if (media_channel == media_channel_) return; - + + RTC_LOG(LS_ERROR) + << "VideoRtpReceiver::SetMediaChannel_w: " << typeid(media_channel).name(); + bool encoded_sink_enabled = saved_encoded_sink_enabled_; if (encoded_sink_enabled && media_channel_) { // Turn off the old sink, if any. SetEncodedSinkEnabled(false); } + media_channel ? worker_thread_safety_->SetAlive() + : worker_thread_safety_->SetNotAlive(); media_channel_ = static_cast(media_channel); if (media_channel_) { diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index b5381860b3..a94026d20c 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -43,7 +43,8 @@ namespace webrtc { -class VideoRtpReceiver : public RtpReceiverInternal { +class VideoRtpReceiver : public RtpReceiverInternal, + public ObserverInterface { public: // An SSRC of 0 will create a receiver that will match the first SSRC it // sees. Must be called on signaling thread. @@ -61,6 +62,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { rtc::scoped_refptr video_track() const { return track_; } + // ObserverInterface implementation + void OnChanged() override; + // RtpReceiverInterface implementation rtc::scoped_refptr track() const override { return track_; @@ -111,6 +115,9 @@ class VideoRtpReceiver : public RtpReceiverInternal { std::vector GetSources() const override; private: + + void StartMediaChannel(); + void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc); void SetSink(rtc::VideoSinkInterface* sink) RTC_RUN_ON(worker_thread_); @@ -165,6 +172,8 @@ class VideoRtpReceiver : public RtpReceiverInternal { RTC_GUARDED_BY(&signaling_thread_checker_) = nullptr; bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; + + bool cached_track_enabled_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); @@ -180,6 +189,7 @@ class VideoRtpReceiver : public RtpReceiverInternal { // or switched. bool saved_generate_keyframe_ RTC_GUARDED_BY(worker_thread_) = false; bool saved_encoded_sink_enabled_ RTC_GUARDED_BY(worker_thread_) = false; + const rtc::scoped_refptr worker_thread_safety_; }; } // namespace webrtc diff --git a/pc/video_track.cc b/pc/video_track.cc index d0246faa87..f50424e621 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -19,6 +19,7 @@ #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/ref_counted_object.h" +#include "rtc_base/logging.h" namespace webrtc { @@ -53,7 +54,7 @@ void VideoTrack::AddOrUpdateSink(rtc::VideoSinkInterface* sink, RTC_DCHECK_RUN_ON(worker_thread_); VideoSourceBaseGuarded::AddOrUpdateSink(sink, wants); rtc::VideoSinkWants modified_wants = wants; - modified_wants.black_frames = !enabled(); + modified_wants.black_frames = !enabled_w_; video_source_->AddOrUpdateSink(sink, modified_wants); } @@ -69,12 +70,12 @@ VideoTrackSourceInterface* VideoTrack::GetSource() const { } VideoTrackInterface::ContentHint VideoTrack::content_hint() const { - RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; } void VideoTrack::set_content_hint(ContentHint hint) { - RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK_RUN_ON(&signaling_thread_); if (content_hint_ == hint) return; content_hint_ = hint; @@ -82,17 +83,29 @@ void VideoTrack::set_content_hint(ContentHint hint) { } bool VideoTrack::set_enabled(bool enable) { - RTC_DCHECK_RUN_ON(worker_thread_); - for (auto& sink_pair : sink_pairs()) { - rtc::VideoSinkWants modified_wants = sink_pair.wants; - modified_wants.black_frames = !enable; - video_source_->AddOrUpdateSink(sink_pair.sink, modified_wants); - } - return MediaStreamTrack::set_enabled(enable); + RTC_DCHECK_RUN_ON(&signaling_thread_); + + bool ret = MediaStreamTrack::set_enabled(enable); + + worker_thread_->Invoke(RTC_FROM_HERE, [&]() { + RTC_DCHECK_RUN_ON(worker_thread_); + enabled_w_ = enable; + for (auto& sink_pair : sink_pairs()) { + rtc::VideoSinkWants modified_wants = sink_pair.wants; + modified_wants.black_frames = !enable; + video_source_->AddOrUpdateSink(sink_pair.sink, modified_wants); + } + }); + + return ret; } bool VideoTrack::enabled() const { - RTC_DCHECK_RUN_ON(worker_thread_); + if (worker_thread_->IsCurrent()) { + RTC_DCHECK_RUN_ON(worker_thread_); + return enabled_w_; + } + RTC_DCHECK_RUN_ON(&signaling_thread_); return MediaStreamTrack::enabled(); } diff --git a/pc/video_track.h b/pc/video_track.h index 49deaee76a..cd2adba276 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -60,7 +60,12 @@ class VideoTrack : public MediaStreamTrack, RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker signaling_thread_; rtc::Thread* const worker_thread_; const rtc::scoped_refptr video_source_; - ContentHint content_hint_ RTC_GUARDED_BY(worker_thread_); + ContentHint content_hint_ RTC_GUARDED_BY(signaling_thread_); + // Cached `enabled` state for the worker thread. This is kept in sync with + // the state maintained on the signaling thread via set_enabled() but can + // be queried without blocking on the worker thread by callers that don't + // use an api proxy to call the `enabled()` method. + bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; }; } // namespace webrtc diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 0776a40b35..fcb6d3ed75 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -337,6 +337,8 @@ void VideoReceiveStream2::Start() { return; } + frame_buffer_.reset( + new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_)); const bool protected_by_fec = config_.rtp.protected_by_flexfec || rtp_video_stream_receiver_.IsUlpfecEnabled(); From b0c4de3e52a83d66b6c7b34976686b170a9fea5c Mon Sep 17 00:00:00 2001 From: David Liu Date: Sat, 7 May 2022 00:19:05 +0900 Subject: [PATCH 2/8] move set_state call to signalling thread as well. https://source.chromium.org/chromium/_/webrtc/src.git/+/dfd69c22100e1d2e83295f33d06fa9916caa5c53 --- pc/rtp_sender.cc | 7 +++++-- pc/video_track.cc | 12 +++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pc/rtp_sender.cc b/pc/rtp_sender.cc index d4286371be..8f025e491a 100644 --- a/pc/rtp_sender.cc +++ b/pc/rtp_sender.cc @@ -584,10 +584,13 @@ VideoRtpSender::~VideoRtpSender() { } void VideoRtpSender::OnChanged() { + // Running on the signaling thread. TRACE_EVENT0("webrtc", "VideoRtpSender::OnChanged"); RTC_DCHECK(!stopped_); - if (cached_track_content_hint_ != video_track()->content_hint()) { - cached_track_content_hint_ = video_track()->content_hint(); + + auto content_hint = video_track()->content_hint(); + if (cached_track_content_hint_ != content_hint) { + cached_track_content_hint_ = content_hint; if (can_send_track()) { SetSend(); } diff --git a/pc/video_track.cc b/pc/video_track.cc index f50424e621..143fac47d7 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -116,15 +116,9 @@ MediaStreamTrackInterface::TrackState VideoTrack::state() const { void VideoTrack::OnChanged() { RTC_DCHECK_RUN_ON(&signaling_thread_); - worker_thread_->Invoke( - RTC_FROM_HERE, [this, state = video_source_->state()]() { - // TODO(tommi): Calling set_state() this way isn't ideal since we're - // currently blocking the signaling thread and set_state() may - // internally fire notifications via `FireOnChanged()` which may further - // amplify the blocking effect on the signaling thread. - rtc::Thread::ScopedDisallowBlockingCalls no_blocking_calls; - set_state(state == MediaSourceInterface::kEnded ? kEnded : kLive); - }); + rtc::Thread::ScopedDisallowBlockingCalls no_blocking_calls; + MediaSourceInterface::SourceState state = video_source_->state(); + set_state(state == MediaSourceInterface::kEnded ? kEnded : kLive); } rtc::scoped_refptr VideoTrack::Create( From 398b7ce85930783015b286389336a40215074c10 Mon Sep 17 00:00:00 2001 From: David Liu Date: Mon, 9 May 2022 20:56:35 +0900 Subject: [PATCH 3/8] separate should_receive from enabled for now --- api/media_stream_interface.cc | 4 ++++ api/media_stream_interface.h | 2 ++ pc/media_stream_track_proxy.h | 2 ++ pc/video_rtp_receiver.cc | 12 +++++++----- pc/video_rtp_receiver.h | 2 +- pc/video_track.cc | 13 +++++++++++++ pc/video_track.h | 4 ++++ sdk/android/api/org/webrtc/VideoTrack.java | 20 ++++++++++++++++++++ sdk/android/src/jni/video_track.cc | 11 +++++++++++ 9 files changed, 64 insertions(+), 6 deletions(-) diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index e07907917b..5362522262 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -18,6 +18,10 @@ const char* const MediaStreamTrackInterface::kVideoKind = const char* const MediaStreamTrackInterface::kAudioKind = cricket::kMediaTypeAudio; +bool VideoTrackInterface::should_receive() const { + return true; +} + VideoTrackInterface::ContentHint VideoTrackInterface::content_hint() const { return ContentHint::kNone; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index ad497d943b..65685bfead 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -188,6 +188,8 @@ class RTC_EXPORT VideoTrackInterface virtual VideoTrackSourceInterface* GetSource() const = 0; + virtual void set_should_receive(bool should_receive) {} + virtual bool should_receive() const; virtual ContentHint content_hint() const; virtual void set_content_hint(ContentHint hint) {} diff --git a/pc/media_stream_track_proxy.h b/pc/media_stream_track_proxy.h index e2841154e9..2f741f9ff3 100644 --- a/pc/media_stream_track_proxy.h +++ b/pc/media_stream_track_proxy.h @@ -54,6 +54,8 @@ PROXY_SECONDARY_METHOD2(void, const rtc::VideoSinkWants&) PROXY_SECONDARY_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) BYPASS_PROXY_CONSTMETHOD0(VideoTrackSourceInterface*, GetSource) +PROXY_CONSTMETHOD0(bool, should_receive) +PROXY_METHOD1(void, set_should_receive, bool) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 91d348cdf0..176a4a8118 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -47,7 +47,7 @@ VideoRtpReceiver::VideoRtpReceiver( worker_thread, source_), worker_thread))), - cached_track_enabled_(track_->enabled()), + cached_track_should_receive_(track_->should_receive()), attachment_id_(GenerateUniqueId()), worker_thread_safety_(PendingTaskSafetyFlag::CreateDetachedInactive()) { RTC_DCHECK(worker_thread_); @@ -143,14 +143,16 @@ void VideoRtpReceiver::StopAndEndTrack() { } void VideoRtpReceiver::OnChanged() { + RTC_LOG(LS_ERROR) << "VideoRtpReceiver::OnChanged::id" << id() << ": signaling: " << (&signaling_thread_checker_)->IsCurrent(); + RTC_LOG(LS_ERROR) << "VideoRtpReceiver::OnChanged::id" << id() << ": worker: " << (worker_thread_)->IsCurrent(); RTC_DCHECK_RUN_ON(&signaling_thread_checker_); - if (cached_track_enabled_ != track_->enabled()) { - cached_track_enabled_ = track_->enabled(); + if (cached_track_should_receive_ != track_->should_receive()) { + cached_track_should_receive_ = track_->should_receive(); worker_thread_->PostTask(ToQueuedTask( worker_thread_safety_, - [this, enabled = cached_track_enabled_]() { + [this, receive = cached_track_should_receive_]() { RTC_DCHECK_RUN_ON(worker_thread_); - if(enabled) { + if(receive) { StartMediaChannel(); } else { StopMediaChannel(); diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index a94026d20c..f35a46079d 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -173,7 +173,7 @@ class VideoRtpReceiver : public RtpReceiverInternal, bool received_first_packet_ RTC_GUARDED_BY(&signaling_thread_checker_) = false; - bool cached_track_enabled_ RTC_GUARDED_BY(&signaling_thread_checker_); + bool cached_track_should_receive_ RTC_GUARDED_BY(&signaling_thread_checker_); const int attachment_id_; rtc::scoped_refptr frame_decryptor_ RTC_GUARDED_BY(worker_thread_); diff --git a/pc/video_track.cc b/pc/video_track.cc index 143fac47d7..f6c6974a35 100644 --- a/pc/video_track.cc +++ b/pc/video_track.cc @@ -69,6 +69,19 @@ VideoTrackSourceInterface* VideoTrack::GetSource() const { return video_source_.get(); } +void VideoTrack::set_should_receive(bool receive) { + RTC_DCHECK_RUN_ON(&signaling_thread_); + if (should_receive_ == receive) + return; + should_receive_ = receive; + Notifier::FireOnChanged(); +} + +bool VideoTrack::should_receive() const { + RTC_DCHECK_RUN_ON(&signaling_thread_); + return should_receive_; +} + VideoTrackInterface::ContentHint VideoTrack::content_hint() const { RTC_DCHECK_RUN_ON(&signaling_thread_); return content_hint_; diff --git a/pc/video_track.h b/pc/video_track.h index cd2adba276..2e93fc2604 100644 --- a/pc/video_track.h +++ b/pc/video_track.h @@ -40,6 +40,9 @@ class VideoTrack : public MediaStreamTrack, void RemoveSink(rtc::VideoSinkInterface* sink) override; VideoTrackSourceInterface* GetSource() const override; + void set_should_receive(bool should_receive) override; + bool should_receive() const override; + ContentHint content_hint() const override; void set_content_hint(ContentHint hint) override; bool set_enabled(bool enable) override; @@ -66,6 +69,7 @@ class VideoTrack : public MediaStreamTrack, // be queried without blocking on the worker thread by callers that don't // use an api proxy to call the `enabled()` method. bool enabled_w_ RTC_GUARDED_BY(worker_thread_) = true; + bool should_receive_ RTC_GUARDED_BY(signaling_thread_) = true; }; } // namespace webrtc diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 5593d424f3..63c0136ba4 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -54,6 +54,24 @@ public void removeSink(VideoSink sink) { } } + /** + * For a remote video track, starts receiving the video stream. + * + * If the VideoTrack was already receiving, this is a no-op. + */ + public void setShouldReceive(boolean shouldReceive){ + nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); + } + + /** + * For a remote video track, stops receiving the video stream. + * + * If the VideoTrack was already receiving, this is a no-op. + */ + public boolean shouldReceive(){ + return nativeGetShouldReceive(getNativeMediaStreamTrack()); + } + @Override public void dispose() { for (long nativeSink : sinks.values()) { @@ -73,4 +91,6 @@ long getNativeVideoTrack() { private static native void nativeRemoveSink(long track, long nativeSink); private static native long nativeWrapSink(VideoSink sink); private static native void nativeFreeSink(long sink); + private static native void nativeSetShouldReceive(long track, boolean shouldReceive); + private static native boolean nativeGetShouldReceive(long track); } diff --git a/sdk/android/src/jni/video_track.cc b/sdk/android/src/jni/video_track.cc index 70bedc12cf..783ede0d44 100644 --- a/sdk/android/src/jni/video_track.cc +++ b/sdk/android/src/jni/video_track.cc @@ -45,5 +45,16 @@ static void JNI_VideoTrack_FreeSink(JNIEnv* jni, delete reinterpret_cast*>(j_native_sink); } +static void JNI_VideoTrack_SetShouldReceive(JNIEnv* jni, + jlong j_native_track, + jboolean should_receive) { + reinterpret_cast(j_native_track)->set_should_receive(should_receive); +} + +static jboolean JNI_VideoTrack_GetShouldReceive(JNIEnv* jni, + jlong j_native_track) { + return reinterpret_cast(j_native_track)->should_receive(); +} + } // namespace jni } // namespace webrtc From a8308994fb4f2b5065bd85cd0b02cde4a9f1eaa0 Mon Sep 17 00:00:00 2001 From: David Liu Date: Mon, 9 May 2022 21:02:20 +0900 Subject: [PATCH 4/8] ios hooks --- sdk/android/api/org/webrtc/VideoTrack.java | 8 ++++---- sdk/objc/api/peerconnection/RTCVideoTrack.h | 3 +++ sdk/objc/api/peerconnection/RTCVideoTrack.mm | 8 ++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sdk/android/api/org/webrtc/VideoTrack.java b/sdk/android/api/org/webrtc/VideoTrack.java index 63c0136ba4..bd5022f39a 100644 --- a/sdk/android/api/org/webrtc/VideoTrack.java +++ b/sdk/android/api/org/webrtc/VideoTrack.java @@ -55,18 +55,18 @@ public void removeSink(VideoSink sink) { } /** - * For a remote video track, starts receiving the video stream. + * For a remote video track, starts/stops receiving the video stream. * - * If the VideoTrack was already receiving, this is a no-op. + * If this is a local video track, this is a no-op. */ public void setShouldReceive(boolean shouldReceive){ nativeSetShouldReceive(getNativeMediaStreamTrack(), shouldReceive); } /** - * For a remote video track, stops receiving the video stream. + * The current receive status for a remote video track. * - * If the VideoTrack was already receiving, this is a no-op. + * This has no meaning for a local video track. */ public boolean shouldReceive(){ return nativeGetShouldReceive(getNativeMediaStreamTrack()); diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.h b/sdk/objc/api/peerconnection/RTCVideoTrack.h index 5382b7169f..56d25c1568 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.h +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.h @@ -25,6 +25,9 @@ RTC_OBJC_EXPORT /** The video source for this video track. */ @property(nonatomic, readonly) RTC_OBJC_TYPE(RTCVideoSource) *source; +/** The receive state, if this is a remote video track. */ +@property(nonatomic, assign) BOOL shouldReceive; + - (instancetype)init NS_UNAVAILABLE; /** Register a renderer that will render all frames received on this track. */ diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index 3f38dd51a9..149051ba43 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -69,6 +69,14 @@ - (void)dealloc { return _source; } +- (BOOL)shouldReceive { + return _nativeTrack->shouldReceive(); +} + +- (void)setShouldReceive:(BOOL)shouldReceive { + _nativeTrack->set_should_receive(shouldReceive); +} + - (void)addRenderer:(id)renderer { // Make sure we don't have this renderer yet. for (RTCVideoRendererAdapter *adapter in _adapters) { From 936412753a9ac146d029ff19f0c2f013f5ddac19 Mon Sep 17 00:00:00 2001 From: davidliu Date: Mon, 9 May 2022 22:47:59 +0900 Subject: [PATCH 5/8] Fix ios compile errors --- sdk/objc/api/peerconnection/RTCVideoTrack.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/objc/api/peerconnection/RTCVideoTrack.mm b/sdk/objc/api/peerconnection/RTCVideoTrack.mm index 149051ba43..a06fa59ae5 100644 --- a/sdk/objc/api/peerconnection/RTCVideoTrack.mm +++ b/sdk/objc/api/peerconnection/RTCVideoTrack.mm @@ -70,11 +70,11 @@ - (void)dealloc { } - (BOOL)shouldReceive { - return _nativeTrack->shouldReceive(); + return self.nativeVideoTrack->should_receive(); } - (void)setShouldReceive:(BOOL)shouldReceive { - _nativeTrack->set_should_receive(shouldReceive); + self.nativeVideoTrack->set_should_receive(shouldReceive); } - (void)addRenderer:(id)renderer { From 8952a5a8854644a4742a7df84c072e815a20dd88 Mon Sep 17 00:00:00 2001 From: David Liu Date: Mon, 9 May 2022 22:59:08 +0900 Subject: [PATCH 6/8] code cleanup --- media/engine/webrtc_video_engine.cc | 4 ---- pc/video_rtp_receiver.cc | 12 +----------- pc/video_rtp_receiver.h | 1 - 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 2baee37d67..39e179ae13 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -951,7 +951,6 @@ void WebRtcVideoChannel::RequestEncoderSwitch( void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { RTC_DCHECK_RUN_ON(&thread_checker_); - RTC_LOG(LS_WARNING) << "WebRtcVideoChannel::StartReceive"; WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); if(!stream) { return; @@ -961,7 +960,6 @@ void WebRtcVideoChannel::StartReceive(uint32_t ssrc) { void WebRtcVideoChannel::StopReceive(uint32_t ssrc) { RTC_DCHECK_RUN_ON(&thread_checker_); - RTC_LOG(LS_WARNING) << "WebRtcVideoChannel::StopReceive"; WebRtcVideoReceiveStream* stream = FindReceiveStream(ssrc); if(!stream) { return; @@ -3038,13 +3036,11 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } void WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream(){ - RTC_LOG(LS_ERROR) << "WebRtcVideoChannel::WebRtcVideoReceiveStream::StartStream"; if (stream_) { stream_->Start(); } } void WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream(){ - RTC_LOG(LS_ERROR) << "WebRtcVideoChannel::WebRtcVideoReceiveStream::StopStream"; if (stream_) { stream_->Stop(); } diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 176a4a8118..8a87383e98 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -53,7 +53,6 @@ VideoRtpReceiver::VideoRtpReceiver( RTC_DCHECK(worker_thread_); SetStreams(streams); track_->RegisterObserver(this); - RTC_LOG(LS_ERROR) << "VideoRtpReceiver::VideoRtpReceiver called, id = " << receiver_id; RTC_DCHECK_EQ(source_->state(), MediaSourceInterface::kLive); } @@ -143,8 +142,6 @@ void VideoRtpReceiver::StopAndEndTrack() { } void VideoRtpReceiver::OnChanged() { - RTC_LOG(LS_ERROR) << "VideoRtpReceiver::OnChanged::id" << id() << ": signaling: " << (&signaling_thread_checker_)->IsCurrent(); - RTC_LOG(LS_ERROR) << "VideoRtpReceiver::OnChanged::id" << id() << ": worker: " << (worker_thread_)->IsCurrent(); RTC_DCHECK_RUN_ON(&signaling_thread_checker_); if (cached_track_should_receive_ != track_->should_receive()) { cached_track_should_receive_ = track_->should_receive(); @@ -161,9 +158,7 @@ void VideoRtpReceiver::OnChanged() { } } -void VideoRtpReceiver::StartMediaChannel() { - RTC_LOG(LS_ERROR) << "VideoRtpReceiver::StartMediaChannel id = " << id(); - +void VideoRtpReceiver::StartMediaChannel() { RTC_DCHECK_RUN_ON(worker_thread_); if (!media_channel_) { return; @@ -172,7 +167,6 @@ void VideoRtpReceiver::StartMediaChannel() { OnGenerateKeyFrame(); } void VideoRtpReceiver::StopMediaChannel() { - RTC_LOG(LS_ERROR) << "VideoRtpReceiver::StopMediaChannelid = " << id(); RTC_DCHECK_RUN_ON(worker_thread_); if (!media_channel_) { return; @@ -278,10 +272,6 @@ void VideoRtpReceiver::set_transport( void VideoRtpReceiver::SetStreams( const std::vector>& streams) { RTC_DCHECK_RUN_ON(&signaling_thread_checker_); - - for (size_t i = 0; i < streams.size(); ++i) { - RTC_LOG(LS_ERROR) << id() << ": stream_id: " << streams[i]->id(); - } // Remove remote track from any streams that are going away. for (const auto& existing_stream : streams_) { diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index f35a46079d..67c8effef2 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -115,7 +115,6 @@ class VideoRtpReceiver : public RtpReceiverInternal, std::vector GetSources() const override; private: - void StartMediaChannel(); void StopMediaChannel(); void RestartMediaChannel(absl::optional ssrc); From 916cb8b8744d060600a711dfb86eac42e4b574f7 Mon Sep 17 00:00:00 2001 From: David Liu Date: Mon, 9 May 2022 23:00:00 +0900 Subject: [PATCH 7/8] clean up --- pc/video_rtp_receiver.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 8a87383e98..ae249d15ad 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -339,10 +339,6 @@ void VideoRtpReceiver::SetMediaChannel(cricket::MediaChannel* media_channel) { void VideoRtpReceiver::SetMediaChannel_w(cricket::MediaChannel* media_channel) { if (media_channel == media_channel_) return; - - RTC_LOG(LS_ERROR) - << "VideoRtpReceiver::SetMediaChannel_w: " << typeid(media_channel).name(); - bool encoded_sink_enabled = saved_encoded_sink_enabled_; if (encoded_sink_enabled && media_channel_) { // Turn off the old sink, if any. From 4c4764db6e0af5cb7a6ad61b01a5e55c9f3fe680 Mon Sep 17 00:00:00 2001 From: David Liu Date: Mon, 9 May 2022 23:20:13 +0900 Subject: [PATCH 8/8] reuse frame buffers instead --- modules/video_coding/frame_buffer2.cc | 7 +++++++ modules/video_coding/frame_buffer2.h | 2 ++ video/video_receive_stream2.cc | 3 +-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index b7ae0f3e94..de1631bee8 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -356,6 +356,13 @@ void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { protection_mode_ = mode; } +void FrameBuffer::Start() { + TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); + MutexLock lock(&mutex_); + if (!stopped_) + return; + stopped_ = false; +} void FrameBuffer::Stop() { TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); MutexLock lock(&mutex_); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index c2a3394bad..47a080ba4a 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -78,6 +78,8 @@ class FrameBuffer { // Stop the frame buffer, causing any sleeping thread in NextFrame to // return immediately. void Stop(); + // Unstop the frame buffer, re-allowing new frames to be inserted and read. + void Start(); // Updates the RTT for jitter buffer estimation. void UpdateRtt(int64_t rtt_ms); diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index fcb6d3ed75..120644a6fe 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -337,8 +337,7 @@ void VideoReceiveStream2::Start() { return; } - frame_buffer_.reset( - new video_coding::FrameBuffer(clock_, timing_.get(), &stats_proxy_)); + frame_buffer_->Start(); const bool protected_by_fec = config_.rtp.protected_by_flexfec || rtp_video_stream_receiver_.IsUlpfecEnabled();