From 2ae885c551872606e6c0ee1e0044c51edff41f35 Mon Sep 17 00:00:00 2001 From: Niklas P Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Fri, 6 Dec 2024 11:08:59 +0100 Subject: [PATCH] WIP: frame_flow --- src/modules/html/producer/html_producer.cpp | 269 ++++++++++---------- 1 file changed, 131 insertions(+), 138 deletions(-) diff --git a/src/modules/html/producer/html_producer.cpp b/src/modules/html/producer/html_producer.cpp index 9a024bc94d..7409d3b081 100644 --- a/src/modules/html/producer/html_producer.cpp +++ b/src/modules/html/producer/html_producer.cpp @@ -39,8 +39,6 @@ #include #include -#include - #include #include #include @@ -62,6 +60,8 @@ #include #include +#include + #include "../html.h" namespace caspar { namespace html { @@ -75,83 +75,36 @@ inline std::int_least64_t now() struct presentation_frame { - core::mutable_frame frame; - int64_t audio_pts; + std::int_least64_t timestamp; + core::draw_frame frame = core::draw_frame::empty(); + bool is_empty = true; - presentation_frame() - : frame(core::mutable_frame(nullptr, - std::vector>(), - caspar::array(), - core::pixel_format_desc())) - , audio_pts(0) - { - } - - presentation_frame(core::mutable_frame&& frame) - : frame(std::move(frame)) - , audio_pts(0) + explicit presentation_frame(core::draw_frame video = {}, std::int_least64_t ts = now()) { + timestamp = ts; + if (video) { + frame = std::move(video); + is_empty = false; + } } - presentation_frame(presentation_frame&& other) - : frame(std::move(other.frame)) - , audio_pts(other.audio_pts) + presentation_frame(presentation_frame&& other) noexcept + : timestamp(other.timestamp) + , frame(std::move(other.frame)) { } presentation_frame(const presentation_frame&) = delete; presentation_frame& operator=(const presentation_frame&) = delete; - presentation_frame& operator=(presentation_frame&& rhs) + presentation_frame& operator=(presentation_frame&& rhs) noexcept { + timestamp = rhs.timestamp; frame = std::move(rhs.frame); - audio_pts = rhs.audio_pts; return *this; } ~presentation_frame() {} - - void set_video(core::mutable_frame&& data) - { - caspar::array audio(std::move(frame.audio_data())); - frame = std::move(data); - frame.set_audio_data(std::move(audio)); - } - bool has_audio() const { return frame.audio_data().size() > 0; } - bool has_video() const { return frame.pixel_format_desc().format != core::pixel_format::invalid; } - bool is_empty() const { return !has_audio() && !has_video(); } -}; - -struct video_frame_data -{ - int width; - int height; - caspar::array data; - - video_frame_data() = delete; - video_frame_data(int width, int height, const uint8_t* src) - : width(width) - , height(height) - , data(width * height * 4) - { - memcpy(data.begin(), src, width * height * 4); - } - video_frame_data(const video_frame_data& other) - : width(other.width) - , height(other.height) - , data(other.data.size()) - { - memcpy(data.begin(), other.data.begin(), other.data.size()); - } - - video_frame_data& operator=(const video_frame_data& other) - { - width = other.width; - height = other.height; - data = caspar::array(other.data.size()); - memcpy(data.begin(), other.data.begin(), other.data.size()); - return *this; - } }; class html_client @@ -162,16 +115,14 @@ class html_client , public CefLoadHandler , public CefDisplayHandler { - std::wstring url_; - spl::shared_ptr graph_; - core::monitor::state state_; - mutable std::mutex state_mutex_; - caspar::timer tick_timer_; - caspar::timer frame_timer_; - caspar::timer paint_timer_; - caspar::timer test_timer_; - std::unique_ptr audioResampler_; - std::shared_ptr last_video_frame_; + std::wstring url_; + spl::shared_ptr graph_; + core::monitor::state state_; + mutable std::mutex state_mutex_; + caspar::timer tick_timer_; + caspar::timer frame_timer_; + caspar::timer paint_timer_; + caspar::timer test_timer_; spl::shared_ptr frame_factory_; core::video_format_desc format_desc_; @@ -179,10 +130,15 @@ class html_client tbb::concurrent_queue javascript_before_load_; std::atomic loaded_; std::queue frames_; + std::queue audio_frames_; + core::draw_frame last_generated_frame_; mutable std::mutex frames_mutex_; + mutable std::mutex audio_frames_mutex_; const size_t frames_max_size_ = 4; std::atomic closing_; + std::unique_ptr audioResampler_; + core::draw_frame last_frame_; std::int_least64_t last_frame_time_; @@ -241,29 +197,78 @@ class html_client { std::lock_guard lock(frames_mutex_); + core::draw_frame audio_frame; + + { + // CASPAR_LOG(info) << "[html_producer] receive: last_frame_time_: " << last_frame_time_; + std::lock_guard audio_lock(audio_frames_mutex_); + if (!audio_frames_.empty()) { + audio_frame = core::draw_frame(std::move(audio_frames_.front().frame)); + audio_frames_.pop(); + } + } + if (!frames_.empty()) { - // last_frame_time_ = frames_.front(); - last_frame_ = core::draw_frame(std::move(frames_.front().frame)); + /* + * CEF in gpu-enabled mode only sends frames when something changes, and interlaced channels + * consume two frames in a short time span. + * This can interact poorly and cause the second + * field of an animation repeat the first. + * If there is a single field in the buffer, it may + * want delaying to avoid this stutter. + * The hazard here is that sometimes animations will + * start a field later than intended. + */ + if (field == core::video_field::a && frames_.size() == 1) { + auto now_time = now(); + + // Make sure there has been a gap before this pop, of at least a couple of frames + auto follows_gap_in_frames = (now_time - last_frame_time_) > 100; + + // Check if the sole buffered frame is too young to have a partner field generated (with a + // tolerance) + auto time_per_frame = (1000 * 1.5) / format_desc_.fps; + auto front_frame_is_too_young = (now_time - frames_.front().timestamp) < time_per_frame; + + if (follows_gap_in_frames && front_frame_is_too_young) { + return false; + } + } + + last_frame_time_ = frames_.front().timestamp; + last_frame_ = std::move(frames_.front().frame); + frames_.pop(); + if (audio_frame) + last_frame_= core::draw_frame::over(last_frame_, audio_frame); + graph_->set_value("buffered-frames", (double)frames_.size() / frames_max_size_); return true; } + if (audio_frame) { + last_frame_time_ = now(); + last_frame_ = core::draw_frame::over(last_frame_, audio_frame); + return true; + } + return false; } core::draw_frame receive(const core::video_field field) { + if (!try_pop(field)) { graph_->set_tag(diagnostics::tag_severity::SILENT, "late-frame"); + return core::draw_frame::still(last_frame_); + } else { + return last_frame_; } - - return last_frame_; } - core::draw_frame last_frame() const { return last_frame_; } + core::draw_frame last_frame() const { return core::draw_frame::still(last_frame_); } bool is_ready() const { @@ -313,13 +318,6 @@ class html_client } private: - std::int_least64_t now() - { - return std::chrono::duration_cast( - std::chrono::high_resolution_clock::now().time_since_epoch()) - .count(); - } - void GetViewRect(CefRefPtr browser, CefRect& rect) override { CASPAR_ASSERT(CefCurrentlyOn(TID_UI)); @@ -327,21 +325,6 @@ class html_client rect = CefRect(0, 0, format_desc_.square_width, format_desc_.square_height); } - core::mutable_frame create_filled_frame(int width, int height, const void* data) - { - core::pixel_format_desc pixel_desc; - pixel_desc.format = core::pixel_format::bgra; - pixel_desc.planes.push_back(core::pixel_format_desc::plane(width, height, 4)); - auto frame = frame_factory_->create_frame(this, pixel_desc); - auto dst = frame.image_data(0).begin(); - std::memcpy(dst, data, width * height * 4); - return frame; - } - core::mutable_frame create_filled_frame(const video_frame_data& video_frame) - { - return create_filled_frame(video_frame.width, video_frame.height, video_frame.data.begin()); - } - void OnPaint(CefRefPtr browser, PaintElementType type, const RectList& dirtyRects, @@ -359,18 +342,36 @@ class html_client if (type != PET_VIEW) return; - auto frame = create_filled_frame(width, height, buffer); - auto frame_data = std::make_shared(width, height, (const uint8_t*)buffer); + core::pixel_format_desc pixel_desc(core::pixel_format::bgra); + pixel_desc.planes.emplace_back(width, height, 4); + + core::mutable_frame frame = frame_factory_->create_frame(this, pixel_desc); + char* src = (char*)buffer; + char* dst = reinterpret_cast(frame.image_data(0).begin()); + test_timer_.restart(); + +#ifdef WIN32 + if (gpu_enabled_) { + int chunksize = height * width; + tbb::parallel_for(0, 4, [&](int y) { std::memcpy(dst + y * chunksize, src + y * chunksize, chunksize); }); + } else { + std::memcpy(dst, src, width * height * 4); + } +#else + // On my one test linux machine, doing a single memcpy doesn't have the same cost as windows, + // making using tbb excessive + std::memcpy(dst, src, width * height * 4); +#endif + + graph_->set_value("memcpy", test_timer_.elapsed() * format_desc_.fps * 0.5 * 5); { std::lock_guard lock(frames_mutex_); - last_video_frame_ = frame_data; - if (!frames_.empty() && !frames_.back().has_video()) { - frames_.back().set_video(std::move(frame)); - } else { - frames_.push(presentation_frame(std::move(frame))); - } + core::draw_frame new_frame = core::draw_frame(std::move(frame)); + last_generated_frame_ = new_frame; + + frames_.push(presentation_frame(std::move(new_frame))); while (frames_.size() > frames_max_size_) { frames_.pop(); graph_->set_tag(diagnostics::tag_severity::WARNING, "dropped-frame"); @@ -421,7 +422,8 @@ class html_client CefRefPtr GetRenderHandler() override { return this; } - CefRefPtr GetAudioHandler() override { return this; } + CefRefPtr GetAudioHandler() override { return this; } + CefRefPtr GetLifeSpanHandler() override { return this; } CefRefPtr GetLoadHandler() override { return this; } @@ -468,53 +470,44 @@ class html_client return false; } - bool GetAudioParameters(CefRefPtr browser, CefAudioParameters& params) + bool GetAudioParameters(CefRefPtr browser, CefAudioParameters& params) override { params.channel_layout = CEF_CHANNEL_LAYOUT_7_1; params.sample_rate = format_desc_.audio_sample_rate; params.frames_per_buffer = format_desc_.audio_cadence[0]; - return format_desc_.audio_cadence.size() == 1; + return format_desc_.audio_cadence.size() == 1; // TODO - handle 59.94 } - void OnAudioStreamStarted(CefRefPtr browser, const CefAudioParameters& params, int channels) + void OnAudioStreamStarted(CefRefPtr browser, const CefAudioParameters& params, int channels) override { + //48000 + CASPAR_LOG(info) << "[html_producer] OnAudioStreamStarted: sample_rate: " << params.sample_rate; audioResampler_ = std::make_unique(params.sample_rate, AV_SAMPLE_FMT_FLTP); } - void OnAudioStreamPacket(CefRefPtr browser, const float** data, int frames, int64_t pts) + void OnAudioStreamPacket(CefRefPtr browser, const float** data, int samples, int64_t pts) override { - if (audioResampler_) { - auto audio = audioResampler_->convert(frames, reinterpret_cast(data)); - try - { - std::lock_guard lock(frames_mutex_); - if (frames_.empty()) { - if (last_video_frame_) { - frames_.push(presentation_frame(create_filled_frame(*last_video_frame_))); - } else { - frames_.push(presentation_frame()); - } - } + if (!audioResampler_) + return; - if (!frames_.back().has_audio()) { - // frames_.back().frame.set_audio_data(std::move(audio)); - } else { - presentation_frame frame(create_filled_frame(*last_video_frame_)); - CASPAR_LOG(info) << "[html_producer] Frame with audio already queued, copying last video frame and queueing another"; - frame.frame.set_audio_data(std::move(audio)); - frames_.push(std::move(frame)); - } - } - catch(...) { - CASPAR_LOG(info) << "[html_producer] Exception in OnAudioStreamPacket"; + auto audio = audioResampler_->convert(samples, reinterpret_cast(data)); + auto audio_frame = core::mutable_frame(this, {}, std::move(audio), core::pixel_format_desc()); + + { + std::lock_guard lock(audio_frames_mutex_); + while (audio_frames_.size() >= frames_max_size_) { + audio_frames_.pop(); } + // CASPAR_LOG(info) << "[html_producer] OnAudioStreamPacket: pts: " << pts; + audio_frames_.push(presentation_frame(core::draw_frame(std::move(audio_frame)))); } } - void OnAudioStreamStopped(CefRefPtr browser) { audioResampler_ = nullptr; } - void OnAudioStreamError(CefRefPtr browser, const CefString& message) + void OnAudioStreamStopped(CefRefPtr browser) override { audioResampler_ = nullptr; } + void OnAudioStreamError(CefRefPtr browser, const CefString& message) override { CASPAR_LOG(info) << "[html_producer] OnAudioStreamError: \"" << message.ToString() << "\""; audioResampler_ = nullptr; } + void do_execute_javascript(const std::wstring& javascript) { html::begin_invoke([=] {