diff --git a/src/accelerator/ogl/image/image_kernel.cpp b/src/accelerator/ogl/image/image_kernel.cpp index 7674802186..86cea420fe 100644 --- a/src/accelerator/ogl/image/image_kernel.cpp +++ b/src/accelerator/ogl/image/image_kernel.cpp @@ -220,6 +220,42 @@ struct image_kernel::impl coord.vertex_y += f_p[1]; }; + auto const first_plane = params.pix_desc.planes.at(0); + if (params.geometry.mode() != core::frame_geometry::scale_mode::stretch && first_plane.width > 0 && first_plane.height > 0) { + auto width_scale = static_cast(params.target_width) / static_cast(first_plane.width); + auto height_scale = static_cast(params.target_height) / static_cast(first_plane.height); + + double target_scale; + switch (params.geometry.mode()) { + case core::frame_geometry::scale_mode::fit: + target_scale = std::min(width_scale, height_scale); + f_s[0] *= target_scale / width_scale; + f_s[1] *= target_scale / height_scale; + break; + + case core::frame_geometry::scale_mode::fill: + target_scale = std::max(width_scale, height_scale); + f_s[0] *= target_scale / width_scale; + f_s[1] *= target_scale / height_scale; + break; + + case core::frame_geometry::scale_mode::original: + f_s[0] /= width_scale; + f_s[1] /= height_scale; + break; + + case core::frame_geometry::scale_mode::hfill: + f_s[1] *= width_scale / height_scale; + break; + + case core::frame_geometry::scale_mode::vfill: + f_s[0] *= height_scale / width_scale; + break; + + default:; + } + } + int corner = 0; for (auto& coord : coords) { do_crop(coord); diff --git a/src/accelerator/ogl/image/image_kernel.h b/src/accelerator/ogl/image/image_kernel.h index d82f049a9c..cc7a256700 100644 --- a/src/accelerator/ogl/image/image_kernel.h +++ b/src/accelerator/ogl/image/image_kernel.h @@ -49,6 +49,8 @@ struct draw_params final std::shared_ptr local_key; std::shared_ptr layer_key; double aspect_ratio = 1.0; + int target_width; + int target_height; }; class image_kernel final diff --git a/src/accelerator/ogl/image/image_mixer.cpp b/src/accelerator/ogl/image/image_mixer.cpp index 7be6c28938..4c750f3797 100644 --- a/src/accelerator/ogl/image/image_mixer.cpp +++ b/src/accelerator/ogl/image/image_mixer.cpp @@ -95,7 +95,7 @@ class image_renderer return make_ready_future(array(buffer.data(), format_desc.size, true)); } - return flatten(ogl_->dispatch_async([=]() mutable -> std::shared_future> { + return flatten(ogl_->dispatch_async([=, layers = std::move(layers)]() mutable -> std::shared_future> { auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_); draw(target_texture, std::move(layers), format_desc); @@ -142,8 +142,8 @@ class image_renderer local_mix_texture, format_desc); - draw(layer_texture, std::move(local_mix_texture), core::blend_mode::normal); - draw(target_texture, std::move(layer_texture), layer.blend_mode); + draw(layer_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal); + draw(target_texture, std::move(layer_texture), format_desc, layer.blend_mode); } else // fast path { for (auto& item : layer.items) @@ -154,7 +154,7 @@ class image_renderer local_mix_texture, format_desc); - draw(target_texture, std::move(local_mix_texture), core::blend_mode::normal); + draw(target_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal); } layer_key_texture = std::move(local_key_texture); @@ -168,6 +168,8 @@ class image_renderer const core::video_format_desc& format_desc) { draw_params draw_params; + draw_params.target_width = format_desc.square_width; + draw_params.target_height = format_desc.square_height; // TODO: Pass the target color_space draw_params.pix_desc = std::move(item.pix_desc); @@ -180,7 +182,7 @@ class image_renderer draw_params.textures.push_back(spl::make_shared_ptr(future_texture.get())); } - if (item.transform.is_key) { + if (item.transform.is_key) { // A key means we will use it for the next non-key item as a mask local_key_texture = local_key_texture ? local_key_texture : ogl_->create_texture(target_texture->width(), target_texture->height(), 1, depth_); @@ -190,20 +192,21 @@ class image_renderer draw_params.layer_key = nullptr; kernel_.draw(std::move(draw_params)); - } else if (item.transform.is_mix) { + } else if (item.transform.is_mix) { // A mix means precomp the items to a texture, before drawing to the channel local_mix_texture = local_mix_texture ? local_mix_texture : ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_); draw_params.background = local_mix_texture; - draw_params.local_key = std::move(local_key_texture); + draw_params.local_key = std::move(local_key_texture); // Use and reset the key draw_params.layer_key = layer_key_texture; draw_params.keyer = keyer::additive; kernel_.draw(std::move(draw_params)); } else { - draw(target_texture, std::move(local_mix_texture), core::blend_mode::normal); + // If there is a mix, this is the end so draw it and reset + draw(target_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal); draw_params.background = target_texture; draw_params.local_key = std::move(local_key_texture); @@ -214,17 +217,21 @@ class image_renderer } void draw(std::shared_ptr& target_texture, - std::shared_ptr&& source_buffer, + std::shared_ptr&& source_texture, + core::video_format_desc format_desc, core::blend_mode blend_mode = core::blend_mode::normal) { - if (!source_buffer) + if (!source_texture) return; draw_params draw_params; + draw_params.target_width = format_desc.square_width; + draw_params.target_height = format_desc.square_height; draw_params.pix_desc.format = core::pixel_format::bgra; draw_params.pix_desc.planes = { - core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4, source_buffer->depth())}; - draw_params.textures = {spl::make_shared_ptr(source_buffer)}; + core::pixel_format_desc::plane(source_texture->width(), source_texture->height(), 4, source_texture->depth()) + }; + draw_params.textures = { spl::make_shared_ptr(source_texture) }; draw_params.transform = core::image_transform(); draw_params.blend_mode = blend_mode; draw_params.background = target_texture; diff --git a/src/core/frame/geometry.cpp b/src/core/frame/geometry.cpp index 15fc95d920..016abdd3da 100644 --- a/src/core/frame/geometry.cpp +++ b/src/core/frame/geometry.cpp @@ -40,8 +40,8 @@ bool frame_geometry::coord::operator==(const frame_geometry::coord& other) const struct frame_geometry::impl { - impl(frame_geometry::geometry_type type, std::vector data) - : type_(type) + impl(frame_geometry::geometry_type type, frame_geometry::scale_mode mode, std::vector data) : + type_{type}, mode_{mode} { if (type == geometry_type::quad && data.size() != 4) CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("The number of coordinates needs to be 4")); @@ -50,42 +50,78 @@ struct frame_geometry::impl } frame_geometry::geometry_type type_; + frame_geometry::scale_mode mode_; std::vector data_; }; -frame_geometry::frame_geometry(geometry_type type, std::vector data) - : impl_(new impl(type, std::move(data))) -{ -} +frame_geometry::frame_geometry(geometry_type type, scale_mode mode, std::vector data) : + impl_{new impl{type, mode, std::move(data)}} +{ } frame_geometry::geometry_type frame_geometry::type() const { return impl_->type_; } +frame_geometry::scale_mode frame_geometry::mode() const { return impl_->mode_; } const std::vector& frame_geometry::data() const { return impl_->data_; } -const frame_geometry& frame_geometry::get_default() +const frame_geometry frame_geometry::get_default(scale_mode mode) { - static std::vector data = { + std::vector data{ // vertex texture {0.0, 0.0, 0.0, 0.0}, // upper left {1.0, 0.0, 1.0, 0.0}, // upper right {1.0, 1.0, 1.0, 1.0}, // lower right {0.0, 1.0, 0.0, 1.0} // lower left }; - static const frame_geometry g(frame_geometry::geometry_type::quad, data); - - return g; + return frame_geometry{ frame_geometry::geometry_type::quad, mode, std::move(data) }; } -const frame_geometry& frame_geometry::get_default_vflip() +const frame_geometry frame_geometry::get_default_vflip(scale_mode mode) { - static std::vector data = { + std::vector data{ // vertex texture {0.0, 0.0, 0.0, 1.0}, // upper left {1.0, 0.0, 1.0, 1.0}, // upper right {1.0, 1.0, 1.0, 0.0}, // lower right {0.0, 1.0, 0.0, 0.0} // lower left }; - static const frame_geometry g(frame_geometry::geometry_type::quad, data); + return frame_geometry{ frame_geometry::geometry_type::quad, mode, std::move(data) }; +} + +frame_geometry::scale_mode scale_mode_from_string(const std::wstring& str) { + auto str2 = boost::to_lower_copy(str); + if (str2 == L"fit") { + return frame_geometry::scale_mode::fit; + } + else if (str2 == L"fill") { + return frame_geometry::scale_mode::fill; + } + else if (str2 == L"original") { + return frame_geometry::scale_mode::original; + } + else if (str2 == L"hfill") { + return frame_geometry::scale_mode::hfill; + } + else if (str2 == L"vfill") { + return frame_geometry::scale_mode::vfill; + } + else { + return frame_geometry::scale_mode::stretch; + } +} - return g; +std::wstring scale_mode_to_string(frame_geometry::scale_mode mode) { + switch (mode) { + case frame_geometry::scale_mode::fit: + return L"FIT"; + case frame_geometry::scale_mode::fill: + return L"FILL"; + case frame_geometry::scale_mode::original: + return L"ORIGINAL"; + case frame_geometry::scale_mode::hfill: + return L"HFILL"; + case frame_geometry::scale_mode::vfill: + return L"VFILL"; + default: + return L"STRETCH"; + } } }} // namespace caspar::core diff --git a/src/core/frame/geometry.h b/src/core/frame/geometry.h index b003ca3f9b..48ae95a82f 100644 --- a/src/core/frame/geometry.h +++ b/src/core/frame/geometry.h @@ -35,6 +35,16 @@ class frame_geometry quad }; + enum class scale_mode + { + stretch, // default + fit, + fill, + original, + hfill, + vfill, + }; + struct coord { double vertex_x = 0.0; @@ -50,17 +60,21 @@ class frame_geometry bool operator==(const coord& other) const; }; - frame_geometry(geometry_type type, std::vector data); + frame_geometry(geometry_type type, scale_mode, std::vector data); geometry_type type() const; + scale_mode mode() const; const std::vector& data() const; - static const frame_geometry& get_default(); - static const frame_geometry& get_default_vflip(); + static const frame_geometry get_default(scale_mode = scale_mode::stretch); + static const frame_geometry get_default_vflip(scale_mode = scale_mode::stretch); private: struct impl; spl::shared_ptr impl_; }; +frame_geometry::scale_mode scale_mode_from_string(const std::wstring&); +std::wstring scale_mode_to_string(frame_geometry::scale_mode); + }} // namespace caspar::core diff --git a/src/modules/ffmpeg/producer/av_producer.cpp b/src/modules/ffmpeg/producer/av_producer.cpp index 64a06172d0..9cdc62ba47 100644 --- a/src/modules/ffmpeg/producer/av_producer.cpp +++ b/src/modules/ffmpeg/producer/av_producer.cpp @@ -665,6 +665,7 @@ struct AVProducer::Impl std::string vfilter_; int seekable_ = 2; + core::frame_geometry::scale_mode scale_mode_; int64_t frame_count_ = 0; bool frame_flush_ = true; int64_t frame_time_ = AV_NOPTS_VALUE; @@ -694,7 +695,8 @@ struct AVProducer::Impl std::optional seek, std::optional duration, bool loop, - int seekable) + int seekable, + core::frame_geometry::scale_mode scale_mode) : frame_factory_(frame_factory) , format_desc_(format_desc) , format_tb_({format_desc.duration, format_desc.time_scale * format_desc.field_count}) @@ -707,6 +709,7 @@ struct AVProducer::Impl , afilter_(afilter) , vfilter_(vfilter) , seekable_(seekable) + , scale_mode_(scale_mode) , video_executor_(L"video-executor") , audio_executor_(L"audio-executor") { @@ -913,7 +916,8 @@ struct AVProducer::Impl } frame.frame = core::draw_frame( - make_frame(this, *frame_factory_, frame.video, frame.audio, get_color_space(frame.video))); + make_frame(this, *frame_factory_, frame.video, frame.audio, get_color_space(frame.video), scale_mode_) + ); frame.frame_count = frame_count_++; graph_->set_value("decode-time", decode_timer.elapsed() * format_desc_.fps * 0.5); @@ -1237,7 +1241,8 @@ AVProducer::AVProducer(std::shared_ptr frame_factory, std::optional seek, std::optional duration, std::optional loop, - int seekable) + int seekable, + core::frame_geometry::scale_mode scale_mode) : impl_(new Impl(std::move(frame_factory), std::move(format_desc), std::move(name), @@ -1248,7 +1253,8 @@ AVProducer::AVProducer(std::shared_ptr frame_factory, std::move(seek), std::move(duration), std::move(loop.value_or(false)), - seekable)) + seekable, + scale_mode)) { } diff --git a/src/modules/ffmpeg/producer/av_producer.h b/src/modules/ffmpeg/producer/av_producer.h index 87b35dd9de..a5d8946889 100644 --- a/src/modules/ffmpeg/producer/av_producer.h +++ b/src/modules/ffmpeg/producer/av_producer.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -24,7 +25,8 @@ class AVProducer std::optional seek, std::optional duration, std::optional loop, - int seekable); + int seekable, + core::frame_geometry::scale_mode scale_mode); core::draw_frame prev_frame(const core::video_field field); core::draw_frame next_frame(const core::video_field field); diff --git a/src/modules/ffmpeg/producer/ffmpeg_producer.cpp b/src/modules/ffmpeg/producer/ffmpeg_producer.cpp index 57eb7b6179..7c43bdf58b 100644 --- a/src/modules/ffmpeg/producer/ffmpeg_producer.cpp +++ b/src/modules/ffmpeg/producer/ffmpeg_producer.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include @@ -73,7 +74,8 @@ struct ffmpeg_producer : public core::frame_producer std::optional seek, std::optional duration, std::optional loop, - int seekable) + int seekable, + core::frame_geometry::scale_mode scale_mode) : filename_(filename) , frame_factory_(frame_factory) , format_desc_(format_desc) @@ -87,9 +89,9 @@ struct ffmpeg_producer : public core::frame_producer seek, duration, loop, - seekable)) - { - } + seekable, + scale_mode)) + { } ~ffmpeg_producer() { @@ -317,6 +319,8 @@ spl::shared_ptr create_producer(const core::frame_producer auto filter_str = get_param(L"FILTER", params, L""); + auto scale_mode = core::scale_mode_from_string(get_param(L"SCALE_MODE", params, L"STRETCH")); + boost::ireplace_all(filter_str, L"DEINTERLACE_BOB", L"YADIF=1:-1"); boost::ireplace_all(filter_str, L"DEINTERLACE_LQ", L"SEPARATEFIELDS"); boost::ireplace_all(filter_str, L"DEINTERLACE", L"YADIF=0:-1"); @@ -350,7 +354,8 @@ spl::shared_ptr create_producer(const core::frame_producer seek2, duration, loop, - seekable); + seekable, + scale_mode); return core::create_destroy_proxy(std::move(producer)); } catch (...) { CASPAR_LOG_CURRENT_EXCEPTION(); diff --git a/src/modules/ffmpeg/util/av_util.cpp b/src/modules/ffmpeg/util/av_util.cpp index cf88307b40..c719ad3bd1 100644 --- a/src/modules/ffmpeg/util/av_util.cpp +++ b/src/modules/ffmpeg/util/av_util.cpp @@ -47,7 +47,8 @@ core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory, std::shared_ptr video, std::shared_ptr audio, - core::color_space color_space) + core::color_space color_space, + core::frame_geometry::scale_mode scale_mode) { std::vector data_map; // TODO(perf) when using data_map, avoid uploading duplicate planes @@ -57,6 +58,9 @@ core::mutable_frame make_frame(void* tag, : core::pixel_format_desc(core::pixel_format::invalid); auto frame = frame_factory.create_frame(tag, pix_desc); + if (scale_mode != core::frame_geometry::scale_mode::stretch) { + frame.geometry() = core::frame_geometry::get_default(scale_mode); + } tbb::parallel_invoke( [&]() { diff --git a/src/modules/ffmpeg/util/av_util.h b/src/modules/ffmpeg/util/av_util.h index 6680a380b2..84fcdb60cb 100644 --- a/src/modules/ffmpeg/util/av_util.h +++ b/src/modules/ffmpeg/util/av_util.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -31,7 +32,8 @@ core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory, std::shared_ptr video, std::shared_ptr audio, - core::color_space color_space = core::color_space::bt709); + core::color_space color_space = core::color_space::bt709, + core::frame_geometry::scale_mode = core::frame_geometry::scale_mode::stretch); std::shared_ptr make_av_video_frame(const core::const_frame& frame, const core::video_format_desc& format_des); std::shared_ptr make_av_audio_frame(const core::const_frame& frame, const core::video_format_desc& format_des); diff --git a/src/modules/image/producer/image_producer.cpp b/src/modules/image/producer/image_producer.cpp index 291e75e077..051eccb959 100644 --- a/src/modules/image/producer/image_producer.cpp +++ b/src/modules/image/producer/image_producer.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -63,35 +64,30 @@ struct image_producer : public core::frame_producer const uint32_t length_ = 0; core::draw_frame frame_; - image_producer(const spl::shared_ptr& frame_factory, std::wstring description, uint32_t length) - : description_(std::move(description)) - , frame_factory_(frame_factory) - , length_(length) + image_producer(const spl::shared_ptr& frame_factory, std::wstring description, uint32_t length, core::frame_geometry::scale_mode scale_mode) : + description_(std::move(description)), frame_factory_(frame_factory), length_(length) { - load(load_image(description_)); + load(load_image(description_), scale_mode); CASPAR_LOG(info) << print() << L" Initialized"; } - image_producer(const spl::shared_ptr& frame_factory, - const void* png_data, - size_t size, - uint32_t length) - : description_(L"png from memory") - , frame_factory_(frame_factory) - , length_(length) + image_producer(const spl::shared_ptr& frame_factory, const void* png_data, size_t size, uint32_t length, core::frame_geometry::scale_mode scale_mode) : + description_(L"png from memory"), frame_factory_(frame_factory), length_(length) { - load(load_png_from_memory(png_data, size)); + load(load_png_from_memory(png_data, size), scale_mode); CASPAR_LOG(info) << print() << L" Initialized"; } - void load(const std::shared_ptr& bitmap) + void load(const std::shared_ptr& bitmap, core::frame_geometry::scale_mode scale_mode) { FreeImage_FlipVertical(bitmap.get()); core::pixel_format_desc desc(core::pixel_format::bgra); desc.planes.emplace_back(FreeImage_GetWidth(bitmap.get()), FreeImage_GetHeight(bitmap.get()), 4); + auto frame = frame_factory_->create_frame(this, desc); + frame.geometry() = core::frame_geometry::get_default(scale_mode); std::copy_n(FreeImage_GetBits(bitmap.get()), frame.image_data(0).size(), frame.image_data(0).begin()); frame_ = core::draw_frame(std::move(frame)); @@ -120,19 +116,6 @@ struct image_producer : public core::frame_producer core::monitor::state state() const override { return state_; } }; -// class ieq -//{ -// std::wstring test_; -// -// public: -// explicit ieq(std::wstring test) -// : test_(std::move(test)) -// { -// } -// -// bool operator()(const std::wstring& elem) const { return boost::iequals(elem, test_); } -// }; - spl::shared_ptr create_producer(const core::frame_producer_dependencies& dependencies, const std::vector& params) { @@ -141,72 +124,14 @@ spl::shared_ptr create_producer(const core::frame_producer } auto length = get_param(L"LENGTH", params, std::numeric_limits::max()); + auto scale_mode = core::scale_mode_from_string(get_param(L"SCALE_MODE", params, L"STRETCH")); - // if (boost::iequals(params.at(0), L"[IMG_SEQUENCE]")) - //{ - // if (params.size() != 2) - // return core::frame_producer::empty(); - - // auto dir = boost::filesystem::path(env::media_folder() + params.at(1)).parent_path(); - // auto basename = boost::filesystem::basename(params.at(1)); - // std::set files; - // boost::filesystem::directory_iterator end; - - // for (boost::filesystem::directory_iterator it(dir); it != end; ++it) - // { - // auto name = it->path().filename().wstring(); - - // if (!boost::algorithm::istarts_with(name, basename)) - // continue; - - // auto extension = it->path().extension().wstring(); - - // if (std::find_if(supported_extensions().begin(), supported_extensions().end(), ieq(extension)) == - // supported_extensions().end()) continue; - - // files.insert(it->path().wstring()); - // } - - // if (files.empty()) - // return core::frame_producer::empty(); - - // int width = -1; - // int height = -1; - // std::vector frames; - // frames.reserve(files.size()); - - // for (auto& file : files) - // { - // auto frame = load_image(dependencies.frame_factory, file); - - // if (width == -1) - // { - // width = static_cast(frame.second.width.get()); - // height = static_cast(frame.second.height.get()); - // } - - // frames.push_back(std::move(frame.first)); - // } - - // return core::create_const_producer(std::move(frames), width, height); - //} - // else - // if (boost::iequals(params.at(0), L"[PNG_BASE64]")) { - // if (params.size() < 2) - // return core::frame_producer::empty(); - - // auto png_data = from_base64(std::string(params.at(1).begin(), params.at(1).end())); - - // return spl::make_shared(dependencies.frame_factory, png_data.data(), png_data.size(), length); - //} - - std::optional filename = - find_file_within_dir_or_absolute(env::media_folder(), params.at(0), is_valid_file); + auto filename = find_file_within_dir_or_absolute(env::media_folder(), params.at(0), is_valid_file); if (!filename) { return core::frame_producer::empty(); } - return spl::make_shared(dependencies.frame_factory, filename->wstring(), length); + return spl::make_shared(dependencies.frame_factory, filename->wstring(), length, scale_mode); } }} // namespace caspar::image