feat: support for scaling modes (#1566)

src/accelerator/ogl/image/image_kernel.cpp

@@ -220,6 +220,42 @@ struct image_kernel::impl
             coord.vertex_y += f_p[1];
         };
 
+		auto const first_plane = params.pix_desc.planes.at(0);
+		if (params.geometry.mode() != core::frame_geometry::scale_mode::stretch && first_plane.width > 0 && first_plane.height > 0) {
+			auto width_scale = static_cast<double>(params.target_width) / static_cast<double>(first_plane.width);
+			auto height_scale = static_cast<double>(params.target_height) / static_cast<double>(first_plane.height);
+
+            double target_scale;
+			switch (params.geometry.mode()) {
+                case core::frame_geometry::scale_mode::fit:
+                    target_scale = std::min(width_scale, height_scale);
+                    f_s[0] *= target_scale / width_scale;
+                    f_s[1] *= target_scale / height_scale;
+                    break;
+
+                case core::frame_geometry::scale_mode::fill:
+                    target_scale = std::max(width_scale, height_scale);
+                    f_s[0] *= target_scale / width_scale;
+                    f_s[1] *= target_scale / height_scale;
+                    break;
+
+                case core::frame_geometry::scale_mode::original:
+                    f_s[0] /= width_scale;
+                    f_s[1] /= height_scale;
+                    break;
+
+                case core::frame_geometry::scale_mode::hfill:
+                    f_s[1] *= width_scale / height_scale;
+                    break;
+
+                case core::frame_geometry::scale_mode::vfill:
+                    f_s[0] *= height_scale / width_scale;
+                    break;
+
+                default:;
+			}
+		}
+
         int corner = 0;
         for (auto& coord : coords) {
             do_crop(coord);

src/accelerator/ogl/image/image_kernel.h

@@ -49,6 +49,8 @@ struct draw_params final
     std::shared_ptr<class texture>              local_key;
     std::shared_ptr<class texture>              layer_key;
     double                                      aspect_ratio = 1.0;
+    int                                         target_width;
+    int                                         target_height;
 };
 
 class image_kernel final

src/accelerator/ogl/image/image_mixer.cpp

@@ -95,7 +95,7 @@ class image_renderer
             return make_ready_future(array<const std::uint8_t>(buffer.data(), format_desc.size, true));
         }
 
-        return flatten(ogl_->dispatch_async([=]() mutable -> std::shared_future<array<const std::uint8_t>> {
+        return flatten(ogl_->dispatch_async([=, layers = std::move(layers)]() mutable -> std::shared_future<array<const std::uint8_t>> {
             auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);
 
             draw(target_texture, std::move(layers), format_desc);
@@ -142,8 +142,8 @@ class image_renderer
                      local_mix_texture,
                      format_desc);
 
-            draw(layer_texture, std::move(local_mix_texture), core::blend_mode::normal);
-            draw(target_texture, std::move(layer_texture), layer.blend_mode);
+            draw(layer_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal);
+            draw(target_texture, std::move(layer_texture), format_desc, layer.blend_mode);
         } else // fast path
         {
             for (auto& item : layer.items)
@@ -154,7 +154,7 @@ class image_renderer
                      local_mix_texture,
                      format_desc);
 
-            draw(target_texture, std::move(local_mix_texture), core::blend_mode::normal);
+            draw(target_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal);
         }
 
         layer_key_texture = std::move(local_key_texture);
@@ -168,6 +168,8 @@ class image_renderer
               const core::video_format_desc& format_desc)
     {
         draw_params draw_params;
+        draw_params.target_width  = format_desc.square_width;
+        draw_params.target_height = format_desc.square_height;
         // TODO: Pass the target color_space
 
         draw_params.pix_desc  = std::move(item.pix_desc);
@@ -180,7 +182,7 @@ class image_renderer
             draw_params.textures.push_back(spl::make_shared_ptr(future_texture.get()));
         }
 
-        if (item.transform.is_key) {
+        if (item.transform.is_key) { // A key means we will use it for the next non-key item as a mask
             local_key_texture =
                 local_key_texture ? local_key_texture
                                   : ogl_->create_texture(target_texture->width(), target_texture->height(), 1, depth_);
@@ -190,20 +192,21 @@ class image_renderer
             draw_params.layer_key  = nullptr;
 
             kernel_.draw(std::move(draw_params));
-        } else if (item.transform.is_mix) {
+        } else if (item.transform.is_mix) { // A mix means precomp the items to a texture, before drawing to the channel
             local_mix_texture =
                 local_mix_texture ? local_mix_texture
                                   : ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_);
 
             draw_params.background = local_mix_texture;
-            draw_params.local_key  = std::move(local_key_texture);
+            draw_params.local_key  = std::move(local_key_texture); // Use and reset the key
             draw_params.layer_key  = layer_key_texture;
 
             draw_params.keyer = keyer::additive;
 
             kernel_.draw(std::move(draw_params));
         } else {
-            draw(target_texture, std::move(local_mix_texture), core::blend_mode::normal);
+            // If there is a mix, this is the end so draw it and reset
+            draw(target_texture, std::move(local_mix_texture), format_desc, core::blend_mode::normal);
 
             draw_params.background = target_texture;
             draw_params.local_key  = std::move(local_key_texture);
@@ -214,17 +217,21 @@ class image_renderer
     }
 
     void draw(std::shared_ptr<texture>&  target_texture,
-              std::shared_ptr<texture>&& source_buffer,
+              std::shared_ptr<texture>&& source_texture,
+              core::video_format_desc    format_desc,
               core::blend_mode           blend_mode = core::blend_mode::normal)
     {
-        if (!source_buffer)
+        if (!source_texture)
             return;
 
         draw_params draw_params;
+        draw_params.target_width    = format_desc.square_width;
+        draw_params.target_height   = format_desc.square_height;
         draw_params.pix_desc.format = core::pixel_format::bgra;
         draw_params.pix_desc.planes = {
-            core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4, source_buffer->depth())};
-        draw_params.textures   = {spl::make_shared_ptr(source_buffer)};
+            core::pixel_format_desc::plane(source_texture->width(), source_texture->height(), 4, source_texture->depth())
+        };
+        draw_params.textures   = { spl::make_shared_ptr(source_texture) };
         draw_params.transform  = core::image_transform();
         draw_params.blend_mode = blend_mode;
         draw_params.background = target_texture;

src/core/frame/geometry.cpp

@@ -40,8 +40,8 @@ bool frame_geometry::coord::operator==(const frame_geometry::coord& other) const
 
 struct frame_geometry::impl
 {
-    impl(frame_geometry::geometry_type type, std::vector<coord> data)
-        : type_(type)
+    impl(frame_geometry::geometry_type type, frame_geometry::scale_mode mode, std::vector<coord> data) :
+        type_{type}, mode_{mode}
     {
         if (type == geometry_type::quad && data.size() != 4)
             CASPAR_THROW_EXCEPTION(invalid_argument() << msg_info("The number of coordinates needs to be 4"));
@@ -50,42 +50,78 @@ struct frame_geometry::impl
     }
 
     frame_geometry::geometry_type type_;
+    frame_geometry::scale_mode    mode_;
     std::vector<coord>            data_;
 };
 
-frame_geometry::frame_geometry(geometry_type type, std::vector<coord> data)
-    : impl_(new impl(type, std::move(data)))
-{
-}
+frame_geometry::frame_geometry(geometry_type type, scale_mode mode, std::vector<coord> data) :
+    impl_{new impl{type, mode, std::move(data)}}
+{ }
 
 frame_geometry::geometry_type             frame_geometry::type() const { return impl_->type_; }
+frame_geometry::scale_mode                frame_geometry::mode() const { return impl_->mode_; }
 const std::vector<frame_geometry::coord>& frame_geometry::data() const { return impl_->data_; }
 
-const frame_geometry& frame_geometry::get_default()
+const frame_geometry frame_geometry::get_default(scale_mode mode)
 {
-    static std::vector<frame_geometry::coord> data = {
+    std::vector<frame_geometry::coord> data{
         //    vertex    texture
         {0.0, 0.0, 0.0, 0.0}, // upper left
         {1.0, 0.0, 1.0, 0.0}, // upper right
         {1.0, 1.0, 1.0, 1.0}, // lower right
         {0.0, 1.0, 0.0, 1.0}  // lower left
     };
-    static const frame_geometry g(frame_geometry::geometry_type::quad, data);
-
-    return g;
+    return frame_geometry{ frame_geometry::geometry_type::quad, mode, std::move(data) };
 }
-const frame_geometry& frame_geometry::get_default_vflip()
+const frame_geometry frame_geometry::get_default_vflip(scale_mode mode)
 {
-    static std::vector<frame_geometry::coord> data = {
+    std::vector<frame_geometry::coord> data{
         //    vertex    texture
         {0.0, 0.0, 0.0, 1.0}, // upper left
         {1.0, 0.0, 1.0, 1.0}, // upper right
         {1.0, 1.0, 1.0, 0.0}, // lower right
         {0.0, 1.0, 0.0, 0.0}  // lower left
     };
-    static const frame_geometry g(frame_geometry::geometry_type::quad, data);
+    return frame_geometry{ frame_geometry::geometry_type::quad, mode, std::move(data) };
+}
+
+frame_geometry::scale_mode scale_mode_from_string(const std::wstring& str) {
+	auto str2 = boost::to_lower_copy(str);
+	if (str2 == L"fit") {
+		return frame_geometry::scale_mode::fit;
+	}
+	else if (str2 == L"fill") {
+		return frame_geometry::scale_mode::fill;
+	}
+	else if (str2 == L"original") {
+		return frame_geometry::scale_mode::original;
+	}
+	else if (str2 == L"hfill") {
+		return frame_geometry::scale_mode::hfill;
+	}
+	else if (str2 == L"vfill") {
+		return frame_geometry::scale_mode::vfill;
+	}
+	else {
+		return frame_geometry::scale_mode::stretch;
+	}
+}
 
-    return g;
+std::wstring scale_mode_to_string(frame_geometry::scale_mode mode) {
+	switch (mode) {
+	case frame_geometry::scale_mode::fit:
+		return L"FIT";
+	case frame_geometry::scale_mode::fill:
+		return L"FILL";
+	case frame_geometry::scale_mode::original:
+		return L"ORIGINAL";
+	case frame_geometry::scale_mode::hfill:
+		return L"HFILL";
+	case frame_geometry::scale_mode::vfill:
+		return L"VFILL";
+	default:
+		return L"STRETCH";
+	}
 }
 
 }} // namespace caspar::core

src/core/frame/geometry.h

@@ -35,6 +35,16 @@ class frame_geometry
         quad
     };
 
+    enum class scale_mode
+    {
+        stretch, // default
+        fit,
+        fill,
+        original,
+        hfill,
+        vfill,
+    };
+
     struct coord
     {
         double vertex_x  = 0.0;
@@ -50,17 +60,21 @@ class frame_geometry
         bool operator==(const coord& other) const;
     };
 
-    frame_geometry(geometry_type type, std::vector<coord> data);
+    frame_geometry(geometry_type type, scale_mode, std::vector<coord> data);
 
     geometry_type             type() const;
+    scale_mode                mode() const;
     const std::vector<coord>& data() const;
 
-    static const frame_geometry& get_default();
-    static const frame_geometry& get_default_vflip();
+    static const frame_geometry get_default(scale_mode = scale_mode::stretch);
+    static const frame_geometry get_default_vflip(scale_mode = scale_mode::stretch);
 
   private:
     struct impl;
     spl::shared_ptr<impl> impl_;
 };
 
+frame_geometry::scale_mode scale_mode_from_string(const std::wstring&);
+std::wstring scale_mode_to_string(frame_geometry::scale_mode);
+
 }} // namespace caspar::core

src/modules/ffmpeg/producer/av_producer.cpp

@@ -665,6 +665,7 @@ struct AVProducer::Impl
     std::string vfilter_;
 
     int              seekable_       = 2;
+    core::frame_geometry::scale_mode scale_mode_;
     int64_t          frame_count_    = 0;
     bool             frame_flush_    = true;
     int64_t          frame_time_     = AV_NOPTS_VALUE;
@@ -694,7 +695,8 @@ struct AVProducer::Impl
          std::optional<int64_t>               seek,
          std::optional<int64_t>               duration,
          bool                                 loop,
-         int                                  seekable)
+         int                                  seekable,
+         core::frame_geometry::scale_mode     scale_mode)
         : frame_factory_(frame_factory)
         , format_desc_(format_desc)
         , format_tb_({format_desc.duration, format_desc.time_scale * format_desc.field_count})
@@ -707,6 +709,7 @@ struct AVProducer::Impl
         , afilter_(afilter)
         , vfilter_(vfilter)
         , seekable_(seekable)
+        , scale_mode_(scale_mode)
         , video_executor_(L"video-executor")
         , audio_executor_(L"audio-executor")
     {
@@ -913,7 +916,8 @@ struct AVProducer::Impl
             }
 
             frame.frame = core::draw_frame(
-                make_frame(this, *frame_factory_, frame.video, frame.audio, get_color_space(frame.video)));
+                make_frame(this, *frame_factory_, frame.video, frame.audio, get_color_space(frame.video), scale_mode_)
+            );
             frame.frame_count = frame_count_++;
 
             graph_->set_value("decode-time", decode_timer.elapsed() * format_desc_.fps * 0.5);
@@ -1237,7 +1241,8 @@ AVProducer::AVProducer(std::shared_ptr<core::frame_factory> frame_factory,
                        std::optional<int64_t>               seek,
                        std::optional<int64_t>               duration,
                        std::optional<bool>                  loop,
-                       int                                  seekable)
+                       int                                  seekable,
+                       core::frame_geometry::scale_mode     scale_mode)
     : impl_(new Impl(std::move(frame_factory),
                      std::move(format_desc),
                      std::move(name),
@@ -1248,7 +1253,8 @@ AVProducer::AVProducer(std::shared_ptr<core::frame_factory> frame_factory,
                      std::move(seek),
                      std::move(duration),
                      std::move(loop.value_or(false)),
-                     seekable))
+                     seekable,
+                     scale_mode))
 {
 }
 

src/modules/ffmpeg/producer/av_producer.h

@@ -2,6 +2,7 @@
 
 #include <core/frame/draw_frame.h>
 #include <core/frame/frame_factory.h>
+#include <core/frame/geometry.h>
 #include <core/monitor/monitor.h>
 #include <core/video_format.h>
 
@@ -24,7 +25,8 @@ class AVProducer
                std::optional<int64_t>               seek,
                std::optional<int64_t>               duration,
                std::optional<bool>                  loop,
-               int                                  seekable);
+               int                                  seekable,
+               core::frame_geometry::scale_mode     scale_mode);
 
     core::draw_frame prev_frame(const core::video_field field);
     core::draw_frame next_frame(const core::video_field field);