Add support for higher precision pixel formats

src/accelerator/accelerator.cpp

@@ -5,6 +5,8 @@
 
 #include <boost/property_tree/ptree.hpp>
 
+#include <common/bit_depth.h>
+
 #include <core/mixer/image/image_mixer.h>
 
 #include <memory>
@@ -23,10 +25,11 @@ struct accelerator::impl
     {
     }
 
-    std::unique_ptr<core::image_mixer> create_image_mixer(const int channel_id)
+    std::unique_ptr<core::image_mixer>
+    create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space)
     {
         return std::make_unique<ogl::image_mixer>(
-            spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size());
+            spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size(), depth, color_space);
     }
 
     std::shared_ptr<ogl::device> get_device()
@@ -46,9 +49,9 @@ accelerator::accelerator(const core::video_format_repository format_repository)
 
 accelerator::~accelerator() {}
 
-std::unique_ptr<core::image_mixer> accelerator::create_image_mixer(const int channel_id)
+std::unique_ptr<core::image_mixer> accelerator::create_image_mixer(const int channel_id, common::bit_depth depth, core::color_space color_space)
 {
-    return impl_->create_image_mixer(channel_id);
+    return impl_->create_image_mixer(channel_id, depth, color_space);
 }
 
 std::shared_ptr<accelerator_device> accelerator::get_device() const

src/accelerator/accelerator.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <common/bit_depth.h>
+
+#include <core/frame/pixel_format.h>
 #include <core/mixer/mixer.h>
 #include <core/video_format.h>
 
@@ -27,7 +30,8 @@ class accelerator
 
     accelerator& operator=(accelerator&) = delete;
 
-    std::unique_ptr<caspar::core::image_mixer> create_image_mixer(int channel_id);
+    std::unique_ptr<caspar::core::image_mixer>
+    create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space);
 
     std::shared_ptr<accelerator_device> get_device() const;
 

src/accelerator/ogl/image/image_kernel.cpp

@@ -81,6 +81,22 @@ double hypotenuse(double x1, double y1, double x2, double y2)
     return std::sqrt(x * x + y * y);
 }
 
+double get_precision_factor(common::bit_depth depth)
+{
+    switch (depth) {
+        case common::bit_depth::bit8:
+            return 1.0;
+        case common::bit_depth::bit10:
+            return 64.0;
+        case common::bit_depth::bit12:
+            return 16.0;
+        case common::bit_depth::bit16:
+            return 1.0;
+        default:
+            return 1.0;
+    }
+}
+
 double calc_q(double close_diagonal, double distant_diagonal)
 {
     return (close_diagonal + distant_diagonal) / distant_diagonal;
@@ -221,10 +237,13 @@ struct image_kernel::impl
             return;
         }
 
+        double precision_factor[4] = {1, 1, 1, 1};
+
         // Bind textures
 
         for (int n = 0; n < params.textures.size(); ++n) {
             params.textures[n]->bind(n);
+            precision_factor[n] = get_precision_factor(params.textures[n]->depth());
         }
 
         if (params.local_key) {
@@ -234,18 +253,35 @@ struct image_kernel::impl
         if (params.layer_key) {
             params.layer_key->bind(static_cast<int>(texture_id::layer_key));
         }
+
+        const auto is_hd = params.pix_desc.planes.at(0).height > 700;
+        const auto color_space = is_hd ? params.pix_desc.color_space : core::color_space::bt601;
 
-        // Setup shader
+        const float color_matrices[3][9] = {{1.0, 0.0, 1.402, 1.0, -0.344, -0.509, 1.0, 1.772, 0.0},            //bt.601
+                                            {1.0, 0.0, 1.5748, 1.0, -0.1873, -0.4681, 1.0, 1.8556, 0.0},        //bt.709
+                                            {1.0, 0.0, 1.4746, 1.0, -0.16455312684366, -0.57135312684366, 1.0, 1.8814, 0.0}};   //bt.2020
+        const auto color_matrix = color_matrices[static_cast<int>(color_space)];
 
+        const float luma_coefficients[3][3] = {{0.299, 0.587, 0.114},       //bt.601
+                                               {0.2126, 0.7152, 0.0722},    //bt.709
+                                               {0.2627, 0.6780, 0.0593}};   //bt.2020
+        const auto luma_coeff = luma_coefficients[static_cast<int>(color_space)];
+
+        // Setup shader
         shader_->use();
 
         shader_->set("plane[0]", texture_id::plane0);
         shader_->set("plane[1]", texture_id::plane1);
         shader_->set("plane[2]", texture_id::plane2);
         shader_->set("plane[3]", texture_id::plane3);
+        shader_->set("precision_factor[0]", precision_factor[0]);
+        shader_->set("precision_factor[1]", precision_factor[1]);
+        shader_->set("precision_factor[2]", precision_factor[2]);
+        shader_->set("precision_factor[3]", precision_factor[3]);
         shader_->set("local_key", texture_id::local_key);
         shader_->set("layer_key", texture_id::layer_key);
-        shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
+        shader_->set_matrix3("color_matrix", color_matrix);
+        shader_->set("luma_coeff", luma_coeff[0], luma_coeff[1], luma_coeff[2]);
         shader_->set("has_local_key", static_cast<bool>(params.local_key));
         shader_->set("has_layer_key", static_cast<bool>(params.layer_key));
         shader_->set("pixel_format", params.pix_desc.format);

src/accelerator/ogl/image/image_mixer.cpp

@@ -27,6 +27,7 @@
 #include "../util/texture.h"
 
 #include <common/array.h>
+#include <common/bit_depth.h>
 #include <common/future.h>
 #include <common/log.h>
 
@@ -70,12 +71,19 @@ class image_renderer
     spl::shared_ptr<device> ogl_;
     image_kernel            kernel_;
     const size_t            max_frame_size_;
+    common::bit_depth       depth_;
+    core::color_space       color_space_;
 
   public:
-    explicit image_renderer(const spl::shared_ptr<device>& ogl, const size_t max_frame_size)
+    explicit image_renderer(const spl::shared_ptr<device>& ogl,
+                            const size_t                   max_frame_size,
+                            common::bit_depth              depth,
+                            core::color_space              color_space)
         : ogl_(ogl)
         , kernel_(ogl_)
         , max_frame_size_(max_frame_size)
+        , depth_(depth)
+        , color_space_(color_space)
     {
     }
 
@@ -88,14 +96,17 @@ class image_renderer
         }
 
         return flatten(ogl_->dispatch_async([=]() mutable -> std::shared_future<array<const std::uint8_t>> {
-            auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4);
+            auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);
 
             draw(target_texture, std::move(layers), format_desc);
 
             return ogl_->copy_async(target_texture);
         }));
     }
 
+    common::bit_depth depth() const { return depth_; }
+    core::color_space color_space() const { return color_space_; }
+
   private:
     void draw(std::shared_ptr<texture>&      target_texture,
               std::vector<layer>             layers,
@@ -121,7 +132,7 @@ class image_renderer
         std::shared_ptr<texture> local_mix_texture;
 
         if (layer.blend_mode != core::blend_mode::normal) {
-            auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4);
+            auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_);
 
             for (auto& item : layer.items)
                 draw(layer_texture,
@@ -157,6 +168,8 @@ class image_renderer
               const core::video_format_desc& format_desc)
     {
         draw_params draw_params;
+        // TODO: Pass the target color_space 
+
         draw_params.pix_desc  = std::move(item.pix_desc);
         draw_params.transform = std::move(item.transform);
         draw_params.geometry  = item.geometry;
@@ -168,19 +181,19 @@ class image_renderer
         }
 
         if (item.transform.is_key) {
-            local_key_texture = local_key_texture
-                                    ? local_key_texture
-                                    : ogl_->create_texture(target_texture->width(), target_texture->height(), 1);
+            local_key_texture =
+                local_key_texture ? local_key_texture
+                                  : ogl_->create_texture(target_texture->width(), target_texture->height(), 1, depth_);
 
             draw_params.background = local_key_texture;
             draw_params.local_key  = nullptr;
             draw_params.layer_key  = nullptr;
 
             kernel_.draw(std::move(draw_params));
         } else if (item.transform.is_mix) {
-            local_mix_texture = local_mix_texture
-                                    ? local_mix_texture
-                                    : ogl_->create_texture(target_texture->width(), target_texture->height(), 4);
+            local_mix_texture =
+                local_mix_texture ? local_mix_texture
+                                  : ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_);
 
             draw_params.background = local_mix_texture;
             draw_params.local_key  = std::move(local_key_texture);
@@ -210,7 +223,7 @@ class image_renderer
         draw_params draw_params;
         draw_params.pix_desc.format = core::pixel_format::bgra;
         draw_params.pix_desc.planes = {
-            core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4)};
+            core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4, source_buffer->depth())};
         draw_params.textures   = {spl::make_shared_ptr(source_buffer)};
         draw_params.transform  = core::image_transform();
         draw_params.blend_mode = blend_mode;
@@ -232,9 +245,13 @@ struct image_mixer::impl
     std::vector<layer*>                layer_stack_;
 
   public:
-    impl(const spl::shared_ptr<device>& ogl, const int channel_id, const size_t max_frame_size)
+    impl(const spl::shared_ptr<device>& ogl,
+         const int                      channel_id,
+         const size_t                   max_frame_size,
+         common::bit_depth              depth,
+         core::color_space              color_space)
         : ogl_(ogl)
-        , renderer_(ogl, max_frame_size)
+        , renderer_(ogl, max_frame_size, depth, color_space)
         , transform_stack_(1)
     {
         CASPAR_LOG(info) << L"Initialized OpenGL Accelerated GPU Image Mixer for channel " << channel_id;
@@ -281,7 +298,8 @@ struct image_mixer::impl
                 item.textures.emplace_back(ogl_->copy_async(frame.image_data(n),
                                                             item.pix_desc.planes[n].width,
                                                             item.pix_desc.planes[n].height,
-                                                            item.pix_desc.planes[n].stride));
+                                                            item.pix_desc.planes[n].stride,
+                                                            item.pix_desc.planes[n].depth));
             }
         }
 
@@ -300,10 +318,17 @@ struct image_mixer::impl
     }
 
     core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override
+    {
+        return create_frame(tag, desc, common::bit_depth::bit8);
+    }
+
+    core::mutable_frame
+    create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth) override
     {
         std::vector<array<std::uint8_t>> image_data;
         for (auto& plane : desc.planes) {
-            image_data.push_back(ogl_->create_array(plane.size));
+            auto bytes_per_pixel = depth == common::bit_depth::bit8 ? 1 : 2;
+            image_data.push_back(ogl_->create_array(plane.size * bytes_per_pixel));
         }
 
         std::weak_ptr<image_mixer::impl> weak_self = shared_from_this();
@@ -319,16 +344,26 @@ struct image_mixer::impl
                 }
                 std::vector<future_texture> textures;
                 for (int n = 0; n < static_cast<int>(desc.planes.size()); ++n) {
-                    textures.emplace_back(self->ogl_->copy_async(
-                        image_data[n], desc.planes[n].width, desc.planes[n].height, desc.planes[n].stride));
+                    textures.emplace_back(self->ogl_->copy_async(image_data[n],
+                                                                 desc.planes[n].width,
+                                                                 desc.planes[n].height,
+                                                                 desc.planes[n].stride,
+                                                                 desc.planes[n].depth));
                 }
                 return std::make_shared<decltype(textures)>(std::move(textures));
             });
     }
+
+    common::bit_depth depth() const { return renderer_.depth(); }
+    core::color_space color_space() const { return renderer_.color_space(); }
 };
 
-image_mixer::image_mixer(const spl::shared_ptr<device>& ogl, const int channel_id, const size_t max_frame_size)
-    : impl_(std::make_unique<impl>(ogl, channel_id, max_frame_size))
+image_mixer::image_mixer(const spl::shared_ptr<device>& ogl,
+                         const int                      channel_id,
+                         const size_t                   max_frame_size,
+                         common::bit_depth              depth,
+                         core::color_space              color_space)
+    : impl_(std::make_unique<impl>(ogl, channel_id, max_frame_size, depth, color_space))
 {
 }
 image_mixer::~image_mixer() {}
@@ -343,5 +378,13 @@ core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel
 {
     return impl_->create_frame(tag, desc);
 }
+core::mutable_frame
+image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth)
+{
+    return impl_->create_frame(tag, desc, depth);
+}
+
+common::bit_depth image_mixer::depth() const { return impl_->depth(); }
+core::color_space image_mixer::color_space() const { return impl_->color_space(); }
 
 }}} // namespace caspar::accelerator::ogl

src/accelerator/ogl/image/image_mixer.h

@@ -22,6 +22,7 @@
 #pragma once
 
 #include <common/array.h>
+#include <common/bit_depth.h>
 #include <common/memory.h>
 
 #include <core/frame/frame.h>
@@ -36,7 +37,11 @@ namespace caspar { namespace accelerator { namespace ogl {
 class image_mixer final : public core::image_mixer
 {
   public:
-    image_mixer(const spl::shared_ptr<class device>& ogl, int channel_id, const size_t max_frame_size);
+    image_mixer(const spl::shared_ptr<class device>& ogl,
+                int                                  channel_id,
+                const size_t                         max_frame_size,
+                common::bit_depth                    depth,
+                core::color_space                    color_space);
     image_mixer(const image_mixer&) = delete;
 
     ~image_mixer();
@@ -45,12 +50,16 @@ class image_mixer final : public core::image_mixer
 
     std::future<array<const std::uint8_t>> operator()(const core::video_format_desc& format_desc) override;
     core::mutable_frame                    create_frame(const void* tag, const core::pixel_format_desc& desc) override;
+    core::mutable_frame
+    create_frame(const void* video_stream_tag, const core::pixel_format_desc& desc, common::bit_depth depth) override;
 
     // core::image_mixer
 
-    void push(const core::frame_transform& frame) override;
-    void visit(const core::const_frame& frame) override;
-    void pop() override;
+    void              push(const core::frame_transform& frame) override;
+    void              visit(const core::const_frame& frame) override;
+    void              pop() override;
+    common::bit_depth depth() const override;
+    core::color_space color_space() const override;
 
   private:
     struct impl;