Skip to content

Commit

Permalink
Merge pull request CasparCG#1528 from niklaspandersson/16bit
Browse files Browse the repository at this point in the history
  • Loading branch information
@Julusian
Julusian authored May 24, 2024
2 parents d3af0b0 + 9082117 commit 2ffc202
Show file tree
Hide file tree
Showing 51 changed files with 1,122 additions and 361 deletions.
11 changes: 7 additions & 4 deletions src/accelerator/accelerator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

#include <boost/property_tree/ptree.hpp>

#include <common/bit_depth.h>

#include <core/mixer/image/image_mixer.h>

#include <memory>
Expand All @@ -23,10 +25,11 @@ struct accelerator::impl
{
}

std::unique_ptr<core::image_mixer> create_image_mixer(const int channel_id)
std::unique_ptr<core::image_mixer>
create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space)
{
return std::make_unique<ogl::image_mixer>(
spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size());
spl::make_shared_ptr(get_device()), channel_id, format_repository_.get_max_video_format_size(), depth, color_space);
}

std::shared_ptr<ogl::device> get_device()
Expand All @@ -46,9 +49,9 @@ accelerator::accelerator(const core::video_format_repository format_repository)

accelerator::~accelerator() {}

std::unique_ptr<core::image_mixer> accelerator::create_image_mixer(const int channel_id)
std::unique_ptr<core::image_mixer> accelerator::create_image_mixer(const int channel_id, common::bit_depth depth, core::color_space color_space)
{
return impl_->create_image_mixer(channel_id);
return impl_->create_image_mixer(channel_id, depth, color_space);
}

std::shared_ptr<accelerator_device> accelerator::get_device() const
Expand Down
6 changes: 5 additions & 1 deletion src/accelerator/accelerator.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#pragma once

#include <common/bit_depth.h>

#include <core/frame/pixel_format.h>
#include <core/mixer/mixer.h>
#include <core/video_format.h>

Expand Down Expand Up @@ -27,7 +30,8 @@ class accelerator

accelerator& operator=(accelerator&) = delete;

std::unique_ptr<caspar::core::image_mixer> create_image_mixer(int channel_id);
std::unique_ptr<caspar::core::image_mixer>
create_image_mixer(int channel_id, common::bit_depth depth, core::color_space color_space);

std::shared_ptr<accelerator_device> get_device() const;

Expand Down
40 changes: 38 additions & 2 deletions src/accelerator/ogl/image/image_kernel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,22 @@ double hypotenuse(double x1, double y1, double x2, double y2)
return std::sqrt(x * x + y * y);
}

double get_precision_factor(common::bit_depth depth)
{
switch (depth) {
case common::bit_depth::bit8:
return 1.0;
case common::bit_depth::bit10:
return 64.0;
case common::bit_depth::bit12:
return 16.0;
case common::bit_depth::bit16:
return 1.0;
default:
return 1.0;
}
}

double calc_q(double close_diagonal, double distant_diagonal)
{
return (close_diagonal + distant_diagonal) / distant_diagonal;
Expand Down Expand Up @@ -221,10 +237,13 @@ struct image_kernel::impl
return;
}

double precision_factor[4] = {1, 1, 1, 1};

// Bind textures

for (int n = 0; n < params.textures.size(); ++n) {
params.textures[n]->bind(n);
precision_factor[n] = get_precision_factor(params.textures[n]->depth());
}

if (params.local_key) {
Expand All @@ -234,18 +253,35 @@ struct image_kernel::impl
if (params.layer_key) {
params.layer_key->bind(static_cast<int>(texture_id::layer_key));
}

const auto is_hd = params.pix_desc.planes.at(0).height > 700;
const auto color_space = is_hd ? params.pix_desc.color_space : core::color_space::bt601;

// Setup shader
const float color_matrices[3][9] = {{1.0, 0.0, 1.402, 1.0, -0.344, -0.509, 1.0, 1.772, 0.0}, //bt.601
{1.0, 0.0, 1.5748, 1.0, -0.1873, -0.4681, 1.0, 1.8556, 0.0}, //bt.709
{1.0, 0.0, 1.4746, 1.0, -0.16455312684366, -0.57135312684366, 1.0, 1.8814, 0.0}}; //bt.2020
const auto color_matrix = color_matrices[static_cast<int>(color_space)];

const float luma_coefficients[3][3] = {{0.299, 0.587, 0.114}, //bt.601
{0.2126, 0.7152, 0.0722}, //bt.709
{0.2627, 0.6780, 0.0593}}; //bt.2020
const auto luma_coeff = luma_coefficients[static_cast<int>(color_space)];

// Setup shader
shader_->use();

shader_->set("plane[0]", texture_id::plane0);
shader_->set("plane[1]", texture_id::plane1);
shader_->set("plane[2]", texture_id::plane2);
shader_->set("plane[3]", texture_id::plane3);
shader_->set("precision_factor[0]", precision_factor[0]);
shader_->set("precision_factor[1]", precision_factor[1]);
shader_->set("precision_factor[2]", precision_factor[2]);
shader_->set("precision_factor[3]", precision_factor[3]);
shader_->set("local_key", texture_id::local_key);
shader_->set("layer_key", texture_id::layer_key);
shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
shader_->set_matrix3("color_matrix", color_matrix);
shader_->set("luma_coeff", luma_coeff[0], luma_coeff[1], luma_coeff[2]);
shader_->set("has_local_key", static_cast<bool>(params.local_key));
shader_->set("has_layer_key", static_cast<bool>(params.layer_key));
shader_->set("pixel_format", params.pix_desc.format);
Expand Down
79 changes: 61 additions & 18 deletions src/accelerator/ogl/image/image_mixer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "../util/texture.h"

#include <common/array.h>
#include <common/bit_depth.h>
#include <common/future.h>
#include <common/log.h>

Expand Down Expand Up @@ -70,12 +71,19 @@ class image_renderer
spl::shared_ptr<device> ogl_;
image_kernel kernel_;
const size_t max_frame_size_;
common::bit_depth depth_;
core::color_space color_space_;

public:
explicit image_renderer(const spl::shared_ptr<device>& ogl, const size_t max_frame_size)
explicit image_renderer(const spl::shared_ptr<device>& ogl,
const size_t max_frame_size,
common::bit_depth depth,
core::color_space color_space)
: ogl_(ogl)
, kernel_(ogl_)
, max_frame_size_(max_frame_size)
, depth_(depth)
, color_space_(color_space)
{
}

Expand All @@ -88,14 +96,17 @@ class image_renderer
}

return flatten(ogl_->dispatch_async([=]() mutable -> std::shared_future<array<const std::uint8_t>> {
auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4);
auto target_texture = ogl_->create_texture(format_desc.width, format_desc.height, 4, depth_);

draw(target_texture, std::move(layers), format_desc);

return ogl_->copy_async(target_texture);
}));
}

common::bit_depth depth() const { return depth_; }
core::color_space color_space() const { return color_space_; }

private:
void draw(std::shared_ptr<texture>& target_texture,
std::vector<layer> layers,
Expand All @@ -121,7 +132,7 @@ class image_renderer
std::shared_ptr<texture> local_mix_texture;

if (layer.blend_mode != core::blend_mode::normal) {
auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4);
auto layer_texture = ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_);

for (auto& item : layer.items)
draw(layer_texture,
Expand Down Expand Up @@ -157,6 +168,8 @@ class image_renderer
const core::video_format_desc& format_desc)
{
draw_params draw_params;
// TODO: Pass the target color_space

draw_params.pix_desc = std::move(item.pix_desc);
draw_params.transform = std::move(item.transform);
draw_params.geometry = item.geometry;
Expand All @@ -168,19 +181,19 @@ class image_renderer
}

if (item.transform.is_key) {
local_key_texture = local_key_texture
? local_key_texture
: ogl_->create_texture(target_texture->width(), target_texture->height(), 1);
local_key_texture =
local_key_texture ? local_key_texture
: ogl_->create_texture(target_texture->width(), target_texture->height(), 1, depth_);

draw_params.background = local_key_texture;
draw_params.local_key = nullptr;
draw_params.layer_key = nullptr;

kernel_.draw(std::move(draw_params));
} else if (item.transform.is_mix) {
local_mix_texture = local_mix_texture
? local_mix_texture
: ogl_->create_texture(target_texture->width(), target_texture->height(), 4);
local_mix_texture =
local_mix_texture ? local_mix_texture
: ogl_->create_texture(target_texture->width(), target_texture->height(), 4, depth_);

draw_params.background = local_mix_texture;
draw_params.local_key = std::move(local_key_texture);
Expand Down Expand Up @@ -210,7 +223,7 @@ class image_renderer
draw_params draw_params;
draw_params.pix_desc.format = core::pixel_format::bgra;
draw_params.pix_desc.planes = {
core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4)};
core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4, source_buffer->depth())};
draw_params.textures = {spl::make_shared_ptr(source_buffer)};
draw_params.transform = core::image_transform();
draw_params.blend_mode = blend_mode;
Expand All @@ -232,9 +245,13 @@ struct image_mixer::impl
std::vector<layer*> layer_stack_;

public:
impl(const spl::shared_ptr<device>& ogl, const int channel_id, const size_t max_frame_size)
impl(const spl::shared_ptr<device>& ogl,
const int channel_id,
const size_t max_frame_size,
common::bit_depth depth,
core::color_space color_space)
: ogl_(ogl)
, renderer_(ogl, max_frame_size)
, renderer_(ogl, max_frame_size, depth, color_space)
, transform_stack_(1)
{
CASPAR_LOG(info) << L"Initialized OpenGL Accelerated GPU Image Mixer for channel " << channel_id;
Expand Down Expand Up @@ -281,7 +298,8 @@ struct image_mixer::impl
item.textures.emplace_back(ogl_->copy_async(frame.image_data(n),
item.pix_desc.planes[n].width,
item.pix_desc.planes[n].height,
item.pix_desc.planes[n].stride));
item.pix_desc.planes[n].stride,
item.pix_desc.planes[n].depth));
}
}

Expand All @@ -300,10 +318,17 @@ struct image_mixer::impl
}

core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override
{
return create_frame(tag, desc, common::bit_depth::bit8);
}

core::mutable_frame
create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth) override
{
std::vector<array<std::uint8_t>> image_data;
for (auto& plane : desc.planes) {
image_data.push_back(ogl_->create_array(plane.size));
auto bytes_per_pixel = depth == common::bit_depth::bit8 ? 1 : 2;
image_data.push_back(ogl_->create_array(plane.size * bytes_per_pixel));
}

std::weak_ptr<image_mixer::impl> weak_self = shared_from_this();
Expand All @@ -319,16 +344,26 @@ struct image_mixer::impl
}
std::vector<future_texture> textures;
for (int n = 0; n < static_cast<int>(desc.planes.size()); ++n) {
textures.emplace_back(self->ogl_->copy_async(
image_data[n], desc.planes[n].width, desc.planes[n].height, desc.planes[n].stride));
textures.emplace_back(self->ogl_->copy_async(image_data[n],
desc.planes[n].width,
desc.planes[n].height,
desc.planes[n].stride,
desc.planes[n].depth));
}
return std::make_shared<decltype(textures)>(std::move(textures));
});
}

common::bit_depth depth() const { return renderer_.depth(); }
core::color_space color_space() const { return renderer_.color_space(); }
};

image_mixer::image_mixer(const spl::shared_ptr<device>& ogl, const int channel_id, const size_t max_frame_size)
: impl_(std::make_unique<impl>(ogl, channel_id, max_frame_size))
image_mixer::image_mixer(const spl::shared_ptr<device>& ogl,
const int channel_id,
const size_t max_frame_size,
common::bit_depth depth,
core::color_space color_space)
: impl_(std::make_unique<impl>(ogl, channel_id, max_frame_size, depth, color_space))
{
}
image_mixer::~image_mixer() {}
Expand All @@ -343,5 +378,13 @@ core::mutable_frame image_mixer::create_frame(const void* tag, const core::pixel
{
return impl_->create_frame(tag, desc);
}
core::mutable_frame
image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc, common::bit_depth depth)
{
return impl_->create_frame(tag, desc, depth);
}

common::bit_depth image_mixer::depth() const { return impl_->depth(); }
core::color_space image_mixer::color_space() const { return impl_->color_space(); }

}}} // namespace caspar::accelerator::ogl
17 changes: 13 additions & 4 deletions src/accelerator/ogl/image/image_mixer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#pragma once

#include <common/array.h>
#include <common/bit_depth.h>
#include <common/memory.h>

#include <core/frame/frame.h>
Expand All @@ -36,7 +37,11 @@ namespace caspar { namespace accelerator { namespace ogl {
class image_mixer final : public core::image_mixer
{
public:
image_mixer(const spl::shared_ptr<class device>& ogl, int channel_id, const size_t max_frame_size);
image_mixer(const spl::shared_ptr<class device>& ogl,
int channel_id,
const size_t max_frame_size,
common::bit_depth depth,
core::color_space color_space);
image_mixer(const image_mixer&) = delete;

~image_mixer();
Expand All @@ -45,12 +50,16 @@ class image_mixer final : public core::image_mixer

std::future<array<const std::uint8_t>> operator()(const core::video_format_desc& format_desc) override;
core::mutable_frame create_frame(const void* tag, const core::pixel_format_desc& desc) override;
core::mutable_frame
create_frame(const void* video_stream_tag, const core::pixel_format_desc& desc, common::bit_depth depth) override;

// core::image_mixer

void push(const core::frame_transform& frame) override;
void visit(const core::const_frame& frame) override;
void pop() override;
void push(const core::frame_transform& frame) override;
void visit(const core::const_frame& frame) override;
void pop() override;
common::bit_depth depth() const override;
core::color_space color_space() const override;

private:
struct impl;
Expand Down
Loading

0 comments on commit 2ffc202

Please sign in to comment.