From e3289ecac25fca89221f258051222ecdefa08d92 Mon Sep 17 00:00:00 2001 From: Niklas Andersson <3985238+niklaspandersson@users.noreply.github.com> Date: Mon, 1 Apr 2024 08:40:50 +0000 Subject: [PATCH] Add support for higher color depths in ffmpeg producer --- src/modules/ffmpeg/producer/av_producer.cpp | 4 + src/modules/ffmpeg/util/av_util.cpp | 104 +++++++++++--------- src/modules/ffmpeg/util/av_util.h | 1 - 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/src/modules/ffmpeg/producer/av_producer.cpp b/src/modules/ffmpeg/producer/av_producer.cpp index e8c4a0f6a6..f18595bf9e 100644 --- a/src/modules/ffmpeg/producer/av_producer.cpp +++ b/src/modules/ffmpeg/producer/av_producer.cpp @@ -502,7 +502,11 @@ struct Filter AV_PIX_FMT_ABGR, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV422P, + AV_PIX_FMT_YUV422P10, + AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P, + AV_PIX_FMT_YUV420P10, + AV_PIX_FMT_YUV420P12, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUVA444P, AV_PIX_FMT_YUVA422P, diff --git a/src/modules/ffmpeg/util/av_util.cpp b/src/modules/ffmpeg/util/av_util.cpp index fa6a2e4736..b853f5b7db 100644 --- a/src/modules/ffmpeg/util/av_util.cpp +++ b/src/modules/ffmpeg/util/av_util.cpp @@ -1,7 +1,8 @@ #include "av_util.h" - #include "av_assert.h" +#include + #if defined(_MSC_VER) #pragma warning(push) #pragma warning(disable : 4244) @@ -22,6 +23,8 @@ extern "C" { #include #include +#include + namespace caspar { namespace ffmpeg { std::shared_ptr alloc_frame() @@ -93,43 +96,51 @@ core::mutable_frame make_frame(void* tag, return frame; } -core::pixel_format get_pixel_format(AVPixelFormat pix_fmt) +std::tuple get_pixel_format(AVPixelFormat pix_fmt) { switch (pix_fmt) { case AV_PIX_FMT_GRAY8: - return core::pixel_format::gray; + return {core::pixel_format::gray, common::bit_depth::bit8}; case AV_PIX_FMT_RGB24: - return core::pixel_format::rgb; + return {core::pixel_format::rgb, common::bit_depth::bit8}; case AV_PIX_FMT_BGR24: - return core::pixel_format::bgr; + return {core::pixel_format::bgr, common::bit_depth::bit8}; case AV_PIX_FMT_BGRA: - return core::pixel_format::bgra; + return {core::pixel_format::bgra, common::bit_depth::bit8}; case AV_PIX_FMT_ARGB: - return core::pixel_format::argb; + return {core::pixel_format::argb, common::bit_depth::bit8}; case AV_PIX_FMT_RGBA: - return core::pixel_format::rgba; + return {core::pixel_format::rgba, common::bit_depth::bit8}; case AV_PIX_FMT_ABGR: - return core::pixel_format::abgr; + return {core::pixel_format::abgr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV444P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV422P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; + case AV_PIX_FMT_YUV422P10: + return {core::pixel_format::ycbcr, common::bit_depth::bit10}; + case AV_PIX_FMT_YUV422P12: + return {core::pixel_format::ycbcr, common::bit_depth::bit12}; case AV_PIX_FMT_YUV420P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; + case AV_PIX_FMT_YUV420P10: + return {core::pixel_format::ycbcr, common::bit_depth::bit10}; + case AV_PIX_FMT_YUV420P12: + return {core::pixel_format::ycbcr, common::bit_depth::bit12}; case AV_PIX_FMT_YUV411P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUV410P: - return core::pixel_format::ycbcr; + return {core::pixel_format::ycbcr, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA420P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA422P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_YUVA444P: - return core::pixel_format::ycbcra; + return {core::pixel_format::ycbcra, common::bit_depth::bit8}; case AV_PIX_FMT_UYVY422: - return core::pixel_format::uyvy; + return {core::pixel_format::uyvy, common::bit_depth::bit8}; default: - return core::pixel_format::invalid; + return {core::pixel_format::invalid, common::bit_depth::bit8}; } } @@ -139,24 +150,26 @@ core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int int linesizes[4]; av_image_fill_linesizes(linesizes, pix_fmt, width); - core::pixel_format_desc desc = core::pixel_format_desc(get_pixel_format(pix_fmt)); + const auto fmt = get_pixel_format(pix_fmt); + auto desc = core::pixel_format_desc(std::get<0>(fmt)); + auto depth = std::get<1>(fmt); switch (desc.format) { case core::pixel_format::gray: case core::pixel_format::luma: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0], height, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); return desc; } case core::pixel_format::bgr: case core::pixel_format::rgb: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 3, height, 3)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 3, height, 3, depth)); return desc; } case core::pixel_format::bgra: case core::pixel_format::argb: case core::pixel_format::rgba: case core::pixel_format::abgr: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 4, height, 4)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 4, height, 4, depth)); return desc; } case core::pixel_format::ycbcr: @@ -177,20 +190,22 @@ core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int av_image_fill_pointers(dummy_pict_data, pix_fmt, height, NULL, linesizes); auto size2 = static_cast(dummy_pict_data[2] - dummy_pict_data[1]); #endif - auto h2 = size2 / linesizes[1]; + auto h2 = size2 / linesizes[1]; + auto factor1 = linesizes[0] / linesizes[1]; + auto factor2 = linesizes[0] / linesizes[2]; - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0], height, 1)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[1], h2, 1)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[2], h2, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / factor1, h2, 1, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / factor2, h2, 1, depth)); if (desc.format == core::pixel_format::ycbcra) - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[3], height, 1)); + desc.planes.push_back(core::pixel_format_desc::plane(width, height, 1, depth)); return desc; } case core::pixel_format::uyvy: { - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 2, height, 2)); - desc.planes.push_back(core::pixel_format_desc::plane(linesizes[0] / 4, height, 4)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 2, height, 2, depth)); + desc.planes.push_back(core::pixel_format_desc::plane(width / 4, height, 4, depth)); data_map.clear(); data_map.push_back(0); @@ -220,28 +235,29 @@ std::shared_ptr make_av_video_frame(const core::const_frame& frame, con av_frame->width = format_desc.width; av_frame->height = format_desc.height; + const auto is_16bit = planes[0].depth != common::bit_depth::bit8; switch (format) { case core::pixel_format::rgb: - av_frame->format = AVPixelFormat::AV_PIX_FMT_RGB24; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_RGB48 : AVPixelFormat::AV_PIX_FMT_RGB24; break; case core::pixel_format::bgr: - av_frame->format = AVPixelFormat::AV_PIX_FMT_BGR24; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGR48 : AVPixelFormat::AV_PIX_FMT_BGR24; break; case core::pixel_format::rgba: - av_frame->format = AVPixelFormat::AV_PIX_FMT_RGBA; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_RGBA64 : AVPixelFormat::AV_PIX_FMT_RGBA; break; case core::pixel_format::argb: - av_frame->format = AVPixelFormat::AV_PIX_FMT_ARGB; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_ARGB; break; case core::pixel_format::bgra: - av_frame->format = AVPixelFormat::AV_PIX_FMT_BGRA; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_BGRA; break; case core::pixel_format::abgr: - av_frame->format = AVPixelFormat::AV_PIX_FMT_ABGR; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_BGRA64 : AVPixelFormat::AV_PIX_FMT_ABGR; break; case core::pixel_format::gray: case core::pixel_format::luma: - av_frame->format = AVPixelFormat::AV_PIX_FMT_GRAY8; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_GRAY16 : AVPixelFormat::AV_PIX_FMT_GRAY8; break; case core::pixel_format::ycbcr: { int y_w = planes[0].width; @@ -250,27 +266,27 @@ std::shared_ptr make_av_video_frame(const core::const_frame& frame, con int c_h = planes[1].height; if (c_h == y_h && c_w == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV444P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV444P10 : AVPixelFormat::AV_PIX_FMT_YUV444P; else if (c_h == y_h && c_w * 2 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV422P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV422P10 : AVPixelFormat::AV_PIX_FMT_YUV422P; else if (c_h == y_h && c_w * 4 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV411P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV422P10 : AVPixelFormat::AV_PIX_FMT_YUV411P; else if (c_h * 2 == y_h && c_w * 2 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV420P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV420P10 : AVPixelFormat::AV_PIX_FMT_YUV420P; else if (c_h * 2 == y_h && c_w * 4 == y_w) - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUV410P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUV420P10 : AVPixelFormat::AV_PIX_FMT_YUV410P; break; } case core::pixel_format::ycbcra: - av_frame->format = AVPixelFormat::AV_PIX_FMT_YUVA420P; + av_frame->format = is_16bit ? AVPixelFormat::AV_PIX_FMT_YUVA420P10 : AVPixelFormat::AV_PIX_FMT_YUVA420P; break; case core::pixel_format::count: case core::pixel_format::invalid: break; } - FF(av_frame_get_buffer(av_frame.get(), 32)); + FF(av_frame_get_buffer(av_frame.get(), is_16bit ? 64 : 32)); // TODO (perf) Avoid extra memcpy. for (int n = 0; n < planes.size(); ++n) { diff --git a/src/modules/ffmpeg/util/av_util.h b/src/modules/ffmpeg/util/av_util.h index 8a6ceed869..c7e5f2ff62 100644 --- a/src/modules/ffmpeg/util/av_util.h +++ b/src/modules/ffmpeg/util/av_util.h @@ -20,7 +20,6 @@ namespace caspar { namespace ffmpeg { std::shared_ptr alloc_frame(); std::shared_ptr alloc_packet(); -core::pixel_format get_pixel_format(AVPixelFormat pix_fmt); core::pixel_format_desc pixel_format_desc(AVPixelFormat pix_fmt, int width, int height, std::vector& data_map); core::mutable_frame make_frame(void* tag, core::frame_factory& frame_factory,