From 76e875c73d34c92f7e38f5e0f5fd1d31c74f04ba Mon Sep 17 00:00:00 2001
From: Kirill Gavrilov <kirill@sview.ru>
Date: Fri, 17 Jan 2025 09:50:05 +0300
Subject: [PATCH] StAVImage - preserve alpha while converting from planar GBRAP
 pixel formats

---
 StShared/StAVImage.cpp | 13 +++++++++++--
 StShared/stAV.cpp      |  7 +++++++
 include/StAV/stAV.h    | 11 ++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/StShared/StAVImage.cpp b/StShared/StAVImage.cpp
index 29400073..2fa7cbf6 100644
--- a/StShared/StAVImage.cpp
+++ b/StShared/StAVImage.cpp
@@ -639,9 +639,18 @@ bool StAVImage::loadExtra(const StString& theFilePath,
         }
     } else {
         ///ST_DEBUG_LOG("StAVImage, perform conversion from Pixel format '" + avcodec_get_pix_fmt_name(myCodecCtx->pix_fmt) + "' to RGB");
+
+        // TODO - rely on AV_PIX_FMT_FLAG_ALPHA
+        //const AVPixFmtDescriptor* aDesc = av_pix_fmt_desc_get(myCodecCtx->pix_fmt);
+        //const bool hasAlpha = aDesc != NULL && (aDesc->flags & AV_PIX_FMT_FLAG_ALPHA);
+        const bool hasAlpha = myCodecCtx->pix_fmt == stAV::PIX_FMT::GBRAP10
+                           || myCodecCtx->pix_fmt == stAV::PIX_FMT::GBRAP12
+                           || myCodecCtx->pix_fmt == stAV::PIX_FMT::GBRAP14
+                           || myCodecCtx->pix_fmt == stAV::PIX_FMT::GBRAP16;
+
         // initialize software scaler/converter
         SwsContext* pToRgbCtx = sws_getContext(myCodecCtx->width, myCodecCtx->height, myCodecCtx->pix_fmt,    // source
-                                               myCodecCtx->width, myCodecCtx->height, stAV::PIX_FMT::RGB24, // destination
+                                               myCodecCtx->width, myCodecCtx->height, hasAlpha ? stAV::PIX_FMT::RGBA32 : stAV::PIX_FMT::RGB24, // destination
                                                SWS_BICUBIC, NULL, NULL, NULL);
         if(pToRgbCtx == NULL) {
             setState("SWScale library, failed to create SWScaler context");
@@ -651,7 +660,7 @@ bool StAVImage::loadExtra(const StString& theFilePath,
 
         // initialize additional buffer for converted RGB data
         setColorModel(StImage::ImgColor_RGB);
-        changePlane(0).initTrash(StImagePlane::ImgRGB,
+        changePlane(0).initTrash(hasAlpha ? StImagePlane::ImgRGBA : StImagePlane::ImgRGB,
                                  myCodecCtx->width, myCodecCtx->height);
 
         uint8_t* rgbData[4]; stMemZero(rgbData,     sizeof(rgbData));
diff --git a/StShared/stAV.cpp b/StShared/stAV.cpp
index 69973d0b..fbe44ddb 100644
--- a/StShared/stAV.cpp
+++ b/StShared/stAV.cpp
@@ -135,6 +135,13 @@ const AVPixelFormat stAV::PIX_FMT::RGB48      = ST_AV_GETPIXFMT("rgb48");
 const AVPixelFormat stAV::PIX_FMT::BGR48      = ST_AV_GETPIXFMT("bgr48");
 const AVPixelFormat stAV::PIX_FMT::RGBA64     = ST_AV_GETPIXFMT("rgba64");
 const AVPixelFormat stAV::PIX_FMT::BGRA64     = ST_AV_GETPIXFMT("bgra64");
+//const AVPixelFormat stAV::PIX_FMT::GBRP       = ST_AV_GETPIXFMT("gbrp");
+//const AVPixelFormat stAV::PIX_FMT::GBRP16     = ST_AV_GETPIXFMT("gbrp16");
+const AVPixelFormat stAV::PIX_FMT::GBRAP      = ST_AV_GETPIXFMT("gbrap");
+const AVPixelFormat stAV::PIX_FMT::GBRAP10    = ST_AV_GETPIXFMT("gbrap10");
+const AVPixelFormat stAV::PIX_FMT::GBRAP12    = ST_AV_GETPIXFMT("gbrap12");
+const AVPixelFormat stAV::PIX_FMT::GBRAP14    = ST_AV_GETPIXFMT("gbrap14");
+const AVPixelFormat stAV::PIX_FMT::GBRAP16    = ST_AV_GETPIXFMT("gbrap16");
 const AVPixelFormat stAV::PIX_FMT::GBRPF32    = ST_AV_GETPIXFMT("gbrpf32");
 const AVPixelFormat stAV::PIX_FMT::GBRAPF32   = ST_AV_GETPIXFMT("gbrapf32");
 const AVPixelFormat stAV::PIX_FMT::XYZ12      = ST_AV_GETPIXFMT("xyz12");
diff --git a/include/StAV/stAV.h b/include/StAV/stAV.h
index 034feaf1..b00b46c5 100644
--- a/include/StAV/stAV.h
+++ b/include/StAV/stAV.h
@@ -24,6 +24,7 @@ extern "C" {
 
     #include <libavutil/mathematics.h>
     #include <libavutil/opt.h>
+    #include <libavutil/pixdesc.h>
 
     // new stereoscopic info API
 #if(LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 56, 100))
@@ -258,7 +259,7 @@ namespace stAV {
         ST_SHARED_CPPEXPORT AVPixelFormat YUVJ422P;  //!< planar YUV 4:2:2, 16bpp, full scale (JPEG)
         ST_SHARED_CPPEXPORT AVPixelFormat YUVJ444P;  //!< planar YUV 4:4:4, 24bpp, full scale (JPEG)
         ST_SHARED_CPPEXPORT AVPixelFormat YUVJ440P;  //!< planar YUV 4:4:0 full scale (JPEG)
-        // RGB formats
+        // packed RGB formats
         ST_SHARED_CPPEXPORT AVPixelFormat RGB24;     //!< packed RGB 8:8:8, 24bpp, RGBRGB...
         ST_SHARED_CPPEXPORT AVPixelFormat BGR24;     //!< packed RGB 8:8:8, 24bpp, BGRBGR...
         ST_SHARED_CPPEXPORT AVPixelFormat RGB48;
@@ -267,6 +268,14 @@ namespace stAV {
         ST_SHARED_CPPEXPORT AVPixelFormat BGRA32;
         ST_SHARED_CPPEXPORT AVPixelFormat RGBA64;
         ST_SHARED_CPPEXPORT AVPixelFormat BGRA64;
+        // planar GBR(A)
+        //ST_SHARED_CPPEXPORT AVPixelFormat GBRP;
+        //ST_SHARED_CPPEXPORT AVPixelFormat GBRP16;
+        ST_SHARED_CPPEXPORT AVPixelFormat GBRAP;
+        ST_SHARED_CPPEXPORT AVPixelFormat GBRAP10;
+        ST_SHARED_CPPEXPORT AVPixelFormat GBRAP12;
+        ST_SHARED_CPPEXPORT AVPixelFormat GBRAP14;
+        ST_SHARED_CPPEXPORT AVPixelFormat GBRAP16;
         ST_SHARED_CPPEXPORT AVPixelFormat GBRPF32;
         ST_SHARED_CPPEXPORT AVPixelFormat GBRAPF32;
         // XYZ formats