[Feature] Support feature map output for mmsegmentation (#1625)

* add feature map output for mmseg

* update api

* update demo

* fix return

* update format_shape

* fix lint

* update csharp demo

* update python demo && api

* fix coreml build

* fix lint

* better sort

* update

* update cpp demo & add missing header

* change to CHW

* update csharp api

* update isort version to 5.12.0

* fix python api

* fix log

* more detail api docs

* isort support python3.7

* remove isort change

* remove whitespace

* axes check

* remove FormatShapeImpl

* minor

* add permute tc

* remove stride buffer

csrc/mmdeploy/apis/c/mmdeploy/segmentor.cpp

@@ -119,10 +119,17 @@ int mmdeploy_segmentor_get_result(mmdeploy_value_t output, mmdeploy_segmentation
       results_ptr->height = segmentor_output.height;
       results_ptr->width = segmentor_output.width;
       results_ptr->classes = segmentor_output.classes;
-      auto mask_size = results_ptr->height * results_ptr->width;
       auto& mask = segmentor_output.mask;
-      results_ptr->mask = mask.data<int>();
-      buffers[i] = mask.buffer();
+      auto& score = segmentor_output.score;
+      results_ptr->mask = nullptr;
+      results_ptr->score = nullptr;
+      if (mask.shape().size()) {
+        results_ptr->mask = mask.data<int>();
+        buffers[i] = mask.buffer();
+      } else {
+        results_ptr->score = score.data<float>();
+        buffers[i] = score.buffer();
+      }
     }
 
     *results = results_data;

csrc/mmdeploy/apis/c/mmdeploy/segmentor.h

@@ -17,11 +17,14 @@ extern "C" {
 #endif
 
 typedef struct mmdeploy_segmentation_t {
-  int height;   ///< height of \p mask that equals to the input image's height
-  int width;    ///< width of \p mask that equals to the input image's width
-  int classes;  ///< the number of labels in \p mask
-  int* mask;    ///< segmentation mask of the input image, in which mask[i * width + j] indicates
-                ///< the label id of pixel at (i, j)
+  int height;    ///< height of \p mask that equals to the input image's height
+  int width;     ///< width of \p mask that equals to the input image's width
+  int classes;   ///< the number of labels in \p mask
+  int* mask;     ///< segmentation mask of the input image, in which mask[i * width + j] indicates
+                 ///< the label id of pixel at (i, j), this field might be null
+  float* score;  ///< segmentation score map of the input image in CHW format, in which
+                 ///< score[height * width * k + i * width + j] indicates the score
+                 ///< of class k at pixel (i, j), this field might be null
 } mmdeploy_segmentation_t;
 
 typedef struct mmdeploy_segmentor* mmdeploy_segmentor_t;

csrc/mmdeploy/apis/csharp/MMDeploy.sln

@@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.31729.503
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MMDeploy", "MMDeploy\MMDeploy.csproj", "{3DC914EB-A8FB-4A89-A7CF-7DF9CC5284A6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MMDeploy", "MMDeploy\MMDeployCSharp.csproj", "{3DC914EB-A8FB-4A89-A7CF-7DF9CC5284A6}"
 EndProject
 
 Global

csrc/mmdeploy/apis/csharp/MMDeploy/APIs/Segmentor.cs

@@ -10,6 +10,7 @@ internal unsafe struct CSegment
         public int Width;
         public int Classes;
         public int* Mask;
+        public float* Score;
     }
 #pragma warning restore 0649
 
@@ -34,36 +35,83 @@ public struct SegmentorOutput
         public int Classes;
 
         /// <summary>
-        /// Mask data.
+        /// Mask data, mask[i * width + j] indicates the label id of pixel at (i, j).
         /// </summary>
         public int[] Mask;
 
+        /// <summary>
+        /// Score data, score[height * width * k + i * width + j] indicates the score
+        /// of class k at pixel (i, j).
+        /// </summary>
+        public float[] Score;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="SegmentorOutput"/> struct.
         /// </summary>
         /// <param name="height">height.</param>
         /// <param name="width">width.</param>
         /// <param name="classes">classes.</param>
         /// <param name="mask">mask.</param>
-        public SegmentorOutput(int height, int width, int classes, int[] mask)
+        /// <param name="score">score.</param>
+        public SegmentorOutput(int height, int width, int classes, int[] mask, float[] score)
         {
             Height = height;
             Width = width;
             Classes = classes;
-            Mask = new int[Height * Width];
-            Array.Copy(mask, this.Mask, mask.Length);
+            if (mask.Length > 0)
+            {
+                Mask = new int[Height * Width];
+                Array.Copy(mask, this.Mask, mask.Length);
+            }
+            else
+            {
+                Mask = new int[] { };
+            }
+
+            if (score.Length > 0)
+            {
+                Score = new float[Height * Width * Classes];
+                Array.Copy(score, this.Score, score.Length);
+            }
+            else
+            {
+                Score = new float[] { };
+            }
         }
 
         internal unsafe SegmentorOutput(CSegment* result)
         {
             Height = result->Height;
             Width = result->Width;
             Classes = result->Classes;
-            Mask = new int[Height * Width];
-            int nbytes = Height * Width * sizeof(int);
-            fixed (int* data = this.Mask)
+            if (result->Mask != null)
+            {
+                Mask = new int[Height * Width];
+
+                int nbytes = Height * Width * sizeof(int);
+                fixed (int* data = this.Mask)
+                {
+                    Buffer.MemoryCopy(result->Mask, data, nbytes, nbytes);
+                }
+            }
+            else
+            {
+                Mask = new int[] { };
+            }
+
+            if (result->Score != null)
+            {
+                Score = new float[Height * Width * Classes];
+
+                int nbytes = Height * Width * Classes * sizeof(float);
+                fixed (float* data = this.Score)
+                {
+                    Buffer.MemoryCopy(result->Score, data, nbytes, nbytes);
+                }
+            }
+            else
             {
-                Buffer.MemoryCopy(result->Mask, data, nbytes, nbytes);
+                Score = new float[] { };
             }
         }
     }

csrc/mmdeploy/apis/python/segmentor.cpp

@@ -37,12 +37,22 @@ class PySegmentor {
 
     std::vector<py::array> rets(mats.size());
     for (size_t i = 0; i < mats.size(); ++i) {
-      rets[i] = {
-          {segm[i].height, segm[i].width},                                 // shape
-          segm[i].mask,                                                    // data
-          py::capsule(new Sptr(holder),                                    // handle
-                      [](void* p) { delete reinterpret_cast<Sptr*>(p); })  //
-      };
+      if (segm[i].mask != nullptr) {
+        rets[i] = {
+            {segm[i].height, segm[i].width},                                 // shape
+            segm[i].mask,                                                    // mask
+            py::capsule(new Sptr(holder),                                    // handle
+                        [](void* p) { delete reinterpret_cast<Sptr*>(p); })  //
+        };
+      }
+      if (segm[i].score != nullptr) {
+        rets[i] = {
+            {segm[i].classes, segm[i].height, segm[i].width},                // shape
+            segm[i].score,                                                   // score
+            py::capsule(new Sptr(holder),                                    // handle
+                        [](void* p) { delete reinterpret_cast<Sptr*>(p); })  //
+        };
+      }
     }
     return rets;
   }

csrc/mmdeploy/codebase/mmaction/CMakeLists.txt

@@ -4,8 +4,7 @@ project(mmdeploy_mmaction)
 
 file(GLOB SRCS ${CMAKE_CURRENT_SOURCE_DIR} "*.cpp")
 mmdeploy_add_module(${PROJECT_NAME} "${SRCS}")
-add_subdirectory(cpu)
-add_subdirectory(cuda)
+
 target_link_libraries(${PROJECT_NAME} PRIVATE
     mmdeploy_operation
     mmdeploy_transform