[Enhancement] Optimize C++ demos (open-mmlab#1715)

* optimize demos

* show text in image

* optimize demos

* fix minor

* fix minor

* fix minor

* install utils & fix demo file extensions

* rename

* parse empty flags

* antialias

* handle video complications

(cherry picked from commit 2b18596)

csrc/mmdeploy/apis/c/mmdeploy/pose_tracker.h

@@ -36,7 +36,7 @@ typedef struct mmdeploy_pose_tracker_param_t {
   int32_t pose_max_num_bboxes;
   // threshold for visible key-points, default = 0.5
   float pose_kpt_thr;
-  // min number of key-points for valid poses, default = -1
+  // min number of key-points for valid poses (-1 indicates ceil(n_kpts/2)), default = -1
   int32_t pose_min_keypoints;
   // scale for expanding key-points to bbox, default = 1.25
   float pose_bbox_scale;

csrc/mmdeploy/apis/cxx/CMakeLists.txt

@@ -24,4 +24,5 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mmdeploy/common.hpp
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/demo/csrc/ DESTINATION example/cpp
         FILES_MATCHING
         PATTERN "*.cxx"
+        PATTERN "*.h"
         )

csrc/mmdeploy/codebase/mmpose/pose_tracker/utils.h

@@ -22,7 +22,7 @@ using Points = vector<cv::Point2f>;
 using Score = float;
 using Scores = vector<float>;
 
-#define POSE_TRACKER_DEBUG(...) MMDEPLOY_INFO(__VA_ARGS__)
+#define POSE_TRACKER_DEBUG(...) MMDEPLOY_DEBUG(__VA_ARGS__)
 
 // opencv3 can't construct cv::Mat from std::array
 template <size_t N>

demo/csrc/cpp/classifier.cxx

@@ -1,31 +1,43 @@
 
 #include "mmdeploy/classifier.hpp"
 
-#include <string>
-
 #include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "utils/argparse.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "classifier_output.jpg", "Output image path");
 
 int main(int argc, char* argv[]) {
-  if (argc != 4) {
-    fprintf(stderr, "usage:\n  image_classification device_name model_path image_path\n");
-    return 1;
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
   }
-  auto device_name = argv[1];
-  auto model_path = argv[2];
-  auto image_path = argv[3];
-  cv::Mat img = cv::imread(image_path);
-  if (!img.data) {
-    fprintf(stderr, "failed to load image: %s\n", image_path);
-    return 1;
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
   }
 
-  mmdeploy::Model model(model_path);
-  mmdeploy::Classifier classifier(model, mmdeploy::Device{device_name, 0});
+  // construct a classifier instance
+  mmdeploy::Classifier classifier(mmdeploy::Model{ARGS_model}, mmdeploy::Device{FLAGS_device});
 
-  auto res = classifier.Apply(img);
+  // apply the classifier; the result is an array-like class holding references to
+  // `mmdeploy_classification_t`, will be released automatically on destruction
+  mmdeploy::Classifier::Result result = classifier.Apply(img);
+
+  // visualize results
+  utils::Visualize v;
+  auto sess = v.get_session(img);
+  int count = 0;
+  for (const mmdeploy_classification_t& cls : result) {
+    sess.add_label(cls.label_id, cls.score, count++);
+  }
 
-  for (const auto& cls : res) {
-    fprintf(stderr, "label: %d, score: %.4f\n", cls.label_id, cls.score);
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
   }
 
   return 0;

demo/csrc/cpp/det_pose.cxx

@@ -0,0 +1,75 @@
+// Copyright (c) OpenMMLab. All rights reserved.
+
+#include <iostream>
+
+#include "mmdeploy/detector.hpp"
+#include "mmdeploy/pose_detector.hpp"
+#include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "utils/argparse.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(det_model, "Object detection model path");
+DEFINE_ARG_string(pose_model, "Pose estimation model path");
+DEFINE_ARG_string(image, "Input image path");
+
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "det_pose_output.jpg", "Output image path");
+DEFINE_string(skeleton, "coco", R"(Path to skeleton data or name of predefined skeletons: "coco")");
+
+DEFINE_int32(det_label, 0, "Detection label use for pose estimation");
+DEFINE_double(det_thr, .5, "Detection score threshold");
+DEFINE_double(det_min_bbox_size, -1, "Detection minimum bbox size");
+
+DEFINE_double(pose_thr, 0, "Pose key-point threshold");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+
+  mmdeploy::Device device{FLAGS_device};
+  // create object detector
+  mmdeploy::Detector detector(mmdeploy::Model(ARGS_det_model), device);
+  // create pose detector
+  mmdeploy::PoseDetector pose(mmdeploy::Model(ARGS_pose_model), device);
+
+  // apply the detector, the result is an array-like class holding references to
+  // `mmdeploy_detection_t`, will be released automatically on destruction
+  mmdeploy::Detector::Result dets = detector.Apply(img);
+
+  // filter detections and extract bboxes for pose model
+  std::vector<mmdeploy_rect_t> bboxes;
+  for (const mmdeploy_detection_t& det : dets) {
+    if (det.label_id == FLAGS_det_label && det.score > FLAGS_det_thr) {
+      bboxes.push_back(det.bbox);
+    }
+  }
+
+  // apply pose detector, if no bboxes are provided, full image will be used; the result is an
+  // array-like class holding references to `mmdeploy_pose_detection_t`, will be released
+  // automatically on destruction
+  mmdeploy::PoseDetector::Result poses = pose.Apply(img, bboxes);
+
+  assert(bboxes.size() == poses.size());
+
+  // visualize results
+  utils::Visualize v;
+  v.set_skeleton(utils::Skeleton::get(FLAGS_skeleton));
+  auto sess = v.get_session(img);
+  for (size_t i = 0; i < bboxes.size(); ++i) {
+    sess.add_bbox(bboxes[i], -1, -1);
+    sess.add_pose(poses[i].point, poses[i].score, poses[i].length, FLAGS_pose_thr);
+  }
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
+
+  return 0;
+}

demo/csrc/cpp/detector.cxx

@@ -1,69 +1,47 @@
 #include "mmdeploy/detector.hpp"
 
-#include <opencv2/imgcodecs/imgcodecs.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
-#include <string>
+#include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "utils/argparse.h"
+#include "utils/visualize.h"
 
-int main(int argc, char* argv[]) {
-  if (argc != 4) {
-    fprintf(stderr, "usage:\n  object_detection device_name model_path image_path\n");
-    return 1;
-  }
-  auto device_name = argv[1];
-  auto model_path = argv[2];
-  auto image_path = argv[3];
-  cv::Mat img = cv::imread(image_path);
-  if (!img.data) {
-    fprintf(stderr, "failed to load image: %s\n", image_path);
-    return 1;
-  }
-
-  mmdeploy::Model model(model_path);
-  mmdeploy::Detector detector(model, mmdeploy::Device{device_name, 0});
-
-  auto dets = detector.Apply(img);
-
-  fprintf(stdout, "bbox_count=%d\n", (int)dets.size());
-
-  for (int i = 0; i < dets.size(); ++i) {
-    const auto& box = dets[i].bbox;
-    const auto& mask = dets[i].mask;
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "detector_output.jpg", "Output image path");
 
-    fprintf(stdout, "box %d, left=%.2f, top=%.2f, right=%.2f, bottom=%.2f, label=%d, score=%.4f\n",
-            i, box.left, box.top, box.right, box.bottom, dets[i].label_id, dets[i].score);
+DEFINE_double(det_thr, .5, "Detection score threshold");
 
-    // skip detections with invalid bbox size (bbox height or width < 1)
-    if ((box.right - box.left) < 1 || (box.bottom - box.top) < 1) {
-      continue;
-    }
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
 
-    // skip detections less than specified score threshold
-    if (dets[i].score < 0.3) {
-      continue;
-    }
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
 
-    // generate mask overlay if model exports masks
-    if (mask != nullptr) {
-      fprintf(stdout, "mask %d, height=%d, width=%d\n", i, mask->height, mask->width);
+  // construct a detector instance
+  mmdeploy::Detector detector(mmdeploy::Model{ARGS_model}, mmdeploy::Device{FLAGS_device});
 
-      cv::Mat imgMask(mask->height, mask->width, CV_8UC1, &mask->data[0]);
-      auto x0 = std::max(std::floor(box.left) - 1, 0.f);
-      auto y0 = std::max(std::floor(box.top) - 1, 0.f);
-      cv::Rect roi((int)x0, (int)y0, mask->width, mask->height);
+  // apply the detector, the result is an array-like class holding references to
+  // `mmdeploy_detection_t`, will be released automatically on destruction
+  mmdeploy::Detector::Result dets = detector.Apply(img);
 
-      // split the RGB channels, overlay mask to a specific color channel
-      cv::Mat ch[3];
-      split(img, ch);
-      int col = 0;  // int col = i % 3;
-      cv::bitwise_or(imgMask, ch[col](roi), ch[col](roi));
-      merge(ch, 3, img);
+  // visualize
+  utils::Visualize v;
+  auto sess = v.get_session(img);
+  int count = 0;
+  for (const mmdeploy_detection_t& det : dets) {
+    if (det.score > FLAGS_det_thr) {  // filter bboxes
+      sess.add_det(det.bbox, det.label_id, det.score, det.mask, count++);
     }
-
-    cv::rectangle(img, cv::Point{(int)box.left, (int)box.top},
-                  cv::Point{(int)box.right, (int)box.bottom}, cv::Scalar{0, 255, 0});
   }
 
-  cv::imwrite("output_detection.png", img);
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
 
   return 0;
 }