Merge pull request #15 from occ-ai/roy.add_sort_tracker

Add SORT tracker

CMakeLists.txt

@@ -80,7 +80,8 @@ endif()
 # add the vendor folder to the include path
 target_include_directories(${CMAKE_PROJECT_NAME} PRIVATE vendor)
 
-target_sources(${CMAKE_PROJECT_NAME} PRIVATE src/plugin-main.c src/detect-filter.cpp src/detect-filter-info.c
-                                             src/obs-utils/obs-utils.cpp src/edgeyolo/edgeyolo_onnxruntime.cpp)
+target_sources(
+  ${CMAKE_PROJECT_NAME} PRIVATE src/plugin-main.c src/detect-filter.cpp src/detect-filter-info.c
+                                src/obs-utils/obs-utils.cpp src/edgeyolo/edgeyolo_onnxruntime.cpp src/sort/Sort.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})

README.md

@@ -17,7 +17,7 @@ If you like this work, which is given to you completely free of charge, please c
 - https://github.com/sponsors/royshil
 - https://github.com/sponsors/umireon
 
-This work uses the great contributions from [EdgeYOLO-ROS](https://github.com/fateshelled/EdgeYOLO-ROS) and [PINTO-Model-Zoo](https://github.com/PINTO0309/PINTO_model_zoo).
+This work uses the great contributions from [EdgeYOLO-ROS](https://github.com/fateshelled/EdgeYOLO-ROS) and [PINTO-Model-Zoo](https://github.com/PINTO0309/PINTO_model_zoo). The Hungarian algorithm is taken from https://github.com/Gluttton/munkres-cpp under the GPLv2 license.
 
 ## Usage
 

cmake/FetchOpenCV.cmake

@@ -19,7 +19,7 @@ else()
 endif()
 
 if(USE_PREDEFINED_OPENCV)
-  set(OpenCV_VERSION "v4.8.1-1")
+  set(OpenCV_VERSION "v4.9.0-1")
   set(OpenCV_BASEURL "https://github.com/obs-ai/obs-backgroundremoval-dep-opencv/releases/download/${OpenCV_VERSION}")
 
   if(${CMAKE_BUILD_TYPE} STREQUAL Release OR ${CMAKE_BUILD_TYPE} STREQUAL RelWithDebInfo)
@@ -31,26 +31,26 @@ if(USE_PREDEFINED_OPENCV)
   if(APPLE)
     if(OpenCV_BUILD_TYPE STREQUAL Debug)
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Debug.tar.gz")
-      set(OpenCV_HASH SHA256=2930e335a19cc03a3d825e2b76eadd0d5cf08d8baf6537747d43f503dff32454)
+      set(OpenCV_HASH SHA256=BE85C8224F71C52162955BEE4EC9FFBE41CBED636D7989843CA75AD42657B121)
     else()
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Release.tar.gz")
-      set(OpenCV_HASH SHA256=b0c4fe2370b0bd5aa65c408e875b1ab18508ba31b93083805d7e398a3ecafdac)
+      set(OpenCV_HASH SHA256=5DB4FCFBD8C7CDBA136657B4D149821A670DF9A7C71120F5A4D34FA35A58D07B)
     endif()
   elseif(MSVC)
     if(OpenCV_BUILD_TYPE STREQUAL Debug)
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Debug.zip")
-      set(OpenCV_HASH SHA256=0c5ef12cf4b4e4db7ea17a24db156165b6f01759f3f1660b069d0722e5d5dc37)
+      set(OpenCV_HASH SHA256=0A1BBC898DCE5F193427586DA84D7A34BBB783127957633236344E9CCD61B9CE)
     else()
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Release.zip")
-      set(OpenCV_HASH SHA256=5e468f71d41d3a3ea46cc4f247475877f65d3655a2764a2c01074bda3b3e6864)
+      set(OpenCV_HASH SHA256=56A5E042F490B8390B1C1819B2B48C858F10CD64E613BABBF11925A57269C3FA)
     endif()
   else()
     if(OpenCV_BUILD_TYPE STREQUAL Debug)
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Debug.tar.gz")
-      set(OpenCV_HASH SHA256=e2e246d6b4f279be80e7fd0a78cba8a0eeee7b53ae807f2f57428d6876306422)
+      set(OpenCV_HASH SHA256=840A7D80B661CFF7B7300272A2A2992D539672ECECA01836B85F68BD8CAF07F5)
     else()
       set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Release.tar.gz")
-      set(OpenCV_HASH SHA256=809922a7cc9f344a2d82a232ed7b02e122c82e77cba94b4047e666a0527cc00e)
+      set(OpenCV_HASH SHA256=73652C2155B477B5FD95FCD8EA7CE35D313543ECE17BDFA3A2B217A0239D74C6)
     endif()
   endif()
 else()
@@ -68,13 +68,14 @@ add_library(OpenCV INTERFACE)
 if(MSVC)
   target_link_libraries(
     OpenCV
-    INTERFACE ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_imgproc481.lib
-              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_core481.lib
+    INTERFACE ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_imgproc490.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_core490.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_video490.lib
               ${opencv_SOURCE_DIR}/x64/vc17/staticlib/zlib.lib)
   target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include)
 else()
   target_link_libraries(
     OpenCV INTERFACE ${opencv_SOURCE_DIR}/lib/libopencv_imgproc.a ${opencv_SOURCE_DIR}/lib/libopencv_core.a
-                     ${opencv_SOURCE_DIR}/lib/opencv4/3rdparty/libzlib.a)
+                     ${opencv_SOURCE_DIR}/lib/libopencv_video.a ${opencv_SOURCE_DIR}/lib/opencv4/3rdparty/libzlib.a)
   target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include/opencv4)
 endif()

cmake/linux/compilerconfig.cmake

@@ -21,6 +21,7 @@ set(_obs_gcc_c_options
     -Wformat
     -Wformat-security
     -Wno-conversion
+    -Wno-error=shadow
     -Wno-float-conversion
     -Wno-implicit-fallthrough
     -Wno-missing-braces

data/locale/en-US.ini

@@ -29,5 +29,8 @@ ZoomObject="Zoom Object"
 SingleFirst="Single (First)"
 ZoomSpeed="Zoom Speed"
 DetectedObject="Detected Object"
+SORTTracking="Continuous Tracking"
+MaxUnseenFrames="Max Unseen Frames"
 ExternalModel="External Model"
 ModelPath="Model Path"
+ShowUnseenObjects="Show Currently Undetected Objects"

src/FilterData.h

@@ -3,6 +3,7 @@
 
 #include <obs-module.h>
 #include "edgeyolo/edgeyolo_onnxruntime.hpp"
+#include "sort/Sort.h"
 
 /**
   * @brief The filter_data struct
@@ -28,6 +29,11 @@ struct filter_data {
 	obs_source_t *trackingFilter;
 	cv::Rect2f trackingRect;
 	int lastDetectedObjectId;
+	bool sortTracking;
+	bool showUnseenObjects;
+
+	// create SORT tracker
+	Sort tracker;
 
 	obs_source_t *source;
 	gs_texrender_t *texrender;

src/detect-filter.cpp

@@ -247,15 +247,25 @@ obs_properties_t *detect_filter_properties(void *data)
 	obs_property_set_modified_callback(advanced, enable_advanced_settings);
 
 	// add a text input for the currently detected object
-	obs_properties_add_text(props, "detected_object", obs_module_text("DetectedObject"),
-				OBS_TEXT_DEFAULT);
+	obs_property_t *detected_obj_prop = obs_properties_add_text(
+		props, "detected_object", obs_module_text("DetectedObject"), OBS_TEXT_DEFAULT);
 	// disable the text input by default
-	obs_property_set_enabled(obs_properties_get(props, "detected_object"), false);
+	obs_property_set_enabled(detected_obj_prop, false);
 
 	// add threshold slider
 	obs_properties_add_float_slider(props, "threshold", obs_module_text("ConfThreshold"), 0.0,
 					1.0, 0.025);
 
+	// add SORT tracking enabled checkbox
+	obs_properties_add_bool(props, "sort_tracking", obs_module_text("SORTTracking"));
+
+	// add parameter for number of missing frames before a track is considered lost
+	obs_properties_add_int(props, "max_unseen_frames", obs_module_text("MaxUnseenFrames"), 1,
+			       30, 1);
+
+	// add option to show unseen objects
+	obs_properties_add_bool(props, "show_unseen_objects", obs_module_text("ShowUnseenObjects"));
+
 	/* GPU, CPU and performance Props */
 	obs_property_t *p_use_gpu =
 		obs_properties_add_list(props, "useGPU", obs_module_text("InferenceDevice"),
@@ -355,6 +365,9 @@ void detect_filter_defaults(obs_data_t *settings)
 	// Linux
 	obs_data_set_default_string(settings, "useGPU", USEGPU_CPU);
 #endif
+	obs_data_set_default_bool(settings, "sort_tracking", false);
+	obs_data_set_default_int(settings, "max_unseen_frames", 10);
+	obs_data_set_default_bool(settings, "show_unseen_objects", true);
 	obs_data_set_default_int(settings, "numThreads", 1);
 	obs_data_set_default_bool(settings, "preview", true);
 	obs_data_set_default_double(settings, "threshold", 0.5);
@@ -389,6 +402,12 @@ void detect_filter_update(void *data, obs_data_t *settings)
 	tf->zoomFactor = (float)obs_data_get_double(settings, "zoom_factor");
 	tf->zoomSpeedFactor = (float)obs_data_get_double(settings, "zoom_speed_factor");
 	tf->zoomObject = obs_data_get_string(settings, "zoom_object");
+	tf->sortTracking = obs_data_get_bool(settings, "sort_tracking");
+	size_t maxUnseenFrames = (size_t)obs_data_get_int(settings, "max_unseen_frames");
+	if (tf->tracker.getMaxUnseenFrames() != maxUnseenFrames) {
+		tf->tracker.setMaxUnseenFrames(maxUnseenFrames);
+	}
+	tf->showUnseenObjects = obs_data_get_bool(settings, "show_unseen_objects");
 
 	// check if tracking state has changed
 	if (tf->trackingEnabled != newTrackingEnabled) {
@@ -748,6 +767,18 @@ void detect_filter_video_tick(void *data, float seconds)
 		objects = filtered_objects;
 	}
 
+	if (tf->sortTracking) {
+		objects = tf->tracker.update(objects);
+	}
+
+	if (!tf->showUnseenObjects) {
+		objects.erase(std::remove_if(objects.begin(), objects.end(),
+					     [](const edgeyolo_cpp::Object &obj) {
+						     return obj.unseenFrames > 0;
+					     }),
+			      objects.end());
+	}
+
 	if (tf->preview || tf->maskingEnabled) {
 		if (tf->preview && objects.size() > 0) {
 			edgeyolo_cpp::utils::draw_objects(frame, objects, tf->classNames);

src/edgeyolo/core.hpp

@@ -4,6 +4,8 @@
 #include <opencv2/core.hpp>
 #include <opencv2/imgproc.hpp>
 
+#include "types.hpp"
+
 namespace edgeyolo_cpp {
 /**
  * @brief Define names based depends on Unicode path support
@@ -18,18 +20,6 @@ namespace edgeyolo_cpp {
 
 #define imread_t cv::imread
 
-struct Object {
-	cv::Rect_<float> rect;
-	int label;
-	float prob;
-};
-
-struct GridAndStride {
-	int grid0;
-	int grid1;
-	int stride;
-};
-
 class AbcEdgeYOLO {
 public:
 	AbcEdgeYOLO() {}
@@ -243,6 +233,7 @@ class AbcEdgeYOLO {
 			proposals[picked[i]].rect.y = y0;
 			proposals[picked[i]].rect.width = x1 - x0;
 			proposals[picked[i]].rect.height = y1 - y0;
+			proposals[picked[i]].id = objects.size() + 1;
 
 			objects.push_back(proposals[picked[i]]);
 		}

src/edgeyolo/types.hpp

@@ -0,0 +1,26 @@
+#ifndef EDGEYOLO_TYPES_HPP
+#define EDGEYOLO_TYPES_HPP
+
+#include <opencv2/core/types.hpp>
+#include <opencv2/video/tracking.hpp>
+
+namespace edgeyolo_cpp {
+
+struct Object {
+	cv::Rect_<float> rect;
+	int label;
+	float prob;
+	uint64_t id;
+	uint64_t unseenFrames;
+	cv::KalmanFilter kf;
+};
+
+struct GridAndStride {
+	int grid0;
+	int grid1;
+	int stride;
+};
+
+} // namespace edgeyolo_cpp
+
+#endif

src/edgeyolo/utils.hpp

@@ -70,6 +70,11 @@ static void draw_objects(cv::Mat bgr, const std::vector<Object> &objects,
 
 		cv::putText(bgr, text, cv::Point(x, y + label_size.height),
 			    cv::FONT_HERSHEY_SIMPLEX, 0.4, txt_color, 1);
+
+		// write the id of the object
+		snprintf(text, sizeof(text), "ID: %d", (int)obj.id);
+		cv::putText(bgr, text, cv::Point(x, y + label_size.height + 15),
+			    cv::FONT_HERSHEY_SIMPLEX, 0.4, txt_color, 1);
 	}
 }
 } // namespace utils

src/plugin-support.c.in

@@ -21,6 +21,8 @@ with this program. If not, see <https://www.gnu.org/licenses/>
 const char *PLUGIN_NAME = "@CMAKE_PROJECT_NAME@";
 const char *PLUGIN_VERSION = "@CMAKE_PROJECT_VERSION@";
 
+extern void blogva(int log_level, const char *format, va_list args);
+
 void obs_log(int log_level, const char *format, ...)
 {
 	size_t length = 4 + strlen(PLUGIN_NAME) + strlen(format);

src/plugin-support.h

@@ -31,7 +31,6 @@ extern const char *PLUGIN_NAME;
 extern const char *PLUGIN_VERSION;
 
 void obs_log(int log_level, const char *format, ...);
-extern void blogva(int log_level, const char *format, va_list args);
 
 #ifdef __cplusplus
 }