Initial commit

CMakeLists.txt

@@ -34,6 +34,53 @@ if(ENABLE_QT)
                AUTORCC ON)
 endif()
 
-target_sources(${CMAKE_PROJECT_NAME} PRIVATE src/plugin-main.c)
+set(USE_SYSTEM_ONNXRUNTIME
+    OFF
+    CACHE STRING "Use system ONNX Runtime")
+
+set(DISABLE_ONNXRUNTIME_GPU
+    OFF
+    CACHE STRING "Disables GPU support of ONNX Runtime (Only valid on Linux)")
+
+if(DISABLE_ONNXRUNTIME_GPU)
+  target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE DISABLE_ONNXRUNTIME_GPU)
+endif()
+
+if(USE_SYSTEM_ONNXRUNTIME)
+  if(OS_LINUX)
+    find_package(Onnxruntime 1.16.3 REQUIRED)
+    set(Onnxruntime_INCLUDE_PATH
+        ${Onnxruntime_INCLUDE_DIR} ${Onnxruntime_INCLUDE_DIR}/onnxruntime
+        ${Onnxruntime_INCLUDE_DIR}/onnxruntime/core/session ${Onnxruntime_INCLUDE_DIR}/onnxruntime/core/providers/cpu)
+    target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIBRARIES}")
+    target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${Onnxruntime_INCLUDE_PATH}")
+  else()
+    message(FATAL_ERROR "System ONNX Runtime is only supported on Linux!")
+  endif()
+else()
+  include(cmake/FetchOnnxruntime.cmake)
+endif()
+
+set(USE_SYSTEM_OPENCV
+    OFF
+    CACHE STRING "Use system OpenCV")
+if(USE_SYSTEM_OPENCV)
+  if(OS_LINUX)
+    find_package(OpenCV REQUIRED COMPONENTS core imgproc)
+    target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${OpenCV_LIBRARIES}")
+    target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${OpenCV_INCLUDE_DIRS}")
+  else()
+    message(FATAL_ERROR "System OpenCV is only supported on Linux!")
+  endif()
+else()
+  include(cmake/FetchOpenCV.cmake)
+  target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE OpenCV)
+endif()
+
+target_sources(${CMAKE_PROJECT_NAME} PRIVATE src/plugin-main.c
+    src/detect-filter.cpp
+    src/detect-filter-info.c
+    src/obs-utils/obs-utils.cpp
+    src/edgeyolo/edgeyolo_onnxruntime.cpp)
 
 set_target_properties_plugin(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME ${_name})

buildspec.json

@@ -33,14 +33,14 @@
     },
     "platformConfig": {
         "macos": {
-            "bundleId": "com.example.obs-plugintemplate"
+            "bundleId": "com.royshilkrot.obs-detect"
         }
     },
-    "name": "obs-plugintemplate",
-    "displayName": "OBS Plugin Template",
-    "version": "1.0.0",
-    "author": "Your Name Here",
-    "website": "https://example.com",
+    "name": "obs-detect",
+    "displayName": "OBS Object Detection plugin",
+    "version": "0.0.1",
+    "author": "Roy Shilkrot",
+    "website": "https://github.com/occ-ai",
     "email": "[email protected]",
     "uuids": {
         "windowsApp": "00000000-0000-0000-0000-000000000000"

cmake/FetchOnnxruntime.cmake

@@ -0,0 +1,113 @@
+include(FetchContent)
+
+set(CUSTOM_ONNXRUNTIME_URL
+    ""
+    CACHE STRING "URL of a downloaded ONNX Runtime tarball")
+
+set(CUSTOM_ONNXRUNTIME_HASH
+    ""
+    CACHE STRING "Hash of a downloaded ONNX Runtime tarball")
+
+set(Onnxruntime_VERSION "1.17.1")
+
+if(CUSTOM_ONNXRUNTIME_URL STREQUAL "")
+  set(USE_PREDEFINED_ONNXRUNTIME ON)
+else()
+  if(CUSTOM_ONNXRUNTIME_HASH STREQUAL "")
+    message(FATAL_ERROR "Both of CUSTOM_ONNXRUNTIME_URL and CUSTOM_ONNXRUNTIME_HASH must be present!")
+  else()
+    set(USE_PREDEFINED_ONNXRUNTIME OFF)
+  endif()
+endif()
+
+if(USE_PREDEFINED_ONNXRUNTIME)
+  set(Onnxruntime_BASEURL "https://github.com/microsoft/onnxruntime/releases/download/v${Onnxruntime_VERSION}")
+  set(Onnxruntime_WINDOWS_VERSION "v${Onnxruntime_VERSION}-1")
+  set(Onnxruntime_WINDOWS_BASEURL
+      "https://github.com/occ-ai/occ-ai-dep-onnxruntime-static-win/releases/download/${Onnxruntime_WINDOWS_VERSION}")
+
+  if(APPLE)
+    set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-osx-universal2-${Onnxruntime_VERSION}.tgz")
+    set(Onnxruntime_HASH SHA256=9FA57FA6F202A373599377EF75064AE568FDA8DA838632B26A86024C7378D306)
+  elseif(MSVC)
+    set(Onnxruntime_URL "${Onnxruntime_WINDOWS_BASEURL}/onnxruntime-windows-${Onnxruntime_WINDOWS_VERSION}-Release.zip")
+    set(OOnnxruntime_HASH SHA256=39E63850D9762810161AE1B4DEAE5E3C02363521273E4B894A9D9707AB626C38)
+  else()
+    if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+      set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-linux-aarch64-${Onnxruntime_VERSION}.tgz")
+      set(Onnxruntime_HASH SHA256=70B6F536BB7AB5961D128E9DBD192368AC1513BFFB74FE92F97AAC342FBD0AC1)
+    else()
+      set(Onnxruntime_URL "${Onnxruntime_BASEURL}/onnxruntime-linux-x64-gpu-${Onnxruntime_VERSION}.tgz")
+      set(Onnxruntime_HASH SHA256=613C53745EA4960ED368F6B3AB673558BB8561C84A8FA781B4EA7FB4A4340BE4)
+    endif()
+  endif()
+else()
+  set(Onnxruntime_URL "${CUSTOM_ONNXRUNTIME_URL}")
+  set(Onnxruntime_HASH "${CUSTOM_ONNXRUNTIME_HASH}")
+endif()
+
+FetchContent_Declare(
+  onnxruntime
+  URL ${Onnxruntime_URL}
+  URL_HASH ${Onnxruntime_HASH})
+FetchContent_MakeAvailable(onnxruntime)
+
+if(APPLE)
+  set(Onnxruntime_LIB "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.${Onnxruntime_VERSION}.dylib")
+  target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIB}")
+  target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${onnxruntime_SOURCE_DIR}/include")
+  target_sources(${CMAKE_PROJECT_NAME} PRIVATE "${Onnxruntime_LIB}")
+  set_property(SOURCE "${Onnxruntime_LIB}" PROPERTY MACOSX_PACKAGE_LOCATION Frameworks)
+  source_group("Frameworks" FILES "${Onnxruntime_LIB}")
+  add_custom_command(
+    TARGET "${CMAKE_PROJECT_NAME}"
+    POST_BUILD
+    COMMAND
+      ${CMAKE_INSTALL_NAME_TOOL} -change "@rpath/libonnxruntime.${Onnxruntime_VERSION}.dylib"
+      "@loader_path/../Frameworks/libonnxruntime.${Onnxruntime_VERSION}.dylib" $<TARGET_FILE:${CMAKE_PROJECT_NAME}>)
+elseif(MSVC)
+  add_library(Ort INTERFACE)
+  set(Onnxruntime_LIB_NAMES
+      session;providers_shared;providers_dml;optimizer;providers;framework;graph;util;mlas;common;flatbuffers)
+  foreach(lib_name IN LISTS Onnxruntime_LIB_NAMES)
+    add_library(Ort::${lib_name} STATIC IMPORTED)
+    set_target_properties(Ort::${lib_name} PROPERTIES IMPORTED_LOCATION
+                                                      ${onnxruntime_SOURCE_DIR}/lib/onnxruntime_${lib_name}.lib)
+    set_target_properties(Ort::${lib_name} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${onnxruntime_SOURCE_DIR}/include)
+    target_link_libraries(Ort INTERFACE Ort::${lib_name})
+  endforeach()
+
+  set(Onnxruntime_EXTERNAL_LIB_NAMES
+      onnx;onnx_proto;libprotobuf-lite;re2;absl_throw_delegate;absl_hash;absl_city;absl_low_level_hash;absl_raw_hash_set
+  )
+  foreach(lib_name IN LISTS Onnxruntime_EXTERNAL_LIB_NAMES)
+    add_library(Ort::${lib_name} STATIC IMPORTED)
+    set_target_properties(Ort::${lib_name} PROPERTIES IMPORTED_LOCATION ${onnxruntime_SOURCE_DIR}/lib/${lib_name}.lib)
+    target_link_libraries(Ort INTERFACE Ort::${lib_name})
+  endforeach()
+
+  add_library(Ort::DirectML SHARED IMPORTED)
+  set_target_properties(Ort::DirectML PROPERTIES IMPORTED_LOCATION ${onnxruntime_SOURCE_DIR}/bin/DirectML.dll)
+  set_target_properties(Ort::DirectML PROPERTIES IMPORTED_IMPLIB ${onnxruntime_SOURCE_DIR}/bin/DirectML.lib)
+
+  target_link_libraries(Ort INTERFACE Ort::DirectML d3d12.lib dxgi.lib dxguid.lib Dxcore.lib)
+
+  target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE Ort)
+
+  install(IMPORTED_RUNTIME_ARTIFACTS Ort::DirectML DESTINATION "obs-plugins/64bit")
+else()
+  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
+    set(Onnxruntime_LINK_LIBS "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.so.${Onnxruntime_VERSION}")
+    set(Onnxruntime_INSTALL_LIBS ${Onnxruntime_LINK_LIBS})
+  else()
+    set(Onnxruntime_LINK_LIBS "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime.so.${Onnxruntime_VERSION}")
+    set(Onnxruntime_INSTALL_LIBS
+        ${Onnxruntime_LINK_LIBS} "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime_providers_shared.so"
+        "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime_providers_cuda.so"
+        "${onnxruntime_SOURCE_DIR}/lib/libonnxruntime_providers_tensorrt.so")
+  endif()
+  target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE ${Onnxruntime_LINK_LIBS})
+  target_include_directories(${CMAKE_PROJECT_NAME} SYSTEM PUBLIC "${onnxruntime_SOURCE_DIR}/include")
+  install(FILES ${Onnxruntime_INSTALL_LIBS} DESTINATION "${CMAKE_INSTALL_LIBDIR}/obs-plugins/${CMAKE_PROJECT_NAME}")
+  set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/${CMAKE_PROJECT_NAME}")
+endif()

cmake/FetchOpenCV.cmake

@@ -0,0 +1,80 @@
+include(FetchContent)
+
+set(CUSTOM_OPENCV_URL
+    ""
+    CACHE STRING "URL of a downloaded OpenCV static library tarball")
+
+set(CUSTOM_OPENCV_HASH
+    ""
+    CACHE STRING "Hash of a downloaded OpenCV staitc library tarball")
+
+if(CUSTOM_OPENCV_URL STREQUAL "")
+  set(USE_PREDEFINED_OPENCV ON)
+else()
+  if(CUSTOM_OPENCV_HASH STREQUAL "")
+    message(FATAL_ERROR "Both of CUSTOM_OPENCV_URL and CUSTOM_OPENCV_HASH must be present!")
+  else()
+    set(USE_PREDEFINED_OPENCV OFF)
+  endif()
+endif()
+
+if(USE_PREDEFINED_OPENCV)
+  set(OpenCV_VERSION "v4.8.1-1")
+  set(OpenCV_BASEURL "https://github.com/obs-ai/obs-backgroundremoval-dep-opencv/releases/download/${OpenCV_VERSION}")
+
+  if(${CMAKE_BUILD_TYPE} STREQUAL Release OR ${CMAKE_BUILD_TYPE} STREQUAL RelWithDebInfo)
+    set(OpenCV_BUILD_TYPE Release)
+  else()
+    set(OpenCV_BUILD_TYPE Debug)
+  endif()
+
+  if(APPLE)
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Debug.tar.gz")
+      set(OpenCV_HASH SHA256=2930e335a19cc03a3d825e2b76eadd0d5cf08d8baf6537747d43f503dff32454)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-macos-${OpenCV_VERSION}-Release.tar.gz")
+      set(OpenCV_HASH SHA256=b0c4fe2370b0bd5aa65c408e875b1ab18508ba31b93083805d7e398a3ecafdac)
+    endif()
+  elseif(MSVC)
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Debug.zip")
+      set(OpenCV_HASH SHA256=0c5ef12cf4b4e4db7ea17a24db156165b6f01759f3f1660b069d0722e5d5dc37)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-windows-${OpenCV_VERSION}-Release.zip")
+      set(OpenCV_HASH SHA256=5e468f71d41d3a3ea46cc4f247475877f65d3655a2764a2c01074bda3b3e6864)
+    endif()
+  else()
+    if(OpenCV_BUILD_TYPE STREQUAL Debug)
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Debug.tar.gz")
+      set(OpenCV_HASH SHA256=e2e246d6b4f279be80e7fd0a78cba8a0eeee7b53ae807f2f57428d6876306422)
+    else()
+      set(OpenCV_URL "${OpenCV_BASEURL}/opencv-linux-${OpenCV_VERSION}-Release.tar.gz")
+      set(OpenCV_HASH SHA256=809922a7cc9f344a2d82a232ed7b02e122c82e77cba94b4047e666a0527cc00e)
+    endif()
+  endif()
+else()
+  set(OpenCV_URL "${CUSTOM_OPENCV_URL}")
+  set(OpenCV_HASH "${CUSTOM_OPENCV_HASH}")
+endif()
+
+FetchContent_Declare(
+  opencv
+  URL ${OpenCV_URL}
+  URL_HASH ${OpenCV_HASH})
+FetchContent_MakeAvailable(opencv)
+
+add_library(OpenCV INTERFACE)
+if(MSVC)
+  target_link_libraries(
+    OpenCV
+    INTERFACE ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_imgproc481.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/opencv_core481.lib
+              ${opencv_SOURCE_DIR}/x64/vc17/staticlib/zlib.lib)
+  target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include)
+else()
+  target_link_libraries(
+    OpenCV INTERFACE ${opencv_SOURCE_DIR}/lib/libopencv_imgproc.a ${opencv_SOURCE_DIR}/lib/libopencv_core.a
+                     ${opencv_SOURCE_DIR}/lib/opencv4/3rdparty/libzlib.a)
+  target_include_directories(OpenCV SYSTEM INTERFACE ${opencv_SOURCE_DIR}/include/opencv4)
+endif()

cmake/windows/compilerconfig.cmake

@@ -35,7 +35,8 @@ add_compile_options(
   "$<$<COMPILE_LANG_AND_ID:CXX,MSVC>:/MP>"
   "$<$<COMPILE_LANG_AND_ID:C,Clang>:${_obs_clang_c_options}>"
   "$<$<COMPILE_LANG_AND_ID:CXX,Clang>:${_obs_clang_cxx_options}>"
-  $<$<NOT:$<CONFIG:Debug>>:/Gy>)
+  $<$<NOT:$<CONFIG:Debug>>:/Gy>
+  /IGNORE:4099)
 
 add_compile_definitions(UNICODE _UNICODE _CRT_SECURE_NO_WARNINGS _CRT_NONSTDC_NO_WARNINGS $<$<CONFIG:DEBUG>:DEBUG>
                         $<$<CONFIG:DEBUG>:_DEBUG>)
@@ -45,7 +46,8 @@ add_link_options($<$<NOT:$<CONFIG:Debug>>:/OPT:REF>
                  $<$<NOT:$<CONFIG:Debug>>:/OPT:ICF>
                  $<$<NOT:$<CONFIG:Debug>>:/INCREMENTAL:NO>
                  /DEBUG
-                 /Brepro)
+                 /Brepro
+                 /IGNORE:4099)
 # cmake-format: on
 
 if(CMAKE_COMPILE_WARNING_AS_ERROR)

src/FilterData.h

@@ -0,0 +1,41 @@
+#ifndef FILTERDATA_H
+#define FILTERDATA_H
+
+#include <obs-module.h>
+#include "edgeyolo/edgeyolo_onnxruntime.hpp"
+
+/**
+  * @brief The filter_data struct
+  *
+  * This struct is used to store the base data needed for ORT filters.
+  *
+*/
+struct filter_data {
+	std::string useGPU;
+	uint32_t numThreads;
+
+	obs_source_t *source;
+	gs_texrender_t *texrender;
+	gs_stagesurf_t *stagesurface;
+	gs_effect_t *effect;
+
+	cv::Mat inputBGRA;
+	cv::Mat outputPreviewBGRA;
+
+	bool isDisabled;
+	bool preview;
+
+	std::mutex inputBGRALock;
+	std::mutex outputLock;
+	std::mutex modelMutex;
+
+	std::unique_ptr<edgeyolo_cpp::EdgeYOLOONNXRuntime> edgeyolo;
+
+#if _WIN32
+	std::wstring modelFilepath;
+#else
+	std::string modelFilepath;
+#endif
+};
+
+#endif /* FILTERDATA_H */

src/consts.h

@@ -0,0 +1,21 @@
+#ifndef CONSTS_H
+#define CONSTS_H
+
+const char *const USEGPU_CPU = "cpu";
+const char *const USEGPU_DML = "dml";
+const char *const USEGPU_CUDA = "cuda";
+const char *const USEGPU_TENSORRT = "tensorrt";
+const char *const USEGPU_COREML = "coreml";
+
+const char *const EFFECT_PATH = "effects/mask_alpha_filter.effect";
+const char *const KAWASE_BLUR_EFFECT_PATH = "effects/kawase_blur.effect";
+const char *const BLEND_EFFECT_PATH = "effects/blend_images.effect";
+
+const char *const PLUGIN_INFO_TEMPLATE =
+	"<a href=\"https://github.com/occ-ai/obs-detect/\">Detect Plugin</a> (%1) by "
+	"<a href=\"https://github.com/occ-ai\">OCC AI</a> ❤️ "
+	"<a href=\"https://www.patreon.com/RoyShilkrot\">Support & Follow</a>";
+const char *const PLUGIN_INFO_TEMPLATE_UPDATE_AVAILABLE =
+	"<center><a href=\"https://github.com/occ-ai/obs-detect/releases\">🚀 Update available! (%1)</a></center>";
+
+#endif /* CONSTS_H */

src/detect-filter-info.c

@@ -0,0 +1,17 @@
+#include "detect-filter.h"
+
+struct obs_source_info detect_filter_info = {
+	.id = "detect-filter",
+	.type = OBS_SOURCE_TYPE_FILTER,
+	.output_flags = OBS_SOURCE_VIDEO,
+	.get_name = detect_filter_getname,
+	.create = detect_filter_create,
+	.destroy = detect_filter_destroy,
+	.get_defaults = detect_filter_defaults,
+	.get_properties = detect_filter_properties,
+	.update = detect_filter_update,
+	.activate = detect_filter_activate,
+	.deactivate = detect_filter_deactivate,
+	.video_tick = detect_filter_video_tick,
+	.video_render = detect_filter_video_render,
+};