[WIP] ort upgrade from 1.12.0 to 1.20.1

libs/ultra-infer/CMakeLists.txt

@@ -36,10 +36,10 @@ set(THIRD_PARTY_DIR ${PROJECT_SOURCE_DIR}/third_party)
 add_subdirectory(${CSRCS_DIR_NAME}/ultra_infer)
 include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake)
 
-# Set C++11 as standard for the whole project
+# Set C++17 as standard for the whole project
 if(NOT MSVC)
   if(NOT DEFINED CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD 17)
   endif()
   set(CMAKE_CXX_FLAGS "-Wno-format -g0 -O3")
   if(NEED_ABI0)
@@ -70,6 +70,7 @@ option(ENABLE_SOPHGO_BACKEND "Whether to enable SOPHON backend." OFF)
 option(ENABLE_TVM_BACKEND "Whether to enable TVM backend." OFF)
 option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF)
 option(ENABLE_HORIZON_BACKEND "Whether to enable HORIZON backend." OFF)
+option(ENABLE_OM_BACKEND "Whether to enable OM backend." OFF)
 option(ENABLE_VISION "Whether to enable vision models usage." OFF)
 option(ENABLE_TEXT "Whether to enable text models usage." OFF)
 option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF)
@@ -172,6 +173,7 @@ file(GLOB_RECURSE DEPLOY_SOPHGO_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ult
 file(GLOB_RECURSE DEPLOY_TVM_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/runtime/backends/tvm/*.cc)
 file(GLOB_RECURSE DEPLOY_LITE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/runtime/backends/lite/*.cc)
 file(GLOB_RECURSE DEPLOY_PIPELINE_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/pipeline/*.cc)
+file(GLOB_RECURSE DEPLOY_OM_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/runtime/backends/om/*.cc)
 file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/vision/*.cc)
 file(GLOB_RECURSE DEPLOY_TEXT_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/text/*.cc)
 file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/pybind/*.cc ${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}/ultra_infer/*_pybind.cc)
@@ -194,7 +196,7 @@ list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS}
                                  ${DEPLOY_PIPELINE_SRCS} ${DEPLOY_RKNPU2_SRCS}
                                  ${DEPLOY_SOPHGO_SRCS}
                                  ${DEPLOY_HORIZON_SRCS} ${DEPLOY_TVM_SRCS}
-                                 ${DEPLOY_PADDLE_CUSTOM_OP_SRCS})
+                                 ${DEPLOY_PADDLE_CUSTOM_OP_SRCS} ${DEPLOY_OM_SRCS})
 
 
 set(DEPEND_LIBS "")
@@ -328,6 +330,13 @@ if(ENABLE_POROS_BACKEND)
   execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib)
 endif()
 
+if(ENABLE_OM_BACKEND)
+  add_definitions(-DENABLE_OM_BACKEND)
+  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_OM_SRCS})
+  include(${PROJECT_SOURCE_DIR}/cmake/om.cmake)
+  list(APPEND DEPEND_LIBS ${NPU_libs})
+endif()
+
 if(WITH_GPU)
   add_definitions(-DWITH_GPU)
   include_directories(${CUDA_DIRECTORY}/include)

libs/ultra-infer/UltraInfer.cmake.in

@@ -32,6 +32,7 @@ set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@)
 set(ENABLE_OPENVINO_BACKEND @ENABLE_OPENVINO_BACKEND@)
 set(ENABLE_POROS_BACKEND @ENABLE_POROS_BACKEND@)
 set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@)
+set(ENABLE_OM_BACKEND @ENABLE_OM_BACKEND@)
 set(ENABLE_PADDLE2ONNX @ENABLE_PADDLE2ONNX@)
 set(BUILD_PADDLE2ONNX @BUILD_PADDLE2ONNX@)
 
@@ -179,6 +180,32 @@ if(ENABLE_POROS_BACKEND)
   list(APPEND ULTRAINFER_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/poros/include ${TORCH_INCLUDE})
 endif()
 
+if(ENABLE_OM_BACKEND)
+  set(LIB_PATH $ENV{NPU_HOST_LIB})
+  if (NOT DEFINED ENV{NPU_HOST_LIB})
+    set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/lib64")
+    message(STATUS "set default LIB_PATH: ${LIB_PATH}")
+  else()
+    message(STATUS "set LIB_PATH: ${LIB_PATH}")
+  endif ()
+
+  set(INC_PATH $ENV{DDK_PATH})
+  if (NOT DEFINED ENV{DDK_PATH})
+    set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest")
+    message(STATUS "set default INC_PATH: ${INC_PATH}")
+  else()
+    message(STATUS "set INC_PATH: ${INC_PATH}")
+  endif ()
+
+  set(NPU_libs ascendcl stdc++)
+
+  link_directories(${LIB_PATH})
+
+  list(APPEND ULTRAINFER_LIBS ${NPU_libs})
+
+  list(APPEND ULTRAINFER_INCS ${INC_PATH}/runtime/include/)
+endif()
+
 if(WITH_GPU)
   if(NOT CUDA_DIRECTORY)
     set(CUDA_DIRECTORY "/usr/local/cuda")

libs/ultra-infer/cmake/om.cmake

@@ -0,0 +1,29 @@
+set(CMAKE_CXX_FLAGS_DEBUG "-fPIC -O0 -g -Wall")
+set(CMAKE_CXX_FLAGS_RELEASE "-fPIC -O2 -Wall")
+
+set(INC_PATH $ENV{DDK_PATH})
+if (NOT DEFINED ENV{DDK_PATH})
+    set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest")
+    message(STATUS "set default INC_PATH: ${INC_PATH}")
+else()
+    message(STATUS "set INC_PATH: ${INC_PATH}")
+endif ()
+
+set(LIB_PATH $ENV{NPU_HOST_LIB})
+if (NOT DEFINED ENV{NPU_HOST_LIB})
+    set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/x86_64-linux/lib64")
+    message(STATUS "set default LIB_PATH: ${LIB_PATH}")
+else()
+    message(STATUS "set LIB_PATH: ${LIB_PATH}")
+endif ()
+
+
+set(NPU_libs ascendcl stdc++)
+
+include_directories(
+   ${INC_PATH}/runtime/include/
+)
+
+link_directories(
+    ${LIB_PATH}
+)

libs/ultra-infer/cmake/onnxruntime.cmake

@@ -41,8 +41,8 @@ else()
 endif()
 set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${ONNXRUNTIME_LIB_DIR}")
 
-set(ONNXRUNTIME_VERSION "1.12.0")
-set(ONNXRUNTIME_URL_PREFIX "https://bj.bcebos.com/paddle2onnx/libs/")
+set(ONNXRUNTIME_VERSION "1.20.1")
+set(ONNXRUNTIME_URL_PREFIX "https://fastdeploy.bj.bcebos.com/third_libs/")
 
 if(WIN32) 
   if(WITH_GPU)

libs/ultra-infer/python/setup.py

@@ -69,6 +69,7 @@
 setup_configs["ENABLE_POROS_BACKEND"] = os.getenv("ENABLE_POROS_BACKEND", "OFF")
 setup_configs["ENABLE_TRT_BACKEND"] = os.getenv("ENABLE_TRT_BACKEND", "OFF")
 setup_configs["ENABLE_LITE_BACKEND"] = os.getenv("ENABLE_LITE_BACKEND", "OFF")
+setup_configs["ENABLE_OM_BACKEND"] = os.getenv("ENABLE_OM_BACKEND", "OFF")
 setup_configs["ENABLE_PADDLE2ONNX"] = os.getenv("ENABLE_PADDLE2ONNX", "OFF")
 setup_configs["ENABLE_VISION"] = os.getenv("ENABLE_VISION", "OFF")
 setup_configs["ENABLE_FLYCV"] = os.getenv("ENABLE_FLYCV", "OFF")

libs/ultra-infer/python/ultra_infer/__init__.py

@@ -152,6 +152,8 @@ def tensorrt_is_avaliable():
     is_built_with_paddle,
     is_built_with_trt,
     get_default_cuda_directory,
+    is_built_with_openvino,
+    is_built_with_om,
 )
 
 

libs/ultra-infer/python/ultra_infer/c_lib_wrap.py.in

@@ -43,6 +43,10 @@ def is_built_with_openvino() ->bool:
     return True if "@ENABLE_OPENVINO_BACKEND@" == "ON" else False
 
 
+def is_built_with_om() ->bool:
+    return True if "@ENABLE_OM_BACKEND@" == "ON" else False
+
+
 def get_default_cuda_directory() -> str:
     if not is_built_with_gpu():
        return ""

libs/ultra-infer/python/ultra_infer/runtime.py

@@ -321,6 +321,10 @@ def use_paddle_lite_backend(self):
         """Wrapper function of use_lite_backend(), use Paddle Lite backend, support inference Paddle model on ARM CPU."""
         return self.use_lite_backend()
 
+    def use_om_backend(self):
+        """Use Om backend, support inference Om model on NPU"""
+        return self._option.use_om_backend()
+
     def set_lite_context_properties(self, context_properties):
         """Set nnadapter context properties for Paddle Lite backend."""
         logging.warning(

libs/ultra-infer/ultra_infer/core/config.h.in

@@ -84,3 +84,7 @@
 #ifndef ENABLE_TVM_BACKEND
 #cmakedefine ENABLE_TVM_BACKEND
 #endif
+
+#ifndef ENABLE_OM_BACKEND
+#cmakedefine ENABLE_OM_BACKEND
+#endif

libs/ultra-infer/ultra_infer/pybind/main.cc.in

@@ -41,6 +41,8 @@ pybind11::dtype FDDataTypeToNumpyDataType(const FDDataType& fd_dtype) {
     dt = pybind11::dtype::of<int8_t>();
   } else if (fd_dtype == FDDataType::FP16) {
     dt = pybind11::dtype::of<float16>();
+  } else if (fd_dtype == FDDataType::BOOL) {
+    dt = pybind11::dtype::of<bool>();
   } else {
     FDASSERT(false, "The function doesn't support data type of %s.",
                         Str(fd_dtype).c_str());

libs/ultra-infer/ultra_infer/pybind/runtime.cc

@@ -137,15 +137,17 @@ void BindRuntime(pybind11::module &m) {
       .value("RKNPU2", Backend::RKNPU2)
       .value("SOPHGOTPU", Backend::SOPHGOTPU)
       .value("TVM", Backend::TVM)
-      .value("LITE", Backend::LITE);
+      .value("LITE", Backend::LITE)
+      .value("OMONNPU", Backend::OMONNPU);
   pybind11::enum_<ModelFormat>(m, "ModelFormat", pybind11::arithmetic(),
                                "ModelFormat for inference.")
       .value("PADDLE", ModelFormat::PADDLE)
       .value("TORCHSCRIPT", ModelFormat::TORCHSCRIPT)
       .value("RKNN", ModelFormat::RKNN)
       .value("SOPHGO", ModelFormat::SOPHGO)
       .value("ONNX", ModelFormat::ONNX)
-      .value("TVMFormat", ModelFormat::TVMFormat);
+      .value("TVMFormat", ModelFormat::TVMFormat)
+      .value("OM", ModelFormat::OM);
   pybind11::enum_<Device>(m, "Device", pybind11::arithmetic(),
                           "Device for inference.")
       .value("CPU", Device::CPU)