Add TRTIS_ENABLE_ONNXRUNTIME_TENSORRT to allow TensorRT execution pro…

…vider to be disabled (triton-inference-server#1132)
mengkai94 · Feb 26, 2020 · 97aa9b9 · 97aa9b9
1 parent 7d3088a
commit 97aa9b9
Show file tree

Hide file tree

Showing 7 changed files with 27 additions and 7 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -326,6 +326,7 @@ RUN LIBCUDA_FOUND=$(ldconfig -p | grep -v compat | awk '{print $1}' | grep libcu
                   -DTRTIS_ENABLE_TENSORRT=ON \
                   -DTRTIS_ENABLE_CAFFE2=ON \
                   -DTRTIS_ENABLE_ONNXRUNTIME=ON \
+                  -DTRTIS_ENABLE_ONNXRUNTIME_TENSORRT=ON \
                   -DTRTIS_ENABLE_ONNXRUNTIME_OPENVINO=ON \
                   -DTRTIS_ENABLE_PYTORCH=ON \
                   -DTRTIS_ONNXRUNTIME_INCLUDE_PATHS="/opt/tensorrtserver/include/onnxruntime" \

diff --git a/build/CMakeLists.txt b/build/CMakeLists.txt
@@ -36,6 +36,8 @@ option(TRTIS_ENABLE_TENSORRT "Include TensorRT backend in server" OFF)
 option(TRTIS_ENABLE_TENSORFLOW "Include TensorFlow backend in server" OFF)
 option(TRTIS_ENABLE_CAFFE2 "Include Caffe2 backend in server" OFF)
 option(TRTIS_ENABLE_ONNXRUNTIME "Include ONNXRuntime backend in server" OFF)
+option(TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
+  "Enable TensorRT execution provider for ONNXRuntime backend in server" OFF)
 option(TRTIS_ENABLE_ONNXRUNTIME_OPENVINO
   "Enable OpenVINO execution provider for ONNXRuntime backend in server" OFF)
 option(TRTIS_ENABLE_PYTORCH "Include PyTorch backend in server" OFF)
@@ -97,6 +99,13 @@ if (TRTIS_ENABLE_METRICS_GPU AND NOT TRTIS_ENABLE_GPU)
   message(FATAL_ERROR "TRTIS_ENABLE_METRICS_GPU=ON requires TRTIS_ENABLE_GPU ON")
 endif()
 
+if(TRTIS_ENABLE_ONNXRUNTIME_TENSORRT AND NOT TRTIS_ENABLE_ONNXRUNTIME)
+  message(FATAL_ERROR "TRTIS_ENABLE_ONNXRUNTIME_TENSORRT=ON requires TRTIS_ENABLE_ONNXRUNTIME=ON")
+endif()
+if(TRTIS_ENABLE_ONNXRUNTIME_TENSORRT AND NOT TRTIS_ENABLE_TENSORRT)
+  message(FATAL_ERROR "TRTIS_ENABLE_ONNXRUNTIME_TENSORRT=ON requires TRTIS_ENABLE_TENSORRT=ON")
+endif()
+
 if(TRTIS_ENABLE_ONNXRUNTIME_OPENVINO AND NOT TRTIS_ENABLE_ONNXRUNTIME)
   message(FATAL_ERROR "TRTIS_ENABLE_ONNXRUNTIME_OPENVINO=ON requires TRTIS_ENABLE_ONNXRUNTIME=ON")
 endif()
@@ -482,6 +491,7 @@ ExternalProject_Add(trtis
     -DTRTIS_ENABLE_TENSORRT:BOOL=${TRTIS_ENABLE_TENSORRT}
     -DTRTIS_ENABLE_CAFFE2:BOOL=${TRTIS_ENABLE_CAFFE2}
     -DTRTIS_ENABLE_ONNXRUNTIME:BOOL=${TRTIS_ENABLE_ONNXRUNTIME}
+    -DTRTIS_ENABLE_ONNXRUNTIME_TENSORRT:BOOL=${TRTIS_ENABLE_ONNXRUNTIME_TENSORRT}
     -DTRTIS_ENABLE_ONNXRUNTIME_OPENVINO:BOOL=${TRTIS_ENABLE_ONNXRUNTIME_OPENVINO}
     -DTRTIS_ENABLE_PYTORCH:BOOL=${TRTIS_ENABLE_PYTORCH}
     -DTRTIS_ENABLE_CUSTOM:BOOL=${TRTIS_ENABLE_CUSTOM}

diff --git a/build/trtis/CMakeLists.txt b/build/trtis/CMakeLists.txt
@@ -116,6 +116,10 @@ if(${TRTIS_ENABLE_ONNXRUNTIME})
   add_definitions(-DTRTIS_ENABLE_ONNXRUNTIME=1)
 endif() # TRTIS_ENABLE_ONNXRUNTIME
 
+if(${TRTIS_ENABLE_ONNXRUNTIME_TENSORRT})
+  add_definitions(-DTRTIS_ENABLE_ONNXRUNTIME_TENSORRT=1)
+endif() # TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
+
 if(${TRTIS_ENABLE_ONNXRUNTIME_OPENVINO})
   add_definitions(-DTRTIS_ENABLE_ONNXRUNTIME_OPENVINO=1)
 endif() # TRTIS_ENABLE_ONNXRUNTIME_OPENVINO

diff --git a/docs/build.rst b/docs/build.rst
@@ -201,9 +201,12 @@ different build process but you may have build or execution issues.
 Your build should produce the ONNX Runtime library, libonnxruntime.so.
 You can enable the ONNX Runtime backend in the inference server with
 the CMake option -DTRTIS_ENABLE_ONNXRUNTIME=ON as described below. If
-you want to enable OpenVino within the ONNX Runtime you must also
-specify the CMake option TRTIS_ENABLE_ONNXRUNTIME_OPENVINO=ON and
-provide the necessary OpenVino dependencies.
+you want to enable TensorRT within the ONNX Runtime you must also
+specify the CMake option TRTIS_ENABLE_ONNXRUNTIME_TENSORRT=ON and
+provide the necessary TensorRT dependencies. If you want to enable
+OpenVino within the ONNX Runtime you must also specify the CMake
+option TRTIS_ENABLE_ONNXRUNTIME_OPENVINO=ON and provide the necessary
+OpenVino dependencies.
 
 You can install the library in a system library path or you can
 specify the path with the CMake option TRTIS_EXTRA_LIB_PATHS. Multiple

diff --git a/qa/L0_grpc_v2/test.sh b/qa/L0_grpc_v2/test.sh
@@ -68,6 +68,7 @@ fi
         -DTRTIS_ENABLE_TENSORFLOW=ON \
         -DTRTIS_ENABLE_CAFFE2=OFF \
         -DTRTIS_ENABLE_ONNXRUNTIME=OFF \
+        -DTRTIS_ENABLE_ONNXRUNTIME_TENSORRT=OFF \
         -DTRTIS_ENABLE_ONNXRUNTIME_OPENVINO=OFF \
         -DTRTIS_ENABLE_PYTORCH=OFF \
         -DTRTIS_ENABLE_GPU=OFF \

diff --git a/qa/L0_http_v2/test.sh b/qa/L0_http_v2/test.sh
@@ -67,6 +67,7 @@ fi
         -DTRTIS_ENABLE_TENSORFLOW=ON \
         -DTRTIS_ENABLE_CAFFE2=OFF \
         -DTRTIS_ENABLE_ONNXRUNTIME=OFF \
+        -DTRTIS_ENABLE_ONNXRUNTIME_TENSORRT=OFF \
         -DTRTIS_ENABLE_ONNXRUNTIME_OPENVINO=OFF \
         -DTRTIS_ENABLE_PYTORCH=OFF \
         -DTRTIS_ENABLE_GPU=OFF \

diff --git a/src/backends/onnx/onnx_backend.cc b/src/backends/onnx/onnx_backend.cc
@@ -43,9 +43,9 @@
 #include <cuda_runtime_api.h>
 #endif  // TRTIS_ENABLE_GPU
 
-#ifdef TRTIS_ENABLE_TENSORRT
+#ifdef TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
 #include <tensorrt_provider_factory.h>
-#endif  // TRTIS_ENABLE_TENSORRT
+#endif  // TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
 
 #ifdef TRTIS_ENABLE_ONNXRUNTIME_OPENVINO
 #include <openvino_provider_factory.h>
@@ -226,14 +226,14 @@ OnnxBackend::CreateExecutionContext(
                .optimization()
                .execution_accelerators()
                .gpu_execution_accelerator()) {
-#ifdef TRTIS_ENABLE_TENSORRT
+#ifdef TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
         if (execution_accelerator.name() == kTensorRTExecutionAccelerator) {
           RETURN_IF_ORT_ERROR(OrtSessionOptionsAppendExecutionProvider_Tensorrt(
               session_options, gpu_device));
           LOG_VERBOSE(1) << "TensorRT Execution Accelerator is set for "
                          << instance_name << " on device " << gpu_device;
         } else
-#endif  // TRTIS_ENABLE_TENSORRT
+#endif  // TRTIS_ENABLE_ONNXRUNTIME_TENSORRT
         {
           return Status(
               RequestStatusCode::INVALID_ARG,