From f6676723ec14abbb61c18b04459d043df5a99080 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 23 Apr 2024 14:42:02 -0700 Subject: [PATCH 1/5] update --- cmake/onnxruntime_providers_tensorrt.cmake | 61 ++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 15ffc29e79ff4..4a8ba538ce115 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -40,15 +40,70 @@ HINTS ${TENSORRT_ROOT} PATH_SUFFIXES include) MESSAGE(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") - find_library(TENSORRT_LIBRARY_INFER nvinfer + + # For TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows. + if (WIN32) + file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) + string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") + string(REGEX MATCH "define NV_TENSORRT_PATCH * +([0-9]+)" NV_TENSORRT_PATCH "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_PATCH * +([0-9]+)" "\\1" NV_TENSORRT_PATCH "${NV_TENSORRT_PATCH}") + math(EXPR NV_TENSORRT_MAJOR_INT "${NV_TENSORRT_MAJOR}") + math(EXPR NV_TENSORRT_PATCH_INT "${NV_TENSORRT_PATCH}") + + if (NV_TENSORRT_MAJOR) + MESSAGE(STATUS "NV_TENSORRT_MAJOR is ${NV_TENSORRT_MAJOR}") + else() + MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") + endif() + + # Here we only check TRT version > TRT 10 GA + # Note: TRT 10 EA is 10.0.0.6 but with no major version appended to the end + if ((NV_TENSORT_MAJOR_INT GREATER 10) OR (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) + set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") + set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") + set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") + endif() + endif() + + if (NOT NVINFER_LIB) + set(NVINFER_LIB "nvinfer") + endif() + + if (NOT NVINFER_PLUGIN_LIB) + set(NVINFER_PLUGIN_LIB "nvinfer_plugin") + endif() + + if (NOT PARSER_LIB) + set(PARSER_LIB "nvonnxparser") + endif() + + MESSAGE(STATUS "Search for ${NVINFER_LIB}, ${NVINFER_PLUGIN_LIB} and ${PARSER_LIB}") + + find_library(TENSORRT_LIBRARY_INFER ${NVINFER_LIB} HINTS ${TENSORRT_ROOT} PATH_SUFFIXES lib lib64 lib/x64) - find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin + + if (NOT TENSORRT_LIBRARY_INFER) + MESSAGE(STATUS "Can't find ${NVINFER_LIB}") + endif() + + find_library(TENSORRT_LIBRARY_INFER_PLUGIN ${NVINFER_PLUGIN_LIB} HINTS ${TENSORRT_ROOT} PATH_SUFFIXES lib lib64 lib/x64) - find_library(TENSORRT_LIBRARY_NVONNXPARSER nvonnxparser + + if (NOT TENSORRT_LIBRARY_INFER_PLUGIN) + MESSAGE(STATUS "Can't find ${NVINFER_PLUGIN_LIB}") + endif() + + find_library(TENSORRT_LIBRARY_NVONNXPARSER ${PARSER_LIB} HINTS ${TENSORRT_ROOT} PATH_SUFFIXES lib lib64 lib/x64) + + if (NOT TENSORRT_LIBRARY_NVONNXPARSER) + MESSAGE(STATUS "Can't find ${PARSER_LIB}") + endif() + set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_NVONNXPARSER}) MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}") else() From 92e11f16599a549b790a3e6d2fcb62889078e1c0 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Tue, 23 Apr 2024 14:44:26 -0700 Subject: [PATCH 2/5] update --- cmake/onnxruntime_providers_tensorrt.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 4a8ba538ce115..37479987d8a9f 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -41,7 +41,8 @@ PATH_SUFFIXES include) MESSAGE(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") - # For TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows. + # For TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, + # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... if (WIN32) file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") From 2d456346d2b9ce449e52356df6f56d1e4b3b469e Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 24 Apr 2024 09:24:34 -0700 Subject: [PATCH 3/5] update --- cmake/onnxruntime_providers_tensorrt.cmake | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 37479987d8a9f..265f1ffb22ff2 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -41,15 +41,18 @@ PATH_SUFFIXES include) MESSAGE(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") - # For TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, + # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... if (WIN32) file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") + string(REGEX MATCH "define NV_TENSORRT_MINOR * +([0-9]+)" NV_TENSORRT_MINOR "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_MINOR * +([0-9]+)" "\\1" NV_TENSORRT_MINOR "${NV_TENSORRT_MINOR}") string(REGEX MATCH "define NV_TENSORRT_PATCH * +([0-9]+)" NV_TENSORRT_PATCH "${NVINFER_VER_CONTENT}") string(REGEX REPLACE "define NV_TENSORRT_PATCH * +([0-9]+)" "\\1" NV_TENSORRT_PATCH "${NV_TENSORRT_PATCH}") math(EXPR NV_TENSORRT_MAJOR_INT "${NV_TENSORRT_MAJOR}") + math(EXPR NV_TENSORRT_MINOR_INT "${NV_TENSORRT_MINOR}") math(EXPR NV_TENSORRT_PATCH_INT "${NV_TENSORRT_PATCH}") if (NV_TENSORRT_MAJOR) @@ -58,9 +61,10 @@ MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") endif() - # Here we only check TRT version > TRT 10 GA - # Note: TRT 10 EA is 10.0.0.6 but with no major version appended to the end - if ((NV_TENSORT_MAJOR_INT GREATER 10) OR (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) + # Check whether TRT version > TRT 10 GA (Note: TRT 10 EA is 10.0.0.6 but with no major version appended to the end) + if ((NV_TENSORRT_MAJOR_INT GREATER 10) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_MINOR_INT GREATER 0) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") @@ -78,7 +82,6 @@ if (NOT PARSER_LIB) set(PARSER_LIB "nvonnxparser") endif() - MESSAGE(STATUS "Search for ${NVINFER_LIB}, ${NVINFER_PLUGIN_LIB} and ${PARSER_LIB}") find_library(TENSORRT_LIBRARY_INFER ${NVINFER_LIB} From 17ac8b42e3f78dd904af9a862f0725b478a95302 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 24 Apr 2024 12:12:31 -0700 Subject: [PATCH 4/5] update per reviewer's comment --- cmake/onnxruntime_providers_tensorrt.cmake | 92 +++++++++++----------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 265f1ffb22ff2..1af5560e2ef40 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -34,54 +34,54 @@ MESSAGE(STATUS "[Note] There is an issue when running \"Debug build\" TRT EP with \"Release build\" TRT built-in parser on Windows. This build will use tensorrt oss parser instead.") endif() + # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, + # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... + if (WIN32) + file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) + string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") + string(REGEX MATCH "define NV_TENSORRT_MINOR * +([0-9]+)" NV_TENSORRT_MINOR "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_MINOR * +([0-9]+)" "\\1" NV_TENSORRT_MINOR "${NV_TENSORRT_MINOR}") + string(REGEX MATCH "define NV_TENSORRT_PATCH * +([0-9]+)" NV_TENSORRT_PATCH "${NVINFER_VER_CONTENT}") + string(REGEX REPLACE "define NV_TENSORRT_PATCH * +([0-9]+)" "\\1" NV_TENSORRT_PATCH "${NV_TENSORRT_PATCH}") + math(EXPR NV_TENSORRT_MAJOR_INT "${NV_TENSORRT_MAJOR}") + math(EXPR NV_TENSORRT_MINOR_INT "${NV_TENSORRT_MINOR}") + math(EXPR NV_TENSORRT_PATCH_INT "${NV_TENSORRT_PATCH}") + + if (NV_TENSORRT_MAJOR) + MESSAGE(STATUS "NV_TENSORRT_MAJOR is ${NV_TENSORRT_MAJOR}") + else() + MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") + endif() + + # Check TRT version >= 10.0.1.6 (Note: TRT 10 EA is 10.0.0.6 but with no major version appended to the end) + if ((NV_TENSORRT_MAJOR_INT GREATER 10) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_MINOR_INT GREATER 0) OR + (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) + set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") + set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") + set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") + endif() + endif() + + if (NOT NVINFER_LIB) + set(NVINFER_LIB "nvinfer") + endif() + + if (NOT NVINFER_PLUGIN_LIB) + set(NVINFER_PLUGIN_LIB "nvinfer_plugin") + endif() + + if (NOT PARSER_LIB) + set(PARSER_LIB "nvonnxparser") + endif() + if (onnxruntime_USE_TENSORRT_BUILTIN_PARSER) # Add TensorRT library find_path(TENSORRT_INCLUDE_DIR NvInfer.h HINTS ${TENSORRT_ROOT} PATH_SUFFIXES include) MESSAGE(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") - - # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, - # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... - if (WIN32) - file(READ ${TENSORRT_INCLUDE_DIR}/NvInferVersion.h NVINFER_VER_CONTENT) - string(REGEX MATCH "define NV_TENSORRT_MAJOR * +([0-9]+)" NV_TENSORRT_MAJOR "${NVINFER_VER_CONTENT}") - string(REGEX REPLACE "define NV_TENSORRT_MAJOR * +([0-9]+)" "\\1" NV_TENSORRT_MAJOR "${NV_TENSORRT_MAJOR}") - string(REGEX MATCH "define NV_TENSORRT_MINOR * +([0-9]+)" NV_TENSORRT_MINOR "${NVINFER_VER_CONTENT}") - string(REGEX REPLACE "define NV_TENSORRT_MINOR * +([0-9]+)" "\\1" NV_TENSORRT_MINOR "${NV_TENSORRT_MINOR}") - string(REGEX MATCH "define NV_TENSORRT_PATCH * +([0-9]+)" NV_TENSORRT_PATCH "${NVINFER_VER_CONTENT}") - string(REGEX REPLACE "define NV_TENSORRT_PATCH * +([0-9]+)" "\\1" NV_TENSORRT_PATCH "${NV_TENSORRT_PATCH}") - math(EXPR NV_TENSORRT_MAJOR_INT "${NV_TENSORRT_MAJOR}") - math(EXPR NV_TENSORRT_MINOR_INT "${NV_TENSORRT_MINOR}") - math(EXPR NV_TENSORRT_PATCH_INT "${NV_TENSORRT_PATCH}") - - if (NV_TENSORRT_MAJOR) - MESSAGE(STATUS "NV_TENSORRT_MAJOR is ${NV_TENSORRT_MAJOR}") - else() - MESSAGE(STATUS "Can't find NV_TENSORRT_MAJOR macro") - endif() - - # Check whether TRT version > TRT 10 GA (Note: TRT 10 EA is 10.0.0.6 but with no major version appended to the end) - if ((NV_TENSORRT_MAJOR_INT GREATER 10) OR - (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_MINOR_INT GREATER 0) OR - (NV_TENSORRT_MAJOR_INT EQUAL 10 AND NV_TENSORRT_PATCH_INT GREATER 0)) - set(NVINFER_LIB "nvinfer_${NV_TENSORRT_MAJOR}") - set(NVINFER_PLUGIN_LIB "nvinfer_plugin_${NV_TENSORRT_MAJOR}") - set(PARSER_LIB "nvonnxparser_${NV_TENSORRT_MAJOR}") - endif() - endif() - - if (NOT NVINFER_LIB) - set(NVINFER_LIB "nvinfer") - endif() - - if (NOT NVINFER_PLUGIN_LIB) - set(NVINFER_PLUGIN_LIB "nvinfer_plugin") - endif() - - if (NOT PARSER_LIB) - set(PARSER_LIB "nvonnxparser") - endif() MESSAGE(STATUS "Search for ${NVINFER_LIB}, ${NVINFER_PLUGIN_LIB} and ${PARSER_LIB}") find_library(TENSORRT_LIBRARY_INFER ${NVINFER_LIB} @@ -132,11 +132,11 @@ unset(PROTOBUF_LIBRARY) unset(OLD_CMAKE_CXX_FLAGS) unset(OLD_CMAKE_CUDA_FLAGS) - set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199") - target_compile_options(nvonnxparser_static PRIVATE /FIio.h /wd4100) - target_compile_options(nvonnxparser PRIVATE /FIio.h /wd4100) + set_target_properties(${PARSER_LIB} PROPERTIES LINK_FLAGS "/ignore:4199") + target_compile_options(${PARSER_LIB}_static PRIVATE /FIio.h /wd4100) + target_compile_options(${PARSER_LIB} PRIVATE /FIio.h /wd4100) endif() - set(onnxparser_link_libs nvonnxparser_static) + set(onnxparser_link_libs ${PARSER_LIB}_static) endif() include_directories(${TENSORRT_INCLUDE_DIR}) From 295896a088a3c098613c2b387cc82aae111f6b85 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Wed, 24 Apr 2024 14:25:32 -0700 Subject: [PATCH 5/5] onnx parser static library are just nvonnxparser_static on all platforms --- cmake/onnxruntime_providers_tensorrt.cmake | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cmake/onnxruntime_providers_tensorrt.cmake b/cmake/onnxruntime_providers_tensorrt.cmake index 1af5560e2ef40..42275873ceeb0 100644 --- a/cmake/onnxruntime_providers_tensorrt.cmake +++ b/cmake/onnxruntime_providers_tensorrt.cmake @@ -34,6 +34,10 @@ MESSAGE(STATUS "[Note] There is an issue when running \"Debug build\" TRT EP with \"Release build\" TRT built-in parser on Windows. This build will use tensorrt oss parser instead.") endif() + find_path(TENSORRT_INCLUDE_DIR NvInfer.h + HINTS ${TENSORRT_ROOT} + PATH_SUFFIXES include) + # TensorRT 10 GA onwards, the TensorRT libraries will have major version appended to the end on Windows, # for example, nvinfer_10.dll, nvinfer_plugin_10.dll, nvonnxparser_10.dll ... if (WIN32) @@ -78,10 +82,6 @@ if (onnxruntime_USE_TENSORRT_BUILTIN_PARSER) # Add TensorRT library - find_path(TENSORRT_INCLUDE_DIR NvInfer.h - HINTS ${TENSORRT_ROOT} - PATH_SUFFIXES include) - MESSAGE(STATUS "Found TensorRT headers at ${TENSORRT_INCLUDE_DIR}") MESSAGE(STATUS "Search for ${NVINFER_LIB}, ${NVINFER_PLUGIN_LIB} and ${PARSER_LIB}") find_library(TENSORRT_LIBRARY_INFER ${NVINFER_LIB} @@ -132,11 +132,12 @@ unset(PROTOBUF_LIBRARY) unset(OLD_CMAKE_CXX_FLAGS) unset(OLD_CMAKE_CUDA_FLAGS) - set_target_properties(${PARSER_LIB} PROPERTIES LINK_FLAGS "/ignore:4199") - target_compile_options(${PARSER_LIB}_static PRIVATE /FIio.h /wd4100) - target_compile_options(${PARSER_LIB} PRIVATE /FIio.h /wd4100) + set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199") + target_compile_options(nvonnxparser_static PRIVATE /FIio.h /wd4100) + target_compile_options(nvonnxparser PRIVATE /FIio.h /wd4100) endif() - set(onnxparser_link_libs ${PARSER_LIB}_static) + # Static libraries are just nvonnxparser_static on all platforms + set(onnxparser_link_libs nvonnxparser_static) endif() include_directories(${TENSORRT_INCLUDE_DIR})