From 0df507230b430076d1aad5b1c699e42fe2db2633 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Sat, 13 May 2023 09:00:54 -0700 Subject: [PATCH 1/2] Changes to support 23.05 (#34) * Update AWS SDK version * Disabling S3 * Revert it back * Update version 'nlohmann-json' * Revert "Update version 'nlohmann-json'" This reverts commit 1d84f4cfb0be6b81c6c7a3f9853459e219bc36b5. * Ignore Azure deprecated warning * Fix up * Set higher version for 'nlohmann-json' * Change version to nlohmann-json * Eanble CXX STD 17 to fix absl linking * Apply same CXX_STANDARD for related third party library (#33) --------- Co-authored-by: GuanLuo Co-authored-by: GuanLuo <41310872+GuanLuo@users.noreply.github.com> --- CMakeLists.txt | 41 +++++++++++---------------------- tools/azure_no_deprecated.patch | 13 +++++++++++ 2 files changed, 26 insertions(+), 28 deletions(-) create mode 100644 tools/azure_no_deprecated.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 27fc562c..222d40ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,10 +23,8 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - cmake_minimum_required(VERSION 3.17) project(tritonthirdparty LANGUAGES C CXX) - # # ExternalProject Dependencies # @@ -38,32 +36,27 @@ project(tritonthirdparty LANGUAGES C CXX) # allows a good point to perform the source install. # include(ExternalProject) - if(NOT TRITON_THIRD_PARTY_INSTALL_PREFIX) message(FATAL_ERROR "TRITON_THIRD_PARTY_INSTALL_PREFIX must be set") endif() # TRITON_THIRD_PARTY_INSTALL_PREFIX - if(NOT TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX) message(STATUS "TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX not set") set(INSTALL_SRC_DEST_ARG "") else() set(INSTALL_SRC_DEST_ARG --dest ${TRITON_THIRD_PARTY_SRC_INSTALL_PREFIX}) endif() # TRITON_THIRD_PARTY_INSTALL_PREFIX - # If CMAKE_TOOLCHAIN_FILE is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "") if (CMAKE_TOOLCHAIN_FILE) set(_CMAKE_ARGS_CMAKE_TOOLCHAIN_FILE "-DCMAKE_TOOLCHAIN_FILE:PATH=${CMAKE_TOOLCHAIN_FILE}") endif() - # If VCPKG_TARGET_TRIPLET is set, propagate that hint path to the external # projects. set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "") if (VCPKG_TARGET_TRIPLET) set(_CMAKE_ARGS_VCPKG_TARGET_TRIPLET "-DVCPKG_TARGET_TRIPLET:STRING=${VCPKG_TARGET_TRIPLET}") endif() - # If OPENSSL_ROOT_DIR is set, propagate that hint path to the external # projects with OpenSSL dependency. set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "") @@ -71,6 +64,13 @@ if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() +# [FIXME] apply the same version to all third party, and pick a Triton default +if (NOT DEFINED CMAKE_CXX_STANDARD) + set(THIRD_PARTY_CMAKE_CXX_STANDARD 17) +else() +set(THIRD_PARTY_CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +endif() + # # Build libcurl # @@ -98,7 +98,6 @@ ExternalProject_Add(curl ${LIBCURL_EXTRA_BUILD_ARGS} PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - ExternalProject_Add(grpc-repo PREFIX grpc-repo GIT_REPOSITORY "https://github.com/grpc/grpc.git" @@ -111,14 +110,13 @@ ExternalProject_Add(grpc-repo TEST_COMMAND "" PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} --dest-basename=grpc_1.48.0 ) - # # Build nlohmann/json # ExternalProject_Add(nlohmann-json PREFIX nlohmann-json GIT_REPOSITORY "https://github.com/nlohmann/json.git" - GIT_TAG "v3.9.0" + GIT_TAG "v3.10.5" SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/json" EXCLUDE_FROM_ALL ON CMAKE_CACHE_ARGS @@ -130,7 +128,6 @@ ExternalProject_Add(nlohmann-json -H. -Bcmake-out PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} --dest-basename=nlohmann-json ) - # # Build protobuf project from grpc-repo, but apply a patch that is available # in 3.20.0, to resolve a linking issue on Windows @@ -163,7 +160,6 @@ ExternalProject_Add(protobuf PATCH_COMMAND git cherry-pick -n f180289c4670ca1afde5865bb8a7f2b61a3efcc5 DEPENDS grpc-repo ) - # # Build absl (abseil) project from grpc-repo # @@ -176,13 +172,13 @@ ExternalProject_Add(absl EXCLUDE_FROM_ALL ON DOWNLOAD_COMMAND "" CMAKE_CACHE_ARGS + -DCMAKE_CXX_STANDARD:STRING=${THIRD_PARTY_CMAKE_CXX_STANDARD} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DBUILD_TESTING:BOOL=OFF DEPENDS grpc-repo ) - # # Build googletest project from grpc-repo # @@ -201,7 +197,6 @@ ExternalProject_Add(googletest -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/googletest DEPENDS grpc-repo ) - # Location where protobuf-config.cmake will be installed varies by # platform if (WIN32) @@ -209,7 +204,6 @@ if (WIN32) else() set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/lib/cmake/protobuf") endif() - # # Build c-ares project from grpc-repo # @@ -231,7 +225,6 @@ ExternalProject_Add(c-ares -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares DEPENDS grpc-repo ) - # # Build GRPC # @@ -244,6 +237,7 @@ ExternalProject_Add(grpc EXCLUDE_FROM_ALL ON DOWNLOAD_COMMAND "" CMAKE_CACHE_ARGS + -DCMAKE_CXX_STANDARD:STRING=${THIRD_PARTY_CMAKE_CXX_STANDARD} -DgRPC_INSTALL:BOOL=ON -DgRPC_BUILD_TESTS:BOOL=OFF -DgRPC_PROTOBUF_PROVIDER:STRING=package @@ -260,7 +254,6 @@ ExternalProject_Add(grpc -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc DEPENDS grpc-repo c-ares protobuf absl ) - # # Build libevent # @@ -278,7 +271,6 @@ ExternalProject_Add(libevent -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - # # Build patched libevhtp # @@ -325,7 +317,6 @@ ExternalProject_Add(libevhtp DEPENDS libevent ) endif() - # # Build Prometheus C++ library # @@ -351,7 +342,6 @@ ExternalProject_Add(prometheus-cpp -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/prometheus-cpp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - # # Build crc32c # @@ -376,7 +366,6 @@ ExternalProject_Add(crc32c -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/crc32c PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - # This is necessary because google-cloud-cpp # does not pass protobuf_DIR and gRPC_DIR etc to its # external projects but expects them in CMAKE_PREFIX_PATH. @@ -390,7 +379,6 @@ set(GCS_CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/grpc/lib/cmake/grpc ${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/lib/cmake/nlohmann_json ${_FINDPACKAGE_PROTOBUF_CONFIG_DIR}) - # # Build google-cloud-cpp # @@ -414,7 +402,6 @@ ExternalProject_Add(google-cloud-cpp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} DEPENDS grpc c-ares protobuf crc32c absl nlohmann-json ) - # # Build azure-storage-cpplite # @@ -433,10 +420,9 @@ ExternalProject_Add(azure-storage-cpplite -DCURL_INCLUDE_DIR:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/include -DCURL_LIBRARY:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/lib -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/azure-storage-cpplite - PATCH_COMMAND git cherry-pick --no-commit a3a9c21de3fa9932dcfceb2b6617757a511c75c3 && python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} + PATCH_COMMAND git cherry-pick --no-commit a3a9c21de3fa9932dcfceb2b6617757a511c75c3 && git apply ${CMAKE_CURRENT_SOURCE_DIR}/tools/azure_no_deprecated.patch && python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} DEPENDS curl ) - # # Build CNMeM (CUDA memory management library) # @@ -452,14 +438,13 @@ ExternalProject_Add(cnmem -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/cnmem PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - # # Build aws-sdk-cpp # ExternalProject_Add(aws-sdk-cpp PREFIX aws-sdk-cpp GIT_REPOSITORY "https://github.com/aws/aws-sdk-cpp.git" - GIT_TAG "1.8.186" + GIT_TAG "1.11.60" SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp/src/aws-sdk-cpp" EXCLUDE_FROM_ALL ON CMAKE_CACHE_ARGS @@ -473,7 +458,6 @@ ExternalProject_Add(aws-sdk-cpp -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/aws-sdk-cpp PATCH_COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/install_src.py --src ${INSTALL_SRC_DEST_ARG} ) - # # Build OpenTelemetry C++ library # [FIXME] Remove `NOT WIN32` condition when Windows build works on Triton (JIRA DLIS-4786) @@ -491,6 +475,7 @@ if (NOT WIN32) SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry-cpp/src/opentelemetry-cpp" EXCLUDE_FROM_ALL ON CMAKE_CACHE_ARGS + -DCMAKE_CXX_STANDARD:STRING=${THIRD_PARTY_CMAKE_CXX_STANDARD} -DProtobuf_LIBRARIES:STRING=${PROTO_LIB_PATH} -DProtobuf_INCLUDE_DIR:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/include -DProtobuf_PROTOC_EXECUTABLE:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/bin/protoc diff --git a/tools/azure_no_deprecated.patch b/tools/azure_no_deprecated.patch new file mode 100644 index 00000000..d625abdd --- /dev/null +++ b/tools/azure_no_deprecated.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index dfb441e..af3dd36 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -173,7 +173,7 @@ target_link_libraries(azure-storage-lite Threads::Threads ${CURL_LIBRARIES} ${EX + if(MSVC) + target_compile_options(azure-storage-lite PRIVATE /W4 /WX /MP) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "GNU") +- target_compile_options(azure-storage-lite PRIVATE -Wall -Wextra -Werror -pedantic) ++ target_compile_options(azure-storage-lite PRIVATE -Wall -Wextra -Werror -pedantic -Wno-deprecated-declarations) + endif() + + if(BUILD_ADLS) From 9e9931fd172f034d1d614b8a1390fb92f70f8b11 Mon Sep 17 00:00:00 2001 From: Neelay Shah Date: Mon, 22 May 2023 13:06:59 -0700 Subject: [PATCH 2/2] changed to get RE2 library from system package in grpc build --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 222d40ac..df4ec02a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,6 +245,7 @@ ExternalProject_Add(grpc -DgRPC_ZLIB_PROVIDER:STRING=package -DgRPC_CARES_PROVIDER:STRING=package -DgRPC_SSL_PROVIDER:STRING=package + -DgRPC_RE2_PROVIDER:STRING=package -Dc-ares_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/c-ares/lib/cmake/c-ares -DProtobuf_DIR:PATH=${_FINDPACKAGE_PROTOBUF_CONFIG_DIR} ${_CMAKE_ARGS_OPENSSL_ROOT_DIR}