From 6f483ae8899fd362165ef8a333775a0c2bfc5dbf Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Fri, 12 Jan 2024 16:05:33 -0500 Subject: [PATCH] Update to use rapids-cmake for all deps --- cpp/bench/ann/CMakeLists.txt | 19 +++--- cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h | 4 +- cpp/bench/ann/src/raft/raft_benchmark.cu | 4 +- cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu | 4 +- cpp/cmake/patches/{ggnn.patch => ggnn.diff} | 5 +- .../patches/{hnswlib.patch => hnswlib.diff} | 5 +- cpp/cmake/patches/nlohmann_json.patch | 38 ----------- cpp/cmake/thirdparty/get_ggnn.cmake | 58 +++++++++------- cpp/cmake/thirdparty/get_hnswlib.cmake | 66 ++++++++++--------- 9 files changed, 91 insertions(+), 112 deletions(-) rename cpp/cmake/patches/{ggnn.patch => ggnn.diff} (98%) rename cpp/cmake/patches/{hnswlib.patch => hnswlib.diff} (98%) delete mode 100644 cpp/cmake/patches/nlohmann_json.patch diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt index 16b0f7e1ac..de980e8945 100644 --- a/cpp/bench/ann/CMakeLists.txt +++ b/cpp/bench/ann/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -136,7 +136,7 @@ endif() function(ConfigureAnnBench) set(oneValueArgs NAME) - set(multiValueArgs PATH LINKS CXXFLAGS INCLUDES) + set(multiValueArgs PATH LINKS CXXFLAGS) if(NOT BUILD_CPU_ONLY) set(GPU_BUILD ON) @@ -225,9 +225,11 @@ endfunction() if(RAFT_ANN_BENCH_USE_HNSWLIB) ConfigureAnnBench( - NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp INCLUDES - ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib CXXFLAGS "${HNSW_CXX_FLAGS}" + NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp + LINKS + hnswlib::hnswlib ) + endif() if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ) @@ -278,12 +280,9 @@ if(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB) RAFT_CAGRA_HNSWLIB PATH bench/ann/src/raft/raft_cagra_hnswlib.cu - INCLUDES - ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib LINKS raft::compiled - CXXFLAGS - "${HNSW_CXX_FLAGS}" + hnswlib::hnswlib ) endif() @@ -338,8 +337,8 @@ endif() if(RAFT_ANN_BENCH_USE_GGNN) include(cmake/thirdparty/get_glog.cmake) ConfigureAnnBench( - NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu INCLUDES - ${CMAKE_CURRENT_BINARY_DIR}/_deps/ggnn-src/include LINKS glog::glog + NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu + LINKS glog::glog ggnn::ggnn ) endif() diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h index 2a5177d295..5ddfc58677 100644 --- a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h +++ b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ #include "../common/ann_types.hpp" #include "../common/thread_pool.hpp" -#include +#include namespace raft::bench::ann { diff --git a/cpp/bench/ann/src/raft/raft_benchmark.cu b/cpp/bench/ann/src/raft/raft_benchmark.cu index cfc30bef7d..4ea8babea5 100644 --- a/cpp/bench/ann/src/raft/raft_benchmark.cu +++ b/cpp/bench/ann/src/raft/raft_benchmark.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,7 +44,7 @@ std::unique_ptr> create_algo(const std::string& algo, // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); + [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); std::unique_ptr> ann; if constexpr (std::is_same_v) { diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu index ce6fa255b2..9a275a31f0 100644 --- a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu +++ b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ std::unique_ptr> create_algo(const std::string& algo, // stop compiler warning; not all algorithms support multi-GPU so it may not be used (void)dev_list; - raft::bench::ann::Metric metric = parse_metric(distance); + [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance); std::unique_ptr> ann; if constexpr (std::is_same_v or std::is_same_v) { diff --git a/cpp/cmake/patches/ggnn.patch b/cpp/cmake/patches/ggnn.diff similarity index 98% rename from cpp/cmake/patches/ggnn.patch rename to cpp/cmake/patches/ggnn.diff index 21df3bd738..fc45298803 100644 --- a/cpp/cmake/patches/ggnn.patch +++ b/cpp/cmake/patches/ggnn.diff @@ -1,5 +1,3 @@ -diff --git a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh -index 890420e..d792903 100644 --- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh +++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh @@ -62,7 +62,7 @@ struct SimpleKNNSymCache { @@ -227,3 +225,6 @@ index 49d76a1..eef69e6 100644 #include #include "ggnn/utils/cuda_knn_constants.cuh" +-- +2.43.0 + diff --git a/cpp/cmake/patches/hnswlib.patch b/cpp/cmake/patches/hnswlib.diff similarity index 98% rename from cpp/cmake/patches/hnswlib.patch rename to cpp/cmake/patches/hnswlib.diff index 32c1537c58..0007ed6425 100644 --- a/cpp/cmake/patches/hnswlib.patch +++ b/cpp/cmake/patches/hnswlib.diff @@ -1,5 +1,3 @@ -diff --git a/hnswlib/hnswalg.h b/hnswlib/hnswalg.h -index e95e0b5..f0fe50a 100644 --- a/hnswlib/hnswalg.h +++ b/hnswlib/hnswalg.h @@ -3,6 +3,7 @@ @@ -128,3 +126,6 @@ index 5e1a4a5..4195ebd 100644 numelements = numelements1; mass = new vl_type[numelements]; } +-- +2.43.0 + diff --git a/cpp/cmake/patches/nlohmann_json.patch b/cpp/cmake/patches/nlohmann_json.patch deleted file mode 100644 index 83dd56bc16..0000000000 --- a/cpp/cmake/patches/nlohmann_json.patch +++ /dev/null @@ -1,38 +0,0 @@ ---- nlohmann/json.hpp 2021-05-06 11:40:39.770669693 +0800 -+++ nlohmann/json_patched.hpp 2021-06-02 18:46:43.849334466 +0800 -@@ -16607,6 +16607,21 @@ - } - } - -+ -+ template ::value, int> = 0> -+ bool is_negative_number(NumberType x) -+ { -+ return x < 0; -+ } -+ -+ template < typename NumberType, -+ enable_if_t < std::is_unsigned::value, int > = 0 > -+ bool is_negative_number(NumberType /*unused*/) -+ { -+ return false; -+ } -+ - /*! - @brief dump an integer - -@@ -16649,12 +16664,11 @@ - // use a pointer to fill the buffer - auto buffer_ptr = number_buffer.begin(); // NOLINT(llvm-qualified-auto,readability-qualified-auto,cppcoreguidelines-pro-type-vararg,hicpp-vararg) - -- const bool is_negative = std::is_same::value && !(x >= 0); // see issue #755 - number_unsigned_t abs_value; - - unsigned int n_chars{}; - -- if (is_negative) -+ if (is_negative_number(x)) - { - *buffer_ptr = '-'; - abs_value = remove_sign(static_cast(x)); diff --git a/cpp/cmake/thirdparty/get_ggnn.cmake b/cpp/cmake/thirdparty/get_ggnn.cmake index 708acb6b8d..3ffee1dc62 100644 --- a/cpp/cmake/thirdparty/get_ggnn.cmake +++ b/cpp/cmake/thirdparty/get_ggnn.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,30 +15,44 @@ #============================================================================= function(find_and_configure_ggnn) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) + set(oneValueArgs VERSION REPOSITORY PINNED_TAG EXCLUDE_FROM_ALL) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) - set ( EXTERNAL_INCLUDES_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ ) - if (NOT EXISTS ${EXTERNAL_INCLUDES_DIRECTORY}/_deps/ggnn-src/) - execute_process ( - COMMAND git clone "https://github.com/${PKG_FORK}/ggnn" --branch ${PKG_PINNED_TAG} ggnn-src - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/ ) + set(patch_files_to_run "${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/ggnn.diff") + set(patch_issues_to_ref "fix compile issues") + set(patch_script "${CMAKE_BINARY_DIR}/rapids-cmake/patches/ggnn/patch.cmake") + set(log_file "${CMAKE_BINARY_DIR}/rapids-cmake/patches/ggnn/log") + string(TIMESTAMP current_year "%Y" UTC) + configure_file(${rapids-cmake-dir}/cpm/patches/command_template.cmake.in "${patch_script}" + @ONLY) - message("SOURCE ${CMAKE_CURRENT_SOURCE_DIR}") - execute_process ( - COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/ggnn.patch - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/ggnn-src - ) - endif() + rapids_cpm_find( + ggnn ${PKG_VERSION} + GLOBAL_TARGETS ggnn::ggnn + CPM_ARGS + GIT_REPOSITORY ${PKG_REPOSITORY} + GIT_TAG ${PKG_PINNED_TAG} + GIT_SHALLOW TRUE + DOWNLOAD_ONLY ON + PATCH_COMMAND ${CMAKE_COMMAND} -P ${patch_script} + ) + if(NOT TARGET ggnn::ggnn) + add_library(ggnn INTERFACE) + target_include_directories(ggnn INTERFACE "$") + add_library(ggnn::ggnn ALIAS ggnn) + endif() endfunction() +if(NOT RAFT_GGNN_GIT_TAG) + set(RAFT_GGNN_GIT_TAG release_0.5) +endif() -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_ggnn(VERSION 0.5 - FORK cgtuebingen - PINNED_TAG release_0.5 - EXCLUDE_FROM_ALL YES) +if(NOT RAFT_GGNN_GIT_REPOSITORY) + set(RAFT_GGNN_GIT_REPOSITORY https://github.com/cgtuebingen/ggnn.git) +endif() +find_and_configure_ggnn(VERSION 0.5 + REPOSITORY ${RAFT_GGNN_GIT_REPOSITORY} + PINNED_TAG ${RAFT_GGNN_GIT_TAG} + ) diff --git a/cpp/cmake/thirdparty/get_hnswlib.cmake b/cpp/cmake/thirdparty/get_hnswlib.cmake index a4ceacae38..6a73c52d60 100644 --- a/cpp/cmake/thirdparty/get_hnswlib.cmake +++ b/cpp/cmake/thirdparty/get_hnswlib.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,40 +15,42 @@ #============================================================================= function(find_and_configure_hnswlib) - set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) + set(oneValueArgs VERSION REPOSITORY PINNED_TAG EXCLUDE_FROM_ALL) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) - set ( EXTERNAL_INCLUDES_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) - if( NOT EXISTS ${EXTERNAL_INCLUDES_DIRECTORY}/_deps/hnswlib-src ) + set(patch_files_to_run "${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/hnswlib.diff") + set(patch_issues_to_ref "fix compile issues") + set(patch_script "${CMAKE_BINARY_DIR}/rapids-cmake/patches/hnswlib/patch.cmake") + set(log_file "${CMAKE_BINARY_DIR}/rapids-cmake/patches/hnswlib/log") + string(TIMESTAMP current_year "%Y" UTC) + configure_file(${rapids-cmake-dir}/cpm/patches/command_template.cmake.in "${patch_script}" + @ONLY) - execute_process ( - COMMAND git clone --branch=v0.6.2 https://github.com/nmslib/hnswlib.git hnswlib-src - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps ) - - message("SOURCE ${CMAKE_CURRENT_SOURCE_DIR}") - execute_process ( - COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/hnswlib.patch - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src - ) - endif () + rapids_cpm_find( + hnswlib ${PKG_VERSION} + GLOBAL_TARGETS hnswlib::hnswlib + CPM_ARGS + GIT_REPOSITORY ${PKG_REPOSITORY} + GIT_TAG ${PKG_PINNED_TAG} + GIT_SHALLOW TRUE + EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} + PATCH_COMMAND ${CMAKE_COMMAND} -P ${patch_script} + ) + if(NOT TARGET hnswlib::hnswlib) + add_library(hnswlib::hnswlib ALIAS hnswlib) + endif() +endfunction() - include(cmake/modules/FindAVX.cmake) - set(HNSW_CXX_FLAGS "") - if(CXX_AVX_FOUND) - set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX_FLAGS}") - elseif(CXX_AVX2_FOUND) - set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX2_FLAGS}") - elseif(CXX_AVX512_FOUND) - set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX512_FLAGS}") - endif() -endfunction() +if(NOT RAFT_HNSWLIB_GIT_TAG) + set(RAFT_HNSWLIB_GIT_TAG v0.6.2) +endif() -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_hnswlib(VERSION 0.6.2 - FORK nmslib - PINNED_TAG v0.6.2 +if(NOT RAFT_HNSWLIB_GIT_REPOSITORY) + set(RAFT_HNSWLIB_GIT_REPOSITORY https://github.com/nmslib/hnswlib.git) +endif() +find_and_configure_hnswlib(VERSION 0.6.2 + REPOSITORY ${RAFT_HNSWLIB_GIT_REPOSITORY} + PINNED_TAG ${RAFT_HNSWLIB_GIT_TAG} EXCLUDE_FROM_ALL YES)