Update to use rapids-cmake for all deps (#2096)

Remove manual checkouts via `execute_process` and use `rapids_cpm_find` so that we can use rapids-cmake overrides to support offline builds Authors: - Robert Maynard (https://github.com/robertmaynard) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: #2096
rapidsai · Jan 17, 2024 · b75fbd0 · b75fbd0
1 parent 3c7586f
commit b75fbd0
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 111 deletions.
diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt
@@ -1,5 +1,5 @@
 # =============================================================================
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 # in compliance with the License. You may obtain a copy of the License at
@@ -136,7 +136,7 @@ endif()
 function(ConfigureAnnBench)
 
   set(oneValueArgs NAME)
-  set(multiValueArgs PATH LINKS CXXFLAGS INCLUDES)
+  set(multiValueArgs PATH LINKS CXXFLAGS)
 
   if(NOT BUILD_CPU_ONLY)
     set(GPU_BUILD ON)
@@ -225,9 +225,11 @@ endfunction()
 
 if(RAFT_ANN_BENCH_USE_HNSWLIB)
   ConfigureAnnBench(
-    NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp INCLUDES
-    ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib CXXFLAGS "${HNSW_CXX_FLAGS}"
+    NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp
+    LINKS
+    hnswlib::hnswlib
   )
+
 endif()
 
 if(RAFT_ANN_BENCH_USE_RAFT_IVF_PQ)
@@ -278,12 +280,9 @@ if(RAFT_ANN_BENCH_USE_RAFT_CAGRA_HNSWLIB)
     RAFT_CAGRA_HNSWLIB
     PATH
     bench/ann/src/raft/raft_cagra_hnswlib.cu
-    INCLUDES
-    ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src/hnswlib
     LINKS
     raft::compiled
-    CXXFLAGS
-    "${HNSW_CXX_FLAGS}"
+    hnswlib::hnswlib
   )
 endif()
 
@@ -338,8 +337,8 @@ endif()
 if(RAFT_ANN_BENCH_USE_GGNN)
   include(cmake/thirdparty/get_glog.cmake)
   ConfigureAnnBench(
-    NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu INCLUDES
-    ${CMAKE_CURRENT_BINARY_DIR}/_deps/ggnn-src/include LINKS glog::glog
+    NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu
+    LINKS glog::glog ggnn::ggnn
   )
 endif()
 

diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@
 
 #include "../common/ann_types.hpp"
 #include "../common/thread_pool.hpp"
-#include <hnswlib.h>
+#include <hnswlib/hnswlib.h>
 
 namespace raft::bench::ann {
 

diff --git a/cpp/bench/ann/src/raft/raft_benchmark.cu b/cpp/bench/ann/src/raft/raft_benchmark.cu
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,7 +44,7 @@ std::unique_ptr<raft::bench::ann::ANN<T>> create_algo(const std::string& algo,
   // stop compiler warning; not all algorithms support multi-GPU so it may not be used
   (void)dev_list;
 
-  raft::bench::ann::Metric metric = parse_metric(distance);
+  [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance);
   std::unique_ptr<raft::bench::ann::ANN<T>> ann;
 
   if constexpr (std::is_same_v<T, float>) {

diff --git a/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu b/cpp/bench/ann/src/raft/raft_cagra_hnswlib.cu
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -43,7 +43,7 @@ std::unique_ptr<raft::bench::ann::ANN<T>> create_algo(const std::string& algo,
   // stop compiler warning; not all algorithms support multi-GPU so it may not be used
   (void)dev_list;
 
-  raft::bench::ann::Metric metric = parse_metric(distance);
+  [[maybe_unused]] raft::bench::ann::Metric metric = parse_metric(distance);
   std::unique_ptr<raft::bench::ann::ANN<T>> ann;
 
   if constexpr (std::is_same_v<T, float> or std::is_same_v<T, std::uint8_t>) {

diff --git a/cpp/cmake/patches/ggnn.patch → cpp/cmake/patches/ggnn.diff b/cpp/cmake/patches/ggnn.patch → cpp/cmake/patches/ggnn.diff
@@ -1,5 +1,3 @@
-diff --git a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh
-index 890420e..d792903 100644
 --- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh
 +++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh
 @@ -62,7 +62,7 @@ struct SimpleKNNSymCache {
@@ -227,3 +225,6 @@ index 49d76a1..eef69e6 100644
  #include <cub/cub.cuh>
 
  #include "ggnn/utils/cuda_knn_constants.cuh"
+-- 
+2.43.0
+
diff --git a/cpp/cmake/patches/hnswlib.patch → cpp/cmake/patches/hnswlib.diff b/cpp/cmake/patches/hnswlib.patch → cpp/cmake/patches/hnswlib.diff
@@ -1,5 +1,3 @@
-diff --git a/hnswlib/hnswalg.h b/hnswlib/hnswalg.h
-index e95e0b5..f0fe50a 100644
 --- a/hnswlib/hnswalg.h
 +++ b/hnswlib/hnswalg.h
 @@ -3,6 +3,7 @@
@@ -128,3 +126,6 @@ index 5e1a4a5..4195ebd 100644
              numelements = numelements1;
              mass = new vl_type[numelements];
          }
+-- 
+2.43.0
+
diff --git a/cpp/cmake/patches/nlohmann_json.patch b/cpp/cmake/patches/nlohmann_json.patch
diff --git a/cpp/cmake/thirdparty/get_ggnn.cmake b/cpp/cmake/thirdparty/get_ggnn.cmake
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,30 +15,44 @@
 #=============================================================================
 
 function(find_and_configure_ggnn)
-    set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL)
-    cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
-            "${multiValueArgs}" ${ARGN} )
+  set(oneValueArgs VERSION REPOSITORY PINNED_TAG)
+  cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
+          "${multiValueArgs}" ${ARGN} )
 
-    set ( EXTERNAL_INCLUDES_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ )
-    if (NOT EXISTS ${EXTERNAL_INCLUDES_DIRECTORY}/_deps/ggnn-src/)
 
-        execute_process (
-                COMMAND git clone "https://github.com/${PKG_FORK}/ggnn" --branch ${PKG_PINNED_TAG} ggnn-src
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/ )
+  set(patch_files_to_run "${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/ggnn.diff")
+  set(patch_issues_to_ref "fix compile issues")
+  set(patch_script "${CMAKE_BINARY_DIR}/rapids-cmake/patches/ggnn/patch.cmake")
+  set(log_file "${CMAKE_BINARY_DIR}/rapids-cmake/patches/ggnn/log")
+  string(TIMESTAMP current_year "%Y" UTC)
+  configure_file(${rapids-cmake-dir}/cpm/patches/command_template.cmake.in "${patch_script}"
+                @ONLY)
 
-        message("SOURCE ${CMAKE_CURRENT_SOURCE_DIR}")
-        execute_process (
-                COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/ggnn.patch
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/ggnn-src
-        )
-    endif()
+  rapids_cpm_find(
+    ggnn ${PKG_VERSION}
+    GLOBAL_TARGETS ggnn::ggnn
+    CPM_ARGS
+    GIT_REPOSITORY ${PKG_REPOSITORY}
+    GIT_TAG ${PKG_PINNED_TAG}
+    GIT_SHALLOW TRUE
+    DOWNLOAD_ONLY ON
+    PATCH_COMMAND ${CMAKE_COMMAND} -P ${patch_script}
+  )
+  if(NOT TARGET ggnn::ggnn)
+    add_library(ggnn INTERFACE)
+    target_include_directories(ggnn INTERFACE "$<BUILD_INTERFACE:${ggnn_SOURCE_DIR}/include>")
+    add_library(ggnn::ggnn ALIAS ggnn)
+  endif()
 
 endfunction()
+if(NOT RAFT_GGNN_GIT_TAG)
+  set(RAFT_GGNN_GIT_TAG release_0.5)
+endif()
 
-# Change pinned tag here to test a commit in CI
-# To use a different RAFT locally, set the CMake variable
-# CPM_raft_SOURCE=/path/to/local/raft
-find_and_configure_ggnn(VERSION          0.5
-        FORK             cgtuebingen
-        PINNED_TAG       release_0.5
-        EXCLUDE_FROM_ALL YES)
+if(NOT RAFT_GGNN_GIT_REPOSITORY)
+  set(RAFT_GGNN_GIT_REPOSITORY https://github.com/cgtuebingen/ggnn.git)
+endif()
+find_and_configure_ggnn(VERSION 0.5
+        REPOSITORY       ${RAFT_GGNN_GIT_REPOSITORY}
+        PINNED_TAG       ${RAFT_GGNN_GIT_TAG}
+        )
diff --git a/cpp/cmake/thirdparty/get_hnswlib.cmake b/cpp/cmake/thirdparty/get_hnswlib.cmake
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,40 +15,64 @@
 #=============================================================================
 
 function(find_and_configure_hnswlib)
-    set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL)
-    cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
-            "${multiValueArgs}" ${ARGN} )
+  set(oneValueArgs VERSION REPOSITORY PINNED_TAG EXCLUDE_FROM_ALL)
+  cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
+          "${multiValueArgs}" ${ARGN} )
 
-    set ( EXTERNAL_INCLUDES_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} )
-    if( NOT EXISTS ${EXTERNAL_INCLUDES_DIRECTORY}/_deps/hnswlib-src )
+  set(patch_files_to_run "${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/hnswlib.diff")
+  set(patch_issues_to_ref "fix compile issues")
+  set(patch_script "${CMAKE_BINARY_DIR}/rapids-cmake/patches/hnswlib/patch.cmake")
+  set(log_file "${CMAKE_BINARY_DIR}/rapids-cmake/patches/hnswlib/log")
+  string(TIMESTAMP current_year "%Y" UTC)
+  configure_file(${rapids-cmake-dir}/cpm/patches/command_template.cmake.in "${patch_script}"
+                @ONLY)
 
-        execute_process (
-                COMMAND git clone --branch=v0.6.2 https://github.com/nmslib/hnswlib.git hnswlib-src
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps )
+  rapids_cpm_find(
+    hnswlib ${PKG_VERSION}
+    GLOBAL_TARGETS hnswlib::hnswlib
+    CPM_ARGS
+    GIT_REPOSITORY ${PKG_REPOSITORY}
+    GIT_TAG ${PKG_PINNED_TAG}
+    GIT_SHALLOW TRUE
+    DOWNLOAD_ONLY ON
+    PATCH_COMMAND ${CMAKE_COMMAND} -P ${patch_script}
+  )
+  if(NOT TARGET hnswlib::hnswlib)
+    add_library(hnswlib INTERFACE )
+    add_library(hnswlib::hnswlib ALIAS hnswlib)
+    target_include_directories(hnswlib INTERFACE
+     "$<BUILD_INTERFACE:${hnswlib_SOURCE_DIR}>"
+     "$<INSTALL_INTERFACE:include>")
 
-        message("SOURCE ${CMAKE_CURRENT_SOURCE_DIR}")
-        execute_process (
-                COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patches/hnswlib.patch
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/_deps/hnswlib-src
-        )
-    endif ()
-
-    include(cmake/modules/FindAVX.cmake)
+    if(NOT PKG_EXCLUDE_FROM_ALL)
+      install(TARGETS hnswlib EXPORT hnswlib-exports)
+      install(DIRECTORY "${hnswlib_SOURCE_DIR}/hnswlib/" DESTINATION include/hnswlib)
 
-    set(HNSW_CXX_FLAGS "")
-    if(CXX_AVX_FOUND)
-        set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX_FLAGS}")
-    elseif(CXX_AVX2_FOUND)
-        set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX2_FLAGS}")
-    elseif(CXX_AVX512_FOUND)
-        set(HNSW_CXX_FLAGS "${HNSW_CXX_FLAGS} ${CXX_AVX512_FLAGS}")
+      # write export rules
+      rapids_export(
+        BUILD hnswlib
+        EXPORT_SET hnswlib-exports
+        GLOBAL_TARGETS hnswlib
+        NAMESPACE hnswlib::)
+      rapids_export(
+        INSTALL hnswlib
+        EXPORT_SET hnswlib-exports
+        GLOBAL_TARGETS hnswlib
+        NAMESPACE hnswlib::)
     endif()
+  endif()
 endfunction()
 
-# Change pinned tag here to test a commit in CI
-# To use a different RAFT locally, set the CMake variable
-# CPM_raft_SOURCE=/path/to/local/raft
-find_and_configure_hnswlib(VERSION  0.6.2
-        FORK             nmslib
-        PINNED_TAG       v0.6.2
-        EXCLUDE_FROM_ALL YES)
+
+if(NOT RAFT_HNSWLIB_GIT_TAG)
+  set(RAFT_HNSWLIB_GIT_TAG v0.6.2)
+endif()
+
+if(NOT RAFT_HNSWLIB_GIT_REPOSITORY)
+  set(RAFT_HNSWLIB_GIT_REPOSITORY https://github.com/nmslib/hnswlib.git)
+endif()
+find_and_configure_hnswlib(VERSION 0.6.2
+        REPOSITORY       ${RAFT_HNSWLIB_GIT_REPOSITORY}
+        PINNED_TAG       ${RAFT_HNSWLIB_GIT_TAG}
+        EXCLUDE_FROM_ALL ON
+        )