Updating cuML to use consolidated RAFT libs

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -21,7 +21,6 @@ dependencies:
 - dask>=2023.1.1
 - distributed>=2023.1.1
 - doxygen=1.8.20
-- faiss-proc=*=cuda
 - gcc_linux-64=11.*
 - graphviz
 - hdbscan
@@ -40,10 +39,8 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
-- libfaiss>=1.7.1
-- libraft-distance=23.04.*
 - libraft-headers=23.04.*
-- libraft-nn=23.04.*
+- libraft=23.04.*
 - nbsphinx
 - ninja
 - nltk

conda/environments/cpp_all_cuda-118_arch-x86_64.yaml

@@ -11,7 +11,6 @@ dependencies:
 - cmake>=3.23.1,!=3.25.0
 - cudatoolkit=11.8
 - cxx-compiler
-- faiss-proc=*=cuda
 - gcc_linux-64=11.*
 - libcublas-dev=11.11.3.6
 - libcublas=11.11.3.6
@@ -24,10 +23,8 @@ dependencies:
 - libcusolver=11.4.1.48
 - libcusparse-dev=11.7.5.86
 - libcusparse=11.7.5.86
-- libfaiss>=1.7.1
-- libraft-distance=23.04.*
 - libraft-headers=23.04.*
-- libraft-nn=23.04.*
+- libraft=23.04.*
 - ninja
 - rmm=23.04.*
 - sysroot_linux-64==2.17

conda/recipes/libcuml/meta.yaml

@@ -55,9 +55,8 @@ requirements:
     - libcusolver-dev {{ libcusolver_host_version }}
     - libcusparse {{ libcusparse_host_version }}
     - libcusparse-dev {{ libcusparse_host_version }}
-    - libraft-distance ={{ minor_version }}
+    - libraft ={{ minor_version }}
     - libraft-headers ={{ minor_version }}
-    - libraft-nn ={{ minor_version }}
     - treelite {{ treelite_version }}
     - libfaiss>=1.7.1
     - faiss-proc=*=cuda
@@ -84,9 +83,8 @@ outputs:
         - libcurand {{ libcurand_run_version }}
         - libcusparse {{ libcusparse_run_version }}
         - libcusparse-dev {{ libcusparse_run_version }}
-        - libraft-distance ={{ minor_version }}
+        - libraft ={{ minor_version }}
         - libraft-headers ={{ minor_version }}
-        - libraft-nn ={{ minor_version }}
         - treelite {{ treelite_version }}
         - libfaiss>=1.7.1
         - faiss-proc=*=cuda

cpp/CMakeLists.txt

@@ -63,7 +63,7 @@ option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF)
 option(USE_CCACHE "Cache build artifacts with ccache" OFF)
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF)
 option(CUML_USE_RAFT_STATIC "Build and statically link the RAFT libraries" OFF)
-option(CUML_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" OFF)
+option(CUML_RAFT_COMPILED "Use libraft shared liibrary" ON)
 option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF)
 option(CUML_EXPORT_TREELITE_LINKAGE "Whether to publicly or privately link treelite to libcuml++" OFF)
 option(CUML_USE_CUMLPRIMS_MG_STATIC "Build and statically link the cumlprims_mg library" OFF)
@@ -170,8 +170,7 @@ include(cmake/modules/ConfigureCUDA.cmake)
 
 ##############################################################################
 # - Set options based on user defined one  -----------------------------------
-set(CUML_USE_RAFT_NN OFF)
-set(CUML_USE_RAFT_DIST OFF)
+set(CUML_RAFT_COMPILED OFF)
 set(LINK_TREELITE OFF)
 set(LINK_CUFFT OFF)
 include(cmake/modules/ConfigureAlgorithms.cmake)

cpp/bench/CMakeLists.txt

@@ -46,9 +46,7 @@ if(BUILD_CUML_BENCH)
       benchmark::benchmark
       ${TREELITE_LIBS}
       raft::raft
-      raft::nn
-      faiss::faiss
-      raft::distance
+      raft::compiled
   )
 
   target_include_directories(${CUML_CPP_BENCH_TARGET}

cpp/bench/sg/svc.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 

cpp/bench/sg/svr.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2020-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 

cpp/cmake/modules/ConfigureAlgorithms.cmake

@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,8 +16,7 @@
 
 
 if(CUML_ALGORITHMS STREQUAL "ALL")
-    set(CUML_USE_RAFT_NN ON)
-    set(CUML_USE_RAFT_DIST ON)
+    set(CUML_RAFT_COMPILED ON)
     set(LINK_TREELITE ON)
     set(LINK_CUFFT ON)
     set(all_algo ON)
@@ -31,8 +30,7 @@ else()
     set(BUILD_CUML_C_LIBRARY OFF)
     set(BUILD_CUML_BENCH OFF)
     set(BUILD_CUML_EXAMPLES OFF)
-    set(CUML_USE_RAFT_NN OFF)
-    set(CUML_USE_RAFT_DIST OFF)
+    set(CUML_RAFT_COMPILED OFF)
 
     foreach(algo ${CUML_ALGORITHMS})
       string(TOLOWER ${algo} lower_algo)
@@ -106,7 +104,7 @@ else()
     endif()
 
     if(knn_algo)
-        set(CUML_USE_RAFT_NN ON)
+        set(CUML_RAFT_COMPILED ON)
     endif()
 
     if(randomforest_algo)
@@ -119,7 +117,7 @@ else()
     endif()
 
     if(metrics_algo)
-      set(CUML_USE_RAFT_DIST ON)
+        set(CUML_RAFT_COMPILED ON)
     endif()
 endif()
 

cpp/cmake/thirdparty/get_raft.cmake

@@ -18,7 +18,7 @@ set(CUML_MIN_VERSION_raft "${CUML_VERSION_MAJOR}.${CUML_VERSION_MINOR}.00")
 set(CUML_BRANCH_VERSION_raft "${CUML_VERSION_MAJOR}.${CUML_VERSION_MINOR}")
 
 function(find_and_configure_raft)
-    set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL USE_RAFT_DIST USE_RAFT_NN USE_RAFT_STATIC USE_FAISS_STATIC CLONE_ON_PIN NVTX)
+	set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL USE_RAFT_STATIC COMPILE_LIBRARY CLONE_ON_PIN NVTX)
     cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
             "${multiValueArgs}" ${ARGN} )
 
@@ -30,54 +30,41 @@ function(find_and_configure_raft)
         set(CPM_DOWNLOAD_raft ON)
     endif()
 
-    if(PKG_USE_RAFT_DIST)
-        string(APPEND RAFT_COMPONENTS "distance")
-    endif()
-
-    if(PKG_USE_RAFT_NN)
-        string(APPEND RAFT_COMPONENTS " nn")
-    endif()
 
     # We need RAFT::distributed for MG tests
     if(BUILD_CUML_MG_TESTS)
         string(APPEND RAFT_COMPONENTS " distributed")
     endif()
 
-    if(PKG_USE_RAFT_DIST AND PKG_USE_RAFT_NN)
-        set(RAFT_COMPILE_LIBRARIES ON)
+
+    if(PKG_COMPILE_LIBRARY AND NOT PKG_USE_RAFT_STATIC)
+      string(APPEND RAFT_COMPONENTS " compiled")
+      set(RAFT_COMPILE_LIBRARY ON)
     else()
-        set(RAFT_COMPILE_LIBRARIES OFF)
+      set(RAFT_COMPILE_LIBRARY OFF)
     endif()
 
     # We need to set this each time so that on subsequent calls to cmake
     # the raft-config.cmake re-evaluates the RAFT_NVTX value
     set(RAFT_NVTX ${PKG_NVTX})
 
-    set(RAFT_BUILD_SHARED_LIBS ON)
-    if(${PKG_USE_RAFT_STATIC})
-        set(RAFT_BUILD_SHARED_LIBS OFF)
-    endif()
-
     message(VERBOSE "CUML: raft FIND_PACKAGE_ARGUMENTS COMPONENTS ${RAFT_COMPONENTS}")
 
     rapids_cpm_find(raft ${PKG_VERSION}
-            GLOBAL_TARGETS      raft::raft
-            BUILD_EXPORT_SET    cuml-exports
-            INSTALL_EXPORT_SET  cuml-exports
-            COMPONENTS          ${RAFT_COMPONENTS}
-            CPM_ARGS
-            GIT_REPOSITORY         https://github.com/${PKG_FORK}/raft.git
-            GIT_TAG                ${PKG_PINNED_TAG}
-            SOURCE_SUBDIR          cpp
-            EXCLUDE_FROM_ALL       ${PKG_EXCLUDE_FROM_ALL}
-            OPTIONS
-            "BUILD_TESTS OFF"
-            "BUILD_SHARED_LIBS ${RAFT_BUILD_SHARED_LIBS}"
-            "RAFT_COMPILE_LIBRARIES ${RAFT_COMPILE_LIBRARIES}"
-            "RAFT_COMPILE_NN_LIBRARY ${PKG_USE_RAFT_NN}"
-            "RAFT_COMPILE_DIST_LIBRARY ${PKG_USE_RAFT_DIST}"
-            "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}"
-            )
+      GLOBAL_TARGETS      raft::raft
+      BUILD_EXPORT_SET    cuml-exports
+      INSTALL_EXPORT_SET  cuml-exports
+      COMPONENTS          ${RAFT_COMPONENTS}
+      CPM_ARGS
+        GIT_REPOSITORY         https://github.com/${PKG_FORK}/raft.git
+        GIT_TAG                ${PKG_PINNED_TAG}
+        SOURCE_SUBDIR          cpp
+        EXCLUDE_FROM_ALL       ${PKG_EXCLUDE_FROM_ALL}
+        OPTIONS
+          "BUILD_TESTS OFF"
+          "RAFT_COMPILE_LIBRARY ${RAFT_COMPILE_LIBRARIES}"
+          "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}"
+    )
 
     if(raft_ADDED)
         message(VERBOSE "CUML: Using RAFT located in ${raft_SOURCE_DIR}")
@@ -92,16 +79,14 @@ endfunction()
 # To use a different RAFT locally, set the CMake variable
 # CPM_raft_SOURCE=/path/to/local/raft
 find_and_configure_raft(VERSION          ${CUML_MIN_VERSION_raft}
-        FORK             rapidsai
-        PINNED_TAG       branch-${CUML_BRANCH_VERSION_raft}
-        EXCLUDE_FROM_ALL ${CUML_EXCLUDE_RAFT_FROM_ALL}
-        # When PINNED_TAG above doesn't match cuml,
-        # force local raft clone in build directory
-        # even if it's already installed.
-        CLONE_ON_PIN     ${CUML_RAFT_CLONE_ON_PIN}
-        USE_RAFT_NN      ${CUML_USE_RAFT_NN}
-        USE_RAFT_DIST    ${CUML_USE_RAFT_DIST}
-        USE_RAFT_STATIC  ${CUML_USE_RAFT_STATIC}
-        USE_FAISS_STATIC ${CUML_USE_FAISS_STATIC}
-        NVTX             ${NVTX}
-        )
+      FORK             rapidsai
+      PINNED_TAG       branch-${CUML_BRANCH_VERSION_raft}
+      EXCLUDE_FROM_ALL ${CUML_EXCLUDE_RAFT_FROM_ALL}
+      # When PINNED_TAG above doesn't match cuml,
+      # force local raft clone in build directory
+      # even if it's already installed.
+      CLONE_ON_PIN     ${CUML_RAFT_CLONE_ON_PIN}
+			COMPILE_LIBRARY  ON
+      USE_RAFT_STATIC  ${CUML_USE_RAFT_STATIC}
+      NVTX             ${NVTX}
+      )

cpp/src/metrics/silhouette_score.cu

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2021-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 #include <cuml/metrics/metrics.hpp>
 #include <raft/core/handle.hpp>
 #include <raft/distance/distance_types.hpp>
-#ifdef RAFT_DISTANCE_COMPILED
+#ifdef RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 #include <raft/stats/silhouette_score.cuh>

cpp/src/metrics/silhouette_score_batched_double.cu

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2021-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 #include <cuml/metrics/metrics.hpp>
 #include <raft/core/handle.hpp>
 #include <raft/distance/distance_types.hpp>
-#ifdef RAFT_DISTANCE_COMPILED
+#ifdef RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 #include <raft/stats/silhouette_score.cuh>

cpp/src/metrics/silhouette_score_batched_float.cu

@@ -1,6 +1,6 @@
 
 /*
- * Copyright (c) 2021-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2021-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 #include <cuml/metrics/metrics.hpp>
 #include <raft/core/handle.hpp>
 #include <raft/distance/distance_types.hpp>
-#ifdef RAFT_DISTANCE_COMPILED
+#ifdef RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 #include <raft/stats/silhouette_score.cuh>

cpp/src/metrics/trustworthiness.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2018-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,11 +19,11 @@
 #include <cuml/metrics/metrics.hpp>
 #include <raft/core/handle.hpp>
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 
-#if defined RAFT_NN_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/spatial/knn/specializations.cuh>
 #endif
 

cpp/src/svm/linear.cu

@@ -17,7 +17,7 @@
 #include <random>
 #include <type_traits>
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 

cpp/src/svm/svc.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 
 #include <iostream>
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif
 

cpp/src/svm/svr.cu

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2022, NVIDIA CORPORATION.
+ * Copyright (c) 2019-2023, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 
 #include <iostream>
 
-#if defined RAFT_DISTANCE_COMPILED
+#if defined RAFT_COMPILED
 #include <raft/distance/specializations.cuh>
 #endif