Move faiss_mr from raft (#5281)

The faiss_mr file in RAFT is only currently used by the reachability_faiss code in cuml. Move this file over to allow us to completely remove the faiss dependency from RAFT. Authors: - Ben Frederickson (https://github.com/benfred) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Joseph (https://github.com/jolorunyomi) URL: #5281
rapidsai · Mar 18, 2023 · 222121d · 222121d
1 parent 8e62206
commit 222121d
Show file tree

Hide file tree

Showing 9 changed files with 791 additions and 50 deletions.
diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py
@@ -36,6 +36,7 @@
 ]
 FILES_TO_EXCLUDE = [
     re.compile(r"cpp/src/tsne/cannylab/bh\.cu"),
+    re.compile(r"cpp/src/hdbscan/detail/faiss_mr\.hpp"),
 ]
 
 # this will break starting at year 10000, which is probably OK :)

diff --git a/conda/recipes/libcuml/meta.yaml b/conda/recipes/libcuml/meta.yaml
@@ -59,6 +59,8 @@ requirements:
     - libraft-headers ={{ minor_version }}
     - libraft-nn ={{ minor_version }}
     - treelite {{ treelite_version }}
+    - libfaiss>=1.7.1
+    - faiss-proc=*=cuda
 
 outputs:
   - name: libcuml

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -67,10 +67,13 @@ option(CUML_USE_FAISS_STATIC "Build and statically link the FAISS library for ne
 option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF)
 option(CUML_EXPORT_TREELITE_LINKAGE "Whether to publicly or privately link treelite to libcuml++" OFF)
 option(CUML_USE_CUMLPRIMS_MG_STATIC "Build and statically link the cumlprims_mg library" OFF)
+option(CUML_ENABLE_NN_DEPENDENCIES "Whether to enable FAISS dependency" ON)
+
 # The options below allow incorporating libcuml into another build process
 # without installing all its components. This is useful if total file size is
 # at a premium and we do not expect other consumers to use any APIs of the
 # dependency except those that are directly linked to by the dependent library.
+option(CUML_EXCLUDE_FAISS_FROM_ALL "Exclude FAISS targets from RAFT's 'all' target" ON)
 option(CUML_EXCLUDE_RAFT_FROM_ALL "Exclude RAFT targets from cuML's 'all' target" OFF)
 option(CUML_EXCLUDE_TREELITE_FROM_ALL "Exclude Treelite targets from cuML's 'all' target" OFF)
 option(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL "Exclude cumlprims_mg targets from cuML's 'all' target" OFF)
@@ -220,6 +223,10 @@ endif()
 
 include(cmake/thirdparty/get_raft.cmake)
 
+if(CUML_USE_RAFT_NN)
+  include(cmake/thirdparty/get_faiss.cmake)
+endif()
+
 if(LINK_TREELITE)
   include(cmake/thirdparty/get_treelite.cmake)
 endif()
@@ -556,10 +563,6 @@ if(BUILD_CUML_CPP_LIBRARY)
   if(CUML_USE_RAFT_STATIC AND (TARGET raft::raft))
     copy_interface_excludes(INCLUDED_TARGET raft::raft TARGET ${CUML_CPP_TARGET})
 
-    if(CUML_USE_RAFT_NN AND (TARGET faiss::faiss))
-      copy_interface_excludes(INCLUDED_TARGET faiss::faiss TARGET ${CUML_CPP_TARGET})
-    endif()
-
     if(CUML_USE_RAFT_DIST AND (TARGET cuco::cuco))
       list(APPEND _cuml_cpp_private_libs cuco::cuco)
     endif()
@@ -602,6 +605,7 @@ if(BUILD_CUML_CPP_LIBRARY)
   # INTERFACE target.
   list(APPEND ${_cuml_cpp_libs_var_name}
     raft::raft
+          $<$<BOOL:${CUML_USE_RAFT_NN}>:faiss>
     $<$<BOOL:${CUML_USE_RAFT_NN}>:raft::nn>
     $<$<BOOL:${CUML_USE_RAFT_DIST}>:raft::distance>
     $<TARGET_NAME_IF_EXISTS:cumlprims_mg::cumlprims_mg>

diff --git a/cpp/bench/CMakeLists.txt b/cpp/bench/CMakeLists.txt
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.
+# Copyright (c) 2019-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,6 +47,7 @@ if(BUILD_CUML_BENCH)
       ${TREELITE_LIBS}
       raft::raft
       raft::nn
+      faiss::faiss
       raft::distance
   )
 

diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake
@@ -0,0 +1,89 @@
+#=============================================================================
+# Copyright (c) 2021-2023, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+function(find_and_configure_faiss)
+    set(oneValueArgs VERSION REPOSITORY PINNED_TAG BUILD_STATIC_LIBS EXCLUDE_FROM_ALL)
+    cmake_parse_arguments(PKG "${options}" "${oneValueArgs}"
+            "${multiValueArgs}" ${ARGN} )
+
+    if(CUML_USE_RAFT_NN)
+        rapids_find_generate_module(faiss
+                HEADER_NAMES  faiss/IndexFlat.h
+                LIBRARY_NAMES faiss
+                )
+
+        set(BUILD_SHARED_LIBS ON)
+        if (PKG_BUILD_STATIC_LIBS)
+            set(BUILD_SHARED_LIBS OFF)
+            set(CPM_DOWNLOAD_faiss ON)
+        endif()
+
+        rapids_cpm_find(faiss ${PKG_VERSION}
+                GLOBAL_TARGETS     faiss::faiss
+                CPM_ARGS
+                GIT_REPOSITORY   ${PKG_REPOSITORY}
+                GIT_TAG          ${PKG_PINNED_TAG}
+                EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL}
+                OPTIONS
+                "FAISS_ENABLE_PYTHON OFF"
+                "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}"
+                "FAISS_ENABLE_GPU ON"
+                "BUILD_TESTING OFF"
+                "CMAKE_MESSAGE_LOG_LEVEL VERBOSE"
+                "FAISS_USE_CUDA_TOOLKIT_STATIC ${CUDA_STATIC_RUNTIME}"
+                )
+
+        if(TARGET faiss AND NOT TARGET faiss::faiss)
+            add_library(faiss::faiss ALIAS faiss)
+        endif()
+
+        if(faiss_ADDED)
+            rapids_export(BUILD faiss
+                    EXPORT_SET faiss-targets
+                    GLOBAL_TARGETS faiss
+                    NAMESPACE faiss::)
+        endif()
+    endif()
+
+    # We generate the faiss-config files when we built faiss locally, so always do `find_dependency`
+    rapids_export_package(BUILD OpenMP cuml-exports) # faiss uses openMP but doesn't export a need for it
+    rapids_export_package(BUILD faiss cuml-exports GLOBAL_TARGETS faiss::faiss faiss)
+    rapids_export_package(INSTALL faiss cuml-exports GLOBAL_TARGETS faiss::faiss faiss)
+
+    # Tell cmake where it can find the generated faiss-config.cmake we wrote.
+    include("${rapids-cmake-dir}/export/find_package_root.cmake")
+    rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] cuml-exports)
+endfunction()
+
+if(NOT CUML_FAISS_GIT_TAG)
+    # TODO: Remove this once faiss supports FAISS_USE_CUDA_TOOLKIT_STATIC
+    # (https://github.com/facebookresearch/faiss/pull/2446)
+    set(CUML_FAISS_GIT_TAG fea/statically-link-ctk-v1.7.0)
+    # set(RAFT_FAISS_GIT_TAG bde7c0027191f29c9dadafe4f6e68ca0ee31fb30)
+endif()
+
+if(NOT CUML_FAISS_GIT_REPOSITORY)
+    # TODO: Remove this once faiss supports FAISS_USE_CUDA_TOOLKIT_STATIC
+    # (https://github.com/facebookresearch/faiss/pull/2446)
+    set(CUML_FAISS_GIT_REPOSITORY https://github.com/trxcllnt/faiss.git)
+    # set(RAFT_FAISS_GIT_REPOSITORY https://github.com/facebookresearch/faiss.git)
+endif()
+
+find_and_configure_faiss(VERSION    1.7.0
+        REPOSITORY  ${CUML_FAISS_GIT_REPOSITORY}
+        PINNED_TAG  ${CUML_FAISS_GIT_TAG}
+        BUILD_STATIC_LIBS ${CUML_USE_FAISS_STATIC}
+        EXCLUDE_FROM_ALL ${CUML_EXCLUDE_FAISS_FROM_ALL})
diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake
@@ -1,5 +1,5 @@
 #=============================================================================
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.
+# Copyright (c) 2021-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,30 +23,30 @@ function(find_and_configure_raft)
             "${multiValueArgs}" ${ARGN} )
 
     if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${CUML_BRANCH_VERSION_raft}")
-      message(STATUS "CUML: RAFT pinned tag found: ${PKG_PINNED_TAG}. Cloning raft locally.")
-      set(CPM_DOWNLOAD_raft ON)
+        message(STATUS "CUML: RAFT pinned tag found: ${PKG_PINNED_TAG}. Cloning raft locally.")
+        set(CPM_DOWNLOAD_raft ON)
     elseif(PKG_USE_RAFT_STATIC AND (NOT CPM_raft_SOURCE))
-      message(STATUS "CUML: Cloning raft locally to build static libraries.")
-      set(CPM_DOWNLOAD_raft ON)
+        message(STATUS "CUML: Cloning raft locally to build static libraries.")
+        set(CPM_DOWNLOAD_raft ON)
     endif()
 
     if(PKG_USE_RAFT_DIST)
-      string(APPEND RAFT_COMPONENTS "distance")
+        string(APPEND RAFT_COMPONENTS "distance")
     endif()
 
     if(PKG_USE_RAFT_NN)
-      string(APPEND RAFT_COMPONENTS " nn")
+        string(APPEND RAFT_COMPONENTS " nn")
     endif()
 
     # We need RAFT::distributed for MG tests
     if(BUILD_CUML_MG_TESTS)
-      string(APPEND RAFT_COMPONENTS " distributed")
+        string(APPEND RAFT_COMPONENTS " distributed")
     endif()
 
     if(PKG_USE_RAFT_DIST AND PKG_USE_RAFT_NN)
-      set(RAFT_COMPILE_LIBRARIES ON)
+        set(RAFT_COMPILE_LIBRARIES ON)
     else()
-      set(RAFT_COMPILE_LIBRARIES OFF)
+        set(RAFT_COMPILE_LIBRARIES OFF)
     endif()
 
     # We need to set this each time so that on subsequent calls to cmake
@@ -55,29 +55,29 @@ function(find_and_configure_raft)
 
     set(RAFT_BUILD_SHARED_LIBS ON)
     if(${PKG_USE_RAFT_STATIC})
-      set(RAFT_BUILD_SHARED_LIBS OFF)
+        set(RAFT_BUILD_SHARED_LIBS OFF)
     endif()
 
     message(VERBOSE "CUML: raft FIND_PACKAGE_ARGUMENTS COMPONENTS ${RAFT_COMPONENTS}")
 
     rapids_cpm_find(raft ${PKG_VERSION}
-      GLOBAL_TARGETS      raft::raft
-      BUILD_EXPORT_SET    cuml-exports
-      INSTALL_EXPORT_SET  cuml-exports
-      COMPONENTS          ${RAFT_COMPONENTS}
-      CPM_ARGS
-        GIT_REPOSITORY         https://github.com/${PKG_FORK}/raft.git
-        GIT_TAG                ${PKG_PINNED_TAG}
-        SOURCE_SUBDIR          cpp
-        EXCLUDE_FROM_ALL       ${PKG_EXCLUDE_FROM_ALL}
-        OPTIONS
-          "BUILD_TESTS OFF"
-          "BUILD_SHARED_LIBS ${RAFT_BUILD_SHARED_LIBS}"
-          "RAFT_COMPILE_LIBRARIES ${RAFT_COMPILE_LIBRARIES}"
-          "RAFT_COMPILE_NN_LIBRARY ${PKG_USE_RAFT_NN}"
-          "RAFT_COMPILE_DIST_LIBRARY ${PKG_USE_RAFT_DIST}"
-          "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}"
-    )
+            GLOBAL_TARGETS      raft::raft
+            BUILD_EXPORT_SET    cuml-exports
+            INSTALL_EXPORT_SET  cuml-exports
+            COMPONENTS          ${RAFT_COMPONENTS}
+            CPM_ARGS
+            GIT_REPOSITORY         https://github.com/${PKG_FORK}/raft.git
+            GIT_TAG                ${PKG_PINNED_TAG}
+            SOURCE_SUBDIR          cpp
+            EXCLUDE_FROM_ALL       ${PKG_EXCLUDE_FROM_ALL}
+            OPTIONS
+            "BUILD_TESTS OFF"
+            "BUILD_SHARED_LIBS ${RAFT_BUILD_SHARED_LIBS}"
+            "RAFT_COMPILE_LIBRARIES ${RAFT_COMPILE_LIBRARIES}"
+            "RAFT_COMPILE_NN_LIBRARY ${PKG_USE_RAFT_NN}"
+            "RAFT_COMPILE_DIST_LIBRARY ${PKG_USE_RAFT_DIST}"
+            "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}"
+            )
 
     if(raft_ADDED)
         message(VERBOSE "CUML: Using RAFT located in ${raft_SOURCE_DIR}")
@@ -92,16 +92,16 @@ endfunction()
 # To use a different RAFT locally, set the CMake variable
 # CPM_raft_SOURCE=/path/to/local/raft
 find_and_configure_raft(VERSION          ${CUML_MIN_VERSION_raft}
-                        FORK             rapidsai
-                        PINNED_TAG       branch-${CUML_BRANCH_VERSION_raft}
-                        EXCLUDE_FROM_ALL ${CUML_EXCLUDE_RAFT_FROM_ALL}
-                        # When PINNED_TAG above doesn't match cuml,
-                        # force local raft clone in build directory
-                        # even if it's already installed.
-                        CLONE_ON_PIN     ${CUML_RAFT_CLONE_ON_PIN}
-                        USE_RAFT_NN      ${CUML_USE_RAFT_NN}
-                        USE_RAFT_DIST    ${CUML_USE_RAFT_DIST}
-                        USE_RAFT_STATIC  ${CUML_USE_RAFT_STATIC}
-                        USE_FAISS_STATIC ${CUML_USE_FAISS_STATIC}
-                        NVTX             ${NVTX}
-                        )
+        FORK             rapidsai
+        PINNED_TAG       branch-${CUML_BRANCH_VERSION_raft}
+        EXCLUDE_FROM_ALL ${CUML_EXCLUDE_RAFT_FROM_ALL}
+        # When PINNED_TAG above doesn't match cuml,
+        # force local raft clone in build directory
+        # even if it's already installed.
+        CLONE_ON_PIN     ${CUML_RAFT_CLONE_ON_PIN}
+        USE_RAFT_NN      ${CUML_USE_RAFT_NN}
+        USE_RAFT_DIST    ${CUML_USE_RAFT_DIST}
+        USE_RAFT_STATIC  ${CUML_USE_RAFT_STATIC}
+        USE_FAISS_STATIC ${CUML_USE_FAISS_STATIC}
+        NVTX             ${NVTX}
+        )