diff --git a/README.md b/README.md index a178d90008..ff1066492e 100755 --- a/README.md +++ b/README.md @@ -263,12 +263,12 @@ find_and_configure_raft(VERSION ${RAFT_VERSION}.00 Several CMake targets can be made available by adding components in the table below to the `RAFT_COMPONENTS` list above, separated by spaces. The `raft::raft` target will always be available. RAFT headers require, at a minimum, the CUDA toolkit libraries and RMM dependencies. -| Component | Target | Description | Base Dependencies | -| --- | --- | --- | --- | +| Component | Target | Description | Base Dependencies | +| --- | --- | --- |------------------------------------------------------------------| | n/a | `raft::raft` | Full RAFT header library | CUDA toolkit library, RMM, Thrust (optional), NVTools (optional) | -| distance | `raft::distance` | Pre-compiled template specializations for raft::distance | raft::raft, cuCollections (optional) | -| nn | `raft::nn` | Pre-compiled template specializations for raft::neighbors | raft::raft, FAISS (optional) | -| distributed | `raft::distributed` | No specializations | raft::raft, UCX, NCCL | +| distance | `raft::distance` | Pre-compiled template specializations for raft::distance | raft::raft, cuCollections (optional) | +| nn | `raft::nn` | Pre-compiled template specializations for raft::neighbors | raft::raft | +| distributed | `raft::distributed` | No specializations | raft::raft, UCX, NCCL | ### Source diff --git a/build.sh b/build.sh index 575f6bdaa1..7215b8199a 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean --uninstall -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean --uninstall -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --minimal-deps" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -45,7 +45,6 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=\\\" - pass arbitrary list of CMake configuration options (escape all quotes in argument) @@ -69,11 +68,9 @@ BUILD_ALL_GPU_ARCH=0 BUILD_TESTS=OFF BUILD_TYPE=Release BUILD_BENCH=OFF -BUILD_STATIC_FAISS=OFF COMPILE_LIBRARIES=OFF COMPILE_NN_LIBRARY=OFF COMPILE_DIST_LIBRARY=OFF -ENABLE_NN_DEPENDENCIES=OFF INSTALL_TARGET=install TEST_TARGETS="CLUSTER_TEST;CORE_TEST;DISTANCE_TEST;LABEL_TEST;LINALG_TEST;MATRIX_TEST;RANDOM_TEST;SOLVERS_TEST;SPARSE_TEST;SPARSE_DIST_TEST;SPARSE_NEIGHBORS_TEST;NEIGHBORS_TEST;STATS_TEST;UTILS_TEST" @@ -278,7 +275,6 @@ if hasArg --compile-libs || (( ${NUMARGS} == 0 )); then fi if hasArg --compile-nn || hasArg --compile-libs || (( ${NUMARGS} == 0 )); then - ENABLE_NN_DEPENDENCIES=ON COMPILE_NN_LIBRARY=ON CMAKE_TARGET="${CMAKE_TARGET};raft_nn_lib" fi @@ -299,7 +295,6 @@ if hasArg tests || (( ${NUMARGS} == 0 )); then $CMAKE_TARGET == *"NEIGHBORS_TEST"* || \ $CMAKE_TARGET == *"STATS_TEST"* ]]; then echo "-- Enabling nearest neighbors lib for gtests" - ENABLE_NN_DEPENDENCIES=ON COMPILE_NN_LIBRARY=ON fi @@ -324,7 +319,6 @@ if hasArg bench || (( ${NUMARGS} == 0 )); then if [[ $CMAKE_TARGET == *"CLUSTER_BENCH"* || \ $CMAKE_TARGET == *"NEIGHBORS_BENCH"* ]]; then echo "-- Enabling nearest neighbors lib for benchmarks" - ENABLE_NN_DEPENDENCIES=ON COMPILE_NN_LIBRARY=ON fi @@ -338,9 +332,6 @@ if hasArg bench || (( ${NUMARGS} == 0 )); then fi -if hasArg --buildfaiss; then - BUILD_STATIC_FAISS=ON -fi if hasArg --no-nvtx; then NVTX=OFF fi @@ -402,7 +393,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ - -DRAFT_ENABLE_NN_DEPENDENCIES=${ENABLE_NN_DEPENDENCIES} \ -DRAFT_NVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \ -DBUILD_TESTS=${BUILD_TESTS} \ @@ -410,7 +400,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \ -DRAFT_COMPILE_NN_LIBRARY=${COMPILE_NN_LIBRARY} \ -DRAFT_COMPILE_DIST_LIBRARY=${COMPILE_DIST_LIBRARY} \ - -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} \ -DRAFT_ENABLE_thrust_DEPENDENCY=${ENABLE_thrust_DEPENDENCY} \ ${CACHE_ARGS} \ ${EXTRA_CMAKE_ARGS} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 47af29d9d2..39f1fef4d5 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -22,7 +22,6 @@ dependencies: - dask>=2023.1.1 - distributed>=2023.1.1 - doxygen>=1.8.20 -- faiss-proc=*=cuda - gcc_linux-64=11.* - graphviz - ipython @@ -34,7 +33,6 @@ dependencies: - libcusolver=11.4.1.48 - libcusparse-dev=11.7.5.86 - libcusparse=11.7.5.86 -- libfaiss>=1.7.1=cuda* - ninja - numpydoc - pydata-sphinx-theme diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index ca213dc317..e1079f4db8 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -19,9 +19,6 @@ nccl_version: gtest_version: - "=1.10.0" -libfaiss_version: - - "1.7.2 *_cuda" - # The CTK libraries below are missing from the conda-forge::cudatoolkit # package. The "*_host_*" version specifiers correspond to `11.8` packages and the # "*_run_*" version specifiers correspond to `11.x` packages. diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 771c7d55b8..158016ddf0 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -130,7 +130,6 @@ outputs: host: - {{ pin_subpackage('libraft-headers', exact=True) }} - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - faiss-proc=*=cuda - lapack - libcublas {{ libcublas_host_version }} - libcublas-dev {{ libcublas_host_version }} @@ -140,10 +139,7 @@ outputs: - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} - - libfaiss {{ libfaiss_version }} run: - - faiss-proc=*=cuda - - libfaiss {{ libfaiss_version }} - {{ pin_subpackage('libraft-headers', exact=True) }} about: home: https://rapids.ai/ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2999045a0c..3889c39e6e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,9 +71,6 @@ option( option(RAFT_COMPILE_DIST_LIBRARY "Enable building raft distant shared library instantiations" ${RAFT_COMPILE_LIBRARIES} ) -option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" - ${RAFT_COMPILE_NN_LIBRARY} -) option(RAFT_ENABLE_thrust_DEPENDENCY "Enable Thrust dependency" ON) @@ -89,17 +86,6 @@ if(BUILD_TESTS AND NOT RAFT_ENABLE_thrust_DEPENDENCY) set(RAFT_ENABLE_thrust_DEPENDENCY ON) endif() -option(RAFT_EXCLUDE_FAISS_FROM_ALL "Exclude FAISS targets from RAFT's 'all' target" ON) - -include(CMakeDependentOption) -cmake_dependent_option( - RAFT_USE_FAISS_STATIC - "Build and statically link the FAISS library for nearest neighbors search on GPU" - ON - RAFT_COMPILE_LIBRARIES - OFF -) - message(VERBOSE "RAFT: Building optional components: ${raft_FIND_COMPONENTS}") message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Building raft C++ benchmarks: ${BUILD_BENCH}") @@ -183,7 +169,6 @@ rapids_cpm_init() # thrust before rmm/cuco so we get the right version of thrust/cub include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) -include(cmake/thirdparty/get_faiss.cmake) include(cmake/thirdparty/get_cutlass.cmake) if(RAFT_ENABLE_cuco_DEPENDENCY) @@ -502,7 +487,7 @@ if(RAFT_COMPILE_NN_LIBRARY) target_link_libraries( raft_nn_lib - PUBLIC faiss::faiss raft::raft + PUBLIC raft::raft PRIVATE nvidia::cutlass::cutlass ) target_compile_options( @@ -710,12 +695,6 @@ endif() if(nn IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) - - if(TARGET faiss AND (NOT TARGET faiss::faiss)) - add_library(faiss::faiss ALIAS faiss) - elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) - add_library(faiss ALIAS faiss::faiss) - endif() endif() ]=] ) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake deleted file mode 100644 index e6f06a00a5..0000000000 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ /dev/null @@ -1,89 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_faiss) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG BUILD_STATIC_LIBS EXCLUDE_FROM_ALL) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - if(RAFT_ENABLE_NN_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) - rapids_find_generate_module(faiss - HEADER_NAMES faiss/IndexFlat.h - LIBRARY_NAMES faiss - ) - - set(BUILD_SHARED_LIBS ON) - if (PKG_BUILD_STATIC_LIBS) - set(BUILD_SHARED_LIBS OFF) - set(CPM_DOWNLOAD_faiss ON) - endif() - - rapids_cpm_find(faiss ${PKG_VERSION} - GLOBAL_TARGETS faiss::faiss - CPM_ARGS - GIT_REPOSITORY ${PKG_REPOSITORY} - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} - OPTIONS - "FAISS_ENABLE_PYTHON OFF" - "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" - "FAISS_ENABLE_GPU ON" - "BUILD_TESTING OFF" - "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" - "FAISS_USE_CUDA_TOOLKIT_STATIC ${CUDA_STATIC_RUNTIME}" - ) - - if(TARGET faiss AND NOT TARGET faiss::faiss) - add_library(faiss::faiss ALIAS faiss) - endif() - - if(faiss_ADDED) - rapids_export(BUILD faiss - EXPORT_SET faiss-targets - GLOBAL_TARGETS faiss - NAMESPACE faiss::) - endif() - endif() - - # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` - rapids_export_package(BUILD OpenMP raft-nn-lib-exports) # faiss uses openMP but doesn't export a need for it - rapids_export_package(BUILD faiss raft-nn-lib-exports GLOBAL_TARGETS faiss::faiss faiss) - rapids_export_package(INSTALL faiss raft-nn-lib-exports GLOBAL_TARGETS faiss::faiss faiss) - - # Tell cmake where it can find the generated faiss-config.cmake we wrote. - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-nn-lib-exports) -endfunction() - -if(NOT RAFT_FAISS_GIT_TAG) - # TODO: Remove this once faiss supports FAISS_USE_CUDA_TOOLKIT_STATIC - # (https://github.com/facebookresearch/faiss/pull/2446) - set(RAFT_FAISS_GIT_TAG fea/statically-link-ctk-v1.7.0) - # set(RAFT_FAISS_GIT_TAG bde7c0027191f29c9dadafe4f6e68ca0ee31fb30) -endif() - -if(NOT RAFT_FAISS_GIT_REPOSITORY) - # TODO: Remove this once faiss supports FAISS_USE_CUDA_TOOLKIT_STATIC - # (https://github.com/facebookresearch/faiss/pull/2446) - set(RAFT_FAISS_GIT_REPOSITORY https://github.com/trxcllnt/faiss.git) - # set(RAFT_FAISS_GIT_REPOSITORY https://github.com/facebookresearch/faiss.git) -endif() - -find_and_configure_faiss(VERSION 1.7.0 - REPOSITORY ${RAFT_FAISS_GIT_REPOSITORY} - PINNED_TAG ${RAFT_FAISS_GIT_TAG} - BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} - EXCLUDE_FROM_ALL ${RAFT_EXCLUDE_FAISS_FROM_ALL}) diff --git a/dependencies.yaml b/dependencies.yaml index 93893d07af..9fbf26bcd1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -179,8 +179,6 @@ dependencies: - ucx-py=0.31.* - ucx-proc=*=gpu - rmm=23.04 - - libfaiss>=1.7.1=cuda* - - faiss-proc=*=cuda - dask-cuda=23.04 test_python: common: diff --git a/docs/source/build.md b/docs/source/build.md index 70b07f4e81..5ba1e75fad 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -47,7 +47,6 @@ In addition to the libraries included with cudatoolkit 11.0+, there are some oth - [cuCollections](https://github.com/NVIDIA/cuCollections) - Used in `raft::sparse::distance` API. - [Libcu++](https://github.com/NVIDIA/libcudacxx) v1.7.0 - Used by cuCollections - [CUTLASS](https://github.com/NVIDIA/cutlass) v2.9.1 - Used in `raft::distance` API. -- [FAISS](https://github.com/facebookresearch/faiss) v1.7.0 - Used in `raft::neighbors` API. - [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `raft-dask`. - [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `raft-dask`. - [Googletest](https://github.com/google/googletest) - Needed to build tests @@ -60,14 +59,14 @@ The recommended way to build and install RAFT is to use the `build.sh` script in ### Header-only C++ -`build.sh` uses [rapids-cmake](https://github.com/rapidsai/rapids-cmake), which will automatically download any dependencies which are not already installed. It's important to note that while all the headers will be installed and available, some parts of the RAFT API depend on libraries like `FAISS`, which will need to be explicitly enabled in `build.sh`. +`build.sh` uses [rapids-cmake](https://github.com/rapidsai/rapids-cmake), which will automatically download any dependencies which are not already installed. It's important to note that while all the headers will be installed and available, some parts of the RAFT API depend on libraries like CUTLASS, which will need to be explicitly enabled in `build.sh`. The following example will download the needed dependencies and install the RAFT headers into `$INSTALL_PREFIX/include/raft`. ```bash ./build.sh libraft ``` -The `-n` flag can be passed to just have the build download the needed dependencies. Since RAFT is primarily used at build-time, the dependencies will never be installed by the RAFT build, with the exception of building FAISS statically into the shared libraries. +The `-n` flag can be passed to just have the build download the needed dependencies. Since RAFT is primarily used at build-time, the dependencies will never be installed by the RAFT build. ```bash ./build.sh libraft -n ``` @@ -167,15 +166,13 @@ RAFT's cmake has the following configurable flags available:. | RAFT_COMPILE_LIBRARIES | ON, OFF | ON if either BUILD_TESTS or BUILD_BENCH is ON; otherwise OFF | Compiles all `libraft` shared libraries (these are required for Googletests) | | RAFT_COMPILE_NN_LIBRARY | ON, OFF | OFF | Compiles the `libraft-nn` shared library | | RAFT_COMPILE_DIST_LIBRARY | ON, OFF | OFF | Compiles the `libraft-distance` shared library | -| RAFT_ENABLE_NN_DEPENDENCIES | ON, OFF | OFF | Searches for dependencies of nearest neighbors API, such as FAISS, and compiles them if not found. Needed for `raft::spatial::knn` | -| RAFT_USE_FAISS_STATIC | ON, OFF | OFF | Statically link FAISS into `libraft-nn` | | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | | RAFT_NVTX | ON, OFF | OFF | Enable NVTX Markers | | CUDA_ENABLE_KERNELINFO | ON, OFF | OFF | Enables `kernelinfo` in nvcc. This is useful for `compute-sanitizer` | | CUDA_ENABLE_LINEINFO | ON, OFF | OFF | Enable the -lineinfo option for nvcc | | CUDA_STATIC_RUNTIME | ON, OFF | OFF | Statically link the CUDA runtime | -Currently, shared libraries are provided for the `libraft-nn` and `libraft-distance` components. The `libraft-nn` component depends upon [FAISS](https://github.com/facebookresearch/faiss) and the `RAFT_ENABLE_NN_DEPENDENCIES` option will build it from source if it is not already installed. +Currently, shared libraries are provided for the `libraft-nn` and `libraft-distance` components. ### Python @@ -277,7 +274,7 @@ If RAFT has already been installed, such as by using the `build.sh` script, use ### Using C++ pre-compiled shared libraries -Use `find_package(raft COMPONENTS nn distance)` to enable the shared libraries and transitively pass dependencies through separate targets for each component. In this example, the `raft::distance` and `raft::nn` targets will be available for configuring linking paths in addition to `raft::raft`. These targets will also pass through any transitive dependencies (such as FAISS for the `nn` package). +Use `find_package(raft COMPONENTS nn distance)` to enable the shared libraries and transitively pass dependencies through separate targets for each component. In this example, the `raft::distance` and `raft::nn` targets will be available for configuring linking paths in addition to `raft::raft`. These targets will also pass through any transitive dependencies (such as CUTLASS for the `distance` package). The pre-compiled libraries contain template specializations for commonly used types, such as single- and double-precision floating-point. In order to use the symbols in the pre-compiled libraries, the compiler needs to be told not to instantiate templates that are already contained in the shared libraries. By convention, these header files are named `specializations.cuh` and located in the base directory for the packages that contain specializations. @@ -302,8 +299,8 @@ set(RAFT_FORK "rapidsai") set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG USE_FAISS_STATIC - COMPILE_LIBRARIES ENABLE_NN_DEPENDENCIES CLONE_ON_PIN + set(oneValueArgs VERSION FORK PINNED_TAG + COMPILE_LIBRARIES CLONE_ON_PIN USE_NN_LIBRARY USE_DISTANCE_LIBRARY ENABLE_thrust_DEPENDENCY) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" @@ -346,8 +343,6 @@ function(find_and_configure_raft) OPTIONS "BUILD_TESTS OFF" "BUILD_BENCH OFF" - "RAFT_ENABLE_NN_DEPENDENCIES ${PKG_ENABLE_NN_DEPENDENCIES}" - "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}" "RAFT_COMPILE_LIBRARIES ${PKG_COMPILE_LIBRARIES}" "RAFT_ENABLE_thrust_DEPENDENCY ${PKG_ENABLE_thrust_DEPENDENCY}" ) @@ -369,8 +364,6 @@ find_and_configure_raft(VERSION ${RAFT_VERSION}.00 COMPILE_LIBRARIES NO USE_NN_LIBRARY NO USE_DISTANCE_LIBRARY NO - ENABLE_NN_DEPENDENCIES NO # This builds FAISS if not installed - USE_FAISS_STATIC NO ENABLE_thrust_DEPENDENCY YES ) ```