diff --git a/BUILD.md b/BUILD.md index 5a053650a5..f2186b9740 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,15 +1,17 @@ # RAFT Build and Development Guide -- [Building and running tests](#building-and-running-tests) -- [Usage of RAFT by downstream projects](#usage-of-raft-by-downstream-projects) - - [C++ Integration](#c-integration) - - [Python/Cython Integration](#pythoncython-integration) -- [CI Process](#ci-process) -- [Developer Guide](#developer-guide) - - [Local Development](#local-development) - - [Submitting PRs](#submitting-prs) +- [Building and installing RAFT](#build_install) +- [Using RAFT in downstream projects](#use_raft) + - [C++ Integration](#cxx_integration) + - [Building RAFT C++ from source](#build_cxx_source) + - [Python/Cython Integration](#py_integration) -## Building and installing RAFT +## Building and installing RAFT + +### CUDA/GPU Requirements +- CUDA 11.0+ +- NVIDIA driver 450.80.02+ +- Pascal architecture of better (Compute capability >= 6.0) C++ RAFT is a header-only library but provides the option of building shared libraries with template instantiations for common types to speed up compile times for larger projects. The recommended way to build and install RAFT is to use the `build.sh` script in the root of the repository. This script can build both the C++ and Python code and provides options for building and installing the shared libraries. @@ -28,7 +30,7 @@ python -m pytest raft To build manually, you can also use `CMake` and setup.py directly. -For C++, the `RAFT_COMPILE_LIBRARIES` option can be used to compile the shared libraries. Shared libraries are provided for the `nn` and `distance` packages currently. The `nn` package requires FAISS, which will be built from source if it is not already installed. FAISS can optionally be statically compiled into the `nn` shared library with the `RAFT_USE_FAISS_STATIC` option. +For C++, the `RAFT_COMPILE_LIBRARIES` option can be used to compile the shared libraries. Shared libraries are provided for the `nn` and `distance` packages currently. The `nn` package requires FAISS, which will be built from source if it is not already installed. [FAISS](https://github.com/facebookresearch/faiss) can optionally be statically compiled into the `nn` shared library with the `RAFT_USE_FAISS_STATIC` option. To install RAFT into a specific location, use `CMAKE_INSTALL_PREFIX`. The snippet below will install it into the current conda environment. ```bash @@ -47,22 +49,22 @@ python setup.py build_ext --inplace python setup.py install ``` -## Using RAFT in downstream projects +## Using RAFT in downstream projects -### C++ Integration +### C++ Integration Use RAFT in cmake projects with `find_package(raft)` for header-only operation and the `raft::raft` target will be available for configuring linking and `RAFT_INCLUDE_DIR` will be available for includes. Note that if any packages are used which require downstream dependencies, such as the `nn` package requiring FAISS, these dependencies will have be installed and configured in cmake independently. Use `find_package(raft COMPONENTS nn, distance)` to enable the shared libraries and pass dependencies through separate targets for each component. In this example, `raft::distance` and `raft::nn` targets will be available for configuring linking paths. These targets will also pass through any transitive dependencies (such as FAISS in the case of the `nn` package). -### Building RAFT C++ from source +### Building RAFT C++ from source -RAFT uses the [RAPIDS cmake](https://github.com/rapidsai/rapids-cmake) library, so it can be easily included into downstream projects. RAPIDS cmake provides a convenience layer around the [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake). The following example is similar to building RAFT itself from source but allows it to be done in cmake, providing the `raft::raft` target for includes by default. The `COMPILE_LIBRARIES` option enables the building of the shared libraries +RAFT uses the [RAPIDS cmake](https://github.com/rapidsai/rapids-cmake) library, so it can be easily included into downstream projects. RAPIDS cmake provides a convenience layer around the [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake). The following example is similar to building RAFT itself from source but allows it to be done in cmake, providing the `raft::raft` link target and `RAFT_INCLUDE_DIR` for includes. The `COMPILE_LIBRARIES` option enables the building of the shared libraries ```cmake function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG USE_RAFT_NN USE_FAISS_STATIC COMPILE_LIBRARIES) + set(oneValueArgs VERSION FORK PINNED_TAG USE_FAISS_STATIC COMPILE_LIBRARIES ENABLE_NN_DEPENDENCIES) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -77,10 +79,9 @@ function(find_and_configure_raft) FIND_PACKAGE_ARGUMENTS "COMPONENTS ${RAFT_COMPONENTS}" OPTIONS "BUILD_TESTS OFF" + "RAFT_ENABLE_NN_DEPENDENCIES ${PKG_ENABLE_NN_DEPENDENCIES}" "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}" - "NVTX ${NVTX}" - "RAFT_COMPILE_LIBRARIES ${COMPILE_LIBRARIES}" - + "RAFT_COMPILE_LIBRARIES ${PKG_COMPILE_LIBRARIES}" ) endfunction() @@ -91,12 +92,12 @@ endfunction() find_and_configure_raft(VERSION 22.02.00 FORK rapidsai PINNED_TAG branch-22.02 - USE_RAFT_NN NO - USE_FAISS_STATIC NO - COMPILE_LIBRARIES NO + COMPILE_LIBRARIES NO + ENABLE_NN_DEPENDENCIES NO + USE_FAISS_STATIC NO ) ``` -### Python/Cython Integration +### Python/Cython Integration Once installed, RAFT's Python library can be imported and used directly. \ No newline at end of file diff --git a/README.md b/README.md index c0eeab75e5..5f0b9a109f 100755 --- a/README.md +++ b/README.md @@ -18,17 +18,18 @@ the maintenance burden by maximizing reuse across projects. RAFT relies on the [ like other projects in the RAPIDS ecosystem, eases the burden of configuring different allocation strategies globally across the libraries that use it. RMM also provides RAII wrappers around device arrays that handle the allocation and cleanup. -## RAFT's primary goals are to be fast, simple, reusable, composable, and comprehensive. +RAFT's primary goals are to be fast, simple, reusable, composable, and comprehensive. -## Getting started - -Refer to the [Build](BUILD.md) instructions for details on building and including the RAFT library in downstream projects. The [Developer Guide](DEVELOPER_GUIDE.md) contains details on the developer guidelines, workflows, and principals. If you are interested in contributing to the RAFT project, please read our [Contributing guidelines](CONTRIBUTING.md). +## Build/Install RAFT -Most of the primitives in RAFT accept a `raft::handle_t` object for the management of resources which are expensive to create, such CUDA streams, stream pools, and handles to other CUDA libraries like `cublas` and `cusolver`. +Refer to the [Build](BUILD.md) instructions for details on building and including the RAFT library in downstream projects. +## Getting started ### C++ Example +Most of the primitives in RAFT accept a `raft::handle_t` object for the management of resources which are expensive to create, such CUDA streams, stream pools, and handles to other CUDA libraries like `cublas` and `cusolver`. + The example below demonstrates creating a RAFT handle and using it with RMM's `device_uvector` to allocate memory on device and compute pairwise Euclidean distances: ```c++ @@ -55,8 +56,6 @@ raft::distance::pairwise_distance(handle, input.data(), input.data(), ``` - - ## Folder Structure and Contents The folder structure mirrors other RAPIDS repos (cuDF, cuML, cuGraph...), with the following folders: @@ -66,3 +65,7 @@ The folder structure mirrors other RAPIDS repos (cuDF, cuML, cuGraph...), with t - `cpp`: Source code for all C++ code. The code is currently header-only, therefore it is in the `include` folder (with no `src`). - `docs`: Source code and scripts for building library documentation - `python`: Source code for all Python source code. + +## Contributing + +If you are interested in contributing to the RAFT project, please read our [Contributing guidelines](CONTRIBUTING.md). Refer to the [Developer Guide](DEVELOPER_GUIDE.md) for details on the developer guidelines, workflows, and principals. \ No newline at end of file diff --git a/build.sh b/build.sh index 94d0b5e812..1346270e45 100755 --- a/build.sh +++ b/build.sh @@ -18,24 +18,26 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean cppraft pyraft docs -v -g --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" +VALIDARGS="clean libraft pyraft docs -v -g --compilelibs --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" HELP="$0 [ ...] [ ...] where is: clean - remove all existing build artifacts and configuration (start over) - cppraft - build the cuml C++ code only. Also builds the C-wrapper library + libraft - build the raft C++ code only. Also builds the C-wrapper library around the C++ code. pyraft - build the cuml Python package docs - build the documentation + and is: -v - verbose build mode -g - build for debug + --compilelibs - compile shared libraries --allgpuarch - build for all supported GPU architectures --buildfaiss - build faiss statically into raft --nvtx - Enable nvtx for profiling support --show_depr_warn - show cmake deprecation warnings -h - print this text - default action (no args) is to build both cppraft and pyraft targets + default action (no args) is to build both libraft and pyraft targets " CPP_RAFT_BUILD_DIR=${REPODIR}/cpp/build SPHINX_BUILD_DIR=${REPODIR}/docs @@ -47,8 +49,10 @@ BUILD_DIRS="${CPP_RAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PYTHON_DEPS_CLONE}" CMAKE_LOG_LEVEL="" VERBOSE_FLAG="" BUILD_ALL_GPU_ARCH=0 -BUILD_GTEST=OFF +BUILD_TESTS=ON BUILD_STATIC_FAISS=OFF +COMPILE_LIBRARIES=OFF +ENABLE_NN_DEPENDENCIES=ON SINGLEGPU="" NVTX=OFF CLEAN=0 @@ -93,6 +97,10 @@ if hasArg -g; then BUILD_TYPE=Debug fi +if hasArg --compilelibs; then + COMPILE_LIBRARIES=ON +fi + if hasArg --allgpuarch; then BUILD_ALL_GPU_ARCH=1 fi @@ -137,7 +145,7 @@ fi ################################################################################ # Configure for building all C++ targets -if (( ${NUMARGS} == 0 )) || hasArg cppraft || hasArg docs; then +if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then RAFT_CMAKE_CUDA_ARCHITECTURES="NATIVE" echo "Building for the architecture of the GPU in the system..." @@ -149,18 +157,20 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft || hasArg docs; then cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} ${CMAKE_LOG_LEVEL} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ + -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ + -DRAFT_ENABLE_NN_DEPENDENCIES=${ENABLE_NN_DEPENDENCIES} \ -DNVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ - -DBUILD_GTEST=${BUILD_GTEST} \ - -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} + -DBUILD_TESTS=${BUILD_TESTS} \ + -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} \ + .. - if hasArg cppraft; then + if (( ${NUMARGS} == 0 )) || hasArg libraft; then # Run all c++ targets at once - cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${MAKE_TARGETS} ${VERBOSE_FLAG} + cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} fi fi - # Build and (optionally) install the cuml Python package if (( ${NUMARGS} == 0 )) || hasArg pyraft || hasArg docs; then @@ -168,7 +178,7 @@ if (( ${NUMARGS} == 0 )) || hasArg pyraft || hasArg docs; then if [[ ${INSTALL_TARGET} != "" ]]; then python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace ${SINGLEGPU} else - python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace --library-dir=${LIBCUML_BUILD_DIR} ${SINGLEGPU} + python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace --library-dir=${LIBRAFT_BUILD_DIR} ${SINGLEGPU} fi fi diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 145624500c..45d1e03881 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -89,7 +89,7 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -"$WORKSPACE/build.sh" cppraft pyraft -v +"$WORKSPACE/build.sh" libraft pyraft -v gpuci_logger "Building docs" "$WORKSPACE/build.sh" docs -v