From 9389108b7f7f100c883092a46f09738f23ab8151 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 24 Mar 2023 13:42:03 -0400 Subject: [PATCH] RAFT skeleton project template (#1312) This is a copy and modification of a user's project but I think this is going to be generally useful to users as the same types of challenges are going to come up again. In this case, the user wasn't able to build/link because they weren't using `rapids-cmake` to propagate important configuration settings. I think having a skeleton project available that we build in CI and keep up to date will help new users build more applications on RAFT. TODO: - [x] Make building the template optional - [x] Verify this can build in CMake and reuse already built/installed bits - [x] Add to docs / readme and reference in README.md - [x] Add a little example of invoking an API (maybe `pairwise_distances`?) to `main()` Authors: - Corey J. Nolet (https://github.com/cjnolet) - Ben Frederickson (https://github.com/benfred) Approvers: - Micka (https://github.com/lowener) - Dante Gama Dessavre (https://github.com/dantegd) - Divye Gala (https://github.com/divyegala) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/raft/pull/1312 --- .github/workflows/build.yaml | 1 + README.md | 61 ++++-------------- build.sh | 15 ++++- .../recipes/libraft/build_libraft_template.sh | 5 ++ conda/recipes/libraft/meta.yaml | 36 +++++++++++ cpp/template/CMakeLists.txt | 38 +++++++++++ cpp/template/README.md | 18 ++++++ cpp/template/build.sh | 41 ++++++++++++ .../cmake/thirdparty/fetch_rapids.cmake | 21 ++++++ cpp/template/cmake/thirdparty/get_raft.cmake | 62 ++++++++++++++++++ cpp/template/src/test_distance.cu | 42 ++++++++++++ docs/source/build.md | 64 ++++++++----------- python/pylibraft/setup.cfg | 38 +++++++++++ setup.cfg | 55 ++++++++++++++++ 14 files changed, 409 insertions(+), 88 deletions(-) create mode 100644 conda/recipes/libraft/build_libraft_template.sh create mode 100644 cpp/template/CMakeLists.txt create mode 100644 cpp/template/README.md create mode 100755 cpp/template/build.sh create mode 100644 cpp/template/cmake/thirdparty/fetch_rapids.cmake create mode 100644 cpp/template/cmake/thirdparty/get_raft.cmake create mode 100644 cpp/template/src/test_distance.cu create mode 100644 python/pylibraft/setup.cfg create mode 100644 setup.cfg diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 41b6a639d8..32aab5656b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,7 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + skip_upload_pkgs: libraft-template docs-build: if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build diff --git a/README.md b/README.md index 8c6e817bf9..81973ff82e 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ #
 RAFT: Reusable Accelerated Functions and Tools
- ![Navigating the canyons of accelerated possibilities](img/raft.png) ## Resources @@ -85,9 +84,9 @@ raft::device_resources handle; int n_samples = 5000; int n_features = 50; -auto input = raft::make_device_matrix(handle, n_samples, n_features); -auto labels = raft::make_device_vector(handle, n_samples); -auto output = raft::make_device_matrix(handle, n_samples, n_samples); +auto input = raft::make_device_matrix(handle, n_samples, n_features); +auto labels = raft::make_device_vector(handle, n_samples); +auto output = raft::make_device_matrix(handle, n_samples, n_samples); raft::random::make_blobs(handle, input.view(), labels.view()); @@ -222,52 +221,15 @@ pip install raft-dask-cu11 --extra-index-url=https://pypi.ngc.nvidia.com ### CMake & CPM -RAFT uses the [RAPIDS-CMake](https://github.com/rapidsai/rapids-cmake) library, which makes it simple to include in downstream cmake projects. RAPIDS CMake provides a convenience layer around CPM. - -After [installing](https://github.com/rapidsai/rapids-cmake#installation) rapids-cmake in your project, you can begin using RAFT by placing the code snippet below in a file named `get_raft.cmake` and including it in your cmake build with `include(get_raft.cmake)`. This will make available several targets to add to configure the link libraries for your artifacts. - -```cmake - -set(RAFT_VERSION "22.12") -set(RAFT_FORK "rapidsai") -set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") - -function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARIES) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - #----------------------------------------------------- - # Invoke CPM find_package() - #----------------------------------------------------- - - rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET projname-exports - INSTALL_EXPORT_SET projname-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp - OPTIONS - "BUILD_TESTS OFF" - "BUILD_BENCH OFF" - "RAFT_COMPILE_LIBRARIES ${PKG_COMPILE_LIBRARIES}" - ) - -endfunction() - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAFT_VERSION}.00 - FORK ${RAFT_FORK} - PINNED_TAG ${RAFT_PINNED_TAG} - COMPILE_LIBRARIES NO -) -``` +RAFT uses the [RAPIDS-CMake](https://github.com/rapidsai/rapids-cmake) library, which makes it easy to include in downstream cmake projects. RAPIDS-CMake provides a convenience layer around CPM. Please refer to [these instructions](https://github.com/rapidsai/rapids-cmake#installation) to install and use rapids-cmake in your project. + +#### Example Template Project + +You can find an [example RAFT](cpp/template/README.md) project template in the `cpp/template` directory, which demonstrates how to build a new application with RAFT or incorporate RAFT into an existing cmake project. + +#### CMake Targets -Several CMake targets can be made available by adding components in the table below to the `RAFT_COMPONENTS` list above, separated by spaces. The `raft::raft` target will always be available. RAFT headers require, at a minimum, the CUDA toolkit libraries and RMM dependencies. +Additional CMake targets can be made available by adding components in the table below to the `RAFT_COMPONENTS` list above, separated by spaces. The `raft::raft` target will always be available. RAFT headers require, at a minimum, the CUDA toolkit libraries and RMM dependencies. | Component | Target | Description | Base Dependencies | |-------------|---------------------|-----------------------------------------------------------|---------------------------------------| @@ -321,6 +283,7 @@ The folder structure mirrors other RAPIDS repos, with the following folders: - `internal`: A private header-only component that hosts the code shared between benchmarks and tests. - `scripts`: Helpful scripts for development - `src`: Compiled APIs and template specializations for the shared libraries + - `template`: A skeleton template containing the bare-bones file structure and cmake configuration for writing applications with RAFT. - `test`: Googletests source code - `docs`: Source code and scripts for building library documentation (Uses breath, doxygen, & pydocs) - `python`: Source code for Python libraries. diff --git a/build.sh b/build.sh index b5a72f4205..9468d2cab0 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean --uninstall -v -g -n --compile-lib --allgpuarch --no-nvtx --show_depr_warn -h" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench template clean --uninstall -v -g -n --compile-lib --allgpuarch --no-nvtx --show_depr_warn -h" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -29,6 +29,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode @@ -354,13 +355,12 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs || hasArg tests || has -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ - -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ + -DRAFT_COMPILE_LIBRARY=${COMPILE_LIBRARY} \ -DRAFT_NVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNINGS=${DISABLE_DEPRECATION_WARNINGS} \ -DBUILD_TESTS=${BUILD_TESTS} \ -DBUILD_BENCH=${BUILD_BENCH} \ -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} \ - -DRAFT_COMPILE_LIBRARY=${COMPILE_LIBRARY} \ ${CACHE_ARGS} \ ${EXTRA_CMAKE_ARGS} @@ -410,3 +410,12 @@ if hasArg docs; then cd ${SPHINX_BUILD_DIR} sphinx-build -b html source _html fi + +################################################################################ +# Initiate build for example RAFT application template (if needed) + +if hasArg template; then + pushd cpp/template + ./build.sh + popd +fi diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh new file mode 100644 index 0000000000..9759402884 --- /dev/null +++ b/conda/recipes/libraft/build_libraft_template.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +# Just building template so we verify it uses libraft.so and fail if it doesn't build +./build.sh template \ No newline at end of file diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml index 2a724672ab..f911166a9a 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft/meta.yaml @@ -150,3 +150,39 @@ outputs: home: https://rapids.ai/ license: Apache-2.0 summary: libraft tests + - name: libraft-template + version: {{ version }} + script: build_libraft_template.sh + build: + script_env: *script_env + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + ignore_run_exports_from: + - {{ compiler('cuda') }} + requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cuda') }} {{ cuda_version }} + - {{ compiler('cxx') }} + - cmake {{ cmake_version }} + - ninja + - sysroot_{{ target_platform }} {{ sysroot_version }} + host: + - {{ pin_subpackage('libraft', exact=True) }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - libcublas {{ libcublas_host_version }} + - libcublas-dev {{ libcublas_host_version }} + - libcurand {{ libcurand_host_version }} + - libcurand-dev {{ libcurand_host_version }} + - libcusolver {{ libcusolver_host_version }} + - libcusolver-dev {{ libcusolver_host_version }} + - libcusparse {{ libcusparse_host_version }} + - libcusparse-dev {{ libcusparse_host_version }} + run: + - {{ pin_subpackage('libraft', exact=True) }} + - {{ pin_subpackage('libraft-headers', exact=True) }} + about: + home: https://rapids.ai/ + license: Apache-2.0 + summary: libraft template diff --git a/cpp/template/CMakeLists.txt b/cpp/template/CMakeLists.txt new file mode 100644 index 0000000000..501a5c9503 --- /dev/null +++ b/cpp/template/CMakeLists.txt @@ -0,0 +1,38 @@ +# ============================================================================= +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR) + +# ------------- configure rapids-cmake --------------# + +include(cmake/thirdparty/fetch_rapids.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +# ------------- configure project --------------# + +rapids_cuda_init_architectures(test_raft) + +project(test_raft LANGUAGES CXX CUDA) + +# ------------- configure raft -----------------# + +rapids_cpm_init() +include(cmake/thirdparty/get_raft.cmake) + +# -------------- compile tasks ----------------- # +add_executable(TEST_RAFT src/test_distance.cu) +target_link_libraries(TEST_RAFT PRIVATE raft::raft raft::compiled) diff --git a/cpp/template/README.md b/cpp/template/README.md new file mode 100644 index 0000000000..348dff270a --- /dev/null +++ b/cpp/template/README.md @@ -0,0 +1,18 @@ +# Example RAFT Project Template + +This template project provides a drop-in sample to either start building a new application with, or using RAFT in an existing CMake project. + +First, please refer to our [installation docs](https://docs.rapids.ai/api/raft/stable/build.html#cuda-gpu-requirements) for the minimum requirements to use RAFT. + +Once the minimum requirements are satisfied, this example template application can be built with the provided `build.sh` script. This is a bash script that calls the appropriate CMake commands, so you can look into it to see the typical CMake based build workflow. + +This directory (`RAFT_SOURCE/cpp/template`) can be copied directly in order to build a new application with RAFT. + +RAFT can be integrated into an existing CMake project by copying the contents in the `configure rapids-cmake` and `configure raft` sections of the provided `CMakeLists.txt` into your project, along with `cmake/thirdparty/get_raft.cmake`. + +Make sure to link against the appropriate Cmake targets. Use `raft::raft`to add make the headers available and `raft::compiled` when utilizing the shared library. + +```cmake +target_link_libraries(your_app_target PRIVATE raft::raft raft::compiled) +``` + diff --git a/cpp/template/build.sh b/cpp/template/build.sh new file mode 100755 index 0000000000..3ac00fc9af --- /dev/null +++ b/cpp/template/build.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright (c) 2023, NVIDIA CORPORATION. + +# raft empty project template build script + +# Abort script on first error +set -e + +PARALLEL_LEVEL=${PARALLEL_LEVEL:=`nproc`} + +BUILD_TYPE=Release +BUILD_DIR=build/ + +RAFT_REPO_REL="" +EXTRA_CMAKE_ARGS="" +set -e + + +if [[ ${RAFT_REPO_REL} != "" ]]; then + RAFT_REPO_PATH="`readlink -f \"${RAFT_REPO_REL}\"`" + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCPM_raft_SOURCE=${RAFT_REPO_PATH}" +fi + +if [ "$1" == "clean" ]; then + rm -rf build + exit 0 +fi + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +cmake \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DRAFT_NVTX=OFF \ + -DCMAKE_CUDA_ARCHITECTURES="NATIVE" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + ${EXTRA_CMAKE_ARGS} \ + ../ + +cmake --build . -j${PARALLEL_LEVEL} diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/template/cmake/thirdparty/fetch_rapids.cmake new file mode 100644 index 0000000000..40ba83be9e --- /dev/null +++ b/cpp/template/cmake/thirdparty/fetch_rapids.cmake @@ -0,0 +1,21 @@ +# ============================================================================= +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use this variable to update RAPIDS and RAFT versions +set(RAPIDS_VERSION "23.04") + +if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake + ${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) +endif() +include(${CMAKE_CURRENT_BINARY_DIR}/RAFT_RAPIDS.cmake) diff --git a/cpp/template/cmake/thirdparty/get_raft.cmake b/cpp/template/cmake/thirdparty/get_raft.cmake new file mode 100644 index 0000000000..5463942adf --- /dev/null +++ b/cpp/template/cmake/thirdparty/get_raft.cmake @@ -0,0 +1,62 @@ +# ============================================================================= +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake +set(RAFT_VERSION "${RAPIDS_VERSION}") +set(RAFT_FORK "rapidsai") +set(RAFT_PINNED_TAG "branch-${RAPIDS_VERSION}") + +function(find_and_configure_raft) + set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + set(RAFT_COMPONENTS "") + if(PKG_COMPILE_LIBRARY) + string(APPEND RAFT_COMPONENTS " compiled") + endif() + + if(PKG_ENABLE_MNMG_DEPENDENCIES) + string(APPEND RAFT_COMPONENTS " distributed") + endif() + + #----------------------------------------------------- + # Invoke CPM find_package() + #----------------------------------------------------- + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET raft-template-exports + INSTALL_EXPORT_SET raft-template-exports + COMPONENTS ${RAFT_COMPONENTS} + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "BUILD_BENCH OFF" + "RAFT_NVTX ${ENABLE_NVTX}" + "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" + ) +endfunction() + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${RAFT_VERSION}.00 + FORK ${RAFT_FORK} + PINNED_TAG ${RAFT_PINNED_TAG} + COMPILE_LIBRARY ON + ENABLE_MNMG_DEPENDENCIES OFF + ENABLE_NVTX OFF +) diff --git a/cpp/template/src/test_distance.cu b/cpp/template/src/test_distance.cu new file mode 100644 index 0000000000..b86dde70e5 --- /dev/null +++ b/cpp/template/src/test_distance.cu @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022-2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#ifdef RAFT_COMPILED +#include +#endif + +int main() +{ + raft::device_resources handle; + + int n_samples = 5000; + int n_features = 50; + + auto input = raft::make_device_matrix(handle, n_samples, n_features); + auto labels = raft::make_device_vector(handle, n_samples); + auto output = raft::make_device_matrix(handle, n_samples, n_samples); + + raft::random::make_blobs(handle, input.view(), labels.view()); + + auto metric = raft::distance::DistanceType::L2SqrtExpanded; + raft::distance::pairwise_distance(handle, input.view(), input.view(), output.view(), metric); +} diff --git a/docs/source/build.md b/docs/source/build.md index bbb454736a..e8e6ac8a14 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -75,7 +75,7 @@ Once installed, `libraft` headers (and dependencies which were downloaded and in ``` -### C++ Shared Libraries (optional) +### C++ Shared Library (optional) A shared library can be built for speeding up compile times. The shared library also contains a runtime API that allows you to invoke RAFT APIs directly from C++ source files (without `nvcc`). The shared library can also significantly improve re-compile times both while developing RAFT and using its APIs to develop applications. Pass the `--compile-lib` flag to `build.sh` to build the library: ```bash @@ -109,7 +109,7 @@ Compile the tests using the `tests` target in `build.sh`. Test compile times can be improved significantly by using the optional shared libraries. If installed, they will be used automatically when building the tests but `--compile-libs` can be used to add additional compilation units and compile them with the tests. ```bash -./build.sh libraft tests --compile-libs +./build.sh libraft tests --compile-lib ``` The tests are broken apart by algorithm category, so you will find several binaries in `cpp/build/` named `*_TEST`. @@ -151,19 +151,17 @@ make -j install RAFT's cmake has the following configurable flags available:. -| Flag | Possible Values | Default Value | Behavior | -| --- | --- | --- | --- | -| BUILD_TESTS | ON, OFF | ON | Compile Googletests | -| BUILD_BENCH | ON, OFF | OFF | Compile benchmarks | -| raft_FIND_COMPONENTS | nn distance | | Configures the optional components as a space-separated list | -| RAFT_COMPILE_LIBRARIES | ON, OFF | ON if either BUILD_TESTS or BUILD_BENCH is ON; otherwise OFF | Compiles all `libraft` shared libraries (these are required for Googletests) | -| RAFT_COMPILE_NN_LIBRARY | ON, OFF | OFF | Compiles the `libraft-nn` shared library | -| RAFT_COMPILE_DIST_LIBRARY | ON, OFF | OFF | Compiles the `libraft-distance` shared library | -| DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | -| RAFT_NVTX | ON, OFF | OFF | Enable NVTX Markers | -| CUDA_ENABLE_KERNELINFO | ON, OFF | OFF | Enables `kernelinfo` in nvcc. This is useful for `compute-sanitizer` | -| CUDA_ENABLE_LINEINFO | ON, OFF | OFF | Enable the -lineinfo option for nvcc | -| CUDA_STATIC_RUNTIME | ON, OFF | OFF | Statically link the CUDA runtime | +| Flag | Possible Values | Default Value | Behavior | +|---------------------------|----------------------| --- | --- | +| BUILD_TESTS | ON, OFF | ON | Compile Googletests | +| BUILD_BENCH | ON, OFF | OFF | Compile benchmarks | +| raft_FIND_COMPONENTS | compiled distributed | | Configures the optional components as a space-separated list | +| RAFT_COMPILE_LIBRARY | ON, OFF | ON if either BUILD_TESTS or BUILD_BENCH is ON; otherwise OFF | Compiles all `libraft` shared libraries (these are required for Googletests) | +| DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | +| RAFT_NVTX | ON, OFF | OFF | Enable NVTX Markers | +| CUDA_ENABLE_KERNELINFO | ON, OFF | OFF | Enables `kernelinfo` in nvcc. This is useful for `compute-sanitizer` | +| CUDA_ENABLE_LINEINFO | ON, OFF | OFF | Enable the -lineinfo option for nvcc | +| CUDA_STATIC_RUNTIME | ON, OFF | OFF | Statically link the CUDA runtime | Currently, shared libraries are provided for the `libraft-nn` and `libraft-distance` components. @@ -248,9 +246,9 @@ PROPERTIES CXX_STANDARD 17 ``` -### C++ header-only integration +### C++ header-only integration (without cmake) -When the needed [build dependencies](#build-dependencies) are already satisfied, RAFT can be trivially integrated into downstream projects by cloning the repository and adding `cpp/include` from RAFT to the include path: +While not a highly suggested method for building against RAFT, when all of the needed [build dependencies](#build-dependencies) are already satisfied, RAFT can be integrated into downstream projects by cloning the repository and adding `cpp/include` from RAFT to the include path: ```cmake set(RAFT_GIT_DIR ${CMAKE_CURRENT_BINARY_DIR}/raft CACHE STRING "Path to RAFT repo") ExternalProject_Add(raft @@ -262,8 +260,12 @@ ExternalProject_Add(raft INSTALL_COMMAND "") set(RAFT_INCLUDE_DIR ${RAFT_GIT_DIR}/raft/cpp/include CACHE STRING "RAFT include variable") ``` +### C++ header-only integration (with cmake) -If RAFT has already been installed, such as by using the `build.sh` script, use `find_package(raft)` and the `raft::raft` target. + +When using cmake, you can install RAFT headers into your environment with `./build.sh libraft`. + +If the RAFT headers have already been installed into your environment with cmake or through conda, such as by using the `build.sh` script, use `find_package(raft)` and the `raft::raft` target. ### Using C++ pre-compiled shared libraries @@ -271,17 +273,19 @@ Use `find_package(raft COMPONENTS compiled distributed)` to enable the shared li The pre-compiled libraries contain template specializations for commonly used types, such as single- and double-precision floating-point. In order to use the symbols in the pre-compiled libraries, the compiler needs to be told not to instantiate templates that are already contained in the shared libraries. By convention, these header files are named `specializations.cuh` and located in the base directory for the packages that contain specializations. -The following example tells the compiler to ignore the pre-compiled templates for the `raft::distance` API so any symbols already compiled into the `libraft` shared library will be used instead: +The following example tells the compiler to ignore the pre-compiled templates for the `raft::distance` API so any symbols already compiled into the `libraft` shared library will be used instead. RAFT's cmake creates a variable `RAFT_COMPILED` which can be used to ignore the pre-compiled template specializations only when the shared library has been enabled through cmake (such as by specifying the `compiled` component in `find_package`): ```c++ +#ifdef RAFT_COMPILED #include #include +#endif ``` ### Building RAFT C++ from source in cmake RAFT uses the [RAPIDS-CMake](https://github.com/rapidsai/rapids-cmake) library so it can be more easily included into downstream projects. RAPIDS cmake provides a convenience layer around the [CMake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake). -The following example is similar to invoking `find_package(raft)` but uses `rapids_cpm_find`, which provides a richer and more flexible configuration landscape by using CPM to fetch any dependencies not already available to the build. The `raft::raft` link target will be made available and it's recommended that it be used as a `PRIVATE` link dependency in downstream projects. The `COMPILE_LIBRARIES` option enables the building the shared libraries. +The following example is similar to invoking `find_package(raft)` but uses `rapids_cpm_find`, which provides a richer and more flexible configuration landscape by using CPM to fetch any dependencies not already available to the build. The `raft::raft` link target will be made available and it's recommended that it be used as a `PRIVATE` link dependency in downstream projects. The `COMPILE_LIBRARY` option enables the building the shared libraries. The following `cmake` snippet enables a flexible configuration of RAFT: @@ -292,19 +296,10 @@ set(RAFT_FORK "rapidsai") set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY CLONE_ON_PIN) + set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - - #----------------------------------------------------- - # Clone RAFT locally if PINNED_TAG has been changed - #----------------------------------------------------- - if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${RAFT_VERSION}") - message("Pinned tag found: ${PKG_PINNED_TAG}. Cloning raft locally.") - set(CPM_DOWNLOAD_raft ON) - set(CMAKE_IGNORE_PATH "${CMAKE_INSTALL_PREFIX}/include/raft;${CMAKE_IGNORE_PATH}) - endif() - + #----------------------------------------------------- # Invoke CPM find_package() #----------------------------------------------------- @@ -332,15 +327,12 @@ endfunction() find_and_configure_raft(VERSION ${RAFT_VERSION}.00 FORK ${RAFT_FORK} PINNED_TAG ${RAFT_PINNED_TAG} - - # When PINNED_TAG above doesn't match cuml, - # force local raft clone in build directory - # even if it's already installed. - CLONE_ON_PIN ON COMPILE_LIBRARY NO ) ``` +You can find a fully-functioning [example template project](../../cpp/template/README.md) in the `cpp/template` directory, which provides everything you need to build a new application with RAFT or incorporate RAFT Into your existing libraries. + ## Uninstall Once built and installed, RAFT can be safely uninstalled using `build.sh` by specifying any or all of the installed components. Please note that since `pylibraft` depends on `libraft`, uninstalling `pylibraft` will also uninstall `libraft`: diff --git a/python/pylibraft/setup.cfg b/python/pylibraft/setup.cfg new file mode 100644 index 0000000000..7d1a0c9065 --- /dev/null +++ b/python/pylibraft/setup.cfg @@ -0,0 +1,38 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +[isort] +line_length=79 +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +combine_as_imports=True +order_by_type=True +known_dask= + dask + distributed + dask_cuda +known_rapids= + nvtext + cudf + cuml + cugraph + dask_cudf + rmm +known_first_party= + raft + pylibraft +default_section=THIRDPARTY +sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER +skip= + thirdparty + .eggs + .git + .hg + .mypy_cache + .tox + .venv + _build + buck-out + build + dist + __init__.py diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..e64641d05b --- /dev/null +++ b/setup.cfg @@ -0,0 +1,55 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +[flake8] +filename = *.py, *.pyx, *.pxd, *.pxi +exclude = __init__.py, *.egg, build, docs, .git +force-check = True +ignore = + # line break before binary operator + W503, + # whitespace before : + E203 +per-file-ignores = + # Rules ignored only in Cython: + # E211: whitespace before '(' (used in multi-line imports) + # E225: Missing whitespace around operators (breaks cython casting syntax like ) + # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) + # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) + # E275: Missing whitespace after keyword (Doesn't work with Cython except?) + # E402: invalid syntax (works for Python, not Cython) + # E999: invalid syntax (works for Python, not Cython) + # W504: line break after binary operator (breaks lines that end with a pointer) + *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 + +[pydocstyle] +# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather +# than include using match-dir. Note that as discussed in +# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, +# unlike the match option above this match-dir will have no effect when +# pydocstyle is invoked from pre-commit. Therefore this exclusion list must +# also be maintained in the pre-commit config file. +match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ +# Allow missing docstrings for docutils +ignore-decorators = .*(docutils|doc_apply|copy_docstring).* +select = + D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418 + # Would like to enable the following rules in the future: + # D200, D202, D205, D400 + +[mypy] +ignore_missing_imports = True +# If we don't specify this, then mypy will check excluded files if +# they are imported by a checked file. +follow_imports = skip + +[codespell] +# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - +# this is only to allow you to run codespell interactively +skip = ./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,.*_skbuild +# ignore short words, and typename parameters like OffsetT +ignore-regex = \b(.{1,4}|[A-Z]\w*T)\b +ignore-words-list = inout,unparseable,numer +builtin = clear +quiet-level = 3