From 7fbff06325e98bed74c63a4e793ff14a4bf50bd8 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 2 Sep 2022 14:06:59 -0400 Subject: [PATCH 01/16] iUpdating readme --- BUILD.md | 16 ++++++++-------- README.md | 8 ++++---- build.sh | 12 ++++++------ ci/cpu/build.sh | 8 ++++---- ci/cpu/upload.sh | 10 +++++----- ci/gpu/build.sh | 12 ++++++------ conda/recipes/{pyraft => dask-raft}/build.sh | 2 +- .../conda_build_config.yaml | 0 conda/recipes/{pyraft => dask-raft}/meta.yaml | 4 ++-- python/{raft => dask-raft}/CMakeLists.txt | 12 ++++++------ .../cmake/thirdparty/get_nccl.cmake | 0 python/{raft => dask-raft}/pyproject.toml | 0 python/{raft => dask-raft}/pytest.ini | 0 python/{raft => dask-raft}/raft/__init__.py | 0 python/{raft => dask-raft}/raft/_version.py | 0 .../raft/common/CMakeLists.txt | 0 .../{raft => dask-raft}/raft/common/__init__.pxd | 0 .../{raft => dask-raft}/raft/common/__init__.py | 0 python/{raft => dask-raft}/raft/common/cuda.pxd | 0 python/{raft => dask-raft}/raft/common/cuda.pyx | 0 .../{raft => dask-raft}/raft/common/handle.pxd | 0 .../{raft => dask-raft}/raft/common/handle.pyx | 0 .../raft/common/interruptible.pxd | 0 .../raft/common/interruptible.pyx | 0 python/{raft => dask-raft}/raft/dask/__init__.py | 0 .../raft/dask/common/CMakeLists.txt | 0 .../raft/dask/common/__init__.py | 0 .../raft/dask/common/comms.py | 0 .../raft/dask/common/comms_utils.pyx | 0 .../raft/dask/common/nccl.pyx | 0 .../{raft => dask-raft}/raft/dask/common/ucx.py | 0 .../raft/dask/common/utils.py | 0 .../raft/include_test/CMakeLists.txt | 0 .../raft/include_test/__init__.py | 0 .../raft/include_test/raft_include_test.pyx | 0 python/{raft => dask-raft}/raft/test/conftest.py | 0 .../{raft => dask-raft}/raft/test/test_comms.py | 0 .../raft/test/test_interruptible.py | 0 .../{raft => dask-raft}/raft/test/test_raft.py | 0 python/{raft => dask-raft}/setup.cfg | 0 python/{raft => dask-raft}/setup.py | 4 ++-- python/{raft => dask-raft}/setuputils.py | 0 python/{raft => dask-raft}/versioneer.py | 0 43 files changed, 44 insertions(+), 44 deletions(-) rename conda/recipes/{pyraft => dask-raft}/build.sh (81%) rename conda/recipes/{pyraft => dask-raft}/conda_build_config.yaml (100%) rename conda/recipes/{pyraft => dask-raft}/meta.yaml (97%) rename python/{raft => dask-raft}/CMakeLists.txt (91%) rename python/{raft => dask-raft}/cmake/thirdparty/get_nccl.cmake (100%) rename python/{raft => dask-raft}/pyproject.toml (100%) rename python/{raft => dask-raft}/pytest.ini (100%) rename python/{raft => dask-raft}/raft/__init__.py (100%) rename python/{raft => dask-raft}/raft/_version.py (100%) rename python/{raft => dask-raft}/raft/common/CMakeLists.txt (100%) rename python/{raft => dask-raft}/raft/common/__init__.pxd (100%) rename python/{raft => dask-raft}/raft/common/__init__.py (100%) rename python/{raft => dask-raft}/raft/common/cuda.pxd (100%) rename python/{raft => dask-raft}/raft/common/cuda.pyx (100%) rename python/{raft => dask-raft}/raft/common/handle.pxd (100%) rename python/{raft => dask-raft}/raft/common/handle.pyx (100%) rename python/{raft => dask-raft}/raft/common/interruptible.pxd (100%) rename python/{raft => dask-raft}/raft/common/interruptible.pyx (100%) rename python/{raft => dask-raft}/raft/dask/__init__.py (100%) rename python/{raft => dask-raft}/raft/dask/common/CMakeLists.txt (100%) rename python/{raft => dask-raft}/raft/dask/common/__init__.py (100%) rename python/{raft => dask-raft}/raft/dask/common/comms.py (100%) rename python/{raft => dask-raft}/raft/dask/common/comms_utils.pyx (100%) rename python/{raft => dask-raft}/raft/dask/common/nccl.pyx (100%) rename python/{raft => dask-raft}/raft/dask/common/ucx.py (100%) rename python/{raft => dask-raft}/raft/dask/common/utils.py (100%) rename python/{raft => dask-raft}/raft/include_test/CMakeLists.txt (100%) rename python/{raft => dask-raft}/raft/include_test/__init__.py (100%) rename python/{raft => dask-raft}/raft/include_test/raft_include_test.pyx (100%) rename python/{raft => dask-raft}/raft/test/conftest.py (100%) rename python/{raft => dask-raft}/raft/test/test_comms.py (100%) rename python/{raft => dask-raft}/raft/test/test_interruptible.py (100%) rename python/{raft => dask-raft}/raft/test/test_raft.py (100%) rename python/{raft => dask-raft}/setup.cfg (100%) rename python/{raft => dask-raft}/setup.py (94%) rename python/{raft => dask-raft}/setuputils.py (100%) rename python/{raft => dask-raft}/versioneer.py (100%) diff --git a/BUILD.md b/BUILD.md index 3c6ad2bf20..b854108efe 100644 --- a/BUILD.md +++ b/BUILD.md @@ -34,8 +34,8 @@ In addition to the libraries included with cudatoolkit 11.0+, there are some oth - [cuCollections](https://github.com/NVIDIA/cuCollections) - Used in `raft::sparse::distance` API. - [Libcu++](https://github.com/NVIDIA/libcudacxx) v1.7.0 - [FAISS](https://github.com/facebookresearch/faiss) v1.7.0 - Used in `raft::spatial::knn` API and needed to build tests. -- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `Pyraft` -- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `Pyraft` +- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `dask-raft` +- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `dask-raft` - [Googletest](https://github.com/google/googletest) - Needed to build tests - [Googlebench](https://github.com/google/benchmark) - Needed to build benchmarks - [Doxygen](https://github.com/doxygen/doxygen) - Needed to build docs @@ -129,8 +129,8 @@ RAFT's cmake has the following configurable flags available:. | RAFT_ENABLE_NN_DEPENDENCIES | ON, OFF | OFF | Searches for dependencies of nearest neighbors API, such as FAISS, and compiles them if not found. Needed for `raft::spatial::knn` | | RAFT_ENABLE_thrust_DEPENDENCY | ON, OFF | ON | Enables the Thrust dependency. This can be disabled when using many simple utilities or to override with a different Thrust version. | | RAFT_ENABLE_mdspan_DEPENDENCY | ON, OFF | ON | Enables the std::mdspan dependency. This can be disabled when using many simple utilities. | -| RAFT_ENABLE_nccl_DEPENDENCY | ON, OFF | OFF | Enables NCCL dependency used by `raft::comms` and needed to build `pyraft` | -| RAFT_ENABLE_ucx_DEPENDENCY | ON, OFF | OFF | Enables UCX dependency used by `raft::comms` and needed to build `pyraft` | +| RAFT_ENABLE_nccl_DEPENDENCY | ON, OFF | OFF | Enables NCCL dependency used by `raft::comms` and needed to build `dask-raft` | +| RAFT_ENABLE_ucx_DEPENDENCY | ON, OFF | OFF | Enables UCX dependency used by `raft::comms` and needed to build `dask-raft` | | RAFT_USE_FAISS_STATIC | ON, OFF | OFF | Statically link FAISS into `libraft-nn` | | RAFT_STATIC_LINK_LIBRARIES | ON, OFF | ON | Build static link libraries instead of shared libraries | | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | @@ -153,12 +153,12 @@ mamba activate raft_env_name The Python APIs can be built using the `build.sh` script: ```bash -./build.sh pyraft pylibraft +./build.sh dask-raft pylibraft ``` `setup.py` can also be used to build the Python APIs manually: ```bash -cd python/raft +cd python/dask-raft python setup.py build_ext --inplace python setup.py install @@ -169,7 +169,7 @@ python setup.py install To run the Python tests: ```bash -cd python/raft +cd python/dask-raft py.test -s -v raft cd python pylibraft @@ -187,7 +187,7 @@ When the needed [build dependencies](#required_depenencies) are already satisfie set(RAFT_GIT_DIR ${CMAKE_CURRENT_BINARY_DIR}/raft CACHE STRING "Path to RAFT repo") ExternalProject_Add(raft GIT_REPOSITORY git@github.com:rapidsai/raft.git - GIT_TAG branch-22.04 + GIT_TAG branch-22.10 PREFIX ${RAFT_GIT_DIR} CONFIGURE_COMMAND "" BUILD_COMMAND "" diff --git a/README.md b/README.md index 2159f128bf..6770dd5e37 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ RAFT provides a header-only C++ library and pre-compiled shared libraries that c RAFT also provides 2 Python libraries: - `pylibraft` - low-level Python wrappers around RAFT algorithms and primitives. -- `pyraft` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. +- `dask-raft` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. ## Getting started @@ -108,11 +108,11 @@ The easiest way to install RAFT is through conda and several packages are provid - `libraft-nn` (optional) contains shared libraries for the nearest neighbors primitives. - `libraft-distance` (optional) contains shared libraries for distance primitives. - `pylibraft` (optional) Python wrappers around RAFT algorithms and primitives -- `pyraft` (optional) contains reusable Python infrastructure and tools to accelerate Python algorithm development. +- `dask-raft` (optional) enables deployment of multi-node multi-GPU algorithms that use RAFT `raft::comms` in Dask clusters. Use the following command to install RAFT with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash -mamba install -c rapidsai libraft-headers libraft-nn libraft-distance pyraft pylibraft +mamba install -c rapidsai libraft-headers libraft-nn libraft-distance dask-raft pylibraft ``` After installing RAFT, `find_package(raft COMPONENTS nn distance)` can be used in your CUDA/C++ cmake build to compile and/or link against needed dependencies in your raft target. `COMPONENTS` are optional and will depend on the packages installed. @@ -181,7 +181,7 @@ mamba env create --name raft_dev_env -f conda/environments/raft_dev_cuda11.5.yml mamba activate raft_dev_env ``` ``` -./build.sh pyraft pylibraft libraft tests bench --compile-libs +./build.sh dask-raft pylibraft libraft tests bench --compile-libs ``` The [build](BUILD.md) instructions contain more details on building RAFT from source and including it in downstream projects. You can also find a more comprehensive version of the above CPM code snippet the [Building RAFT C++ from source](BUILD.md#build_cxx_source) section of the build instructions. diff --git a/build.sh b/build.sh index ce4365985b..fc38b69c89 100755 --- a/build.sh +++ b/build.sh @@ -18,14 +18,14 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pyraft pylibraft docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" +VALIDARGS="clean libraft pylibraft dask-raft docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] where is: clean - remove all existing build artifacts and configuration (start over) libraft - build the raft C++ code only. Also builds the C-wrapper library around the C++ code. - pyraft - build the pyraft Python package pylibraft - build the pylibraft Python package + dask-raft - build the dask-raft Python package docs - build the documentation tests - build the tests bench - build the benchmarks @@ -50,7 +50,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= Date: Fri, 2 Sep 2022 14:15:51 -0400 Subject: [PATCH 02/16] Building `pylibraft.common` as part of pylibraft API --- python/pylibraft/CMakeLists.txt | 1 + .../pylibraft/pylibraft/common/CMakeLists.txt | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 python/pylibraft/pylibraft/common/CMakeLists.txt diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 5f877187eb..6567fdff0f 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -68,5 +68,6 @@ endif() include(rapids-cython) rapids_cython_init() +add_subdirectory(pylibraft/common) add_subdirectory(pylibraft/distance) add_subdirectory(pylibraft/random) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt new file mode 100644 index 0000000000..18dc3b292b --- /dev/null +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -0,0 +1,28 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# Set the list of Cython files to build +set(cython_sources cuda.pyx handle.pyx) +set(linked_libraries raft::raft) + +# Build all of the Cython targets +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" + MODULE_PREFIX common_) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() From fceb45b004b4cc5cd186a96260ad86c7628da6b1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 13:44:41 -0400 Subject: [PATCH 03/16] Fixing dask/common cmake --- python/dask-raft/raft/dask/common/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dask-raft/raft/dask/common/CMakeLists.txt b/python/dask-raft/raft/dask/common/CMakeLists.txt index 1abddae947..7829c821fa 100644 --- a/python/dask-raft/raft/dask/common/CMakeLists.txt +++ b/python/dask-raft/raft/dask/common/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -include(${raft-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) +include(${dask-raft-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) find_package(ucx REQUIRED) set(cython_sources comms_utils.pyx nccl.pyx) From e2dcffb360ee6bf4d05305b85413e9e3e995b836 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 18:12:21 -0400 Subject: [PATCH 04/16] Fixing pylibraft --- build.sh | 2 +- python/dask-raft/raft/common/handle.pxd | 2 +- python/pylibraft/pylibraft/common/handle.pxd | 8 +++++++- python/pylibraft/pylibraft/common/handle.pyx | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index fc38b69c89..e19d17ae59 100755 --- a/build.sh +++ b/build.sh @@ -241,7 +241,7 @@ if (( ${CLEAN} == 1 )); then fi done - cd ${REPODIR}/python/raft + cd ${REPODIR}/python/dask-raft python setup.py clean --all cd ${REPODIR} diff --git a/python/dask-raft/raft/common/handle.pxd b/python/dask-raft/raft/common/handle.pxd index 48ec625370..e810384676 100644 --- a/python/dask-raft/raft/common/handle.pxd +++ b/python/dask-raft/raft/common/handle.pxd @@ -26,7 +26,7 @@ from rmm._lib.cuda_stream_pool cimport cuda_stream_pool from libcpp.memory cimport shared_ptr from libcpp.memory cimport unique_ptr -cdef extern from "raft/handle.hpp" namespace "raft" nogil: +cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: cdef cppclass handle_t: handle_t() except + handle_t(cuda_stream_view stream_view) except + diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index 6504a122f7..e810384676 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ # cython: embedsignature = True # cython: language_level = 3 + from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_view from rmm._lib.cuda_stream_pool cimport cuda_stream_pool @@ -33,3 +34,8 @@ cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: shared_ptr[cuda_stream_pool] stream_pool) except + cuda_stream_view get_stream() except + void sync_stream() except + + +cdef class Handle: + cdef unique_ptr[handle_t] c_obj + cdef shared_ptr[cuda_stream_pool] stream_pool + cdef int n_streams diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index 83a4676076..de3952cc87 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 5b58689a3a80915e6469c36bf3c7ac011e160163 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 20:19:18 -0400 Subject: [PATCH 05/16] Fixing tests for dask-raft --- ci/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 0922e123ea..9dee15dbc6 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -90,7 +90,7 @@ gpuci_logger "GoogleTest for libraft" GTEST_OUTPUT="xml:${WORKSPACE}/test-results/raft_cpp/" $CONDA_PREFIX/bin/libraft/gtests/test_raft gpuci_logger "Python pytest for dask-raft" -cd "$WORKSPACE/python/raft/dask-raft/test" +cd "$WORKSPACE/python/dask-raft/raft/test" pytest --cache-clear --junitxml="$WORKSPACE/junit-dask-raft.xml" -v -s gpuci_logger "Python pytest for pylibraft" From 9c52b9976ed9592552d90e2fbcecd491cc78b58f Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 20:20:55 -0400 Subject: [PATCH 06/16] Fixing incorrect copyright --- python/pylibraft/pylibraft/common/handle.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibraft/pylibraft/common/handle.pyx b/python/pylibraft/pylibraft/common/handle.pyx index de3952cc87..83a4676076 100644 --- a/python/pylibraft/pylibraft/common/handle.pyx +++ b/python/pylibraft/pylibraft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 38b017ba78729780b112e096262abbe20ac244cf Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 20:21:28 -0400 Subject: [PATCH 07/16] Fixing incorrect copyright --- python/pylibraft/pylibraft/common/handle.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index e810384676..d7abe236ba 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 73b81134d1568976f5b917a991d022e37a23ad56 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 20:27:33 -0400 Subject: [PATCH 08/16] Some doc updates --- BUILD.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/BUILD.md b/BUILD.md index b854108efe..a9b0a19ea5 100644 --- a/BUILD.md +++ b/BUILD.md @@ -128,9 +128,6 @@ RAFT's cmake has the following configurable flags available:. | RAFT_COMPILE_DIST_LIBRARY | ON, OFF | OFF | Compiles the `libraft-distance` shared library | | RAFT_ENABLE_NN_DEPENDENCIES | ON, OFF | OFF | Searches for dependencies of nearest neighbors API, such as FAISS, and compiles them if not found. Needed for `raft::spatial::knn` | | RAFT_ENABLE_thrust_DEPENDENCY | ON, OFF | ON | Enables the Thrust dependency. This can be disabled when using many simple utilities or to override with a different Thrust version. | -| RAFT_ENABLE_mdspan_DEPENDENCY | ON, OFF | ON | Enables the std::mdspan dependency. This can be disabled when using many simple utilities. | -| RAFT_ENABLE_nccl_DEPENDENCY | ON, OFF | OFF | Enables NCCL dependency used by `raft::comms` and needed to build `dask-raft` | -| RAFT_ENABLE_ucx_DEPENDENCY | ON, OFF | OFF | Enables UCX dependency used by `raft::comms` and needed to build `dask-raft` | | RAFT_USE_FAISS_STATIC | ON, OFF | OFF | Statically link FAISS into `libraft-nn` | | RAFT_STATIC_LINK_LIBRARIES | ON, OFF | ON | Build static link libraries instead of shared libraries | | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | @@ -219,7 +216,7 @@ The following `cmake` snippet enables a flexible configuration of RAFT: ```cmake -set(RAFT_VERSION "22.04") +set(RAFT_VERSION "22.10") set(RAFT_FORK "rapidsai") set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") @@ -227,7 +224,7 @@ function(find_and_configure_raft) set(oneValueArgs VERSION FORK PINNED_TAG USE_FAISS_STATIC COMPILE_LIBRARIES ENABLE_NN_DEPENDENCIES CLONE_ON_PIN USE_NN_LIBRARY USE_DISTANCE_LIBRARY - ENABLE_thrust_DEPENDENCY ENABLE_mdspan_DEPENDENCY) + ENABLE_thrust_DEPENDENCY) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -272,7 +269,6 @@ function(find_and_configure_raft) "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}" "RAFT_COMPILE_LIBRARIES ${PKG_COMPILE_LIBRARIES}" "RAFT_ENABLE_thrust_DEPENDENCY ${PKG_ENABLE_thrust_DEPENDENCY}" - "RAFT_ENABLE_mdspan_DEPENDENCY ${PKG_ENABLE_mdspan_DEPENDENCY}" ) endfunction() @@ -295,7 +291,6 @@ find_and_configure_raft(VERSION ${RAFT_VERSION}.00 ENABLE_NN_DEPENDENCIES NO # This builds FAISS if not installed USE_FAISS_STATIC NO ENABLE_thrust_DEPENDENCY YES - ENABLE_mdspan_DEPENDENCY YES ) ``` From 0b5b312aba46cd2e43a9ee3f262ab19ebd5e3989 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Sep 2022 20:52:01 -0400 Subject: [PATCH 09/16] Updating docs to use new name --- docs/source/{pyraft_api.rst => dask_raft_api.rst} | 13 +++---------- docs/source/index.rst | 2 +- docs/source/python.rst | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) rename docs/source/{pyraft_api.rst => dask_raft_api.rst} (58%) diff --git a/docs/source/pyraft_api.rst b/docs/source/dask_raft_api.rst similarity index 58% rename from docs/source/pyraft_api.rst rename to docs/source/dask_raft_api.rst index c763c9a0f7..662936eaee 100644 --- a/docs/source/pyraft_api.rst +++ b/docs/source/dask_raft_api.rst @@ -1,18 +1,11 @@ -~~~~~~~~~~~~~~~~~~~~ -PyRAFT API Reference -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~ +Dask-RAFT API Reference +~~~~~~~~~~~~~~~~~~~~~~~ .. role:: py(code) :language: python :class: highlight - -RAFT Handle ------------ - -.. autoclass:: raft.common.handle.Handle - :members: - Dask-based Multi-Node Multi-GPU Communicator -------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 97c616dd8e..55b1f27198 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,7 @@ RAFT contains fundamental widely-used algorithms and primitives for data science :caption: Contents: cpp_api.rst - pyraft_api.rst + dask_raft_api.rst pylibraft_api.rst diff --git a/docs/source/python.rst b/docs/source/python.rst index 0ae9f88398..c59f6cea6b 100644 --- a/docs/source/python.rst +++ b/docs/source/python.rst @@ -5,6 +5,6 @@ RAFT Python APIs :maxdepth: 2 :caption: Contents: - pyraft_api.rst pylibraft_api.rst + dask_raft_api.rst From 8a209ced958dd724146d8ab9ca700e1c3d1cf000 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 10:32:46 -0400 Subject: [PATCH 10/16] Renaming dask_raft -> raft_dask. It's more appropriate given it isn't "official dask" blessed --- BUILD.md | 10 +-- README.md | 8 +- build.sh | 14 +-- conda/recipes/dask-raft/meta.yaml | 10 +-- docs/source/conf.py | 2 +- docs/source/index.rst | 2 +- docs/source/python.rst | 2 +- .../{dask_raft_api.rst => raft_dask_api.rst} | 4 +- python/dask-raft/raft/common/CMakeLists.txt | 20 ----- python/dask-raft/raft/common/__init__.py | 17 ---- python/dask-raft/raft/common/cuda.pxd | 22 ----- python/dask-raft/raft/common/cuda.pyx | 84 ----------------- python/dask-raft/raft/common/handle.pxd | 41 --------- python/dask-raft/raft/common/handle.pyx | 89 ------------------- python/dask-raft/raft/dask/__init__.py | 16 ---- .../pylibraft/pylibraft/common/CMakeLists.txt | 2 +- .../pylibraft}/common/interruptible.pxd | 4 +- .../pylibraft}/common/interruptible.pyx | 0 .../pylibraft}/test/test_interruptible.py | 2 +- python/pylibraft/pyproject.toml | 2 +- .../{dask-raft => raft-dask}/CMakeLists.txt | 17 ++-- .../cmake/thirdparty/get_nccl.cmake | 0 .../{dask-raft => raft-dask}/pyproject.toml | 1 + python/{dask-raft => raft-dask}/pytest.ini | 0 .../raft => raft-dask/raft_dask}/__init__.py | 0 .../raft => raft-dask/raft_dask}/_version.py | 0 .../raft_dask}/common/CMakeLists.txt | 2 +- .../raft_dask}/common/__init__.pxd | 0 .../raft_dask}/common/__init__.py | 0 .../raft_dask}/common/comms.py | 2 +- .../raft_dask}/common/comms_utils.pyx | 2 +- .../raft_dask}/common/nccl.pyx | 0 .../raft_dask}/common/ucx.py | 0 .../raft_dask}/common/utils.py | 0 .../raft_dask}/include_test/CMakeLists.txt | 0 .../raft_dask}/include_test/__init__.py | 0 .../include_test/raft_include_test.pyx | 0 .../raft_dask}/test/conftest.py | 0 .../raft_dask}/test/test_comms.py | 28 +++--- .../raft_dask}/test/test_raft.py | 6 +- python/{dask-raft => raft-dask}/setup.cfg | 8 +- python/{dask-raft => raft-dask}/setup.py | 10 +-- python/{dask-raft => raft-dask}/setuputils.py | 0 python/{dask-raft => raft-dask}/versioneer.py | 0 44 files changed, 68 insertions(+), 359 deletions(-) rename docs/source/{dask_raft_api.rst => raft_dask_api.rst} (77%) delete mode 100644 python/dask-raft/raft/common/CMakeLists.txt delete mode 100644 python/dask-raft/raft/common/__init__.py delete mode 100644 python/dask-raft/raft/common/cuda.pxd delete mode 100644 python/dask-raft/raft/common/cuda.pyx delete mode 100644 python/dask-raft/raft/common/handle.pxd delete mode 100644 python/dask-raft/raft/common/handle.pyx delete mode 100644 python/dask-raft/raft/dask/__init__.py rename python/{dask-raft/raft => pylibraft/pylibraft}/common/interruptible.pxd (90%) rename python/{dask-raft/raft => pylibraft/pylibraft}/common/interruptible.pyx (100%) rename python/{dask-raft/raft => pylibraft/pylibraft}/test/test_interruptible.py (95%) rename python/{dask-raft => raft-dask}/CMakeLists.txt (87%) rename python/{dask-raft => raft-dask}/cmake/thirdparty/get_nccl.cmake (100%) rename python/{dask-raft => raft-dask}/pyproject.toml (97%) rename python/{dask-raft => raft-dask}/pytest.ini (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/__init__.py (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/_version.py (100%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/CMakeLists.txt (93%) rename python/{dask-raft/raft => raft-dask/raft_dask}/common/__init__.pxd (100%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/__init__.py (100%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/comms.py (99%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/comms_utils.pyx (99%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/nccl.pyx (100%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/ucx.py (100%) rename python/{dask-raft/raft/dask => raft-dask/raft_dask}/common/utils.py (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/include_test/CMakeLists.txt (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/include_test/__init__.py (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/include_test/raft_include_test.pyx (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/test/conftest.py (100%) rename python/{dask-raft/raft => raft-dask/raft_dask}/test/test_comms.py (90%) rename python/{dask-raft/raft => raft-dask/raft_dask}/test/test_raft.py (84%) rename python/{dask-raft => raft-dask}/setup.cfg (87%) rename python/{dask-raft => raft-dask}/setup.py (86%) rename python/{dask-raft => raft-dask}/setuputils.py (100%) rename python/{dask-raft => raft-dask}/versioneer.py (100%) diff --git a/BUILD.md b/BUILD.md index a9b0a19ea5..0c7fdd7e82 100644 --- a/BUILD.md +++ b/BUILD.md @@ -34,8 +34,8 @@ In addition to the libraries included with cudatoolkit 11.0+, there are some oth - [cuCollections](https://github.com/NVIDIA/cuCollections) - Used in `raft::sparse::distance` API. - [Libcu++](https://github.com/NVIDIA/libcudacxx) v1.7.0 - [FAISS](https://github.com/facebookresearch/faiss) v1.7.0 - Used in `raft::spatial::knn` API and needed to build tests. -- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `dask-raft` -- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `dask-raft` +- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `raft-dask` +- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `raft-dask` - [Googletest](https://github.com/google/googletest) - Needed to build tests - [Googlebench](https://github.com/google/benchmark) - Needed to build benchmarks - [Doxygen](https://github.com/doxygen/doxygen) - Needed to build docs @@ -150,12 +150,12 @@ mamba activate raft_env_name The Python APIs can be built using the `build.sh` script: ```bash -./build.sh dask-raft pylibraft +./build.sh raft-dask pylibraft ``` `setup.py` can also be used to build the Python APIs manually: ```bash -cd python/dask-raft +cd python/raft-dask python setup.py build_ext --inplace python setup.py install @@ -166,7 +166,7 @@ python setup.py install To run the Python tests: ```bash -cd python/dask-raft +cd python/raft-dask py.test -s -v raft cd python pylibraft diff --git a/README.md b/README.md index 6770dd5e37..7a9434c16d 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ RAFT provides a header-only C++ library and pre-compiled shared libraries that c RAFT also provides 2 Python libraries: - `pylibraft` - low-level Python wrappers around RAFT algorithms and primitives. -- `dask-raft` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. +- `raft-dask` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. ## Getting started @@ -108,11 +108,11 @@ The easiest way to install RAFT is through conda and several packages are provid - `libraft-nn` (optional) contains shared libraries for the nearest neighbors primitives. - `libraft-distance` (optional) contains shared libraries for distance primitives. - `pylibraft` (optional) Python wrappers around RAFT algorithms and primitives -- `dask-raft` (optional) enables deployment of multi-node multi-GPU algorithms that use RAFT `raft::comms` in Dask clusters. +- `raft-dask` (optional) enables deployment of multi-node multi-GPU algorithms that use RAFT `raft::comms` in Dask clusters. Use the following command to install RAFT with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash -mamba install -c rapidsai libraft-headers libraft-nn libraft-distance dask-raft pylibraft +mamba install -c rapidsai libraft-headers libraft-nn libraft-distance raft-dask pylibraft ``` After installing RAFT, `find_package(raft COMPONENTS nn distance)` can be used in your CUDA/C++ cmake build to compile and/or link against needed dependencies in your raft target. `COMPONENTS` are optional and will depend on the packages installed. @@ -181,7 +181,7 @@ mamba env create --name raft_dev_env -f conda/environments/raft_dev_cuda11.5.yml mamba activate raft_dev_env ``` ``` -./build.sh dask-raft pylibraft libraft tests bench --compile-libs +./build.sh raft-dask pylibraft libraft tests bench --compile-libs ``` The [build](BUILD.md) instructions contain more details on building RAFT from source and including it in downstream projects. You can also find a more comprehensive version of the above CPM code snippet the [Building RAFT C++ from source](BUILD.md#build_cxx_source) section of the build instructions. diff --git a/build.sh b/build.sh index e19d17ae59..3b606406cb 100755 --- a/build.sh +++ b/build.sh @@ -18,14 +18,14 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft dask-raft docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] where is: clean - remove all existing build artifacts and configuration (start over) libraft - build the raft C++ code only. Also builds the C-wrapper library around the C++ code. pylibraft - build the pylibraft Python package - dask-raft - build the dask-raft Python package + raft-dask - build the raft-dask Python package docs - build the documentation tests - build the tests bench - build the benchmarks @@ -50,7 +50,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool==0.29,<0.30 - scikit-build>=0.13.1 - rmm {{ minor_version }} - - libraft-headers {{ version }} + - pylibraft {{ version }} - cudatoolkit {{ cuda_version }}.* - cuda-python >=11.5,<11.7.1 - nccl>=2.9.9 @@ -45,7 +45,7 @@ requirements: run: - python x.x - dask-cuda {{ minor_version }} - - libraft-headers {{ version }} + - pylibraft {{ version }} - nccl>=2.9.9 - rmm {{ minor_version }} - ucx >={{ ucx_version }} @@ -61,10 +61,10 @@ tests: # [linux64] requirements: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] imports: # [linux64] - - raft # [linux64] + - raft_dask # [linux64] about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: dask-raft library + summary: raft-dask library diff --git a/docs/source/conf.py b/docs/source/conf.py index f34c1d34cb..a96f86c68d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,7 +24,7 @@ # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. sys.path.insert(0, os.path.abspath("sphinxext")) -sys.path.insert(0, os.path.abspath("../../python/raft")) +sys.path.insert(0, os.path.abspath("../../python/raft-dask")) sys.path.insert(0, os.path.abspath("../../python/pylibraft")) from github_link import make_linkcode_resolve # noqa diff --git a/docs/source/index.rst b/docs/source/index.rst index 55b1f27198..0d7ab295f4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,7 @@ RAFT contains fundamental widely-used algorithms and primitives for data science :caption: Contents: cpp_api.rst - dask_raft_api.rst + raft_dask_api.rst pylibraft_api.rst diff --git a/docs/source/python.rst b/docs/source/python.rst index c59f6cea6b..561ec3d029 100644 --- a/docs/source/python.rst +++ b/docs/source/python.rst @@ -6,5 +6,5 @@ RAFT Python APIs :caption: Contents: pylibraft_api.rst - dask_raft_api.rst + raft_dask_api.rst diff --git a/docs/source/dask_raft_api.rst b/docs/source/raft_dask_api.rst similarity index 77% rename from docs/source/dask_raft_api.rst rename to docs/source/raft_dask_api.rst index 662936eaee..10ba8781a2 100644 --- a/docs/source/dask_raft_api.rst +++ b/docs/source/raft_dask_api.rst @@ -1,5 +1,5 @@ ~~~~~~~~~~~~~~~~~~~~~~~ -Dask-RAFT API Reference +RAFT Dask API Reference ~~~~~~~~~~~~~~~~~~~~~~~ .. role:: py(code) @@ -9,5 +9,5 @@ Dask-RAFT API Reference Dask-based Multi-Node Multi-GPU Communicator -------------------------------------------- -.. autoclass:: raft.dask.common.Comms +.. autoclass:: raft_dask.common.Comms :members: diff --git a/python/dask-raft/raft/common/CMakeLists.txt b/python/dask-raft/raft/common/CMakeLists.txt deleted file mode 100644 index 59b336574e..0000000000 --- a/python/dask-raft/raft/common/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -set(cython_sources cuda.pyx handle.pyx interruptible.pyx) -set(linked_libraries raft::raft) -rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) diff --git a/python/dask-raft/raft/common/__init__.py b/python/dask-raft/raft/common/__init__.py deleted file mode 100644 index 62db7d5831..0000000000 --- a/python/dask-raft/raft/common/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .cuda import Stream -from .handle import Handle \ No newline at end of file diff --git a/python/dask-raft/raft/common/cuda.pxd b/python/dask-raft/raft/common/cuda.pxd deleted file mode 100644 index 0459cb96af..0000000000 --- a/python/dask-raft/raft/common/cuda.pxd +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2019-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from cuda.ccudart cimport cudaStream_t - -cdef class Stream: - cdef cudaStream_t s - - cdef cudaStream_t getStream(self) diff --git a/python/dask-raft/raft/common/cuda.pyx b/python/dask-raft/raft/common/cuda.pyx deleted file mode 100644 index c3c90936aa..0000000000 --- a/python/dask-raft/raft/common/cuda.pyx +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cuda.ccudart cimport( - cudaStream_t, - cudaError_t, - cudaSuccess, - cudaStreamCreate, - cudaStreamDestroy, - cudaStreamSynchronize, - cudaGetLastError, - cudaGetErrorString, - cudaGetErrorName -) - - -class CudaRuntimeError(RuntimeError): - def __init__(self, extraMsg=None): - cdef cudaError_t e = cudaGetLastError() - cdef bytes errMsg = cudaGetErrorString(e) - cdef bytes errName = cudaGetErrorName(e) - msg = "Error! %s reason='%s'" % (errName.decode(), errMsg.decode()) - if extraMsg is not None: - msg += " extraMsg='%s'" % extraMsg - super(CudaRuntimeError, self).__init__(msg) - - -cdef class Stream: - """ - Stream represents a thin-wrapper around cudaStream_t and its operations. - - Examples - -------- - - .. code-block:: python - - from raft.common.cuda import Stream - stream = Stream() - stream.sync() - del stream # optional! - """ - def __cinit__(self): - cdef cudaStream_t stream - cdef cudaError_t e = cudaStreamCreate(&stream) - if e != cudaSuccess: - raise CudaRuntimeError("Stream create") - self.s = stream - - def __dealloc__(self): - self.sync() - cdef cudaError_t e = cudaStreamDestroy(self.s) - if e != cudaSuccess: - raise CudaRuntimeError("Stream destroy") - - def sync(self): - """ - Synchronize on the cudastream owned by this object. Note that this - could raise exception due to issues with previous asynchronous - launches - """ - cdef cudaError_t e = cudaStreamSynchronize(self.s) - if e != cudaSuccess: - raise CudaRuntimeError("Stream sync") - - cdef cudaStream_t getStream(self): - return self.s diff --git a/python/dask-raft/raft/common/handle.pxd b/python/dask-raft/raft/common/handle.pxd deleted file mode 100644 index e810384676..0000000000 --- a/python/dask-raft/raft/common/handle.pxd +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - - -from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view -from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from libcpp.memory cimport shared_ptr -from libcpp.memory cimport unique_ptr - -cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: - cdef cppclass handle_t: - handle_t() except + - handle_t(cuda_stream_view stream_view) except + - handle_t(cuda_stream_view stream_view, - shared_ptr[cuda_stream_pool] stream_pool) except + - cuda_stream_view get_stream() except + - void sync_stream() except + - -cdef class Handle: - cdef unique_ptr[handle_t] c_obj - cdef shared_ptr[cuda_stream_pool] stream_pool - cdef int n_streams diff --git a/python/dask-raft/raft/common/handle.pyx b/python/dask-raft/raft/common/handle.pyx deleted file mode 100644 index de3952cc87..0000000000 --- a/python/dask-raft/raft/common/handle.pyx +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -# import raft -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread -from rmm._lib.cuda_stream_view cimport cuda_stream_view - -from .cuda cimport Stream -from .cuda import CudaRuntimeError - - -cdef class Handle: - """ - Handle is a lightweight python wrapper around the corresponding C++ class - of handle_t exposed by RAFT's C++ interface. Refer to the header file - raft/handle.hpp for interface level details of this struct - - Examples - -------- - - .. code-block:: python - - from raft.common import Stream, Handle - stream = Stream() - handle = Handle(stream) - - # call algos here - - # final sync of all work launched in the stream of this handle - # this is same as `raft.cuda.Stream.sync()` call, but safer in case - # the default stream inside the `handle_t` is being used - handle.sync() - del handle # optional! - """ - - def __cinit__(self, stream: Stream = None, n_streams=0): - self.n_streams = n_streams - if n_streams > 0: - self.stream_pool.reset(new cuda_stream_pool(n_streams)) - - cdef cuda_stream_view c_stream - if stream is None: - # this constructor will construct a "main" handle on - # per-thread default stream, which is non-blocking - self.c_obj.reset(new handle_t(cuda_stream_per_thread, - self.stream_pool)) - else: - # this constructor constructs a handle on user stream - c_stream = cuda_stream_view(stream.getStream()) - self.c_obj.reset(new handle_t(c_stream, - self.stream_pool)) - - def sync(self): - """ - Issues a sync on the stream set for this handle. - """ - self.c_obj.get()[0].sync_stream() - - def getHandle(self): - return self.c_obj.get() - - def __getstate__(self): - return self.n_streams - - def __setstate__(self, state): - self.n_streams = state - if self.n_streams > 0: - self.stream_pool.reset(new cuda_stream_pool(self.n_streams)) - - self.c_obj.reset(new handle_t(cuda_stream_per_thread, - self.stream_pool)) diff --git a/python/dask-raft/raft/dask/__init__.py b/python/dask-raft/raft/dask/__init__.py deleted file mode 100644 index f6a1c28ea8..0000000000 --- a/python/dask-raft/raft/dask/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .common.comms import Comms \ No newline at end of file diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 18dc3b292b..d9ed2d12fc 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -13,7 +13,7 @@ # ============================================================================= # Set the list of Cython files to build -set(cython_sources cuda.pyx handle.pyx) +set(cython_sources cuda.pyx handle.pyx interruptible.pyx) set(linked_libraries raft::raft) # Build all of the Cython targets diff --git a/python/dask-raft/raft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd similarity index 90% rename from python/dask-raft/raft/common/interruptible.pxd rename to python/pylibraft/pylibraft/common/interruptible.pxd index a73e8c1ac7..a4c7d90ac2 100644 --- a/python/dask-raft/raft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -22,11 +22,11 @@ from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_view -cdef extern from "raft/interruptible.hpp" namespace "raft" nogil: +cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: cdef cppclass interruptible: void cancel() -cdef extern from "raft/interruptible.hpp" \ +cdef extern from "raft/core/interruptible.hpp" \ namespace "raft::interruptible" nogil: cdef void inter_synchronize \ "raft::interruptible::synchronize"(cuda_stream_view stream) except+ diff --git a/python/dask-raft/raft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx similarity index 100% rename from python/dask-raft/raft/common/interruptible.pyx rename to python/pylibraft/pylibraft/common/interruptible.pyx diff --git a/python/dask-raft/raft/test/test_interruptible.py b/python/pylibraft/pylibraft/test/test_interruptible.py similarity index 95% rename from python/dask-raft/raft/test/test_interruptible.py rename to python/pylibraft/pylibraft/test/test_interruptible.py index a3559f6476..bda98edd13 100644 --- a/python/dask-raft/raft/test/test_interruptible.py +++ b/python/pylibraft/pylibraft/test/test_interruptible.py @@ -4,7 +4,7 @@ import pytest import signal import time -from raft.common.interruptible import cuda_interruptible, cuda_yield +from pylibraft.common.interruptible import cuda_interruptible, cuda_yield def send_ctrl_c(): diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index e3fe544bef..cb166a4d75 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -20,5 +20,5 @@ requires = [ "cython>=0.29,<0.30", "scikit-build>=0.13.1", "cmake>=3.20.1,!=3.23.0", - "ninja", + "ninja" ] diff --git a/python/dask-raft/CMakeLists.txt b/python/raft-dask/CMakeLists.txt similarity index 87% rename from python/dask-raft/CMakeLists.txt rename to python/raft-dask/CMakeLists.txt index 8fb5fb7186..d2c4a2c2d1 100644 --- a/python/dask-raft/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -14,13 +14,13 @@ cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) -set(dask_raft_version 22.10.00) +set(raft_dask_version 22.10.00) include(../../fetch_rapids.cmake) project( - dask-raft-python - VERSION ${dask_raft_version} + raft-dask-python + VERSION ${raft_dask_version} LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until @@ -33,7 +33,7 @@ option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulti # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${dask_raft_version} REQUIRED) + find_package(raft ${raft_dask_version} REQUIRED) else() set(raft_FOUND OFF) endif() @@ -43,11 +43,11 @@ if(NOT raft_FOUND) # pull in the required languages for the C++ project even if this project # does not require those languges. include(rapids-cuda) - rapids_cuda_init_architectures(dask_raft) + rapids_cuda_init_architectures(raft-dask) enable_language(CUDA) # Since pyraft only enables CUDA optionally we need to manually include the file that # rapids_cuda_init_architectures relies on `project` including. - include("${CMAKE_PROJECT_dask_raft_INCLUDE}") + include("${CMAKE_PROJECT_raft_dask_INCLUDE}") # pyraft doesn't actually use raft libraries, it just needs the headers, so # we can turn off all library compilation and we don't need to install @@ -63,6 +63,5 @@ endif() include(rapids-cython) rapids_cython_init() -add_subdirectory(raft/common) -add_subdirectory(raft/dask/common) -add_subdirectory(raft/include_test) +add_subdirectory(raft_dask/common) +add_subdirectory(raft_dask/include_test) diff --git a/python/dask-raft/cmake/thirdparty/get_nccl.cmake b/python/raft-dask/cmake/thirdparty/get_nccl.cmake similarity index 100% rename from python/dask-raft/cmake/thirdparty/get_nccl.cmake rename to python/raft-dask/cmake/thirdparty/get_nccl.cmake diff --git a/python/dask-raft/pyproject.toml b/python/raft-dask/pyproject.toml similarity index 97% rename from python/dask-raft/pyproject.toml rename to python/raft-dask/pyproject.toml index e3fe544bef..3979c0d7c5 100644 --- a/python/dask-raft/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -21,4 +21,5 @@ requires = [ "scikit-build>=0.13.1", "cmake>=3.20.1,!=3.23.0", "ninja", + "pylibraft" ] diff --git a/python/dask-raft/pytest.ini b/python/raft-dask/pytest.ini similarity index 100% rename from python/dask-raft/pytest.ini rename to python/raft-dask/pytest.ini diff --git a/python/dask-raft/raft/__init__.py b/python/raft-dask/raft_dask/__init__.py similarity index 100% rename from python/dask-raft/raft/__init__.py rename to python/raft-dask/raft_dask/__init__.py diff --git a/python/dask-raft/raft/_version.py b/python/raft-dask/raft_dask/_version.py similarity index 100% rename from python/dask-raft/raft/_version.py rename to python/raft-dask/raft_dask/_version.py diff --git a/python/dask-raft/raft/dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt similarity index 93% rename from python/dask-raft/raft/dask/common/CMakeLists.txt rename to python/raft-dask/raft_dask/common/CMakeLists.txt index 7829c821fa..0e1da015fc 100644 --- a/python/dask-raft/raft/dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -include(${dask-raft-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) +include(${raft-dask-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) find_package(ucx REQUIRED) set(cython_sources comms_utils.pyx nccl.pyx) diff --git a/python/dask-raft/raft/common/__init__.pxd b/python/raft-dask/raft_dask/common/__init__.pxd similarity index 100% rename from python/dask-raft/raft/common/__init__.pxd rename to python/raft-dask/raft_dask/common/__init__.pxd diff --git a/python/dask-raft/raft/dask/common/__init__.py b/python/raft-dask/raft_dask/common/__init__.py similarity index 100% rename from python/dask-raft/raft/dask/common/__init__.py rename to python/raft-dask/raft_dask/common/__init__.py diff --git a/python/dask-raft/raft/dask/common/comms.py b/python/raft-dask/raft_dask/common/comms.py similarity index 99% rename from python/dask-raft/raft/dask/common/comms.py rename to python/raft-dask/raft_dask/common/comms.py index 549ac7fccb..d7260ece9b 100644 --- a/python/dask-raft/raft/dask/common/comms.py +++ b/python/raft-dask/raft_dask/common/comms.py @@ -20,7 +20,7 @@ from .comms_utils import inject_comms_on_handle_coll_only from .utils import parse_host_port -from ...common.handle import Handle +from pylibraft.common.handle import Handle from dask.distributed import get_worker, default_client diff --git a/python/dask-raft/raft/dask/common/comms_utils.pyx b/python/raft-dask/raft_dask/common/comms_utils.pyx similarity index 99% rename from python/dask-raft/raft/dask/common/comms_utils.pyx rename to python/raft-dask/raft_dask/common/comms_utils.pyx index 38c5670372..2014284af1 100644 --- a/python/dask-raft/raft/dask/common/comms_utils.pyx +++ b/python/raft-dask/raft_dask/common/comms_utils.pyx @@ -31,7 +31,7 @@ cdef extern from "nccl.h": cdef struct ncclComm ctypedef ncclComm *ncclComm_t -cdef extern from "raft/handle.hpp" namespace "raft": +cdef extern from "raft/core/handle.hpp" namespace "raft": cdef cppclass handle_t: handle_t() except + diff --git a/python/dask-raft/raft/dask/common/nccl.pyx b/python/raft-dask/raft_dask/common/nccl.pyx similarity index 100% rename from python/dask-raft/raft/dask/common/nccl.pyx rename to python/raft-dask/raft_dask/common/nccl.pyx diff --git a/python/dask-raft/raft/dask/common/ucx.py b/python/raft-dask/raft_dask/common/ucx.py similarity index 100% rename from python/dask-raft/raft/dask/common/ucx.py rename to python/raft-dask/raft_dask/common/ucx.py diff --git a/python/dask-raft/raft/dask/common/utils.py b/python/raft-dask/raft_dask/common/utils.py similarity index 100% rename from python/dask-raft/raft/dask/common/utils.py rename to python/raft-dask/raft_dask/common/utils.py diff --git a/python/dask-raft/raft/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt similarity index 100% rename from python/dask-raft/raft/include_test/CMakeLists.txt rename to python/raft-dask/raft_dask/include_test/CMakeLists.txt diff --git a/python/dask-raft/raft/include_test/__init__.py b/python/raft-dask/raft_dask/include_test/__init__.py similarity index 100% rename from python/dask-raft/raft/include_test/__init__.py rename to python/raft-dask/raft_dask/include_test/__init__.py diff --git a/python/dask-raft/raft/include_test/raft_include_test.pyx b/python/raft-dask/raft_dask/include_test/raft_include_test.pyx similarity index 100% rename from python/dask-raft/raft/include_test/raft_include_test.pyx rename to python/raft-dask/raft_dask/include_test/raft_include_test.pyx diff --git a/python/dask-raft/raft/test/conftest.py b/python/raft-dask/raft_dask/test/conftest.py similarity index 100% rename from python/dask-raft/raft/test/conftest.py rename to python/raft-dask/raft_dask/test/conftest.py diff --git a/python/dask-raft/raft/test/test_comms.py b/python/raft-dask/raft_dask/test/test_comms.py similarity index 90% rename from python/dask-raft/raft/test/test_comms.py rename to python/raft-dask/raft_dask/test/test_comms.py index 345cdbf037..29b4d963d9 100644 --- a/python/dask-raft/raft/test/test_comms.py +++ b/python/raft-dask/raft_dask/test/test_comms.py @@ -21,20 +21,20 @@ from dask.distributed import wait try: - from raft.dask import Comms - from raft.dask.common import local_handle - from raft.dask.common import perform_test_comms_send_recv - from raft.dask.common import perform_test_comms_device_send_or_recv - from raft.dask.common import perform_test_comms_device_sendrecv - from raft.dask.common import perform_test_comms_device_multicast_sendrecv - from raft.dask.common import perform_test_comms_allreduce - from raft.dask.common import perform_test_comms_bcast - from raft.dask.common import perform_test_comms_reduce - from raft.dask.common import perform_test_comms_allgather - from raft.dask.common import perform_test_comms_gather - from raft.dask.common import perform_test_comms_gatherv - from raft.dask.common import perform_test_comms_reducescatter - from raft.dask.common import perform_test_comm_split + from raft_dask import Comms + from raft_dask.common import local_handle + from raft_dask.common import perform_test_comms_send_recv + from raft_dask.common import perform_test_comms_device_send_or_recv + from raft_dask.common import perform_test_comms_device_sendrecv + from raft_dask.common import perform_test_comms_device_multicast_sendrecv + from raft_dask.common import perform_test_comms_allreduce + from raft_dask.common import perform_test_comms_bcast + from raft_dask.common import perform_test_comms_reduce + from raft_dask.common import perform_test_comms_allgather + from raft_dask.common import perform_test_comms_gather + from raft_dask.common import perform_test_comms_gatherv + from raft_dask.common import perform_test_comms_reducescatter + from raft_dask.common import perform_test_comm_split pytestmark = pytest.mark.mg except ImportError: diff --git a/python/dask-raft/raft/test/test_raft.py b/python/raft-dask/raft_dask/test/test_raft.py similarity index 84% rename from python/dask-raft/raft/test/test_raft.py rename to python/raft-dask/raft_dask/test/test_raft.py index 796a4fface..e1370dccc9 100644 --- a/python/dask-raft/raft/test/test_raft.py +++ b/python/raft-dask/raft_dask/test/test_raft.py @@ -17,15 +17,15 @@ import sys try: - import raft + import raft_dask except ImportError: print("Skipping RAFT tests") pytestmart = pytest.mark.skip pytestmark = pytest.mark.skipif( - 'raft' not in sys.argv, reason="marker to allow integration of RAFT" + 'raft_dask' not in sys.argv, reason="marker to allow integration of RAFT" ) def test_raft(): - assert raft.raft_include_test() + assert raft_dask.raft_include_test() diff --git a/python/dask-raft/setup.cfg b/python/raft-dask/setup.cfg similarity index 87% rename from python/dask-raft/setup.cfg rename to python/raft-dask/setup.cfg index e20ff1bb2b..3bc8f126f6 100644 --- a/python/dask-raft/setup.cfg +++ b/python/raft-dask/setup.cfg @@ -9,10 +9,10 @@ exclude = __init__.py,versioneer.py [versioneer] VCS = git style = pep440 -versionfile_source = raft/_version.py -versionfile_build = raft/_version.py +versionfile_source = raft_dask/_version.py +versionfile_build = raft_dask/_version.py tag_prefix = v -parentdir_prefix = raft- +parentdir_prefix = raft_dask- [isort] line_length=79 @@ -33,7 +33,7 @@ known_rapids= dask_cudf rmm known_first_party= - raft + raft_dask default_section=THIRDPARTY sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER skip= diff --git a/python/dask-raft/setup.py b/python/raft-dask/setup.py similarity index 86% rename from python/dask-raft/setup.py rename to python/raft-dask/setup.py index d46bd1df10..59bcc4a2dc 100644 --- a/python/dask-raft/setup.py +++ b/python/raft-dask/setup.py @@ -20,7 +20,7 @@ import versioneer -setup(name='dask-raft', +setup(name='raft-dask', description="Reusable Accelerated Functions & Tools Dask Infrastructure", version=versioneer.get_version(), classifiers=[ @@ -38,14 +38,12 @@ key: ["*.hpp", "*.pxd"] for key in find_packages( include=[ - "raft.dask.common", - "raft.dask.common.includes", - "raft.common", - "raft.common.includes" + "raft_dask.common", + "raft_dask.common.includes", ] ) }, - packages=find_packages(include=['raft', 'raft.*']), + packages=find_packages(include=['raft_dask', 'raft_dask.*']), license="Apache", cmdclass=versioneer.get_cmdclass(), zip_safe=False diff --git a/python/dask-raft/setuputils.py b/python/raft-dask/setuputils.py similarity index 100% rename from python/dask-raft/setuputils.py rename to python/raft-dask/setuputils.py diff --git a/python/dask-raft/versioneer.py b/python/raft-dask/versioneer.py similarity index 100% rename from python/dask-raft/versioneer.py rename to python/raft-dask/versioneer.py From c340e35d7771bef8125368af8e00d2ba506ae680 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 11:17:10 -0400 Subject: [PATCH 11/16] iInstalling pylibraft before dask-raft in CI --- ci/gpu/build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 9dee15dbc6..f0d08a8b80 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -63,9 +63,13 @@ conda list --show-channel-urls gpuci_logger "Build and install Python targets" CONDA_BLD_DIR="$WORKSPACE/.conda-bld" gpuci_mamba_retry install boa -gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/dask-raft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" + +# Install pylibraft first since it's a dependency of dask-raft gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" -gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" dask-raft pylibraft +gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pylibraft + +gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/dask-raft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" +gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" dask-raft ################################################################################ # TEST - Run GoogleTest and py.tests for RAFT From 00896ed9ee24905914d8196cf175b2e57cdd7363 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 11:31:59 -0400 Subject: [PATCH 12/16] iFixing conda recie and ci --- ci/cpu/build.sh | 8 ++++---- ci/gpu/build.sh | 12 ++++++------ conda/recipes/{dask-raft => raft-dask}/build.sh | 2 +- .../{dask-raft => raft-dask}/conda_build_config.yaml | 0 conda/recipes/{dask-raft => raft-dask}/meta.yaml | 0 5 files changed, 11 insertions(+), 11 deletions(-) rename conda/recipes/{dask-raft => raft-dask}/build.sh (81%) rename conda/recipes/{dask-raft => raft-dask}/conda_build_config.yaml (100%) rename conda/recipes/{dask-raft => raft-dask}/meta.yaml (100%) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 850bd3a869..759a941245 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -99,12 +99,12 @@ fi if [ "$BUILD_RAFT" == '1' ]; then gpuci_logger "Building Python conda packages for raft" if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/dask-raft --python=$PYTHON + gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/raft-dask --python=$PYTHON gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pylibraft --python=$PYTHON else - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/dask-raft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON - mkdir -p ${CONDA_BLD_DIR}/dask-raft/work - mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/dask-raft/work + gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/raft-dask -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON + mkdir -p ${CONDA_BLD_DIR}/raft-dask/work + mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/raft-dask/work gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pylibraft -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON mkdir -p ${CONDA_BLD_DIR}/pylibraft/work diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index f0d08a8b80..6d98dd8042 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -64,12 +64,12 @@ gpuci_logger "Build and install Python targets" CONDA_BLD_DIR="$WORKSPACE/.conda-bld" gpuci_mamba_retry install boa -# Install pylibraft first since it's a dependency of dask-raft +# Install pylibraft first since it's a dependency of raft-dask gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/pylibraft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" pylibraft -gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/dask-raft -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" -gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" dask-raft +gpuci_conda_retry mambabuild --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/raft-dask -c "${CONDA_ARTIFACT_PATH}" --python="${PYTHON}" +gpuci_mamba_retry install -y -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" raft-dask ################################################################################ # TEST - Run GoogleTest and py.tests for RAFT @@ -93,9 +93,9 @@ nvidia-smi gpuci_logger "GoogleTest for libraft" GTEST_OUTPUT="xml:${WORKSPACE}/test-results/raft_cpp/" $CONDA_PREFIX/bin/libraft/gtests/test_raft -gpuci_logger "Python pytest for dask-raft" -cd "$WORKSPACE/python/dask-raft/raft/test" -pytest --cache-clear --junitxml="$WORKSPACE/junit-dask-raft.xml" -v -s +gpuci_logger "Python pytest for raft-dask" +cd "$WORKSPACE/python/raft-dask/raft_dask/test" +pytest --cache-clear --junitxml="$WORKSPACE/junit-raft-dask.xml" -v -s gpuci_logger "Python pytest for pylibraft" cd "$WORKSPACE/python/pylibraft/pylibraft/test" diff --git a/conda/recipes/dask-raft/build.sh b/conda/recipes/raft-dask/build.sh similarity index 81% rename from conda/recipes/dask-raft/build.sh rename to conda/recipes/raft-dask/build.sh index 80cafb2730..963433dd8d 100644 --- a/conda/recipes/dask-raft/build.sh +++ b/conda/recipes/raft-dask/build.sh @@ -3,4 +3,4 @@ # Copyright (c) 2022, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory -./build.sh dask-raft --install --no-nvtx +./build.sh raft-dask --install --no-nvtx diff --git a/conda/recipes/dask-raft/conda_build_config.yaml b/conda/recipes/raft-dask/conda_build_config.yaml similarity index 100% rename from conda/recipes/dask-raft/conda_build_config.yaml rename to conda/recipes/raft-dask/conda_build_config.yaml diff --git a/conda/recipes/dask-raft/meta.yaml b/conda/recipes/raft-dask/meta.yaml similarity index 100% rename from conda/recipes/dask-raft/meta.yaml rename to conda/recipes/raft-dask/meta.yaml From 51525144f5c1dbe6d90ad5d45400cd24ae3e97fa Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 11:34:19 -0400 Subject: [PATCH 13/16] Updating final ci script --- ci/cpu/upload.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 9cccc3d050..ec61156bd0 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -38,10 +38,10 @@ if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then fi if [[ "$BUILD_RAFT" == "1" && "$UPLOAD_RAFT" == "1" ]]; then - DASKRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/dask-raft --python=$PYTHON --output) + DASKRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/raft-dask --python=$PYTHON --output) PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output) test -e ${DASKRAFT_FILE} - echo "Upload dask-raft" + echo "Upload raft-dask" echo ${DASKRAFT_FILE} gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASKRAFT_FILE} --no-progress From 0f76811c7a3d4e3d43b251d320732e4fd58157dc Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 13:33:29 -0400 Subject: [PATCH 14/16] Moving pylibraft CI tests before raft-dask --- build.sh | 2 +- ci/gpu/build.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 3b606406cb..3838232fba 100755 --- a/build.sh +++ b/build.sh @@ -25,7 +25,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= Date: Thu, 8 Sep 2022 17:19:07 -0400 Subject: [PATCH 15/16] FIxes based on review feedback --- ci/cpu/upload.sh | 8 ++++---- python/raft-dask/CMakeLists.txt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index ec61156bd0..cce7f4edef 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -38,12 +38,12 @@ if [[ "$BUILD_LIBRAFT" == "1" && "$UPLOAD_LIBRAFT" == "1" ]]; then fi if [[ "$BUILD_RAFT" == "1" && "$UPLOAD_RAFT" == "1" ]]; then - DASKRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/raft-dask --python=$PYTHON --output) + RAFT_DASK_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/raft-dask --python=$PYTHON --output) PYLIBRAFT_FILE=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_LOCAL_CHANNEL} conda/recipes/pylibraft --python=$PYTHON --output) - test -e ${DASKRAFT_FILE} + test -e ${RAFT_DASK_FILE} echo "Upload raft-dask" - echo ${DASKRAFT_FILE} - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASKRAFT_FILE} --no-progress + echo ${RAFT_DASK_FILE} + gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${RAFT_DASK_FILE} --no-progress test -e ${PYLIBRAFT_FILE} echo "Upload pylibraft" diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index d2c4a2c2d1..73d49c462b 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -45,11 +45,11 @@ if(NOT raft_FOUND) include(rapids-cuda) rapids_cuda_init_architectures(raft-dask) enable_language(CUDA) - # Since pyraft only enables CUDA optionally we need to manually include the file that + # Since raft-dask only enables CUDA optionally we need to manually include the file that # rapids_cuda_init_architectures relies on `project` including. include("${CMAKE_PROJECT_raft_dask_INCLUDE}") - # pyraft doesn't actually use raft libraries, it just needs the headers, so + # raft-dask doesn't actually use raft libraries, it just needs the headers, so # we can turn off all library compilation and we don't need to install # anything here. set(BUILD_TESTS OFF) From 0b949de3cef3194916ff839b460cd03b98ab71c1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 8 Sep 2022 17:50:37 -0400 Subject: [PATCH 16/16] Updating pyraft->raft-dask --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 79a9bff3be..4e0ecd8e15 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -34,7 +34,7 @@ function sed_runner() { sed_runner "s/set(RAPIDS_VERSION .*)/set(RAPIDS_VERSION \"${NEXT_SHORT_TAG}\")/g" cpp/CMakeLists.txt sed_runner "s/set(RAFT_VERSION .*)/set(RAFT_VERSION \"${NEXT_FULL_TAG}\")/g" cpp/CMakeLists.txt sed_runner 's/'"pylibraft_version .*)"'/'"pylibraft_version ${NEXT_FULL_TAG})"'/g' python/pylibraft/CMakeLists.txt -sed_runner 's/'"pyraft_version .*)"'/'"pyraft_version ${NEXT_FULL_TAG})"'/g' python/raft/CMakeLists.txt +sed_runner 's/'"raft_dask_version .*)"'/'"raft_dask_version ${NEXT_FULL_TAG})"'/g' python/raft-dask/CMakeLists.txt sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake # Docs update