diff --git a/BUILD.md b/BUILD.md index 3c6ad2bf20..0c7fdd7e82 100644 --- a/BUILD.md +++ b/BUILD.md @@ -34,8 +34,8 @@ In addition to the libraries included with cudatoolkit 11.0+, there are some oth - [cuCollections](https://github.com/NVIDIA/cuCollections) - Used in `raft::sparse::distance` API. - [Libcu++](https://github.com/NVIDIA/libcudacxx) v1.7.0 - [FAISS](https://github.com/facebookresearch/faiss) v1.7.0 - Used in `raft::spatial::knn` API and needed to build tests. -- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `Pyraft` -- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `Pyraft` +- [NCCL](https://github.com/NVIDIA/nccl) - Used in `raft::comms` API and needed to build `raft-dask` +- [UCX](https://github.com/openucx/ucx) - Used in `raft::comms` API and needed to build `raft-dask` - [Googletest](https://github.com/google/googletest) - Needed to build tests - [Googlebench](https://github.com/google/benchmark) - Needed to build benchmarks - [Doxygen](https://github.com/doxygen/doxygen) - Needed to build docs @@ -128,9 +128,6 @@ RAFT's cmake has the following configurable flags available:. | RAFT_COMPILE_DIST_LIBRARY | ON, OFF | OFF | Compiles the `libraft-distance` shared library | | RAFT_ENABLE_NN_DEPENDENCIES | ON, OFF | OFF | Searches for dependencies of nearest neighbors API, such as FAISS, and compiles them if not found. Needed for `raft::spatial::knn` | | RAFT_ENABLE_thrust_DEPENDENCY | ON, OFF | ON | Enables the Thrust dependency. This can be disabled when using many simple utilities or to override with a different Thrust version. | -| RAFT_ENABLE_mdspan_DEPENDENCY | ON, OFF | ON | Enables the std::mdspan dependency. This can be disabled when using many simple utilities. | -| RAFT_ENABLE_nccl_DEPENDENCY | ON, OFF | OFF | Enables NCCL dependency used by `raft::comms` and needed to build `pyraft` | -| RAFT_ENABLE_ucx_DEPENDENCY | ON, OFF | OFF | Enables UCX dependency used by `raft::comms` and needed to build `pyraft` | | RAFT_USE_FAISS_STATIC | ON, OFF | OFF | Statically link FAISS into `libraft-nn` | | RAFT_STATIC_LINK_LIBRARIES | ON, OFF | ON | Build static link libraries instead of shared libraries | | DETECT_CONDA_ENV | ON, OFF | ON | Enable detection of conda environment for dependencies | @@ -153,12 +150,12 @@ mamba activate raft_env_name The Python APIs can be built using the `build.sh` script: ```bash -./build.sh pyraft pylibraft +./build.sh raft-dask pylibraft ``` `setup.py` can also be used to build the Python APIs manually: ```bash -cd python/raft +cd python/raft-dask python setup.py build_ext --inplace python setup.py install @@ -169,7 +166,7 @@ python setup.py install To run the Python tests: ```bash -cd python/raft +cd python/raft-dask py.test -s -v raft cd python pylibraft @@ -187,7 +184,7 @@ When the needed [build dependencies](#required_depenencies) are already satisfie set(RAFT_GIT_DIR ${CMAKE_CURRENT_BINARY_DIR}/raft CACHE STRING "Path to RAFT repo") ExternalProject_Add(raft GIT_REPOSITORY git@github.com:rapidsai/raft.git - GIT_TAG branch-22.04 + GIT_TAG branch-22.10 PREFIX ${RAFT_GIT_DIR} CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -219,7 +216,7 @@ The following `cmake` snippet enables a flexible configuration of RAFT: ```cmake -set(RAFT_VERSION "22.04") +set(RAFT_VERSION "22.10") set(RAFT_FORK "rapidsai") set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") @@ -227,7 +224,7 @@ function(find_and_configure_raft) set(oneValueArgs VERSION FORK PINNED_TAG USE_FAISS_STATIC COMPILE_LIBRARIES ENABLE_NN_DEPENDENCIES CLONE_ON_PIN USE_NN_LIBRARY USE_DISTANCE_LIBRARY - ENABLE_thrust_DEPENDENCY ENABLE_mdspan_DEPENDENCY) + ENABLE_thrust_DEPENDENCY) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -272,7 +269,6 @@ function(find_and_configure_raft) "RAFT_USE_FAISS_STATIC ${PKG_USE_FAISS_STATIC}" "RAFT_COMPILE_LIBRARIES ${PKG_COMPILE_LIBRARIES}" "RAFT_ENABLE_thrust_DEPENDENCY ${PKG_ENABLE_thrust_DEPENDENCY}" - "RAFT_ENABLE_mdspan_DEPENDENCY ${PKG_ENABLE_mdspan_DEPENDENCY}" ) endfunction() @@ -295,7 +291,6 @@ find_and_configure_raft(VERSION ${RAFT_VERSION}.00 ENABLE_NN_DEPENDENCIES NO # This builds FAISS if not installed USE_FAISS_STATIC NO ENABLE_thrust_DEPENDENCY YES - ENABLE_mdspan_DEPENDENCY YES ) ``` diff --git a/README.md b/README.md index 2159f128bf..7a9434c16d 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ RAFT provides a header-only C++ library and pre-compiled shared libraries that c RAFT also provides 2 Python libraries: - `pylibraft` - low-level Python wrappers around RAFT algorithms and primitives. -- `pyraft` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. +- `raft-dask` - reusable infrastructure for building analytics, including tools for building both single-GPU and multi-node multi-GPU algorithms. ## Getting started @@ -108,11 +108,11 @@ The easiest way to install RAFT is through conda and several packages are provid - `libraft-nn` (optional) contains shared libraries for the nearest neighbors primitives. - `libraft-distance` (optional) contains shared libraries for distance primitives. - `pylibraft` (optional) Python wrappers around RAFT algorithms and primitives -- `pyraft` (optional) contains reusable Python infrastructure and tools to accelerate Python algorithm development. +- `raft-dask` (optional) enables deployment of multi-node multi-GPU algorithms that use RAFT `raft::comms` in Dask clusters. Use the following command to install RAFT with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. ```bash -mamba install -c rapidsai libraft-headers libraft-nn libraft-distance pyraft pylibraft +mamba install -c rapidsai libraft-headers libraft-nn libraft-distance raft-dask pylibraft ``` After installing RAFT, `find_package(raft COMPONENTS nn distance)` can be used in your CUDA/C++ cmake build to compile and/or link against needed dependencies in your raft target. `COMPONENTS` are optional and will depend on the packages installed. @@ -181,7 +181,7 @@ mamba env create --name raft_dev_env -f conda/environments/raft_dev_cuda11.5.yml mamba activate raft_dev_env ``` ``` -./build.sh pyraft pylibraft libraft tests bench --compile-libs +./build.sh raft-dask pylibraft libraft tests bench --compile-libs ``` The [build](BUILD.md) instructions contain more details on building RAFT from source and including it in downstream projects. You can also find a more comprehensive version of the above CPM code snippet the [Building RAFT C++ from source](BUILD.md#build_cxx_source) section of the build instructions. diff --git a/build.sh b/build.sh index ce4365985b..3838232fba 100755 --- a/build.sh +++ b/build.sh @@ -18,14 +18,14 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pyraft pylibraft docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean -v -g --install --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] where is: clean - remove all existing build artifacts and configuration (start over) libraft - build the raft C++ code only. Also builds the C-wrapper library around the C++ code. - pyraft - build the pyraft Python package pylibraft - build the pylibraft Python package + raft-dask - build the raft-dask Python package. this also requires pylibraft. docs - build the documentation tests - build the tests bench - build the benchmarks @@ -50,7 +50,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool==0.29,<0.30 - scikit-build>=0.13.1 - rmm {{ minor_version }} - - libraft-headers {{ version }} + - pylibraft {{ version }} - cudatoolkit {{ cuda_version }}.* - cuda-python >=11.5,<11.7.1 - nccl>=2.9.9 @@ -45,7 +45,7 @@ requirements: run: - python x.x - dask-cuda {{ minor_version }} - - libraft-headers {{ version }} + - pylibraft {{ version }} - nccl>=2.9.9 - rmm {{ minor_version }} - ucx >={{ ucx_version }} @@ -61,10 +61,10 @@ tests: # [linux64] requirements: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] imports: # [linux64] - - raft # [linux64] + - raft_dask # [linux64] about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: pyraft library + summary: raft-dask library diff --git a/docs/source/conf.py b/docs/source/conf.py index f34c1d34cb..a96f86c68d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,7 +24,7 @@ # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. sys.path.insert(0, os.path.abspath("sphinxext")) -sys.path.insert(0, os.path.abspath("../../python/raft")) +sys.path.insert(0, os.path.abspath("../../python/raft-dask")) sys.path.insert(0, os.path.abspath("../../python/pylibraft")) from github_link import make_linkcode_resolve # noqa diff --git a/docs/source/index.rst b/docs/source/index.rst index 97c616dd8e..0d7ab295f4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,7 +8,7 @@ RAFT contains fundamental widely-used algorithms and primitives for data science :caption: Contents: cpp_api.rst - pyraft_api.rst + raft_dask_api.rst pylibraft_api.rst diff --git a/docs/source/pyraft_api.rst b/docs/source/pyraft_api.rst deleted file mode 100644 index c763c9a0f7..0000000000 --- a/docs/source/pyraft_api.rst +++ /dev/null @@ -1,20 +0,0 @@ -~~~~~~~~~~~~~~~~~~~~ -PyRAFT API Reference -~~~~~~~~~~~~~~~~~~~~ - -.. role:: py(code) - :language: python - :class: highlight - - -RAFT Handle ------------ - -.. autoclass:: raft.common.handle.Handle - :members: - -Dask-based Multi-Node Multi-GPU Communicator --------------------------------------------- - -.. autoclass:: raft.dask.common.Comms - :members: diff --git a/docs/source/python.rst b/docs/source/python.rst index 0ae9f88398..561ec3d029 100644 --- a/docs/source/python.rst +++ b/docs/source/python.rst @@ -5,6 +5,6 @@ RAFT Python APIs :maxdepth: 2 :caption: Contents: - pyraft_api.rst pylibraft_api.rst + raft_dask_api.rst diff --git a/docs/source/raft_dask_api.rst b/docs/source/raft_dask_api.rst new file mode 100644 index 0000000000..10ba8781a2 --- /dev/null +++ b/docs/source/raft_dask_api.rst @@ -0,0 +1,13 @@ +~~~~~~~~~~~~~~~~~~~~~~~ +RAFT Dask API Reference +~~~~~~~~~~~~~~~~~~~~~~~ + +.. role:: py(code) + :language: python + :class: highlight + +Dask-based Multi-Node Multi-GPU Communicator +-------------------------------------------- + +.. autoclass:: raft_dask.common.Comms + :members: diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 5f877187eb..6567fdff0f 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -68,5 +68,6 @@ endif() include(rapids-cython) rapids_cython_init() +add_subdirectory(pylibraft/common) add_subdirectory(pylibraft/distance) add_subdirectory(pylibraft/random) diff --git a/python/raft/raft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt similarity index 69% rename from python/raft/raft/common/CMakeLists.txt rename to python/pylibraft/pylibraft/common/CMakeLists.txt index 59b336574e..d9ed2d12fc 100644 --- a/python/raft/raft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -12,9 +12,17 @@ # the License. # ============================================================================= +# Set the list of Cython files to build set(cython_sources cuda.pyx handle.pyx interruptible.pyx) set(linked_libraries raft::raft) + +# Build all of the Cython targets rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" - CXX) + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" + MODULE_PREFIX common_) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/pylibraft/pylibraft/common/handle.pxd b/python/pylibraft/pylibraft/common/handle.pxd index 6504a122f7..d7abe236ba 100644 --- a/python/pylibraft/pylibraft/common/handle.pxd +++ b/python/pylibraft/pylibraft/common/handle.pxd @@ -19,6 +19,7 @@ # cython: embedsignature = True # cython: language_level = 3 + from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_view from rmm._lib.cuda_stream_pool cimport cuda_stream_pool @@ -33,3 +34,8 @@ cdef extern from "raft/core/handle.hpp" namespace "raft" nogil: shared_ptr[cuda_stream_pool] stream_pool) except + cuda_stream_view get_stream() except + void sync_stream() except + + +cdef class Handle: + cdef unique_ptr[handle_t] c_obj + cdef shared_ptr[cuda_stream_pool] stream_pool + cdef int n_streams diff --git a/python/raft/raft/common/interruptible.pxd b/python/pylibraft/pylibraft/common/interruptible.pxd similarity index 90% rename from python/raft/raft/common/interruptible.pxd rename to python/pylibraft/pylibraft/common/interruptible.pxd index a73e8c1ac7..a4c7d90ac2 100644 --- a/python/raft/raft/common/interruptible.pxd +++ b/python/pylibraft/pylibraft/common/interruptible.pxd @@ -22,11 +22,11 @@ from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_view -cdef extern from "raft/interruptible.hpp" namespace "raft" nogil: +cdef extern from "raft/core/interruptible.hpp" namespace "raft" nogil: cdef cppclass interruptible: void cancel() -cdef extern from "raft/interruptible.hpp" \ +cdef extern from "raft/core/interruptible.hpp" \ namespace "raft::interruptible" nogil: cdef void inter_synchronize \ "raft::interruptible::synchronize"(cuda_stream_view stream) except+ diff --git a/python/raft/raft/common/interruptible.pyx b/python/pylibraft/pylibraft/common/interruptible.pyx similarity index 100% rename from python/raft/raft/common/interruptible.pyx rename to python/pylibraft/pylibraft/common/interruptible.pyx diff --git a/python/raft/raft/test/test_interruptible.py b/python/pylibraft/pylibraft/test/test_interruptible.py similarity index 95% rename from python/raft/raft/test/test_interruptible.py rename to python/pylibraft/pylibraft/test/test_interruptible.py index a3559f6476..bda98edd13 100644 --- a/python/raft/raft/test/test_interruptible.py +++ b/python/pylibraft/pylibraft/test/test_interruptible.py @@ -4,7 +4,7 @@ import pytest import signal import time -from raft.common.interruptible import cuda_interruptible, cuda_yield +from pylibraft.common.interruptible import cuda_interruptible, cuda_yield def send_ctrl_c(): diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index e3fe544bef..cb166a4d75 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -20,5 +20,5 @@ requires = [ "cython>=0.29,<0.30", "scikit-build>=0.13.1", "cmake>=3.20.1,!=3.23.0", - "ninja", + "ninja" ] diff --git a/python/raft/CMakeLists.txt b/python/raft-dask/CMakeLists.txt similarity index 81% rename from python/raft/CMakeLists.txt rename to python/raft-dask/CMakeLists.txt index 6330e1ef7f..73d49c462b 100644 --- a/python/raft/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -14,13 +14,13 @@ cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) -set(pyraft_version 22.10.00) +set(raft_dask_version 22.10.00) include(../../fetch_rapids.cmake) project( - raft-python - VERSION ${pyraft_version} + raft-dask-python + VERSION ${raft_dask_version} LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C # language to be enabled here. The test project that is built in scikit-build to verify # various linking options for the python library is hardcoded to build with C, so until @@ -33,7 +33,7 @@ option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulti # If the user requested it we attempt to find RAFT. if(FIND_RAFT_CPP) - find_package(raft ${pyraft_version} REQUIRED) + find_package(raft ${raft_dask_version} REQUIRED) else() set(raft_FOUND OFF) endif() @@ -43,13 +43,13 @@ if(NOT raft_FOUND) # pull in the required languages for the C++ project even if this project # does not require those languges. include(rapids-cuda) - rapids_cuda_init_architectures(pyraft) + rapids_cuda_init_architectures(raft-dask) enable_language(CUDA) - # Since pyraft only enables CUDA optionally we need to manually include the file that + # Since raft-dask only enables CUDA optionally we need to manually include the file that # rapids_cuda_init_architectures relies on `project` including. - include("${CMAKE_PROJECT_pyraft_INCLUDE}") + include("${CMAKE_PROJECT_raft_dask_INCLUDE}") - # pyraft doesn't actually use raft libraries, it just needs the headers, so + # raft-dask doesn't actually use raft libraries, it just needs the headers, so # we can turn off all library compilation and we don't need to install # anything here. set(BUILD_TESTS OFF) @@ -63,6 +63,5 @@ endif() include(rapids-cython) rapids_cython_init() -add_subdirectory(raft/common) -add_subdirectory(raft/dask/common) -add_subdirectory(raft/include_test) +add_subdirectory(raft_dask/common) +add_subdirectory(raft_dask/include_test) diff --git a/python/raft/cmake/thirdparty/get_nccl.cmake b/python/raft-dask/cmake/thirdparty/get_nccl.cmake similarity index 100% rename from python/raft/cmake/thirdparty/get_nccl.cmake rename to python/raft-dask/cmake/thirdparty/get_nccl.cmake diff --git a/python/raft/pyproject.toml b/python/raft-dask/pyproject.toml similarity index 97% rename from python/raft/pyproject.toml rename to python/raft-dask/pyproject.toml index e3fe544bef..3979c0d7c5 100644 --- a/python/raft/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -21,4 +21,5 @@ requires = [ "scikit-build>=0.13.1", "cmake>=3.20.1,!=3.23.0", "ninja", + "pylibraft" ] diff --git a/python/raft/pytest.ini b/python/raft-dask/pytest.ini similarity index 100% rename from python/raft/pytest.ini rename to python/raft-dask/pytest.ini diff --git a/python/raft/raft/__init__.py b/python/raft-dask/raft_dask/__init__.py similarity index 100% rename from python/raft/raft/__init__.py rename to python/raft-dask/raft_dask/__init__.py diff --git a/python/raft/raft/_version.py b/python/raft-dask/raft_dask/_version.py similarity index 100% rename from python/raft/raft/_version.py rename to python/raft-dask/raft_dask/_version.py diff --git a/python/raft/raft/dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt similarity index 93% rename from python/raft/raft/dask/common/CMakeLists.txt rename to python/raft-dask/raft_dask/common/CMakeLists.txt index 1abddae947..0e1da015fc 100644 --- a/python/raft/raft/dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -12,7 +12,7 @@ # the License. # ============================================================================= -include(${raft-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) +include(${raft-dask-python_SOURCE_DIR}/cmake/thirdparty/get_nccl.cmake) find_package(ucx REQUIRED) set(cython_sources comms_utils.pyx nccl.pyx) diff --git a/python/raft/raft/common/__init__.pxd b/python/raft-dask/raft_dask/common/__init__.pxd similarity index 100% rename from python/raft/raft/common/__init__.pxd rename to python/raft-dask/raft_dask/common/__init__.pxd diff --git a/python/raft/raft/dask/common/__init__.py b/python/raft-dask/raft_dask/common/__init__.py similarity index 100% rename from python/raft/raft/dask/common/__init__.py rename to python/raft-dask/raft_dask/common/__init__.py diff --git a/python/raft/raft/dask/common/comms.py b/python/raft-dask/raft_dask/common/comms.py similarity index 99% rename from python/raft/raft/dask/common/comms.py rename to python/raft-dask/raft_dask/common/comms.py index 549ac7fccb..d7260ece9b 100644 --- a/python/raft/raft/dask/common/comms.py +++ b/python/raft-dask/raft_dask/common/comms.py @@ -20,7 +20,7 @@ from .comms_utils import inject_comms_on_handle_coll_only from .utils import parse_host_port -from ...common.handle import Handle +from pylibraft.common.handle import Handle from dask.distributed import get_worker, default_client diff --git a/python/raft/raft/dask/common/comms_utils.pyx b/python/raft-dask/raft_dask/common/comms_utils.pyx similarity index 99% rename from python/raft/raft/dask/common/comms_utils.pyx rename to python/raft-dask/raft_dask/common/comms_utils.pyx index 38c5670372..2014284af1 100644 --- a/python/raft/raft/dask/common/comms_utils.pyx +++ b/python/raft-dask/raft_dask/common/comms_utils.pyx @@ -31,7 +31,7 @@ cdef extern from "nccl.h": cdef struct ncclComm ctypedef ncclComm *ncclComm_t -cdef extern from "raft/handle.hpp" namespace "raft": +cdef extern from "raft/core/handle.hpp" namespace "raft": cdef cppclass handle_t: handle_t() except + diff --git a/python/raft/raft/dask/common/nccl.pyx b/python/raft-dask/raft_dask/common/nccl.pyx similarity index 100% rename from python/raft/raft/dask/common/nccl.pyx rename to python/raft-dask/raft_dask/common/nccl.pyx diff --git a/python/raft/raft/dask/common/ucx.py b/python/raft-dask/raft_dask/common/ucx.py similarity index 100% rename from python/raft/raft/dask/common/ucx.py rename to python/raft-dask/raft_dask/common/ucx.py diff --git a/python/raft/raft/dask/common/utils.py b/python/raft-dask/raft_dask/common/utils.py similarity index 100% rename from python/raft/raft/dask/common/utils.py rename to python/raft-dask/raft_dask/common/utils.py diff --git a/python/raft/raft/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt similarity index 100% rename from python/raft/raft/include_test/CMakeLists.txt rename to python/raft-dask/raft_dask/include_test/CMakeLists.txt diff --git a/python/raft/raft/include_test/__init__.py b/python/raft-dask/raft_dask/include_test/__init__.py similarity index 100% rename from python/raft/raft/include_test/__init__.py rename to python/raft-dask/raft_dask/include_test/__init__.py diff --git a/python/raft/raft/include_test/raft_include_test.pyx b/python/raft-dask/raft_dask/include_test/raft_include_test.pyx similarity index 100% rename from python/raft/raft/include_test/raft_include_test.pyx rename to python/raft-dask/raft_dask/include_test/raft_include_test.pyx diff --git a/python/raft/raft/test/conftest.py b/python/raft-dask/raft_dask/test/conftest.py similarity index 100% rename from python/raft/raft/test/conftest.py rename to python/raft-dask/raft_dask/test/conftest.py diff --git a/python/raft/raft/test/test_comms.py b/python/raft-dask/raft_dask/test/test_comms.py similarity index 90% rename from python/raft/raft/test/test_comms.py rename to python/raft-dask/raft_dask/test/test_comms.py index 345cdbf037..29b4d963d9 100644 --- a/python/raft/raft/test/test_comms.py +++ b/python/raft-dask/raft_dask/test/test_comms.py @@ -21,20 +21,20 @@ from dask.distributed import wait try: - from raft.dask import Comms - from raft.dask.common import local_handle - from raft.dask.common import perform_test_comms_send_recv - from raft.dask.common import perform_test_comms_device_send_or_recv - from raft.dask.common import perform_test_comms_device_sendrecv - from raft.dask.common import perform_test_comms_device_multicast_sendrecv - from raft.dask.common import perform_test_comms_allreduce - from raft.dask.common import perform_test_comms_bcast - from raft.dask.common import perform_test_comms_reduce - from raft.dask.common import perform_test_comms_allgather - from raft.dask.common import perform_test_comms_gather - from raft.dask.common import perform_test_comms_gatherv - from raft.dask.common import perform_test_comms_reducescatter - from raft.dask.common import perform_test_comm_split + from raft_dask import Comms + from raft_dask.common import local_handle + from raft_dask.common import perform_test_comms_send_recv + from raft_dask.common import perform_test_comms_device_send_or_recv + from raft_dask.common import perform_test_comms_device_sendrecv + from raft_dask.common import perform_test_comms_device_multicast_sendrecv + from raft_dask.common import perform_test_comms_allreduce + from raft_dask.common import perform_test_comms_bcast + from raft_dask.common import perform_test_comms_reduce + from raft_dask.common import perform_test_comms_allgather + from raft_dask.common import perform_test_comms_gather + from raft_dask.common import perform_test_comms_gatherv + from raft_dask.common import perform_test_comms_reducescatter + from raft_dask.common import perform_test_comm_split pytestmark = pytest.mark.mg except ImportError: diff --git a/python/raft/raft/test/test_raft.py b/python/raft-dask/raft_dask/test/test_raft.py similarity index 84% rename from python/raft/raft/test/test_raft.py rename to python/raft-dask/raft_dask/test/test_raft.py index 796a4fface..e1370dccc9 100644 --- a/python/raft/raft/test/test_raft.py +++ b/python/raft-dask/raft_dask/test/test_raft.py @@ -17,15 +17,15 @@ import sys try: - import raft + import raft_dask except ImportError: print("Skipping RAFT tests") pytestmart = pytest.mark.skip pytestmark = pytest.mark.skipif( - 'raft' not in sys.argv, reason="marker to allow integration of RAFT" + 'raft_dask' not in sys.argv, reason="marker to allow integration of RAFT" ) def test_raft(): - assert raft.raft_include_test() + assert raft_dask.raft_include_test() diff --git a/python/raft/setup.cfg b/python/raft-dask/setup.cfg similarity index 87% rename from python/raft/setup.cfg rename to python/raft-dask/setup.cfg index e20ff1bb2b..3bc8f126f6 100644 --- a/python/raft/setup.cfg +++ b/python/raft-dask/setup.cfg @@ -9,10 +9,10 @@ exclude = __init__.py,versioneer.py [versioneer] VCS = git style = pep440 -versionfile_source = raft/_version.py -versionfile_build = raft/_version.py +versionfile_source = raft_dask/_version.py +versionfile_build = raft_dask/_version.py tag_prefix = v -parentdir_prefix = raft- +parentdir_prefix = raft_dask- [isort] line_length=79 @@ -33,7 +33,7 @@ known_rapids= dask_cudf rmm known_first_party= - raft + raft_dask default_section=THIRDPARTY sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER skip= diff --git a/python/raft/setup.py b/python/raft-dask/setup.py similarity index 83% rename from python/raft/setup.py rename to python/raft-dask/setup.py index 3ba1c9b1e3..59bcc4a2dc 100644 --- a/python/raft/setup.py +++ b/python/raft-dask/setup.py @@ -20,8 +20,8 @@ import versioneer -setup(name='raft', - description="RAPIDS Analytics Frameworks Toolset", +setup(name='raft-dask', + description="Reusable Accelerated Functions & Tools Dask Infrastructure", version=versioneer.get_version(), classifiers=[ "Intended Audience :: Developers", @@ -38,14 +38,12 @@ key: ["*.hpp", "*.pxd"] for key in find_packages( include=[ - "raft.dask.common", - "raft.dask.common.includes", - "raft.common", - "raft.common.includes" + "raft_dask.common", + "raft_dask.common.includes", ] ) }, - packages=find_packages(include=['raft', 'raft.*']), + packages=find_packages(include=['raft_dask', 'raft_dask.*']), license="Apache", cmdclass=versioneer.get_cmdclass(), zip_safe=False diff --git a/python/raft/setuputils.py b/python/raft-dask/setuputils.py similarity index 100% rename from python/raft/setuputils.py rename to python/raft-dask/setuputils.py diff --git a/python/raft/versioneer.py b/python/raft-dask/versioneer.py similarity index 100% rename from python/raft/versioneer.py rename to python/raft-dask/versioneer.py diff --git a/python/raft/raft/common/__init__.py b/python/raft/raft/common/__init__.py deleted file mode 100644 index 62db7d5831..0000000000 --- a/python/raft/raft/common/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .cuda import Stream -from .handle import Handle \ No newline at end of file diff --git a/python/raft/raft/common/cuda.pxd b/python/raft/raft/common/cuda.pxd deleted file mode 100644 index 0459cb96af..0000000000 --- a/python/raft/raft/common/cuda.pxd +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2019-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from cuda.ccudart cimport cudaStream_t - -cdef class Stream: - cdef cudaStream_t s - - cdef cudaStream_t getStream(self) diff --git a/python/raft/raft/common/cuda.pyx b/python/raft/raft/common/cuda.pyx deleted file mode 100644 index c3c90936aa..0000000000 --- a/python/raft/raft/common/cuda.pyx +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cuda.ccudart cimport( - cudaStream_t, - cudaError_t, - cudaSuccess, - cudaStreamCreate, - cudaStreamDestroy, - cudaStreamSynchronize, - cudaGetLastError, - cudaGetErrorString, - cudaGetErrorName -) - - -class CudaRuntimeError(RuntimeError): - def __init__(self, extraMsg=None): - cdef cudaError_t e = cudaGetLastError() - cdef bytes errMsg = cudaGetErrorString(e) - cdef bytes errName = cudaGetErrorName(e) - msg = "Error! %s reason='%s'" % (errName.decode(), errMsg.decode()) - if extraMsg is not None: - msg += " extraMsg='%s'" % extraMsg - super(CudaRuntimeError, self).__init__(msg) - - -cdef class Stream: - """ - Stream represents a thin-wrapper around cudaStream_t and its operations. - - Examples - -------- - - .. code-block:: python - - from raft.common.cuda import Stream - stream = Stream() - stream.sync() - del stream # optional! - """ - def __cinit__(self): - cdef cudaStream_t stream - cdef cudaError_t e = cudaStreamCreate(&stream) - if e != cudaSuccess: - raise CudaRuntimeError("Stream create") - self.s = stream - - def __dealloc__(self): - self.sync() - cdef cudaError_t e = cudaStreamDestroy(self.s) - if e != cudaSuccess: - raise CudaRuntimeError("Stream destroy") - - def sync(self): - """ - Synchronize on the cudastream owned by this object. Note that this - could raise exception due to issues with previous asynchronous - launches - """ - cdef cudaError_t e = cudaStreamSynchronize(self.s) - if e != cudaSuccess: - raise CudaRuntimeError("Stream sync") - - cdef cudaStream_t getStream(self): - return self.s diff --git a/python/raft/raft/common/handle.pxd b/python/raft/raft/common/handle.pxd deleted file mode 100644 index 48ec625370..0000000000 --- a/python/raft/raft/common/handle.pxd +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - - -from libcpp.memory cimport shared_ptr -from rmm._lib.cuda_stream_view cimport cuda_stream_view -from rmm._lib.cuda_stream_pool cimport cuda_stream_pool -from libcpp.memory cimport shared_ptr -from libcpp.memory cimport unique_ptr - -cdef extern from "raft/handle.hpp" namespace "raft" nogil: - cdef cppclass handle_t: - handle_t() except + - handle_t(cuda_stream_view stream_view) except + - handle_t(cuda_stream_view stream_view, - shared_ptr[cuda_stream_pool] stream_pool) except + - cuda_stream_view get_stream() except + - void sync_stream() except + - -cdef class Handle: - cdef unique_ptr[handle_t] c_obj - cdef shared_ptr[cuda_stream_pool] stream_pool - cdef int n_streams diff --git a/python/raft/raft/common/handle.pyx b/python/raft/raft/common/handle.pyx deleted file mode 100644 index de3952cc87..0000000000 --- a/python/raft/raft/common/handle.pyx +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -# import raft -from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread -from rmm._lib.cuda_stream_view cimport cuda_stream_view - -from .cuda cimport Stream -from .cuda import CudaRuntimeError - - -cdef class Handle: - """ - Handle is a lightweight python wrapper around the corresponding C++ class - of handle_t exposed by RAFT's C++ interface. Refer to the header file - raft/handle.hpp for interface level details of this struct - - Examples - -------- - - .. code-block:: python - - from raft.common import Stream, Handle - stream = Stream() - handle = Handle(stream) - - # call algos here - - # final sync of all work launched in the stream of this handle - # this is same as `raft.cuda.Stream.sync()` call, but safer in case - # the default stream inside the `handle_t` is being used - handle.sync() - del handle # optional! - """ - - def __cinit__(self, stream: Stream = None, n_streams=0): - self.n_streams = n_streams - if n_streams > 0: - self.stream_pool.reset(new cuda_stream_pool(n_streams)) - - cdef cuda_stream_view c_stream - if stream is None: - # this constructor will construct a "main" handle on - # per-thread default stream, which is non-blocking - self.c_obj.reset(new handle_t(cuda_stream_per_thread, - self.stream_pool)) - else: - # this constructor constructs a handle on user stream - c_stream = cuda_stream_view(stream.getStream()) - self.c_obj.reset(new handle_t(c_stream, - self.stream_pool)) - - def sync(self): - """ - Issues a sync on the stream set for this handle. - """ - self.c_obj.get()[0].sync_stream() - - def getHandle(self): - return self.c_obj.get() - - def __getstate__(self): - return self.n_streams - - def __setstate__(self, state): - self.n_streams = state - if self.n_streams > 0: - self.stream_pool.reset(new cuda_stream_pool(self.n_streams)) - - self.c_obj.reset(new handle_t(cuda_stream_per_thread, - self.stream_pool)) diff --git a/python/raft/raft/dask/__init__.py b/python/raft/raft/dask/__init__.py deleted file mode 100644 index f6a1c28ea8..0000000000 --- a/python/raft/raft/dask/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .common.comms import Comms \ No newline at end of file