From 869056a9555152e1b83dbd571118190e9edeea57 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Dec 2022 12:27:24 -0500 Subject: [PATCH 1/5] Adding uninstall option to build.sh --- build.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/build.sh b/build.sh index 0708c1b89e..6e88f61fac 100755 --- a/build.sh +++ b/build.sh @@ -29,6 +29,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode @@ -288,8 +289,29 @@ if hasArg clean; then fi if hasArg uninstall; then UNINSTALL=1 + + # If clean or uninstall targets given, run them prior to any other steps + if [[ "$INSTALL_PREFIX" != "" ]]; then + rm -rf ${INSTALL_PREFIX}/include/raft + rm -rf ${INSTALL_PREFIX}/include/raft_runtime + rm -f ${INSTALL_PREFIX}/lib/libraft_distance.so + rm -f ${INSTALL_PREFIX}/lib/libraft_nn.so + rm -rf ${INSTALL_PREFIX}/lib/cmake/raft + fi + # This may be redundant given the above, but can also be used in case + # there are other installed files outside of the locations above. + if [ -e ${LIBRAFT_BUILD_DIR}/install_manifest.txt ]; then + xargs rm -f < ${LIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 + fi + # uninstall raft-dask and pylibraft installed from a prior "setup.py install" + # FIXME: if multiple versions of these packages are installed, this only + # removes the latest one and leaves the others installed. build.sh uninstall + # can be run multiple times to remove all of them, but that is not obvious. + pip uninstall -y raft-dask pylibraft fi + + if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi From c5a008e45df6a998502a7a0f65de637d7021bf22 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Dec 2022 13:40:32 -0500 Subject: [PATCH 2/5] Adjusting uninstall command to solely use cmake's `install_manifest.txt` files. --- README.md | 2 +- build.sh | 70 ++++++++++++++++++++--------------- docs/source/build.md | 59 +++++++++++++++++++++++++++-- docs/source/cpp_api.rst | 6 +-- docs/source/pylibraft_api.rst | 6 +-- 5 files changed, 103 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index e48a1b6193..0f796f3a39 100755 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ pairwise_distance(in1, in2, out=output, metric="euclidean") ## Installing -RAFT itself can be installed through conda, [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake), pip, or by building the repository from source. Please refer to the [build instructions](docs/source/build.md) for more a comprehensive guide on building RAFT and using it in downstream projects. +RAFT itself can be installed through conda, [Cmake Package Manager (CPM)](https://github.com/cpm-cmake/CPM.cmake), pip, or by building the repository from source. Please refer to the [build instructions](docs/source/build.md) for more a comprehensive guide on installing and building RAFT and using it in downstream projects. ### Conda diff --git a/build.sh b/build.sh index 6e88f61fac..39e080cdcd 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" +VALIDARGS="clean libraft pylibraft raft-dask docs tests bench clean --uninstall -v -g -n --compile-libs --compile-nn --compile-dist --allgpuarch --no-nvtx --show_depr_warn -h --buildfaiss --minimal-deps" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -29,12 +29,12 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode -g - build for debug -n - no install step + --uninstall - uninstall files for specified targets which were built and installed prior --compile-libs - compile shared libraries for all components --compile-nn - compile shared library for nn component --compile-dist - compile shared library for distance and current random components @@ -57,9 +57,9 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= /dev/null 2>&1 + fi + + fi + + if hasArg pylibraft || (( ${NUMARGS} == 1 )); then + + echo "Removing pylibraft..." + if [ -e ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt ]; then + xargs rm -fv < ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 + fi + fi + + if hasArg raft-dask; then + + echo "Removing raft-dask..." + if [ -e ${RAFT_DASK_BUILD_DIR}/install_manifest.txt ]; then + xargs rm -fv < ${RAFT_DASK_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 + fi + fi + + echo "Done." + exit 0 +fi + + # Process flags if hasArg -n; then INSTALL_TARGET="" @@ -287,28 +321,6 @@ fi if hasArg clean; then CLEAN=1 fi -if hasArg uninstall; then - UNINSTALL=1 - - # If clean or uninstall targets given, run them prior to any other steps - if [[ "$INSTALL_PREFIX" != "" ]]; then - rm -rf ${INSTALL_PREFIX}/include/raft - rm -rf ${INSTALL_PREFIX}/include/raft_runtime - rm -f ${INSTALL_PREFIX}/lib/libraft_distance.so - rm -f ${INSTALL_PREFIX}/lib/libraft_nn.so - rm -rf ${INSTALL_PREFIX}/lib/cmake/raft - fi - # This may be redundant given the above, but can also be used in case - # there are other installed files outside of the locations above. - if [ -e ${LIBRAFT_BUILD_DIR}/install_manifest.txt ]; then - xargs rm -f < ${LIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 - fi - # uninstall raft-dask and pylibraft installed from a prior "setup.py install" - # FIXME: if multiple versions of these packages are installed, this only - # removes the latest one and leaves the others installed. build.sh uninstall - # can be run multiple times to remove all of them, but that is not obvious. - pip uninstall -y raft-dask pylibraft -fi @@ -392,7 +404,7 @@ if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then fi cd ${REPODIR}/python/raft-dask - python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${LIBRAFT_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} + python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${RAFT_DASK_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} if [[ ${INSTALL_TARGET} != "" ]]; then python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS} fi @@ -406,7 +418,7 @@ if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then fi cd ${REPODIR}/python/pylibraft - python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${LIBRAFT_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} + python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH="${RAFT_DASK_BUILD_DIR};${INSTALL_PREFIX}" -DCMAKE_LIBRARY_PATH=${LIBRAFT_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} if [[ ${INSTALL_TARGET} != "" ]]; then python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} ${EXTRA_CMAKE_ARGS} fi diff --git a/docs/source/build.md b/docs/source/build.md index 4e692f85c5..21140ccc5d 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -1,4 +1,30 @@ -# Install Guide +# Installation + +### Conda + +The easiest way to install RAFT is through conda and several packages are provided. +- `libraft-headers` RAFT headers +- `libraft-nn` (optional) contains shared libraries for the nearest neighbors primitives. +- `libraft-distance` (optional) contains shared libraries for distance primitives. +- `pylibraft` (optional) Python wrappers around RAFT algorithms and primitives. +- `raft-dask` (optional) enables deployment of multi-node multi-GPU algorithms that use RAFT `raft::comms` in Dask clusters. + +Use the following command to install all of the RAFT packages with conda (replace `rapidsai` with `rapidsai-nightly` to install more up-to-date but less stable nightly packages). `mamba` is preferred over the `conda` command. +```bash +mamba install -c rapidsai -c conda-forge -c nvidia raft-dask pylibraft +``` + +You can also install the `libraft-*` conda packages individually using the `mamba` command above. + +After installing RAFT, `find_package(raft COMPONENTS nn distance)` can be used in your CUDA/C++ cmake build to compile and/or link against needed dependencies in your raft target. `COMPONENTS` are optional and will depend on the packages installed. + +### Pip + +pylibraft and raft-dask both have experimental packages that can be [installed through pip](https://rapids.ai/pip.html#install): +```bash +pip install pylibraft-cu11 --extra-index-url=https://pypi.ngc.nvidia.com +pip install raft-dask-cu11 --extra-index-url=https://pypi.ngc.nvidia.com +``` ## Building and installing RAFT @@ -46,6 +72,12 @@ The `-n` flag can be passed to just have the build download the needed dependenc ./build.sh libraft -n ``` +Once installed, `libraft` headers (and dependencies which were downloaded and installed using `rapids-cmake`) can be uninstalled also using `build.sh`: +```bash +./build.sh libraft --uninstall +``` + + ### C++ Shared Libraries (optional) For larger projects which make heavy use of the pairwise distances or nearest neighbors APIs, shared libraries can be built to speed up compile times. These shared libraries can also significantly improve re-compile times both while developing RAFT and developing against the APIs. Build all of the available shared libraries by passing `--compile-libs` flag to `build.sh`: @@ -60,6 +92,12 @@ Individual shared libraries have their own flags and multiple can be used (thoug In above example the shared libraries are installed by default into `$INSTALL_PREFIX/lib`. To disable this, pass `-n` flag. +Once installed, the shared libraries, headers (and any dependencies downloaded and installed via `rapids-cmake`) can be uninstalled using `build.sh`: +```bash +./build.sh libraft --uninstall +``` + + ### ccache and sccache `ccache` and `sccache` can be used to better cache parts of the build when rebuilding frequently, such as when working on a new feature. You can also use `ccache` or `sccache` with `build.sh`: @@ -108,7 +146,7 @@ It can take sometime to compile all of the benchmarks. You can build individual ./build.sh libraft bench --limit-bench=NEIGHBORS_BENCH;DISTANCE_BENCH;LINALG_BENCH ``` -### C++ Using Cmake +### C++ Using Cmake Directly Use `CMAKE_INSTALL_PREFIX` to install RAFT into a specific location. The snippet below will install it into the current conda environment: ```bash @@ -179,6 +217,11 @@ cd python/pylibraft py.test -s -v ``` +The Python packages can also be uninstalled using the `build.sh` script: +```bash +./build.sh pylibraft raft-dask --uninstall +``` + ### Documentation The documentation requires that the C++ headers and python packages have been built and installed. @@ -335,6 +378,14 @@ find_and_configure_raft(VERSION ${RAFT_VERSION}.00 If using the nearest neighbors APIs without the shared libraries, set `ENABLE_NN_DEPENDENCIES=ON` and keep `USE_NN_LIBRARY=OFF` -### Python/Cython Integration +## Uninstall + +Once built and installed, RAFT can be safely uninstalled using `build.sh` by specifying any or all of the installed components. Please note that since `pylibraft` depends on `libraft`, uninstalling `pylibraft` will also uninstall `libraft`: +```bash +./build.sh libraft pylibraft raft-dask --uninstall +``` -Once installed, RAFT's Python library can be added to downstream conda recipes, imported and used directly. +Leaving off the installed components will uninstall everything that's been installed: +```bash +./build.sh --uninstall +``` \ No newline at end of file diff --git a/docs/source/cpp_api.rst b/docs/source/cpp_api.rst index 569fd64061..e1ef1a7160 100644 --- a/docs/source/cpp_api.rst +++ b/docs/source/cpp_api.rst @@ -1,6 +1,6 @@ -~~~~~~~~~~~~~~~~~ -C++ API Reference -~~~~~~~~~~~~~~~~~ +~~~~~~~ +C++ API +~~~~~~~ .. _api: diff --git a/docs/source/pylibraft_api.rst b/docs/source/pylibraft_api.rst index 5c44c5f419..955f036c66 100644 --- a/docs/source/pylibraft_api.rst +++ b/docs/source/pylibraft_api.rst @@ -1,6 +1,6 @@ -~~~~~~~~~~~~~~~~~~~~~~~ -PyLibRAFT API Reference -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~ +PyLibRAFT API +~~~~~~~~~~~~~ .. _api: From 435c2c7da7da89a2045868b22434b1593021e635 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Dec 2022 14:00:11 -0500 Subject: [PATCH 3/5] Using pip or conda to uninstall python packages (conditionally) --- build.sh | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 39e080cdcd..cdcdca0d92 100755 --- a/build.sh +++ b/build.sh @@ -197,30 +197,55 @@ if hasArg --uninstall; then if hasArg pylibraft || hasArg libraft || (( ${NUMARGS} == 1 )); then - echo "Removing libraft..." + echo "Removing libraft files..." if [ -e ${LIBRAFT_BUILD_DIR}/install_manifest.txt ]; then xargs rm -fv < ${LIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 fi - fi if hasArg pylibraft || (( ${NUMARGS} == 1 )); then - - echo "Removing pylibraft..." + echo "Uninstalling pylibraft package..." if [ -e ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt ]; then xargs rm -fv < ${PYLIBRAFT_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 fi + + # Try to uninstall via pip if it is installed + if [ -x "$(command -v pip)" ]; then + echo "Using pip to uninstall pylibraft" + pip uninstall -y pylibraft + + # Otherwise, try to uninstall through conda if that's where things are installed + elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" eq "$CONDA_PREFIX" ]; then + echo "Using conda to uninstall pylibraft" + conda uninstall -y pylibraft + + # Otherwise, fail + else + echo "Could not uninstall pylibraft from pip or conda. pylibraft package will need to be manually uninstalled" + fi fi if hasArg raft-dask; then - - echo "Removing raft-dask..." + echo "Uninstalling raft-dask package..." if [ -e ${RAFT_DASK_BUILD_DIR}/install_manifest.txt ]; then xargs rm -fv < ${RAFT_DASK_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1 fi - fi - echo "Done." + # Try to uninstall via pip if it is installed + if [ -x "$(command -v pip)" ]; then + echo "Using pip to uninstall raft-dask" + pip uninstall -y raft-dask + + # Otherwise, try to uninstall through conda if that's where things are installed + elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" == "$CONDA_PREFIX" ]; then + echo "Using conda to uninstall raft-dask" + conda uninstall -y raft-dask + + # Otherwise, fail + else + echo "Could not uninstall raft-dask from pip or conda. raft-dask package will need to be manually uninstalled." + fi + fi exit 0 fi From 348288ebdfc215961ed2df64f9e9ebdef900f881 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Dec 2022 14:00:58 -0500 Subject: [PATCH 4/5] Fixing string equality --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index cdcdca0d92..a03b917dec 100755 --- a/build.sh +++ b/build.sh @@ -215,7 +215,7 @@ if hasArg --uninstall; then pip uninstall -y pylibraft # Otherwise, try to uninstall through conda if that's where things are installed - elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" eq "$CONDA_PREFIX" ]; then + elif [ -x "$(command -v conda)" ] && [ "$INSTALL_PREFIX" == "$CONDA_PREFIX" ]; then echo "Using conda to uninstall pylibraft" conda uninstall -y pylibraft From 43e4740c150c9b230f9d5e1b7650aa2a8b302e81 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 7 Dec 2022 20:06:00 -0500 Subject: [PATCH 5/5] Final updaqte --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index a03b917dec..34dcd3a2db 100755 --- a/build.sh +++ b/build.sh @@ -225,7 +225,7 @@ if hasArg --uninstall; then fi fi - if hasArg raft-dask; then + if hasArg raft-dask || (( ${NUMARGS} == 1 )); then echo "Uninstalling raft-dask package..." if [ -e ${RAFT_DASK_BUILD_DIR}/install_manifest.txt ]; then xargs rm -fv < ${RAFT_DASK_BUILD_DIR}/install_manifest.txt > /dev/null 2>&1