From 7f96a97eaa77d3baf8931a005745641f3ed4338a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 20 Aug 2024 09:51:02 -0400 Subject: [PATCH 1/6] Use dynamic CUDA libraries --- ci/build_wheel.sh | 18 +++++++++++++++++- ci/build_wheel_cuvs.sh | 11 ++++++++++- dependencies.yaml | 19 +++++++++++++++++++ python/cuvs/CMakeLists.txt | 22 ++++++++++++++++++++++ python/cuvs/pyproject.toml | 4 ++++ 5 files changed, 72 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 5d48bab22..d1030276f 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,10 +16,26 @@ rapids-generate-version > ./VERSION cd "${package_dir}" +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXCLUDE_ARGS=( + --exclude "libcublas.so.12" + --exclude "libcublasLt.so.12" + --exclude "libcurand.so.10" + --exclude "libcusolver.so.11" + --exclude "libcusparse.so.12" + --exclude "libnvJitLink.so.12" + ) + ;; + 11.*) + EXCLUDE_ARGS=() + ;; +esac + # Hardcode the output dir python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* +python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index 0fe28c2f7..e03da9f19 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -3,7 +3,16 @@ set -euo pipefail +case "${RAPIDS_CUDA_VERSION}" in + 12.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" + ;; + 11.*) + EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=OFF" + ;; +esac + # Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF -DFIND_CUVS_CPP=OFF" +export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUVS_CPP=OFF${EXTRA_CMAKE_ARGS}" ci/build_wheel.sh cuvs python/cuvs diff --git a/dependencies.yaml b/dependencies.yaml index 98c9043aa..702fc47b4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -78,6 +78,7 @@ files: extras: table: project includes: + - cuda - run_py_cuvs py_test_py_cuvs: output: pyproject @@ -310,6 +311,24 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + - output_types: pyproject + matrices: + - matrix: + cuda: "12.*" + packages: + - nvidia-cublas-cu12 + - nvidia-curand-cu12 + - nvidia-cusolver-cu12 + - nvidia-cusparse-cu12 + - matrix: + cuda: "11.*" + packages: + - matrix: + packages: + - nvidia-cublas + - nvidia-curand + - nvidia-cusolver + - nvidia-cusparse cupy: common: diff --git a/python/cuvs/CMakeLists.txt b/python/cuvs/CMakeLists.txt index 4d24682cf..100b5e163 100644 --- a/python/cuvs/CMakeLists.txt +++ b/python/cuvs/CMakeLists.txt @@ -37,6 +37,7 @@ project( option(FIND_CUVS_CPP "Search for existing CUVS C++ installations before defaulting to local files" OFF ) +option(USE_CUDA_MATH_WHEELS "Use the CUDA math wheels instead of the system libraries" OFF) message( "CUVS_PY: Searching for existing cuVS C/C++ installations before defaulting to local files: ${FIND_CUVS_CPP}" @@ -62,6 +63,8 @@ else() endif() if(NOT cuvs_FOUND) + find_package(CUDAToolkit REQUIRED) + set(BUILD_TESTS OFF) set(BUILD_C_LIBRARY ON) @@ -70,8 +73,27 @@ if(NOT cuvs_FOUND) set(CUDA_STATIC_MATH_LIBRARIES ON) set(CUVS_USE_RAFT_STATIC ON) + if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12.0) + set(CUDA_STATIC_MATH_LIBRARIES OFF) + elseif(USE_CUDA_MATH_WHEELS) + message(FATAL_ERROR "Cannot use CUDA math wheels with CUDA < 12.0") + endif() + add_subdirectory(../../cpp cuvs-cpp EXCLUDE_FROM_ALL) + if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) + set(rpaths + "$ORIGIN/../nvidia/cublas/lib" + "$ORIGIN/../nvidia/cufft/lib" + "$ORIGIN/../nvidia/curand/lib" + "$ORIGIN/../nvidia/cusolver/lib" + "$ORIGIN/../nvidia/cusparse/lib" + "$ORIGIN/../nvidia/nvjitlink/lib" + ) + set_property(TARGET cuvs PROPERTY INSTALL_RPATH ${rpaths}) + set_property(TARGET cuvs_c PROPERTY INSTALL_RPATH ${rpaths}) + endif() + set(cython_lib_dir cuvs) install(TARGETS cuvs cuvs_c DESTINATION ${cython_lib_dir}) endif() diff --git a/python/cuvs/pyproject.toml b/python/cuvs/pyproject.toml index cc2aa6a7f..50db46910 100644 --- a/python/cuvs/pyproject.toml +++ b/python/cuvs/pyproject.toml @@ -33,6 +33,10 @@ requires-python = ">=3.9" dependencies = [ "cuda-python", "numpy>=1.23,<2.0a0", + "nvidia-cublas", + "nvidia-curand", + "nvidia-cusolver", + "nvidia-cusparse", "pylibraft==24.10.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 2c3e70bfffb295ca347ee4501faba3e16e08670b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 20 Aug 2024 10:05:33 -0400 Subject: [PATCH 2/6] Append rpath --- python/cuvs/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cuvs/CMakeLists.txt b/python/cuvs/CMakeLists.txt index 100b5e163..21e07e0fd 100644 --- a/python/cuvs/CMakeLists.txt +++ b/python/cuvs/CMakeLists.txt @@ -90,8 +90,8 @@ if(NOT cuvs_FOUND) "$ORIGIN/../nvidia/cusparse/lib" "$ORIGIN/../nvidia/nvjitlink/lib" ) - set_property(TARGET cuvs PROPERTY INSTALL_RPATH ${rpaths}) - set_property(TARGET cuvs_c PROPERTY INSTALL_RPATH ${rpaths}) + set_property(TARGET cuvs PROPERTY INSTALL_RPATH ${rpaths} APPEND) + set_property(TARGET cuvs_c PROPERTY INSTALL_RPATH ${rpaths} APPEND) endif() set(cython_lib_dir cuvs) From dd5d96085f9d16fa480a33ed7da6fce2a00effec Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 20 Aug 2024 15:20:00 -0400 Subject: [PATCH 3/6] Remove cufft from rpath --- python/cuvs/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cuvs/CMakeLists.txt b/python/cuvs/CMakeLists.txt index 21e07e0fd..7d2f8dcf9 100644 --- a/python/cuvs/CMakeLists.txt +++ b/python/cuvs/CMakeLists.txt @@ -84,7 +84,6 @@ if(NOT cuvs_FOUND) if(NOT CUDA_STATIC_MATH_LIBRARIES AND USE_CUDA_MATH_WHEELS) set(rpaths "$ORIGIN/../nvidia/cublas/lib" - "$ORIGIN/../nvidia/cufft/lib" "$ORIGIN/../nvidia/curand/lib" "$ORIGIN/../nvidia/cusolver/lib" "$ORIGIN/../nvidia/cusparse/lib" From 1eaf14be291536d32c078970b8f0108ba05721d1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 14:03:57 -0400 Subject: [PATCH 4/6] Add use_cuda_wheels matrix entry --- dependencies.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 702fc47b4..f97654340 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -78,7 +78,7 @@ files: extras: table: project includes: - - cuda + - cuda_wheels - run_py_cuvs py_test_py_cuvs: output: pyproject @@ -311,10 +311,13 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 + cuda_wheels: + specific: - output_types: pyproject matrices: - matrix: cuda: "12.*" + use_cuda_wheels: "true" packages: - nvidia-cublas-cu12 - nvidia-curand-cu12 @@ -322,7 +325,15 @@ dependencies: - nvidia-cusparse-cu12 - matrix: cuda: "11.*" + use_cuda_wheels: "true" + packages: + # if use_cuda_wheels=false is provided, do not add dependencies on any CUDA wheels + # (e.g. for DLFW and pip devcontainers) + - matrix: + use_cuda_wheels: "false" packages: + # if no matching matrix selectors passed, list the unsuffixed packages + # (just as a source of documentation, as this populates pyproject.toml in source control) - matrix: packages: - nvidia-cublas From 6f6f8a2f1a444c2875640de870215e23a2fabb88 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 14:30:48 -0400 Subject: [PATCH 5/6] Add use_cuda_wheels=true to pyproject.toml --- python/cuvs/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cuvs/pyproject.toml b/python/cuvs/pyproject.toml index 50db46910..0a3c30a7c 100644 --- a/python/cuvs/pyproject.toml +++ b/python/cuvs/pyproject.toml @@ -132,7 +132,7 @@ requires = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" -matrix-entry = "cuda_suffixed=true" +matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true" [tool.pytest.ini_options] filterwarnings = [ From d51641e12e97e260895f2caf83326404d4f785b6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Wed, 21 Aug 2024 17:03:37 -0400 Subject: [PATCH 6/6] Add note about CUDA 11's lack of wheels --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index f97654340..f39e30770 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -323,6 +323,7 @@ dependencies: - nvidia-curand-cu12 - nvidia-cusolver-cu12 - nvidia-cusparse-cu12 + # CUDA 11 does not provide wheels, so use the system libraries instead - matrix: cuda: "11.*" use_cuda_wheels: "true"