raft: Build CUDA 12 packages (#1388)

First attempt at building libraft with CUDA 12 Closes #1278 Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Corey J. Nolet (https://github.com/cjnolet) - Dante Gama Dessavre (https://github.com/dantegd) - Tamas Bela Feher (https://github.com/tfeher) - Bradley Dice (https://github.com/bdice) Approvers: - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) - https://github.com/jakirkham - Corey J. Nolet (https://github.com/cjnolet) URL: #1388
rapidsai · Jul 5, 2023 · 32b3dba · 32b3dba
1 parent cc1d1ed
commit 32b3dba
Show file tree

Hide file tree

Showing 18 changed files with 362 additions and 121 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -28,7 +28,7 @@ concurrency:
 jobs:
   cpp-build:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -37,7 +37,7 @@ jobs:
   python-build:
     needs: [cpp-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -46,30 +46,27 @@ jobs:
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       skip_upload_pkgs: libraft-template
   docs-build:
-    if: github.ref_type == 'branch'
+    if: github.ref_type == 'branch' && github.event_name == 'push'
     needs: python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
     with:
+      build_type: branch
+      node_type: "gpu-latest-1"
       arch: "amd64"
-      branch: ${{ inputs.branch }}
-      build_type: ${{ inputs.build_type || 'branch' }}
       container_image: "rapidsai/ci:latest"
-      date: ${{ inputs.date }}
-      node_type: "gpu-v100-latest-1"
       run_script: "ci/build_docs.sh"
-      sha: ${{ inputs.sha }}
   wheel-build-pylibraft:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -78,10 +75,11 @@ jobs:
       package-name: pylibraft
       package-dir: python/pylibraft
       skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
+      uses-setup-env-vars: false
   wheel-publish-pylibraft:
     needs: wheel-build-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -91,7 +89,7 @@ jobs:
   wheel-build-raft-dask:
     needs: wheel-publish-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -100,10 +98,11 @@ jobs:
       package-name: raft_dask
       package-dir: python/raft-dask
       skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF"
+      uses-setup-env-vars: false
   wheel-publish-raft-dask:
     needs: wheel-build-raft-dask
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -23,41 +23,41 @@ jobs:
       - wheel-build-raft-dask
       - wheel-tests-raft-dask
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120
   checks:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120
     with:
       enable_check_generated_files: false
   conda-cpp-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120
     with:
       build_type: pull-request
       node_type: cpu16
   conda-cpp-tests:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
     with:
       build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
     with:
       build_type: pull-request
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -67,7 +67,7 @@ jobs:
   wheel-build-pylibraft:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: pull-request
       package-name: pylibraft
@@ -76,7 +76,7 @@ jobs:
   wheel-tests-pylibraft:
     needs: wheel-build-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: pull-request
       package-name: pylibraft
@@ -85,7 +85,7 @@ jobs:
   wheel-build-raft-dask:
     needs: wheel-tests-pylibraft
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120
     with:
       build_type: pull-request
       package-name: raft_dask
@@ -95,7 +95,7 @@ jobs:
   wheel-tests-raft-dask:
     needs: wheel-build-raft-dask
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: pull-request
       package-name: raft_dask

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,33 +16,33 @@ on:
 jobs:
   conda-cpp-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-tests-pylibraft:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       package-name: pylibraft
-      test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test"
+      test-unittest: "python -m pytest -v ./python/pylibraft/pylibraft/test"
   wheel-tests-raft-dask:
     secrets: inherit
-    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
+    uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
@@ -51,4 +51,4 @@ jobs:
       package-name: raft_dask
       test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
       test-before-arm64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]"
-      test-unittest: "python -m pytest ./python/raft-dask/raft_dask/test"
+      test-unittest: "python -m pytest -v ./python/raft-dask/raft_dask/test"
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -13,8 +13,9 @@ dependencies:
 - clang=16.0.1
 - cmake>=3.23.1,!=3.25.0
 - cuda-profiler-api=11.8.86
-- cuda-python>=11.7.1,<12.0
-- cudatoolkit=11.8
+- cuda-python>=11.7.1,<12.0a0
+- cuda-version=11.8
+- cudatoolkit
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=0.29,<0.30

diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml
@@ -0,0 +1,56 @@
+# This file is generated by `rapids-dependency-file-generator`.
+# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+channels:
+- rapidsai
+- rapidsai-nightly
+- dask/label/dev
+- conda-forge
+- nvidia
+dependencies:
+- breathe
+- c-compiler
+- clang-tools=16.0.1
+- clang=16.0.1
+- cmake>=3.23.1,!=3.25.0
+- cuda-cudart-dev
+- cuda-profiler-api
+- cuda-python>=12.0,<13.0a0
+- cuda-version=12.0
+- cupy>=12.0.0
+- cxx-compiler
+- cython>=0.29,<0.30
+- dask-core>=2023.5.1
+- dask-cuda==23.8.*
+- dask>=2023.5.1
+- distributed>=2023.5.1
+- doxygen>=1.8.20
+- gcc_linux-64=11.*
+- gmock>=1.13.0
+- graphviz
+- gtest>=1.13.0
+- ipython
+- joblib>=0.11
+- libcublas-dev
+- libcurand-dev
+- libcusolver-dev
+- libcusparse-dev
+- nccl>=2.9.9
+- ninja
+- numba>=0.57
+- numpy>=1.21
+- numpydoc
+- pydata-sphinx-theme
+- pytest
+- pytest-cov
+- recommonmark
+- rmm==23.8.*
+- scikit-build>=0.13.1
+- scikit-learn
+- scipy
+- sphinx-copybutton
+- sphinx-markdown-tables
+- sysroot_linux-64==2.17
+- ucx-proc=*=gpu
+- ucx-py==0.33.*
+- ucx>=1.13.0
+name: all_cuda-120_arch-x86_64
diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml
@@ -12,7 +12,8 @@ dependencies:
 - clang=16.0.1
 - cmake>=3.23.1,!=3.25.0
 - cuda-profiler-api=11.8.86
-- cudatoolkit=11.8
+- cuda-version=11.8
+- cudatoolkit
 - cxx-compiler
 - cython>=0.29,<0.30
 - faiss-proc=*=cuda

diff --git a/conda/recipes/libraft/build_libraft_template.sh b/conda/recipes/libraft/build_libraft_template.sh
@@ -2,4 +2,4 @@
 # Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 # Just building template so we verify it uses libraft.so and fail if it doesn't build
-./build.sh template
+./build.sh template
diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml
@@ -5,6 +5,9 @@ cxx_compiler_version:
   - 11
 
 cuda_compiler:
+  - cuda-nvcc
+
+cuda11_compiler:
   - nvcc
 
 sysroot_version:
@@ -31,40 +34,40 @@ h5py_version:
 nlohmann_json_version:
   - ">=3.11.2"
 
-# The CTK libraries below are missing from the conda-forge::cudatoolkit
-# package. The "*_host_*" version specifiers correspond to `11.8` packages and the
-# "*_run_*" version specifiers correspond to `11.x` packages.
+# The CTK libraries below are missing from the conda-forge::cudatoolkit package
+# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
+# and the "*_run_*" version specifiers correspond to `11.x` packages.
 
-libcublas_host_version:
+cuda11_libcublas_host_version:
   - "=11.11.3.6"
 
-libcublas_run_version:
+cuda11_libcublas_run_version:
   - ">=11.5.2.43,<12.0.0"
 
-libcurand_host_version:
+cuda11_libcurand_host_version:
   - "=10.3.0.86"
 
-libcurand_run_version:
+cuda11_libcurand_run_version:
   - ">=10.2.5.43,<10.3.1"
 
-libcusolver_host_version:
+cuda11_libcusolver_host_version:
   - "=11.4.1.48"
 
-libcusolver_run_version:
+cuda11_libcusolver_run_version:
   - ">=11.2.0.43,<11.4.2"
 
-libcusparse_host_version:
+cuda11_libcusparse_host_version:
   - "=11.7.5.86"
 
-libcusparse_run_version:
+cuda11_libcusparse_run_version:
   - ">=11.6.0.43,<12.0.0"
 
 # `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all
 # architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the
 # "*_run_*" version specifiers correspond to `11.x` packages.
 
-cuda_profiler_api_host_version:
+cuda11_cuda_profiler_api_host_version:
   - "=11.8.86"
 
-cuda_profiler_api_run_version:
+cuda11_cuda_profiler_api_run_version:
   - ">=11.4.240,<12"