From 893ec13b4d5256bfd8f0f10d1054d2a827caf51f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 16:37:31 -0700 Subject: [PATCH 01/32] Switch to using CUDA 12 workflows --- .github/workflows/build.yaml | 16 ++++++++-------- .github/workflows/pr.yaml | 24 ++++++++++++------------ .github/workflows/test.yaml | 8 ++++---- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f1cb664fe0d..bfa50e69b1d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -56,7 +56,7 @@ jobs: if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 with: build_type: branch node_type: "gpu-latest-1" @@ -65,7 +65,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -86,7 +86,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -96,7 +96,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -117,7 +117,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5c1506ddec3..2447863ab86 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,41 +24,41 @@ jobs: - wheel-build-cugraph - wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 with: build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 with: build_type: pull-request node_type: "gpu-latest-1" @@ -68,7 +68,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 with: build_type: pull-request node_type: "gpu-latest-1" @@ -78,7 +78,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: pull-request package-name: pylibcugraph @@ -91,7 +91,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: pull-request package-name: pylibcugraph @@ -102,7 +102,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 with: build_type: pull-request package-name: cugraph @@ -116,7 +116,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: pull-request package-name: cugraph diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 57ab2c27618..693a45afd25 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} @@ -44,7 +44,7 @@ jobs: test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 with: build_type: nightly branch: ${{ inputs.branch }} From 067f95dbfdaa1ee19a22be213a66f2678dbc473c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 16:38:43 -0700 Subject: [PATCH 02/32] Add nvcc flags needed to work with broken nvidia packages --- conda/recipes/libcugraph/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh index 3119853df2a..96818fcb2fc 100644 --- a/conda/recipes/libcugraph/build.sh +++ b/conda/recipes/libcugraph/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory @@ -8,4 +8,5 @@ # openmpi build dependencies). The conda package does NOT include these test # binaries or extra dependencies, but these are built here for use in CI runs. ++export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libcugraph libcugraph_etl cpp-mgtests -n -v --allgpuarch From e0547bd56e7985db27bf172343233730ee304558 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 16:57:36 -0700 Subject: [PATCH 03/32] Update recipe for CUDA 12 --- .../libcugraph/conda_build_config.yaml | 71 ++++++++++++++++-- conda/recipes/libcugraph/meta.yaml | 74 +++++++++++++++---- 2 files changed, 124 insertions(+), 21 deletions(-) diff --git a/conda/recipes/libcugraph/conda_build_config.yaml b/conda/recipes/libcugraph/conda_build_config.yaml index 83a383236a4..b7929bd56b4 100644 --- a/conda/recipes/libcugraph/conda_build_config.yaml +++ b/conda/recipes/libcugraph/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc cmake_version: @@ -19,23 +22,77 @@ nccl_version: gtest_version: - "=1.10.0" -cuda_profiler_api_version: - - ">=11.8.86,<12" - sysroot_version: - "2.17" +# CUDA 11 versions + # The CTK libraries below are missing from the conda-forge::cudatoolkit # package. The "*_host_*" version specifiers correspond to `11.8` packages. -libcublas_host_version: +cuda11_libcublas_host_version: - "=11.11.3.6" -libcurand_host_version: +cuda11_libcublas_run_version: + - ">=11.5.2.43,<12.0.0" + +cuda11_libcurand_host_version: - "=10.3.0.86" -libcusolver_host_version: +cuda11_libcurand_run_version: + - ">=10.2.5.43,<10.3.1" + +cuda11_libcusolver_host_version: - "=11.4.1.48" -libcusparse_host_version: +cuda11_libcusolver_run_version: + - ">=11.2.0.43,<11.4.2" + +cuda11_libcusparse_host_version: - "=11.7.5.86" + +cuda11_libcusparse_run_version: + - ">=11.6.0.43,<12.0.0" + +# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all +# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the +# "*_run_*" version specifiers correspond to `11.x` packages. + +cuda11_cuda_profiler_api_host_version: + - "=11.8.86" + +cuda11_cuda_profiler_api_run_version: + - ">=11.4.240,<12" + + +# CUDA 12 versions + +libcublas_host_version: + - "=12.0.1.189" + +libcublas_run_version: + - ">=12.0.1.189" + +libcurand_host_version: + - "=10.3.1.50" + +libcurand_run_version: + - ">=10.3.1.50" + +libcusolver_host_version: + - "=11.4.2.57" + +libcusolver_run_version: + - ">=11.4.2.57" + +libcusparse_host_version: + - "=12.0.0.76" + +libcusparse_run_version: + - ">=12.0.0.76" + +cuda_profiler_api_host_version: + - "=12.0.76" + +cuda_profiler_api_run_version: + - ">=12.0.76" diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 5d53d2640b6..58875b1d06e 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -33,7 +33,14 @@ build: requirements: build: - {{ compiler('c') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} - {{ compiler('cuda') }} {{ cuda_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} + {% endif %} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja @@ -41,24 +48,39 @@ requirements: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - cuda-nvtx ={{ cuda_version }} - - cuda-profiler-api {{ cuda_profiler_api_version }} + {% if cuda_major == "11" %} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - cudatoolkit ={{ cuda_version }} - - doxygen {{ doxygen_version }} - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} + - libcublas {{ cuda11_libcublas_host_version }} + - libcublas-dev {{ cuda11_libcublas_host_version }} + - libcurand {{ cuda11_libcurand_host_version }} + - libcurand-dev {{ cuda11_libcurand_host_version }} + - libcusolver {{ cuda11_libcusolver_host_version }} + - libcusolver-dev {{ cuda11_libcusolver_host_version }} + - libcusparse {{ cuda11_libcusparse_host_version }} + - libcusparse-dev {{ cuda11_libcusparse_host_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_host_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} - libcublas {{ libcublas_host_version }} - libcublas-dev {{ libcublas_host_version }} - - libcudf ={{ minor_version }} - - libcugraphops ={{ minor_version }} - libcurand {{ libcurand_host_version }} - libcurand-dev {{ libcurand_host_version }} - libcusolver {{ libcusolver_host_version }} - libcusolver-dev {{ libcusolver_host_version }} - libcusparse {{ libcusparse_host_version }} - libcusparse-dev {{ libcusparse_host_version }} - - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - - librmm ={{ minor_version }} + {% endif %} + - doxygen {{ doxygen_version }} + - gmock {{ gtest_version }} + - gtest {{ gtest_version }} + - libcudf ={{ minor_version }}*=cuda{{ cuda_major }}* + - libcugraphops ={{ minor_version }}*=cuda{{ cuda_major }}* + - libraft ={{ minor_version }}*=cuda{{ cuda_major }}* + - libraft-headers ={{ minor_version }}*=cuda{{ cuda_major }}* + - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -75,11 +97,35 @@ outputs: build: - cmake {{ cmake_version }} run: - - cudatoolkit {{ cuda_spec }} - - libcugraphops ={{ minor_version }} - - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - - librmm ={{ minor_version }} + {% if cuda_major == "11" %} + - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} + - cudatoolkit ={{ cuda_version }} + - libcublas {{ cuda11_libcublas_run_version }} + - libcublas-dev {{ cuda11_libcublas_run_version }} + - libcurand {{ cuda11_libcurand_run_version }} + - libcurand-dev {{ cuda11_libcurand_run_version }} + - libcusolver {{ cuda11_libcusolver_run_version }} + - libcusolver-dev {{ cuda11_libcusolver_run_version }} + - libcusparse {{ cuda11_libcusparse_run_version }} + - libcusparse-dev {{ cuda11_libcusparse_run_version }} + {% else %} + - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-cudart {{ cuda_version }} + - cuda-cudart-dev {{ cuda_version }} + - cuda-cudart-static {{ cuda_version }} + - libcublas {{ libcublas_run_version }} + - libcublas-dev {{ libcublas_run_version }} + - libcurand {{ libcurand_run_version }} + - libcurand-dev {{ libcurand_run_version }} + - libcusolver {{ libcusolver_run_version }} + - libcusolver-dev {{ libcusolver_run_version }} + - libcusparse {{ libcusparse_run_version }} + - libcusparse-dev {{ libcusparse_run_version }} + {% endif %} + - libcugraphops ={{ minor_version }}*=cuda{{ cuda_major }}* + - libraft ={{ minor_version }}*=cuda{{ cuda_major }}* + - libraft-headers ={{ minor_version }}*=cuda{{ cuda_major }}* + - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* - nccl {{ nccl_version }} - ucx-proc=*=gpu about: From 6e69f8ba1beedb4bc0d3453997c431363b7d27e3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:02:13 -0700 Subject: [PATCH 04/32] Pull CUDA 12 builds of rmm, raft, and cugraph-ops --- ci/build_cpp.sh | 7 ++++++- ci/test_cpp.sh | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 3fd57f24c40..a1acb4f472d 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,10 +7,15 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + rapids-print-env rapids-logger "Begin cpp build" -rapids-mamba-retry mambabuild conda/recipes/libcugraph +rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" --channel "${LIBRAFT_CHANNEL}" --channel "${LIBCUGRAPHOPS_CHANNEL}" conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index a6c4cdb4a4f..56a43deef21 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,6 +18,11 @@ set +u conda activate test set -u +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" @@ -26,6 +31,9 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ + --channel "${LIBRMM_CHANNEL}" \ + --channel "${LIBRAFT_CHANNEL}" \ + --channel "${LIBCUGRAPHOPS_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" From d3260a481b9dee89218e1c3d486f94c86b223c25 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:03:35 -0700 Subject: [PATCH 05/32] Patch condarc --- ci/build_cpp.sh | 19 +++++++++++++++++++ ci/test_cpp.sh | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index a1acb4f472d..24e0d6b1bbf 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -12,6 +12,25 @@ LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_ LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then +cat << EOF > /opt/conda/.condarc +auto_update_conda: False +channels: + - rapidsai + - rapidsai-nightly + - dask/label/dev + - pytorch + - nvidia + - conda-forge +always_yes: true +number_channel_notices: 0 +conda_build: + set_build_id: false + root_dir: /tmp/conda-bld-workspace + output_folder: /tmp/conda-bld-output +EOF +fi + rapids-print-env rapids-logger "Begin cpp build" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 56a43deef21..1a7247aba2a 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -27,6 +27,25 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" +if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then +cat << EOF > /opt/conda/.condarc +auto_update_conda: False +channels: + - rapidsai + - rapidsai-nightly + - dask/label/dev + - pytorch + - nvidia + - conda-forge +always_yes: true +number_channel_notices: 0 +conda_build: + set_build_id: false + root_dir: /tmp/conda-bld-workspace + output_folder: /tmp/conda-bld-output +EOF +fi + rapids-print-env rapids-mamba-retry install \ From 0d014f7cfdf30b8201b37b7d1dd2e1a5bd632d6d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:06:10 -0700 Subject: [PATCH 06/32] Update dependencies.yaml --- dependencies.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index f411b458ee7..bfea3e58b69 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -60,6 +60,12 @@ dependencies: specific: - output_types: [conda] matrices: + - matrix: + cuda: "12.0" + packages: + - cuda-cudart=12.0 + - cuda-cudart-dev=12.0 + - cuda-cudart-static=12.0 - matrix: cuda: "11.8" packages: @@ -119,6 +125,16 @@ dependencies: cuda: "11.8" packages: - nvcc_linux-aarch64=11.8 + - matrix: + arch: x86_64 + cuda: "12.0" + packages: + - cuda-nvcc_linux-64=12.0 + - matrix: + arch: aarch64 + cuda: "12.0" + packages: + - cuda-nvcc_linux-aarch64=12.0 docs: common: - output_types: [conda] From 1f9922c436208f9c94d8359b006135e51819cac9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:17:19 -0700 Subject: [PATCH 07/32] Comment out unnecessary builds --- .github/workflows/pr.yaml | 184 +++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2447863ab86..25d103dbad8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,14 +15,14 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - - conda-notebook-tests - - conda-python-build - - conda-python-tests - - docs-build - - wheel-build-pylibcugraph - - wheel-tests-pylibcugraph - - wheel-build-cugraph - - wheel-tests-cugraph + #- conda-notebook-tests + #- conda-python-build + #- conda-python-tests + #- docs-build + #- wheel-build-pylibcugraph + #- wheel-tests-pylibcugraph + #- wheel-build-cugraph + #- wheel-tests-cugraph secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: @@ -43,87 +43,87 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 - with: - build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 - with: - build_type: pull-request - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - with: - build_type: pull-request - node_type: "gpu-latest-1" - arch: "amd64" - container_image: "rapidsai/ci:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - with: - build_type: pull-request - node_type: "gpu-latest-1" - arch: "amd64" - container_image: "rapidsai/ci:latest" - run_script: "ci/build_docs.sh" - wheel-build-pylibcugraph: - needs: checks - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - with: - build_type: pull-request - package-name: pylibcugraph - package-dir: python/pylibcugraph - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false - wheel-tests-pylibcugraph: - needs: wheel-build-pylibcugraph - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: - build_type: pull-request - package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" - wheel-build-cugraph: - needs: wheel-tests-pylibcugraph - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - with: - build_type: pull-request - package-name: cugraph - package-dir: python/cugraph - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false - wheel-tests-cugraph: - needs: wheel-build-cugraph - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - with: - build_type: pull-request - package-name: cugraph - # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - # On arm also need to install cupy from the specific site. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + #conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + # with: + # build_type: pull-request + #conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + # with: + # build_type: pull-request + #conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + # with: + # build_type: pull-request + # node_type: "gpu-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci:latest" + # run_script: "ci/test_notebooks.sh" + #docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + # with: + # build_type: pull-request + # node_type: "gpu-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci:latest" + # run_script: "ci/build_docs.sh" + #wheel-build-pylibcugraph: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: pylibcugraph + # package-dir: python/pylibcugraph + # extra-repo: rapidsai/cugraph-ops + # extra-repo-sha: branch-23.06 + # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + # uses-setup-env-vars: false + #wheel-tests-pylibcugraph: + # needs: wheel-build-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: pylibcugraph + # # On arm also need to install cupy from the specific webpage. + # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + # test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + # test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + #wheel-build-cugraph: + # needs: wheel-tests-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: cugraph + # package-dir: python/cugraph + # extra-repo: rapidsai/cugraph-ops + # extra-repo-sha: branch-23.06 + # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + # before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + # uses-setup-env-vars: false + #wheel-tests-cugraph: + # needs: wheel-build-cugraph + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + # with: + # build_type: pull-request + # package-name: cugraph + # # Always want to test against latest dask/distributed. + # test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + # # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. + # # On arm also need to install cupy from the specific site. + # test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + # test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests" + # test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 3dc5c0609730615f2a85ed80f4c606f17c58d7e0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:21:09 -0700 Subject: [PATCH 08/32] Also install CUDA 12 version of libcudf --- ci/build_cpp.sh | 4 +++- ci/test_cpp.sh | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 24e0d6b1bbf..03d5e0f13db 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,9 +9,11 @@ export CMAKE_GENERATOR=Ninja RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) + if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then cat << EOF > /opt/conda/.condarc auto_update_conda: False @@ -35,6 +37,6 @@ rapids-print-env rapids-logger "Begin cpp build" -rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" --channel "${LIBRAFT_CHANNEL}" --channel "${LIBCUGRAPHOPS_CHANNEL}" conda/recipes/libcugraph +rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" --channel "${LIBCUDF_CHANNEL}" --channel "${LIBRAFT_CHANNEL}" --channel "${LIBCUGRAPHOPS_CHANNEL}" conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 1a7247aba2a..5290381a06d 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -20,6 +20,7 @@ set -u RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) @@ -51,6 +52,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${LIBRMM_CHANNEL}" \ + --channel "${LIBCUDF_CHANNEL}" \ --channel "${LIBRAFT_CHANNEL}" \ --channel "${LIBCUGRAPHOPS_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests From b13edb6c77915faf0b5a0fd17f08b389306b175c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:23:04 -0700 Subject: [PATCH 09/32] Add missing build string filters --- conda/recipes/libcugraph/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 58875b1d06e..8fc6f8c9a70 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -147,8 +147,8 @@ outputs: run: - {{ pin_subpackage('libcugraph', exact=True) }} - cudatoolkit {{ cuda_spec }} - - libcudf ={{ minor_version }} - - librmm ={{ minor_version }} + - libcudf ={{ minor_version }}*=cuda{{ cuda_major }}* + - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* about: home: https://rapids.ai/ license: Apache-2.0 From 0af90b3f17518a94e19335fa21b4ba61f0eece1a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 17:48:40 -0700 Subject: [PATCH 10/32] Update to working version of cugraph-ops --- ci/build_cpp.sh | 2 +- ci/test_cpp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 03d5e0f13db..26aa8aef0ec 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -11,7 +11,7 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/4fe9810/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 5290381a06d..eb98258cbb5 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -22,7 +22,7 @@ RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/9ff8580/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) +LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/4fe9810/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ From 9baa3b7f77d650cb175a3f35d04b953c8c38c819 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 18:14:55 -0700 Subject: [PATCH 11/32] Fix typo --- conda/recipes/libcugraph/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh index 96818fcb2fc..6d9226efd5c 100644 --- a/conda/recipes/libcugraph/build.sh +++ b/conda/recipes/libcugraph/build.sh @@ -8,5 +8,5 @@ # openmpi build dependencies). The conda package does NOT include these test # binaries or extra dependencies, but these are built here for use in CI runs. -+export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers +export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libcugraph libcugraph_etl cpp-mgtests -n -v --allgpuarch From 8881bd6018149bc1813b9f46a92ec62a2d848563 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 6 Apr 2023 20:33:55 -0700 Subject: [PATCH 12/32] Fix remaining cudatoolkit dependencies and ignore_run_exports_from --- conda/recipes/libcugraph/meta.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 8fc6f8c9a70..fcc4f15086a 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -92,7 +92,11 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% else %} - {{ compiler('cuda') }} + {% endif %} requirements: build: - cmake {{ cmake_version }} @@ -140,13 +144,23 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% else %} - {{ compiler('cuda') }} + {% endif %} requirements: build: - cmake {{ cmake_version }} run: - {{ pin_subpackage('libcugraph', exact=True) }} + {% if cuda_major == "11" %} - cudatoolkit {{ cuda_spec }} + {% else %} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} + {% endif %} - libcudf ={{ minor_version }}*=cuda{{ cuda_major }}* - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* about: @@ -161,14 +175,24 @@ outputs: number: {{ GIT_DESCRIBE_NUMBER }} string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} ignore_run_exports_from: + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} + {% else %} - {{ compiler('cuda') }} + {% endif %} requirements: build: - cmake {{ cmake_version }} run: - {{ pin_subpackage('libcugraph_etl', exact=True) }} - {{ pin_subpackage('libcugraph', exact=True) }} + {% if cuda_major == "11" %} - cudatoolkit {{ cuda_spec }} + {% else %} + - cuda-cudart {{ cuda_spec }} + - cuda-cudart-dev {{ cuda_spec }} + - cuda-cudart-static {{ cuda_spec }} + {% endif %} - gmock {{ gtest_version }} - gtest {{ gtest_version }} about: From a2f304f0df86a09bb5368568c093e417b4fd05da Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 13 Jun 2023 15:13:57 -0700 Subject: [PATCH 13/32] Put back commented out jobs --- .github/workflows/pr.yaml | 184 +++++++++++++++++++------------------- 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 25d103dbad8..2447863ab86 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,14 +15,14 @@ jobs: - checks - conda-cpp-build - conda-cpp-tests - #- conda-notebook-tests - #- conda-python-build - #- conda-python-tests - #- docs-build - #- wheel-build-pylibcugraph - #- wheel-tests-pylibcugraph - #- wheel-build-cugraph - #- wheel-tests-cugraph + - conda-notebook-tests + - conda-python-build + - conda-python-tests + - docs-build + - wheel-build-pylibcugraph + - wheel-tests-pylibcugraph + - wheel-build-cugraph + - wheel-tests-cugraph secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 checks: @@ -43,87 +43,87 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 with: build_type: pull-request - #conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 - # with: - # build_type: pull-request - #conda-python-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 - # with: - # build_type: pull-request - #conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - # with: - # build_type: pull-request - # node_type: "gpu-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci:latest" - # run_script: "ci/test_notebooks.sh" - #docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 - # with: - # build_type: pull-request - # node_type: "gpu-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci:latest" - # run_script: "ci/build_docs.sh" - #wheel-build-pylibcugraph: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: pylibcugraph - # package-dir: python/pylibcugraph - # extra-repo: rapidsai/cugraph-ops - # extra-repo-sha: branch-23.06 - # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - # uses-setup-env-vars: false - #wheel-tests-pylibcugraph: - # needs: wheel-build-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: pylibcugraph - # # On arm also need to install cupy from the specific webpage. - # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - # test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - # test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" - #wheel-build-cugraph: - # needs: wheel-tests-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: cugraph - # package-dir: python/cugraph - # extra-repo: rapidsai/cugraph-ops - # extra-repo-sha: branch-23.06 - # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - # before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - # uses-setup-env-vars: false - #wheel-tests-cugraph: - # needs: wheel-build-cugraph - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 - # with: - # build_type: pull-request - # package-name: cugraph - # # Always want to test against latest dask/distributed. - # test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - # # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - # # On arm also need to install cupy from the specific site. - # test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" - # test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests" - # test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + with: + build_type: pull-request + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" + wheel-build-pylibcugraph: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + with: + build_type: pull-request + package-name: pylibcugraph + package-dir: python/pylibcugraph + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-23.06 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + uses-setup-env-vars: false + wheel-tests-pylibcugraph: + needs: wheel-build-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: + build_type: pull-request + package-name: pylibcugraph + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + wheel-build-cugraph: + needs: wheel-tests-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + with: + build_type: pull-request + package-name: cugraph + package-dir: python/cugraph + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-23.06 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + uses-setup-env-vars: false + wheel-tests-cugraph: + needs: wheel-build-cugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + with: + build_type: pull-request + package-name: cugraph + # Always want to test against latest dask/distributed. + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. + # On arm also need to install cupy from the specific site. + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From f3369713b7b1f83c9e9643422d690b382b8fbaf4 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 13:27:49 -0500 Subject: [PATCH 14/32] Minor change to trigger CI. --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2702dcad0b5..ba34a575050 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -82,7 +82,7 @@ jobs: # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + # the clone as-is. extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.06 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY @@ -113,7 +113,7 @@ jobs: # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. + # the clone as-is. extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.06 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From d1c225835caba01fb4b4d813ec1e6898c4bb281d Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 14:08:06 -0500 Subject: [PATCH 15/32] Removes matrix filters to allow CUDA 12 runs, reverts specification of temporary cuda-120 branches to workflow files now that they've been merged to branch-23.08. --- .github/workflows/build.yaml | 18 ++++++++---------- .github/workflows/pr.yaml | 32 ++++++++++++++------------------ .github/workflows/test.yaml | 10 ++++------ 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ba34a575050..23d9c5a0bec 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,9 +28,8 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} @@ -38,9 +37,8 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} date: ${{ inputs.date }} @@ -48,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -58,7 +56,7 @@ jobs: if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: build_type: branch node_type: "gpu-latest-1" @@ -71,7 +69,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -92,7 +90,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -102,7 +100,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -123,7 +121,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 12d49520d35..44cd6776d24 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,65 +24,61 @@ jobs: - wheel-build-cugraph - wheel-tests-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.08 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.08 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: pull-request conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: build_type: pull-request node_type: "gpu-latest-1" arch: "amd64" - container_image: "rapidsai/ci:cuda11.8.0-ubuntu22.04-py3.10" + container_image: "rapidsai/ci:latest" run_script: "ci/test_notebooks.sh" docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: build_type: pull-request node_type: "gpu-latest-1" arch: "amd64" - container_image: "rapidsai/ci:cuda11.8.0-ubuntu22.04-py3.10" + container_image: "rapidsai/ci:latest" run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 with: build_type: pull-request package-name: pylibcugraph @@ -95,7 +91,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 with: build_type: pull-request package-name: pylibcugraph @@ -106,7 +102,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 with: build_type: pull-request package-name: cugraph @@ -120,7 +116,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 with: build_type: pull-request package-name: cugraph diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d4ba92b0dae..d371b24746f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,25 +16,23 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.08 with: - matrix_filter: map(select(.CUDA_VER | startswith("11"))) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} @@ -46,7 +44,7 @@ jobs: test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 with: build_type: nightly branch: ${{ inputs.branch }} From 26535d6e4e4506b4f75b35f4f426cc9cb0988d2d Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 15:03:54 -0500 Subject: [PATCH 16/32] Updates extra-repo-sha from 23.06 to 23.08 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 23d9c5a0bec..ef9eb750026 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -113,7 +113,7 @@ jobs: # the CMake variables in get_cumlprims_mg.cmake since CMake will just use # the clone as-is. extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 + extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" From a6d0cb0e3785d96a548785181b728daa4c141952 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 18 Jul 2023 15:51:53 -0700 Subject: [PATCH 17/32] Fix GH workflows --- .github/workflows/build.yaml | 18 ++++++++---------- .github/workflows/pr.yaml | 23 +++++++++-------------- .github/workflows/test.yaml | 9 +++------ 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ef9eb750026..de9caa0fabe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,13 +53,11 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} docs-build: - if: github.ref_type == 'branch' && github.event_name == 'push' + if: github.ref_type == 'branch' needs: python-build secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: - build_type: branch - node_type: "gpu-latest-1" arch: "amd64" branch: ${{ inputs.branch }} build_type: ${{ inputs.build_type || 'branch' }} @@ -67,6 +65,7 @@ jobs: date: ${{ inputs.date }} node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" + sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08 @@ -80,13 +79,12 @@ jobs: # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as-is. + # the clone as is. extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 + extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -111,13 +109,13 @@ jobs: # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as-is. + # the clone as is. extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 44cd6776d24..629029b806a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,10 +84,9 @@ jobs: package-name: pylibcugraph package-dir: python/pylibcugraph extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 + extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -95,9 +94,7 @@ jobs: with: build_type: pull-request package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph @@ -108,11 +105,10 @@ jobs: package-name: cugraph package-dir: python/cugraph extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.06 + extra-repo-sha: branch-23.08 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - uses-setup-env-vars: false + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit @@ -121,9 +117,8 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests. - # On arm also need to install cupy from the specific site. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -v -m sg ./python/cugraph/cugraph/tests" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d371b24746f..d697b8f1649 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,9 +39,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08 @@ -53,6 +51,5 @@ jobs: package-name: cugraph # Always want to test against latest dask/distributed. test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -v -m sg ./python/cugraph/cugraph/tests" + test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests" From 52ae59c6a6db91f61884eb277a1c6a8b4060cdce Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 19:10:53 -0500 Subject: [PATCH 18/32] Applies updates from code review: shell script/yaml updates to use latest CUDA 12 packaging changes from conda-forge --- ci/build_cpp.sh | 28 +------ ci/test_cpp.sh | 29 ------- conda/recipes/libcugraph/build.sh | 1 - .../libcugraph/conda_build_config.yaml | 38 +-------- conda/recipes/libcugraph/meta.yaml | 83 ++++++++----------- dependencies.yaml | 31 ++++--- 6 files changed, 50 insertions(+), 160 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 26aa8aef0ec..3fd57f24c40 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -7,36 +7,10 @@ source rapids-env-update export CMAKE_GENERATOR=Ninja -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/4fe9810/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - - -if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then -cat << EOF > /opt/conda/.condarc -auto_update_conda: False -channels: - - rapidsai - - rapidsai-nightly - - dask/label/dev - - pytorch - - nvidia - - conda-forge -always_yes: true -number_channel_notices: 0 -conda_build: - set_build_id: false - root_dir: /tmp/conda-bld-workspace - output_folder: /tmp/conda-bld-output -EOF -fi - rapids-print-env rapids-logger "Begin cpp build" -rapids-mamba-retry mambabuild --channel "${LIBRMM_CHANNEL}" --channel "${LIBCUDF_CHANNEL}" --channel "${LIBRAFT_CHANNEL}" --channel "${LIBCUGRAPHOPS_CHANNEL}" conda/recipes/libcugraph +rapids-mamba-retry mambabuild conda/recipes/libcugraph rapids-upload-conda-to-s3 cpp diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 78832a75d76..f02ac748f18 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -18,43 +18,14 @@ set +u conda activate test set -u -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -LIBRMM_CHANNEL=$(rapids-get-artifact ci/rmm/pull-request/1223/72e0c74/rmm_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUDF_CHANNEL=$(rapids-get-artifact ci/cudf/pull-request/12922/28442d2/cudf_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBRAFT_CHANNEL=$(rapids-get-artifact ci/raft/pull-request/1388/7bddaee/raft_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) -LIBCUGRAPHOPS_CHANNEL=$(rapids-get-artifact ci/cugraph-ops/pull-request/464/4fe9810/cugraph-ops_conda_cpp_cuda${RAPIDS_CUDA_MAJOR}_$(arch).tar.gz) - CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -if [ "${RAPIDS_CUDA_MAJOR}" == 12 ]; then -cat << EOF > /opt/conda/.condarc -auto_update_conda: False -channels: - - rapidsai - - rapidsai-nightly - - dask/label/dev - - pytorch - - nvidia - - conda-forge -always_yes: true -number_channel_notices: 0 -conda_build: - set_build_id: false - root_dir: /tmp/conda-bld-workspace - output_folder: /tmp/conda-bld-output -EOF -fi - rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - --channel "${LIBRMM_CHANNEL}" \ - --channel "${LIBCUDF_CHANNEL}" \ - --channel "${LIBRAFT_CHANNEL}" \ - --channel "${LIBCUGRAPHOPS_CHANNEL}" \ libcugraph libcugraph_etl libcugraph-tests rapids-logger "Check GPU usage" diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh index 6d9226efd5c..2236aad41d4 100644 --- a/conda/recipes/libcugraph/build.sh +++ b/conda/recipes/libcugraph/build.sh @@ -8,5 +8,4 @@ # openmpi build dependencies). The conda package does NOT include these test # binaries or extra dependencies, but these are built here for use in CI runs. -export NVCC_PREPEND_FLAGS="${NVCC_PREPEND_FLAGS} -ccbin ${CXX}" # Needed for CUDA 12 nvidia channel compilers ./build.sh libcugraph libcugraph_etl cpp-mgtests -n -v --allgpuarch diff --git a/conda/recipes/libcugraph/conda_build_config.yaml b/conda/recipes/libcugraph/conda_build_config.yaml index 143e0423eee..3acd0166a67 100644 --- a/conda/recipes/libcugraph/conda_build_config.yaml +++ b/conda/recipes/libcugraph/conda_build_config.yaml @@ -25,10 +25,9 @@ gtest_version: sysroot_version: - "2.17" -# CUDA 11 versions - # The CTK libraries below are missing from the conda-forge::cudatoolkit -# package. The "*_host_*" version specifiers correspond to `11.8` packages. +# package. The "*_host_*" version specifiers correspond to `11.8` packages +# and the "*_run_*" version specifiers correspond to `11.x` packages. cuda11_libcublas_host_version: - "=11.11.3.6" @@ -63,36 +62,3 @@ cuda11_cuda_profiler_api_host_version: cuda11_cuda_profiler_api_run_version: - ">=11.4.240,<12" - - -# CUDA 12 versions - -libcublas_host_version: - - "=12.0.1.189" - -libcublas_run_version: - - ">=12.0.1.189" - -libcurand_host_version: - - "=10.3.1.50" - -libcurand_run_version: - - ">=10.3.1.50" - -libcusolver_host_version: - - "=11.4.2.57" - -libcusolver_run_version: - - ">=11.4.2.57" - -libcusparse_host_version: - - "=12.0.0.76" - -libcusparse_run_version: - - ">=12.0.0.76" - -cuda_profiler_api_host_version: - - "=12.0.76" - -cuda_profiler_api_run_version: - - ">=12.0.76" diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 8b475824d16..5fef8929cc7 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -37,21 +37,19 @@ requirements: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} {{ cuda_version }} {% else %} - - {{ compiler('cuda') }} {{ cuda_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} + - {{ compiler('cuda') }} {% endif %} + - cuda-version ={{ cuda_version }} - {{ compiler('cxx') }} - cmake {{ cmake_version }} - ninja - openmpi # Required for building cpp-mgtests (multi-GPU tests) - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cuda-nvtx ={{ cuda_version }} {% if cuda_major == "11" %} + - cudatoolkit + - cuda-nvtx ={{ cuda_version }} - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - - cudatoolkit ={{ cuda_version }} - libcublas {{ cuda11_libcublas_host_version }} - libcublas-dev {{ cuda11_libcublas_host_version }} - libcurand {{ cuda11_libcurand_host_version }} @@ -61,27 +59,22 @@ requirements: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_host_version }} - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} + - cuda-profiler-api + - cuda-cudart-dev + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev {% endif %} + - cuda-version ={{ cuda_version }} - doxygen {{ doxygen_version }} - gmock {{ gtest_version }} - gtest {{ gtest_version }} - - libcudf ={{ minor_version }}*=cuda{{ cuda_major }}* - - libcugraphops ={{ minor_version }}*=cuda{{ cuda_major }}* - - libraft ={{ minor_version }}*=cuda{{ cuda_major }}* - - libraft-headers ={{ minor_version }}*=cuda{{ cuda_major }}* - - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* + - libcudf ={{ minor_version }} + - libcugraphops ={{ minor_version }} + - libraft ={{ minor_version }} + - libraft-headers ={{ minor_version }} + - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -95,42 +88,35 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} {% endif %} requirements: build: - cmake {{ cmake_version }} + host: + - cuda-version ={{ cuda_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - cudatoolkit ={{ cuda_version }} - libcublas {{ cuda11_libcublas_run_version }} - - libcublas-dev {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - - libcurand-dev {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} - - libcusolver-dev {{ cuda11_libcusolver_run_version }} - libcusparse {{ cuda11_libcusparse_run_version }} - - libcusparse-dev {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api {{ cuda_profiler_api_run_version }} - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} - - libcublas {{ libcublas_run_version }} - - libcublas-dev {{ libcublas_run_version }} - - libcurand {{ libcurand_run_version }} - - libcurand-dev {{ libcurand_run_version }} - - libcusolver {{ libcusolver_run_version }} - - libcusolver-dev {{ libcusolver_run_version }} - - libcusparse {{ libcusparse_run_version }} - - libcusparse-dev {{ libcusparse_run_version }} + - libcublas + - libcurand + - libcusolver + - libcusparse {% endif %} - - libcugraphops ={{ minor_version }}*=cuda{{ cuda_major }}* - - libraft ={{ minor_version }}*=cuda{{ cuda_major }}* - - libraft-headers ={{ minor_version }}*=cuda{{ cuda_major }}* - - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* + - libcugraphops ={{ minor_version }} + - libraft ={{ minor_version }} + - libraft-headers ={{ minor_version }} + - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu about: @@ -147,23 +133,20 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} {% endif %} requirements: build: - cmake {{ cmake_version }} + host: + - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libcugraph', exact=True) }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - - cudatoolkit {{ cuda_spec }} - {% else %} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cudatoolkit {% endif %} - - libcudf ={{ minor_version }}*=cuda{{ cuda_major }}* - - librmm ={{ minor_version }}*=cuda{{ cuda_major }}* + - libcudf ={{ minor_version }} + - librmm ={{ minor_version }} about: home: https://rapids.ai/ license: Apache-2.0 diff --git a/dependencies.yaml b/dependencies.yaml index 3ecdafda65c..441fd2ca9df 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -198,25 +198,27 @@ dependencies: - matrix: cuda: "12.0" packages: - - cuda-cudart=12.0 - - cuda-cudart-dev=12.0 - - cuda-cudart-static=12.0 + - cuda-version=12.0 - matrix: cuda: "11.8" packages: - - cudatoolkit=11.8 + - cuda-version=11.8 + - cudatoolkit - matrix: cuda: "11.5" packages: - - cudatoolkit=11.5 + - cuda-version=11.5 + - cudatoolkit - matrix: cuda: "11.4" packages: - - cudatoolkit=11.4 + - cuda-version=11.4 + - cudatoolkit - matrix: cuda: "11.2" packages: - - cudatoolkit=11.2 + - cuda-version=11.2 + - cudatoolkit common_build: common: - output_types: [conda, pyproject] @@ -259,16 +261,11 @@ dependencies: cuda: "11.8" packages: - nvcc_linux-aarch64=11.8 - - matrix: - arch: x86_64 - cuda: "12.0" - packages: - - cuda-nvcc_linux-64=12.0 - - matrix: - arch: aarch64 - cuda: "12.0" - packages: - - cuda-nvcc_linux-aarch64=12.0 + - matrix: + cuda: "12.0" + packages: + - cuda-version =12.0 + - cuda-nvcc docs: common: - output_types: [conda] From 52498d3a4085537a88a9be7134cf3393e3e8027a Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 19:20:12 -0500 Subject: [PATCH 19/32] Replaces TABs with " " --- dependencies.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 441fd2ca9df..54f24d3d011 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -203,22 +203,22 @@ dependencies: cuda: "11.8" packages: - cuda-version=11.8 - - cudatoolkit + - cudatoolkit - matrix: cuda: "11.5" packages: - cuda-version=11.5 - - cudatoolkit + - cudatoolkit - matrix: cuda: "11.4" packages: - cuda-version=11.4 - - cudatoolkit + - cudatoolkit - matrix: cuda: "11.2" packages: - cuda-version=11.2 - - cudatoolkit + - cudatoolkit common_build: common: - output_types: [conda, pyproject] @@ -261,11 +261,11 @@ dependencies: cuda: "11.8" packages: - nvcc_linux-aarch64=11.8 - - matrix: - cuda: "12.0" - packages: - - cuda-version =12.0 - - cuda-nvcc + - matrix: + cuda: "12.0" + packages: + - cuda-version =12.0 + - cuda-nvcc docs: common: - output_types: [conda] From 368208be280aeed01ddfcb94089a49d1d107151d Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 19:53:06 -0500 Subject: [PATCH 20/32] Adds updates from running rapids-dependency-file-generator. --- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 9c428ef9d07..2302ed146b8 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -12,7 +12,8 @@ dependencies: - aiohttp - c-compiler - cmake>=3.26.4 -- cudatoolkit=11.8 +- cuda-version=11.8 +- cudatoolkit - cudf==23.8.* - cupy>=12.0.0 - cxx-compiler From cdc718d11b38da83dac24f1555f1eb883573c0f6 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 20:03:01 -0500 Subject: [PATCH 21/32] Replaces TABs with 8 spaces. --- conda/recipes/libcugraph/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 5fef8929cc7..b3fd7898185 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -141,7 +141,7 @@ outputs: - cuda-version ={{ cuda_version }} run: - {{ pin_subpackage('libcugraph', exact=True) }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit {% endif %} From 28861623245f3cb3714d42c0e119f3906d8140bc Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Tue, 18 Jul 2023 20:24:09 -0500 Subject: [PATCH 22/32] Removes unneeded compiler spec. --- conda/recipes/libcugraph/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index b3fd7898185..6907a69518c 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -161,8 +161,6 @@ outputs: ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} - {% else %} - - {{ compiler('cuda') }} {% endif %} requirements: build: From 43863ea8ae4203f9d3541e3c94a5577ae6d691db Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 19 Jul 2023 08:08:58 -0500 Subject: [PATCH 23/32] Removes undefined cuda_profiler_api_run_version from CUDA 12 run requirements. --- conda/recipes/libcugraph/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 6907a69518c..2cb360a22f4 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -104,7 +104,7 @@ outputs: - libcusolver {{ cuda11_libcusolver_run_version }} - libcusparse {{ cuda11_libcusparse_run_version }} {% else %} - - cuda-profiler-api {{ cuda_profiler_api_run_version }} + - cuda-profiler-api - cuda-cudart {{ cuda_version }} - cuda-cudart-dev {{ cuda_version }} - cuda-cudart-static {{ cuda_version }} From d2d0c04d8856985dbfe784c21eae3771c5235e7e Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 19 Jul 2023 10:09:33 -0500 Subject: [PATCH 24/32] Applies suggestions from code review: changes node_type, removes unneeded static and dev dependencies, removes unneeded cuda_spec pins, adds missing use of cuda-version meta-package. --- .github/workflows/pr.yaml | 4 ++-- conda/recipes/libcugraph/build.sh | 2 +- conda/recipes/libcugraph/meta.yaml | 16 ++++++---------- dependencies.yaml | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 629029b806a..4d52cd26de4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -61,7 +61,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: build_type: pull-request - node_type: "gpu-latest-1" + node_type: "gpu-v100-latest-1" arch: "amd64" container_image: "rapidsai/ci:latest" run_script: "ci/test_notebooks.sh" @@ -71,7 +71,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.08 with: build_type: pull-request - node_type: "gpu-latest-1" + node_type: "gpu-v100-latest-1" arch: "amd64" container_image: "rapidsai/ci:latest" run_script: "ci/build_docs.sh" diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh index 2236aad41d4..3119853df2a 100644 --- a/conda/recipes/libcugraph/build.sh +++ b/conda/recipes/libcugraph/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2019-2023, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 2cb360a22f4..9ce2a4b38ff 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -98,16 +98,13 @@ outputs: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - cudatoolkit ={{ cuda_version }} + - cudatoolkit - libcublas {{ cuda11_libcublas_run_version }} - libcurand {{ cuda11_libcurand_run_version }} - libcusolver {{ cuda11_libcusolver_run_version }} - libcusparse {{ cuda11_libcusparse_run_version }} {% else %} - cuda-profiler-api - - cuda-cudart {{ cuda_version }} - - cuda-cudart-dev {{ cuda_version }} - - cuda-cudart-static {{ cuda_version }} - libcublas - libcurand - libcusolver @@ -140,8 +137,8 @@ outputs: host: - cuda-version ={{ cuda_version }} run: - - {{ pin_subpackage('libcugraph', exact=True) }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + - {{ pin_subpackage('libcugraph', exact=True) }} {% if cuda_major == "11" %} - cudatoolkit {% endif %} @@ -165,15 +162,14 @@ outputs: requirements: build: - cmake {{ cmake_version }} + host: + - cuda-version ={{ cuda_version }} run: + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {{ pin_subpackage('libcugraph_etl', exact=True) }} - {{ pin_subpackage('libcugraph', exact=True) }} {% if cuda_major == "11" %} - - cudatoolkit {{ cuda_spec }} - {% else %} - - cuda-cudart {{ cuda_spec }} - - cuda-cudart-dev {{ cuda_spec }} - - cuda-cudart-static {{ cuda_spec }} + - cudatoolkit {% endif %} - gmock {{ gtest_version }} - gtest {{ gtest_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 54f24d3d011..c067b623c09 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -264,7 +264,7 @@ dependencies: - matrix: cuda: "12.0" packages: - - cuda-version =12.0 + - cuda-version=12.0 - cuda-nvcc docs: common: From 59a24d8cae6ab9ca11a80156517a34bc072339f7 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Wed, 19 Jul 2023 12:22:32 -0500 Subject: [PATCH 25/32] Adds cuda-nvtx-dev to host dependencies, adds cuda 12 to matrix for conda dev env generation. --- conda/recipes/libcugraph/meta.yaml | 1 + dependencies.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 9ce2a4b38ff..2119a44eef7 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -59,6 +59,7 @@ requirements: - libcusparse {{ cuda11_libcusparse_host_version }} - libcusparse-dev {{ cuda11_libcusparse_host_version }} {% else %} + - cuda-nvtx-dev - cuda-profiler-api - cuda-cudart-dev - libcublas-dev diff --git a/dependencies.yaml b/dependencies.yaml index c067b623c09..e1b03793246 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: [conda] matrix: - cuda: ["11.8"] + cuda: ["11.8", "12.0"] arch: [x86_64] includes: - checks From 807dda924bb4d1fc853b4fe510ae212eb2f466a3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 14:13:39 -0500 Subject: [PATCH 26/32] Only build cugraph-service, cugraph-dgl, cugraph-pyg on CUDA 11. --- ci/build_python.sh | 54 ++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 517dda726ee..5125e86d53a 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -26,28 +26,36 @@ rapids-mamba-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph -rapids-mamba-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cugraph-service - -rapids-mamba-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel pyg \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-pyg - -rapids-mamba-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel dglteam \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-dgl +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" + +if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then + # Only one CUDA configuration is needed, so we choose CUDA 11 arbitrarily. + # Nothing in the cugraph-service packages is CUDA-specific. + rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + conda/recipes/cugraph-service + + # Only CUDA 11 is supported right now due to PyTorch requirement. + rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --channel pyg \ + --channel pytorch \ + --channel pytorch-nightly \ + conda/recipes/cugraph-pyg + + # Only CUDA 11 is supported right now due to PyTorch requirement. + rapids-mamba-retry mambabuild \ + --no-test \ + --channel "${CPP_CHANNEL}" \ + --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ + --channel dglteam \ + --channel pytorch \ + --channel pytorch-nightly \ + conda/recipes/cugraph-dgl +fi rapids-upload-conda-to-s3 python From 12ed8d12607307343f0ca956a9ee96a434eca567 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 14:13:51 -0500 Subject: [PATCH 27/32] Remove extraneous cuda_spec declaration. --- conda/recipes/libcugraph/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 2119a44eef7..d52d81366d7 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -4,7 +4,6 @@ {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} -{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 {% set date_string = environ['RAPIDS_DATE_STRING'] %} package: From 6a5f7994e657d1bfc520001c4836834a6e6cf781 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 14:14:00 -0500 Subject: [PATCH 28/32] Update pylibcugraph. --- .../pylibcugraph/conda_build_config.yaml | 20 +++-------- conda/recipes/pylibcugraph/meta.yaml | 34 ++++++++----------- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/conda/recipes/pylibcugraph/conda_build_config.yaml b/conda/recipes/pylibcugraph/conda_build_config.yaml index e3e18d4620b..611e8300953 100644 --- a/conda/recipes/pylibcugraph/conda_build_config.yaml +++ b/conda/recipes/pylibcugraph/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc cmake_version: @@ -14,19 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.33.*" - -# The CTK libraries below are missing from the conda-forge::cudatoolkit -# package. The "*_host_*" version specifiers correspond to `11.8` packages. - -libcublas_host_version: - - "=11.11.3.6" - -libcurand_host_version: - - "=10.3.0.86" - -libcusolver_host_version: - - "=11.4.1.48" - -libcusparse_host_version: - - "=11.7.5.86" + - "0.33.*" \ No newline at end of file diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index de031a6fe94..d92d8a5b5f2 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -39,41 +39,37 @@ build: requirements: build: - {{ compiler('c') }} - - {{ compiler('cuda') }} {{ cuda_version }} - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} + {% endif %} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit ={{ cuda_version }} - - cudf ={{ minor_version }} + - cuda-version ={{ cuda_version }} + {% if cuda_major == "11" %} + - cudatoolkit + {% endif %} - cython >=0.29,<0.30 - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - libcugraph ={{ version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} - - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - - pylibraft ={{ minor_version}} + - pylibraft ={{ minor_version }} - python - - rmm ={{ minor_version }} - scikit-build >=0.13.1 - setuptools - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} + - cudatoolkit + {% endif %} - libcugraph ={{ version }} - python tests: requirements: - - cudatoolkit ={{ cuda_version }} + - cuda-version ={{ cuda_version }} imports: - pylibcugraph From 77f0af340d526ab0d84f2b967a33fdc7a2fc8097 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 14:15:59 -0500 Subject: [PATCH 29/32] Add cuda-version. --- conda/recipes/pylibcugraph/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index d92d8a5b5f2..aa82c20ad44 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -45,6 +45,7 @@ requirements: {% else %} - {{ compiler('cuda') }} {% endif %} + - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} From f01835f40499e155eb71427e685a8090946499a2 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 14:39:45 -0500 Subject: [PATCH 30/32] Update cugraph. --- conda/recipes/cugraph/conda_build_config.yaml | 20 ++-------- conda/recipes/cugraph/meta.yaml | 40 +++++++++---------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/conda/recipes/cugraph/conda_build_config.yaml b/conda/recipes/cugraph/conda_build_config.yaml index e3e18d4620b..611e8300953 100644 --- a/conda/recipes/cugraph/conda_build_config.yaml +++ b/conda/recipes/cugraph/conda_build_config.yaml @@ -5,6 +5,9 @@ cxx_compiler_version: - 11 cuda_compiler: + - cuda-nvcc + +cuda11_compiler: - nvcc cmake_version: @@ -14,19 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.33.*" - -# The CTK libraries below are missing from the conda-forge::cudatoolkit -# package. The "*_host_*" version specifiers correspond to `11.8` packages. - -libcublas_host_version: - - "=11.11.3.6" - -libcurand_host_version: - - "=10.3.0.86" - -libcusolver_host_version: - - "=11.4.1.48" - -libcusparse_host_version: - - "=11.7.5.86" + - "0.33.*" \ No newline at end of file diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml index 7f2dc542538..e2b9d38c181 100644 --- a/conda/recipes/cugraph/meta.yaml +++ b/conda/recipes/cugraph/meta.yaml @@ -39,36 +39,38 @@ build: requirements: build: - {{ compiler('c') }} - - {{ compiler('cuda') }} {{ cuda_version }} - {{ compiler('cxx') }} + {% if cuda_major == "11" %} + - {{ compiler('cuda11') }} {{ cuda_version }} + {% else %} + - {{ compiler('cuda') }} + {% endif %} + - cuda-version ={{ cuda_version }} - cmake {{ cmake_version }} - ninja - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - cudatoolkit ={{ cuda_version }} + - cuda-version ={{ cuda_version }} + {% if cuda_major == "11" %} + - cudatoolkit + {% endif %} - cudf ={{ minor_version }} - cython >=0.29,<0.30 - - libcublas {{ libcublas_host_version }} - - libcublas-dev {{ libcublas_host_version }} - libcugraph ={{ version }} - - libcurand {{ libcurand_host_version }} - - libcurand-dev {{ libcurand_host_version }} - - libcusolver {{ libcusolver_host_version }} - - libcusolver-dev {{ libcusolver_host_version }} - - libcusparse {{ libcusparse_host_version }} - - libcusparse-dev {{ libcusparse_host_version }} - - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - - pylibraft ={{ minor_version}} + - pylibraft ={{ minor_version }} - python - raft-dask ={{ minor_version }} + - rmm ={{ minor_version }} - scikit-build >=0.13.1 - setuptools - - ucx-proc=*=gpu - - ucx-py {{ ucx_py_version }} run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - cuda-python >=11.7.1,<12.0 + - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} + {% if cuda_major == "11" %} + - cudatoolkit + - cuda-python >=11.7.1,<12.0a0 + {% else %} + - cuda-python >=12.0,<13.0a0 + {% endif %} - cudf ={{ minor_version }} - cupy >=12.0.0 - dask-cuda ={{ minor_version }} @@ -77,8 +79,6 @@ requirements: - dask-core >=2023.5.1 - distributed >=2023.5.1 - libcugraph ={{ version }} - - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - pylibcugraph ={{ version }} - pylibraft ={{ minor_version }} - python @@ -88,7 +88,7 @@ requirements: tests: requirements: - - cudatoolkit ={{ cuda_version }} + - cuda-version ={{ cuda_version }} imports: - cugraph From 90ca8e563ab309a8a62fd13ffe70292a2f23265c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 16:10:10 -0500 Subject: [PATCH 31/32] Add workaround for pylibraft's missing dependency on rmm. --- conda/recipes/pylibcugraph/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index aa82c20ad44..a5731dfdf36 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -58,6 +58,8 @@ requirements: - libcugraph ={{ version }} - pylibraft ={{ minor_version }} - python + # TODO: Remove rmm once https://github.com/rapidsai/raft/pull/1656 is merged (fixes pylibraft use of rmm) + - rmm ={{ minor_version }} - scikit-build >=0.13.1 - setuptools run: From 048dc4115c5532a41b504c394e53c2576f3ccabc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 19 Jul 2023 17:49:10 -0500 Subject: [PATCH 32/32] Remove raft workaround. --- conda/recipes/cugraph/conda_build_config.yaml | 2 +- conda/recipes/pylibcugraph/conda_build_config.yaml | 2 +- conda/recipes/pylibcugraph/meta.yaml | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/conda/recipes/cugraph/conda_build_config.yaml b/conda/recipes/cugraph/conda_build_config.yaml index 611e8300953..4530a4c942d 100644 --- a/conda/recipes/cugraph/conda_build_config.yaml +++ b/conda/recipes/cugraph/conda_build_config.yaml @@ -17,4 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.33.*" \ No newline at end of file + - "0.33.*" diff --git a/conda/recipes/pylibcugraph/conda_build_config.yaml b/conda/recipes/pylibcugraph/conda_build_config.yaml index 611e8300953..4530a4c942d 100644 --- a/conda/recipes/pylibcugraph/conda_build_config.yaml +++ b/conda/recipes/pylibcugraph/conda_build_config.yaml @@ -17,4 +17,4 @@ sysroot_version: - "2.17" ucx_py_version: - - "0.33.*" \ No newline at end of file + - "0.33.*" diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index a5731dfdf36..aa82c20ad44 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -58,8 +58,6 @@ requirements: - libcugraph ={{ version }} - pylibraft ={{ minor_version }} - python - # TODO: Remove rmm once https://github.com/rapidsai/raft/pull/1656 is merged (fixes pylibraft use of rmm) - - rmm ={{ minor_version }} - scikit-build >=0.13.1 - setuptools run: