From 19883d54ac8ae30af88adf4ae578ee9c5bd5deef Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 30 Dec 2022 12:18:33 -0500 Subject: [PATCH 01/64] Build pip wheels alongside conda CI --- .github/workflows/build.yaml | 73 ++++++++++++++++++++++++++++ .github/workflows/pr.yaml | 51 +++++++++++++++++++ .github/workflows/test.yaml | 26 ++++++++++ .github/workflows/wheels.yml | 94 ------------------------------------ ci/release/update-version.sh | 3 ++ 5 files changed, 153 insertions(+), 94 deletions(-) delete mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0dcd76d86ce..ddf93a64234 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,6 +20,12 @@ on: build_type: type: string default: nightly + wheel_versioneer_override: + type: string + default: '' + wheel_build_tag: + type: string + default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -55,3 +61,70 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-build-pylibcugraph: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: pylibcugraph + package-dir: python/pylibcugraph + + # Note that this approach to cloning repos obviates any modification to + # the CMake variables in get_cumlprims_mg.cmake since CMake will just use + # the clone as is. + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-22.12 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + + versioneer-override: ${{ inputs.wheel_versioneer_override }} + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + wheel-publish-pylibcugraph: + needs: wheel-build-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@upkeep/manylinux-improvements + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: pylibcugraph + build-tag: ${{ inputs.wheel_build_tag }} + wheel-build-cugraph: + needs: wheel-publish-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph + package-dir: python/cugraph + versioneer-override: ${{ inputs.wheel_versioneer_override }} + + # Note that this approach to cloning repos obviates any modification to + # the CMake variables in get_cumlprims_mg.cmake since CMake will just use + # the clone as is. + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-22.12 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + wheel-publish-cugraph: + needs: wheel-build-cugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@upkeep/manylinux-improvements + with: + build_type: ${{ inputs.build_type || 'branch' }} + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cugraph + build-tag: ${{ inputs.wheel_build_tag }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ca2e2356c06..638fe74afb0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,10 @@ jobs: - conda-cpp-tests - conda-python-build - conda-python-tests + - wheel-build-pylibcugraph + - wheel-tests-pylibcugraph + - wheel-build-cugraph + - wheel-tests-cugraph secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main checks: @@ -47,3 +51,50 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main with: build_type: pull-request + wheel-build-pylibcugraph: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + with: + build_type: pull-request + package-name: pylibcugraph + package-dir: python/pylibcugraph + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-22.12 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + wheel-tests-pylibcugraph: + needs: wheel-build-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + with: + build_type: pull-request + package-name: pylibcugraph + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-smoketest: "" #"python ci/wheel_smoke_test_pylibcugraph.py" + wheel-build-cugraph: + needs: wheel-tests-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + with: + build_type: pull-request + package-name: cugraph + package-dir: python/cugraph + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-22.12 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + wheel-tests-cugraph: + needs: wheel-build-cugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + with: + build_type: pull-request + package-name: cugraph + # Always want to test against latest dask/distributed. + test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" + test-smoketest: "" #"python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6cc9e5d77a1..47ea050003b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,3 +32,29 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-tests-pylibcugraph: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + with: + build_type: nightly + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + package-name: pylibcugraph + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + wheel-tests-cugraph: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + with: + build_type: nightly + repo: rapidsai/cugraph + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + package-name: raft_dask + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml deleted file mode 100644 index 05123de1d65..00000000000 --- a/.github/workflows/wheels.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: cuGraph wheels - -on: - workflow_call: - inputs: - versioneer-override: - type: string - default: '' - build-tag: - type: string - default: '' - branch: - required: true - type: string - date: - required: true - type: string - sha: - required: true - type: string - build-type: - type: string - default: nightly - -concurrency: - group: "cugraph-${{ github.workflow }}-${{ github.ref }}" - cancel-in-progress: true - -jobs: - pylibcugraph-wheel: - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux.yml@main - with: - repo: rapidsai/cugraph - - build-type: ${{ inputs.build-type }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - - package-dir: python/pylibcugraph - package-name: pylibcugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - python-package-versioneer-override: ${{ inputs.versioneer-override }} - python-package-build-tag: ${{ inputs.build-tag }} - - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-extras: test - - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - secrets: inherit - - cugraph-wheel: - needs: pylibcugraph-wheel - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux.yml@main - with: - repo: rapidsai/cugraph - - build-type: ${{ inputs.build-type }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - - package-dir: python/cugraph - package-name: cugraph - - # Note that this approach to cloning repos obviates any modification to - # the CMake variables in get_cumlprims_mg.cmake since CMake will just use - # the clone as is. - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - - python-package-versioneer-override: ${{ inputs.versioneer-override }} - python-package-build-tag: ${{ inputs.build-tag }} - - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - - # Always want to test against latest dask/distributed. - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-extras: test - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" - secrets: inherit diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 1124fa80281..4d3ce3048b0 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -78,3 +78,6 @@ sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERS sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/cugraph/conda_build_config.yaml sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/cugraph-service/conda_build_config.yaml sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/pylibcugraph/conda_build_config.yaml + +# Wheel builds clone cugraph-ops, update its branch +sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml From cbd6bb9ba17d56fd55e75cbc75f0eaba8311dfcc Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 5 Jan 2023 15:42:36 -0500 Subject: [PATCH 02/64] make wheel builds depend on `checks` --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c79545d24dd..ac04f1fc9ef 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -63,6 +63,7 @@ jobs: container_image: "rapidsai/ci:latest" run_script: "ci/test_notebooks.sh" wheel-build-pylibcugraph: + needs: checks secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements with: From 54f9daedeac10b45631227c820b9effeda78ba5d Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 5 Jan 2023 15:42:53 -0500 Subject: [PATCH 03/64] update copyright headers --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 4d3ce3048b0..d4ba1543d17 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at From 874c7f585649b257bdaba9f53b22ba043ebd82d5 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 6 Jan 2023 13:04:53 -0500 Subject: [PATCH 04/64] Removed some wheel parameters --- .github/workflows/build.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 64e36adf2f9..ba9485cdab2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,12 +20,6 @@ on: build_type: type: string default: nightly - wheel_versioneer_override: - type: string - default: '' - wheel_build_tag: - type: string - default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -77,7 +71,6 @@ jobs: extra-repo-sha: branch-22.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - versioneer-override: ${{ inputs.wheel_versioneer_override }} skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph @@ -90,7 +83,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: pylibcugraph - build-tag: ${{ inputs.wheel_build_tag }} wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit @@ -103,7 +95,6 @@ jobs: date: ${{ inputs.date }} package-name: cugraph package-dir: python/cugraph - versioneer-override: ${{ inputs.wheel_versioneer_override }} # Note that this approach to cloning repos obviates any modification to # the CMake variables in get_cumlprims_mg.cmake since CMake will just use @@ -124,4 +115,3 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cugraph - build-tag: ${{ inputs.wheel_build_tag }} From 497daef66a0568dae29ad3e1cf962d1079e9b3c7 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 6 Jan 2023 13:05:45 -0500 Subject: [PATCH 05/64] Update versions for cugraph-ops and dask-cuda --- .github/workflows/build.yaml | 4 ++-- .github/workflows/pr.yaml | 4 ++-- ci/release/update-version.sh | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ba9485cdab2..a1f9c02d7db 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,7 +68,7 @@ jobs: # the CMake variables in get_cumlprims_mg.cmake since CMake will just use # the clone as is. extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 + extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" @@ -100,7 +100,7 @@ jobs: # the CMake variables in get_cumlprims_mg.cmake since CMake will just use # the clone as is. extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 + extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ac04f1fc9ef..cd903c8f6cb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,7 +71,7 @@ jobs: package-name: pylibcugraph package-dir: python/pylibcugraph extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 + extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-pylibcugraph: @@ -94,7 +94,7 @@ jobs: package-name: cugraph package-dir: python/cugraph extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-22.12 + extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d4ba1543d17..d8b1edf76b7 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -81,3 +81,6 @@ sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/ # Wheel builds clone cugraph-ops, update its branch sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml + +# Wheel builds install dask-cuda from source, update its branch +sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml From fec76565cd25a06b0b6b939d8e4a0e776f43535b Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 6 Jan 2023 15:38:29 -0500 Subject: [PATCH 06/64] Use relative path to datasets --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cd903c8f6cb..cda4a5e182e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,7 +83,7 @@ jobs: package-name: pylibcugraph # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" test-smoketest: "" #"python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph @@ -108,5 +108,5 @@ jobs: test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" test-smoketest: "" #"python ci/wheel_smoke_test_cugraph.py" From 6c9b7cd6ac0acfc270b6c903f0781892149e954b Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 6 Jan 2023 17:56:38 -0500 Subject: [PATCH 07/64] Use main version of improved wheel workflows --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 8 ++++---- .github/workflows/test.yaml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a1f9c02d7db..4183bb77218 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} repo: rapidsai/cugraph @@ -75,7 +75,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} repo: rapidsai/cugraph @@ -86,7 +86,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} repo: rapidsai/cugraph @@ -107,7 +107,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} repo: rapidsai/cugraph diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cda4a5e182e..c6ba8c3d280 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: pull-request package-name: cugraph @@ -100,7 +100,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: pull-request package-name: cugraph diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 008a285c114..7044c8e0c62 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: nightly repo: rapidsai/cugraph @@ -45,7 +45,7 @@ jobs: test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@upkeep/manylinux-improvements + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: nightly repo: rapidsai/cugraph From 230bd07777324adc06d8ee2b578b37363cdeff5c Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Tue, 10 Jan 2023 16:13:39 -0500 Subject: [PATCH 08/64] Remove unused repo: param --- .github/workflows/build.yaml | 4 ---- .github/workflows/test.yaml | 2 -- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4183bb77218..5a20796dfcd 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -57,7 +57,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} - repo: rapidsai/cugraph branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} @@ -78,7 +77,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} - repo: rapidsai/cugraph branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} @@ -89,7 +87,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} - repo: rapidsai/cugraph branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} @@ -110,7 +107,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main with: build_type: ${{ inputs.build_type || 'branch' }} - repo: rapidsai/cugraph branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7044c8e0c62..26663c97d3d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,7 +35,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: nightly - repo: rapidsai/cugraph branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} @@ -48,7 +47,6 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: nightly - repo: rapidsai/cugraph branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} From 5a1dbb6e884cc0de5ab0747ce86f877b4d7d918d Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Tue, 10 Jan 2023 16:15:00 -0500 Subject: [PATCH 09/64] Add workflow-local pylibcugraph dependency to cugraph --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c6ba8c3d280..9595061af73 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -96,6 +96,8 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" + cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From 5c1b5d75efb12c75e1f4e26cde4244274ee964ca Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 16 Jan 2023 16:40:24 -0600 Subject: [PATCH 10/64] Adds smoke tests for pylibcugraph and cugraph --- ci/wheel_smoke_test_cugraph.py | 36 ++++++++++++++++++++++ ci/wheel_smoke_test_pylibcugraph.py | 47 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 ci/wheel_smoke_test_cugraph.py create mode 100644 ci/wheel_smoke_test_pylibcugraph.py diff --git a/ci/wheel_smoke_test_cugraph.py b/ci/wheel_smoke_test_cugraph.py new file mode 100644 index 00000000000..80ec4b08bb9 --- /dev/null +++ b/ci/wheel_smoke_test_cugraph.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cudf +import cugraph + +edgelist = cudf.DataFrame({"source": ["a", "b", "c"], "destination": ["b", "c", "d"]}) + +# directed graph +G = cugraph.Graph(directed=True) +G.from_cudf_edgelist(edgelist, store_transposed=True) +result_df = cugraph.pagerank(G) + +assert(result_df["pagerank"].sum() == 1.0) +assert(result_df.sort_values(by="pagerank")["vertex"].values_host.tolist() + == ["a", "b", "c", "d"]) + +# undirected graph +G = cugraph.Graph(directed=False) +G.from_cudf_edgelist(edgelist, store_transposed=True) +result_df = cugraph.pagerank(G) + +assert(result_df["pagerank"].sum() == 1.0) +result_df.set_index("vertex", inplace=True) +assert(result_df.loc["a", "pagerank"] == result_df.loc["d", "pagerank"]) +assert(result_df.loc["b", "pagerank"] == result_df.loc["c", "pagerank"]) diff --git a/ci/wheel_smoke_test_pylibcugraph.py b/ci/wheel_smoke_test_pylibcugraph.py new file mode 100644 index 00000000000..b16bb77635e --- /dev/null +++ b/ci/wheel_smoke_test_pylibcugraph.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import cupy +from pylibcugraph import ResourceHandle, GraphProperties, SGGraph, pagerank + +src_array = cupy.asarray([100, 201, 302], dtype="int32") +dst_array = cupy.asarray([201, 302, 403], dtype="int32") +wgt_array = cupy.asarray([1.0, 1.0, 1.0], dtype="float32") + +resource_handle = ResourceHandle() + +G = SGGraph(resource_handle, + GraphProperties(is_symmetric=False, is_multigraph=False), + src_array, + dst_array, + wgt_array, + store_transposed=True, + renumber=True, + do_expensive_check=True, + ) + +(vertices, pageranks) = pagerank(resource_handle=resource_handle, + graph=G, + precomputed_vertex_out_weight_vertices=None, + precomputed_vertex_out_weight_sums=None, + initial_guess_vertices=None, + initial_guess_values=None, + alpha=0.85, + epsilon=1.0e-6, + max_iterations=500, + do_expensive_check=True, + ) + +assert(pageranks.sum() == 1.0) +results = dict(zip(vertices.tolist(),pageranks.tolist())) +assert(results[100] < results[201] < results[302] < results[403]) From 189926bb6669554d78a81036c10ab1c32c45d661 Mon Sep 17 00:00:00 2001 From: Rick Ratzel Date: Mon, 16 Jan 2023 16:44:43 -0600 Subject: [PATCH 11/64] Adds proper check for running as a script --- ci/wheel_smoke_test_cugraph.py | 33 ++++++++--------- ci/wheel_smoke_test_pylibcugraph.py | 55 +++++++++++++++-------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/ci/wheel_smoke_test_cugraph.py b/ci/wheel_smoke_test_cugraph.py index 80ec4b08bb9..9b014d9ce95 100644 --- a/ci/wheel_smoke_test_cugraph.py +++ b/ci/wheel_smoke_test_cugraph.py @@ -14,23 +14,24 @@ import cudf import cugraph -edgelist = cudf.DataFrame({"source": ["a", "b", "c"], "destination": ["b", "c", "d"]}) +if __name__ == "__main__": + edgelist = cudf.DataFrame({"source": ["a", "b", "c"], "destination": ["b", "c", "d"]}) -# directed graph -G = cugraph.Graph(directed=True) -G.from_cudf_edgelist(edgelist, store_transposed=True) -result_df = cugraph.pagerank(G) + # directed graph + G = cugraph.Graph(directed=True) + G.from_cudf_edgelist(edgelist, store_transposed=True) + result_df = cugraph.pagerank(G) -assert(result_df["pagerank"].sum() == 1.0) -assert(result_df.sort_values(by="pagerank")["vertex"].values_host.tolist() - == ["a", "b", "c", "d"]) + assert(result_df["pagerank"].sum() == 1.0) + assert(result_df.sort_values(by="pagerank")["vertex"].values_host.tolist() + == ["a", "b", "c", "d"]) -# undirected graph -G = cugraph.Graph(directed=False) -G.from_cudf_edgelist(edgelist, store_transposed=True) -result_df = cugraph.pagerank(G) + # undirected graph + G = cugraph.Graph(directed=False) + G.from_cudf_edgelist(edgelist, store_transposed=True) + result_df = cugraph.pagerank(G) -assert(result_df["pagerank"].sum() == 1.0) -result_df.set_index("vertex", inplace=True) -assert(result_df.loc["a", "pagerank"] == result_df.loc["d", "pagerank"]) -assert(result_df.loc["b", "pagerank"] == result_df.loc["c", "pagerank"]) + assert(result_df["pagerank"].sum() == 1.0) + result_df.set_index("vertex", inplace=True) + assert(result_df.loc["a", "pagerank"] == result_df.loc["d", "pagerank"]) + assert(result_df.loc["b", "pagerank"] == result_df.loc["c", "pagerank"]) diff --git a/ci/wheel_smoke_test_pylibcugraph.py b/ci/wheel_smoke_test_pylibcugraph.py index b16bb77635e..09c48ff93b3 100644 --- a/ci/wheel_smoke_test_pylibcugraph.py +++ b/ci/wheel_smoke_test_pylibcugraph.py @@ -14,34 +14,35 @@ import cupy from pylibcugraph import ResourceHandle, GraphProperties, SGGraph, pagerank -src_array = cupy.asarray([100, 201, 302], dtype="int32") -dst_array = cupy.asarray([201, 302, 403], dtype="int32") -wgt_array = cupy.asarray([1.0, 1.0, 1.0], dtype="float32") +if __name__ == "__main__": + src_array = cupy.asarray([100, 201, 302], dtype="int32") + dst_array = cupy.asarray([201, 302, 403], dtype="int32") + wgt_array = cupy.asarray([1.0, 1.0, 1.0], dtype="float32") -resource_handle = ResourceHandle() + resource_handle = ResourceHandle() -G = SGGraph(resource_handle, - GraphProperties(is_symmetric=False, is_multigraph=False), - src_array, - dst_array, - wgt_array, - store_transposed=True, - renumber=True, - do_expensive_check=True, - ) + G = SGGraph(resource_handle, + GraphProperties(is_symmetric=False, is_multigraph=False), + src_array, + dst_array, + wgt_array, + store_transposed=True, + renumber=True, + do_expensive_check=True, + ) -(vertices, pageranks) = pagerank(resource_handle=resource_handle, - graph=G, - precomputed_vertex_out_weight_vertices=None, - precomputed_vertex_out_weight_sums=None, - initial_guess_vertices=None, - initial_guess_values=None, - alpha=0.85, - epsilon=1.0e-6, - max_iterations=500, - do_expensive_check=True, - ) + (vertices, pageranks) = pagerank(resource_handle=resource_handle, + graph=G, + precomputed_vertex_out_weight_vertices=None, + precomputed_vertex_out_weight_sums=None, + initial_guess_vertices=None, + initial_guess_values=None, + alpha=0.85, + epsilon=1.0e-6, + max_iterations=500, + do_expensive_check=True, + ) -assert(pageranks.sum() == 1.0) -results = dict(zip(vertices.tolist(),pageranks.tolist())) -assert(results[100] < results[201] < results[302] < results[403]) + assert(pageranks.sum() == 1.0) + results = dict(zip(vertices.tolist(),pageranks.tolist())) + assert(results[100] < results[201] < results[302] < results[403]) From 38b584b1cf48db000f96751ab9fd5518652db301 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Tue, 17 Jan 2023 07:51:37 -0500 Subject: [PATCH 12/64] Use smoketests for PR arm64 wheel builds --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9595061af73..43f80625500 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,7 +84,7 @@ jobs: # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "" #"python ci/wheel_smoke_test_pylibcugraph.py" + test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit @@ -111,4 +111,4 @@ jobs: # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-smoketest: "" #"python ci/wheel_smoke_test_cugraph.py" + test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 763357347f8588dbf085d1ef3f023e42311e7c9a Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Tue, 17 Jan 2023 13:42:51 -0500 Subject: [PATCH 13/64] Debug arm smoke test with ldconfig --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 43f80625500..e5cc029ef05 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -111,4 +111,4 @@ jobs: # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + test-smoketest: "ldconfig -v | grep libcusolver ; python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26663c97d3d..88a1c730ed8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,7 +50,7 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - package-name: raft_dask + package-name: cugraph # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" From 970be342dc6fc2073baeb823f45ad5994aa1eede Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 12:03:43 -0500 Subject: [PATCH 14/64] Add scanelf debug command --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e5cc029ef05..1ef675f7723 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -111,4 +111,4 @@ jobs: # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-smoketest: "ldconfig -v | grep libcusolver ; python ci/wheel_smoke_test_cugraph.py" + test-smoketest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 88a1c730ed8..1dee760f7e0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -51,6 +51,8 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: cugraph + # Always want to test against latest dask/distributed. + test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" From bb22d1b090cc24e487f5b01773dd3a729b1f340e Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 13:35:37 -0500 Subject: [PATCH 15/64] More debugging --- .github/workflows/pr.yaml | 6 +++--- ci/wheel_smoke_test_pylibcugraph.py | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1ef675f7723..dec15571605 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,8 +83,8 @@ jobs: package-name: pylibcugraph # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + test-unittest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-smoketest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit @@ -110,5 +110,5 @@ jobs: test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" + test-unittest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" test-smoketest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; python ci/wheel_smoke_test_cugraph.py" diff --git a/ci/wheel_smoke_test_pylibcugraph.py b/ci/wheel_smoke_test_pylibcugraph.py index 09c48ff93b3..738e7061490 100644 --- a/ci/wheel_smoke_test_pylibcugraph.py +++ b/ci/wheel_smoke_test_pylibcugraph.py @@ -14,6 +14,12 @@ import cupy from pylibcugraph import ResourceHandle, GraphProperties, SGGraph, pagerank +# an import statement that reveals a problem with cusolver +from pylibcugraph.components._connectivity import ( + strongly_connected_components, +) + + if __name__ == "__main__": src_array = cupy.asarray([100, 201, 302], dtype="int32") dst_array = cupy.asarray([201, 302, 403], dtype="int32") From a8b3e77ccccccea0e5017e2b057d265cbe6fcf17 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 14:46:16 -0500 Subject: [PATCH 16/64] Remove scanelf debugging --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dec15571605..43f80625500 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -83,8 +83,8 @@ jobs: package-name: pylibcugraph # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-unittest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; python ci/wheel_smoke_test_pylibcugraph.py" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit @@ -110,5 +110,5 @@ jobs: test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-smoketest: "scanelf -l -s 'cusolverDnDestroy' | grep cusolverDnDestroy ; python ci/wheel_smoke_test_cugraph.py" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 9470e9c95f5d0e454aa5601dd8340c116b2e8c98 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 15:55:27 -0500 Subject: [PATCH 17/64] Try FIND_CUGRAPH_CPP=OFF --- .github/workflows/build.yaml | 4 ++-- .github/workflows/pr.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5a20796dfcd..1908df56101 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -70,7 +70,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -100,7 +100,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 43f80625500..85c72754b2c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -73,7 +73,7 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -98,7 +98,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From ee1b3e28bb02c20ec609965ca05d2deab6a0ad87 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 17:21:35 -0500 Subject: [PATCH 18/64] Use PR branch of raft with cutlass static ctk fix --- cpp/cmake/thirdparty/get_raft.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 09650533c2d..c58f459f69a 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -67,7 +67,8 @@ endfunction() # CPM_raft_SOURCE=/path/to/local/raft find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft} FORK rapidsai - PINNED_TAG branch-${CUGRAPH_BRANCH_VERSION_raft} + #PINNED_TAG branch-${CUGRAPH_BRANCH_VERSION_raft} + PINNED_TAG pull-request/1155 # When PINNED_TAG above doesn't match cugraph, # force local raft clone in build directory From 399c27dae05bae85cd645643980bb211ac937877 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 18 Jan 2023 17:32:34 -0500 Subject: [PATCH 19/64] Copyright update for get_raft.cmake --- cpp/cmake/thirdparty/get_raft.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index c58f459f69a..363fa153773 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From ef06778527d18fae6ce076816b5c4349a09fc384 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Thu, 19 Jan 2023 09:08:35 -0500 Subject: [PATCH 20/64] Add ctk_static_suffix --- cpp/CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3427bc064b0..16d24a65672 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -452,16 +452,17 @@ if(CUDA_STATIC_RUNTIME) target_include_directories(cugraph_c PUBLIC ${_includes}) # Add CTK include paths because we make our CTK library links private below target_include_directories(cugraph_c SYSTEM PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) + set(_ctk_static_suffix "_static") endif() ################################################################################ # - C-API link libraries ------------------------------------------------------- target_link_libraries(cugraph_c PUBLIC - CUDA::cublas - CUDA::curand - CUDA::cusolver - CUDA::cusparse + CUDA::cublas${_ctk_static_suffix} + CUDA::curand${_ctk_static_suffix} + CUDA::cusolver${_ctk_static_suffix} + CUDA::cusparse${_ctk_static_suffix} rmm::rmm $<$>:raft::raft> $<$>:raft::distance> From 7e9698e51312efc7ce3d2bb4955f93e819eac321 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Thu, 19 Jan 2023 13:51:43 -0500 Subject: [PATCH 21/64] Add strace debugging --- .github/workflows/pr.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 85c72754b2c..a0ca083159e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,8 +107,9 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02 && apt-get update -y && apt-get install -y strace" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_cugraph.py" + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02 && apt-get update -y && apt-get install -y strace" + #test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" + test-unittest: "strace -f -e openat python ci/wheel_smoke_test_cugraph.py" + test-smoketest: "strace -f -e openat python ci/wheel_smoke_test_cugraph.py" From 2a5c8f750044829050eb42a980eb83e972ae18f6 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 23 Jan 2023 09:35:22 -0500 Subject: [PATCH 22/64] Check if CUDA 118 can leapfrog cusolver issues --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a0ca083159e..68f38a9a379 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: cugraph @@ -102,7 +102,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: pull-request package-name: cugraph From 107ad4ee11f0065eef85aa1f87b74cc6ea2772a3 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 23 Jan 2023 09:38:37 -0500 Subject: [PATCH 23/64] Use ctk118-py39 branch first --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 68f38a9a379..82e40ee8394 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-py39 with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118-py39 with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-py39 with: build_type: pull-request package-name: cugraph @@ -102,7 +102,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118-py39 with: build_type: pull-request package-name: cugraph From c0c96ce8f023e896e795d3cb82ccf4ce21ff5e34 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 23 Jan 2023 09:48:58 -0500 Subject: [PATCH 24/64] Revert "Use PR branch of raft with cutlass static ctk fix" This reverts commit ee1b3e28bb02c20ec609965ca05d2deab6a0ad87. --- cpp/cmake/thirdparty/get_raft.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 363fa153773..43226664e5d 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -67,8 +67,7 @@ endfunction() # CPM_raft_SOURCE=/path/to/local/raft find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft} FORK rapidsai - #PINNED_TAG branch-${CUGRAPH_BRANCH_VERSION_raft} - PINNED_TAG pull-request/1155 + PINNED_TAG branch-${CUGRAPH_BRANCH_VERSION_raft} # When PINNED_TAG above doesn't match cugraph, # force local raft clone in build directory From ca5f3b6c8befd65b5d58f8af7617e0d87157b050 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 23 Jan 2023 23:02:30 -0500 Subject: [PATCH 25/64] Install pylibcugraph built during same workflow run --- .github/workflows/pr.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 82e40ee8394..926643cca05 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,9 +107,8 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02 && apt-get update -y && apt-get install -y strace" + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02 && apt-get update -y && apt-get install -y strace" - #test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" - test-unittest: "strace -f -e openat python ci/wheel_smoke_test_cugraph.py" - test-smoketest: "strace -f -e openat python ci/wheel_smoke_test_cugraph.py" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From abdd76d512becfef00c0250b46dc31a7791d58fc Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 23 Jan 2023 23:03:44 -0500 Subject: [PATCH 26/64] Go back to @main wheel workflows --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 926643cca05..7a0f4e76af9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-py39 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118-py39 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-py39 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main with: build_type: pull-request package-name: cugraph @@ -102,7 +102,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118-py39 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main with: build_type: pull-request package-name: cugraph From 43c2ffcb91ee38ae3cbfcfe4dc4181ea17f31691 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Tue, 24 Jan 2023 10:14:03 -0500 Subject: [PATCH 27/64] Install local pylibcugraph without deps --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7a0f4e76af9..9f022ecd46f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -107,8 +107,8 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 13912f07410330110c87d26815be454b146e21a5 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 25 Jan 2023 09:55:04 -0500 Subject: [PATCH 28/64] Skip multi-gpu cugraph tests --- .github/workflows/pr.yaml | 2 +- .github/workflows/test.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9f022ecd46f..ba1bab08cc2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -110,5 +110,5 @@ jobs: test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=tests/mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1dee760f7e0..451b50f74c6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,7 +41,7 @@ jobs: package-name: pylibcugraph # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main @@ -55,4 +55,4 @@ jobs: test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets pytest -v ./python/cugraph/cugraph/tests" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=tests/mg" From 6edd7632db8b69c8d69e2e9e53680e7ef1fcc5fb Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 25 Jan 2023 18:48:06 -0500 Subject: [PATCH 29/64] Fix mg ignore pattern --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f21ae8554a..5360de7561b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -110,5 +110,5 @@ jobs: test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=tests/mg" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 9baad70913e4dbd00adeac5df430629dc78d988d Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 25 Jan 2023 20:35:06 -0500 Subject: [PATCH 30/64] Use CTK 118/cp310 branch of wheel workflows --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 8 ++++---- .github/workflows/test.yaml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f09f2041644..09db2d6f5de 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -74,7 +74,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -84,7 +84,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -104,7 +104,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5360de7561b..67c01d909f0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: cugraph @@ -102,7 +102,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: pull-request package-name: cugraph diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 00fcde7a9b4..bbb52769a9f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -44,7 +44,7 @@ jobs: test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@main + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} From f9d151d1bfc637ca4ed248fff9104ffc15c51709 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Thu, 26 Jan 2023 09:10:35 -0500 Subject: [PATCH 31/64] Debug why cugraph build is not accepting locally installed pylibcugraph wheel dependency --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 67c01d909f0..61b7dc1fb9f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -97,7 +97,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" - cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" + cibw-before-build: "which pip && pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl && pip freeze | grep pylibcugraph" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From 2e89e9d1fc1753966501c684edf802308fdf4db4 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 09:29:02 -0500 Subject: [PATCH 32/64] Add local pylibcugraph dependency path for build-time cugraph dep --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 61b7dc1fb9f..664c7eb001c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -97,7 +97,8 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" - cibw-before-build: "which pip && pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl && pip freeze | grep pylibcugraph" + cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" + cibw-environment: "PIP_EXTRA_INDEX_URL=file://./local-pylibcugraph-dep/" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From d8b492436958b7e8d917be3eae0fd3ffa9b01328 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 10:26:58 -0500 Subject: [PATCH 33/64] Use absolute path to local dep --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 664c7eb001c..77ed8136bef 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,7 +98,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" - cibw-environment: "PIP_EXTRA_INDEX_URL=file://./local-pylibcugraph-dep/" + cibw-environment: "PIP_EXTRA_INDEX_URL=file:///project/local-pylibcugraph-dep/" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From a6c1922a49e41beb32fc5f2e2f7b35ea7263ade4 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 11:48:14 -0500 Subject: [PATCH 34/64] Improve local dependency installation method --- .github/workflows/pr.yaml | 5 ++--- ci/release/update-version.sh | 6 ++++++ python/cugraph/_custom_build/backend.py | 6 +++--- python/cugraph/setup.py | 10 +++++----- python/pylibcugraph/_custom_build/backend.py | 4 ++-- python/pylibcugraph/setup.py | 4 ++-- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 77ed8136bef..b7e8b87ed56 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -96,9 +96,8 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep" - cibw-before-build: "pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl" - cibw-environment: "PIP_EXTRA_INDEX_URL=file:///project/local-pylibcugraph-dep/" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + #cibw-before-build: "pip install --no-deps ./local-wheelhouse/pylibcugraph*.whl" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d8b1edf76b7..86d58ec3ef8 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -84,3 +84,9 @@ sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG} # Wheel builds install dask-cuda from source, update its branch sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml + +# Wheel builds install intra-RAPIDS dependencies from same release +sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/cugraph/setup.py +sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/cugraph/_custom_build/backend.py +sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/pylibcugraph/setup.py +sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/pylibcugraph/_custom_build/backend.py diff --git a/python/cugraph/_custom_build/backend.py b/python/cugraph/_custom_build/backend.py index b468b1c88be..7d51d0128de 100644 --- a/python/cugraph/_custom_build/backend.py +++ b/python/cugraph/_custom_build/backend.py @@ -21,9 +21,9 @@ def wrapper(config_settings=None): orig_list = getattr(_orig, func.__name__)(config_settings) cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") append_list = [ - f"rmm{cuda_suffix}", - f"raft-dask{cuda_suffix}", - f"pylibcugraph{cuda_suffix}", + f"rmm{cuda_suffix}==23.02.*", + f"raft-dask{cuda_suffix}==23.02.*", + f"pylibcugraph{cuda_suffix}==23.02.*", ] return orig_list + append_list diff --git a/python/cugraph/setup.py b/python/cugraph/setup.py index 858c9594fa5..7e3558b6e33 100644 --- a/python/cugraph/setup.py +++ b/python/cugraph/setup.py @@ -24,11 +24,11 @@ INSTALL_REQUIRES = [ "numba", "dask-cuda", - f"rmm{cuda_suffix}", - f"cudf{cuda_suffix}", - f"raft-dask{cuda_suffix}", - f"dask-cudf{cuda_suffix}", - f"pylibcugraph{cuda_suffix}", + f"rmm{cuda_suffix}==23.02.*", + f"cudf{cuda_suffix}==23.02.*", + f"raft-dask{cuda_suffix}==23.02.*", + f"dask-cudf{cuda_suffix}==23.02.*", + f"pylibcugraph{cuda_suffix}==23.02.*", "cupy-cuda11x", ] diff --git a/python/pylibcugraph/_custom_build/backend.py b/python/pylibcugraph/_custom_build/backend.py index 707add312a0..e2e88298906 100644 --- a/python/pylibcugraph/_custom_build/backend.py +++ b/python/pylibcugraph/_custom_build/backend.py @@ -21,8 +21,8 @@ def wrapper(config_settings=None): orig_list = getattr(_orig, func.__name__)(config_settings) cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") append_list = [ - f"rmm{cuda_suffix}", - f"pylibraft{cuda_suffix}", + f"rmm{cuda_suffix}==23.02.*", + f"pylibraft{cuda_suffix}==23.02.*", ] return orig_list + append_list diff --git a/python/pylibcugraph/setup.py b/python/pylibcugraph/setup.py index c27a6692908..64cd13e6ad3 100644 --- a/python/pylibcugraph/setup.py +++ b/python/pylibcugraph/setup.py @@ -93,8 +93,8 @@ def get_versions(): package_data={key: ["*.pxd"] for key in find_packages(include=["pylibcugraph*"])}, include_package_data=True, install_requires=[ - f"pylibraft{cuda_suffix}", - f"rmm{cuda_suffix}", + f"pylibraft{cuda_suffix}==23.02.*", + f"rmm{cuda_suffix}==23.02.*", ], extras_require={ "test": [ From 0e75dc37e3af3e33d23996850457b64b8d03792c Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 11:56:52 -0500 Subject: [PATCH 35/64] Fix copyright year --- python/cugraph/_custom_build/backend.py | 2 +- python/pylibcugraph/_custom_build/backend.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph/_custom_build/backend.py b/python/cugraph/_custom_build/backend.py index 7d51d0128de..1a0ecb1c913 100644 --- a/python/cugraph/_custom_build/backend.py +++ b/python/cugraph/_custom_build/backend.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. """Custom build backend for cugraph to get versioned requirements. diff --git a/python/pylibcugraph/_custom_build/backend.py b/python/pylibcugraph/_custom_build/backend.py index e2e88298906..5bcc0c741d7 100644 --- a/python/pylibcugraph/_custom_build/backend.py +++ b/python/pylibcugraph/_custom_build/backend.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. """Custom build backend for pylibcugraph to get versioned requirements. From 86ef2e19ccbf1a35f6be940ace8e13e9e113f2a1 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 12:25:43 -0500 Subject: [PATCH 36/64] Adjust versions string to be normalized for PEP440 --- python/cugraph/_custom_build/backend.py | 6 +++--- python/cugraph/setup.py | 10 +++++----- python/pylibcugraph/_custom_build/backend.py | 4 ++-- python/pylibcugraph/setup.py | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/cugraph/_custom_build/backend.py b/python/cugraph/_custom_build/backend.py index 1a0ecb1c913..68991f0a66a 100644 --- a/python/cugraph/_custom_build/backend.py +++ b/python/cugraph/_custom_build/backend.py @@ -21,9 +21,9 @@ def wrapper(config_settings=None): orig_list = getattr(_orig, func.__name__)(config_settings) cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") append_list = [ - f"rmm{cuda_suffix}==23.02.*", - f"raft-dask{cuda_suffix}==23.02.*", - f"pylibcugraph{cuda_suffix}==23.02.*", + f"rmm{cuda_suffix}==23.2.*", + f"raft-dask{cuda_suffix}==23.2.*", + f"pylibcugraph{cuda_suffix}==23.2.*", ] return orig_list + append_list diff --git a/python/cugraph/setup.py b/python/cugraph/setup.py index 7e3558b6e33..fab399d72f7 100644 --- a/python/cugraph/setup.py +++ b/python/cugraph/setup.py @@ -24,11 +24,11 @@ INSTALL_REQUIRES = [ "numba", "dask-cuda", - f"rmm{cuda_suffix}==23.02.*", - f"cudf{cuda_suffix}==23.02.*", - f"raft-dask{cuda_suffix}==23.02.*", - f"dask-cudf{cuda_suffix}==23.02.*", - f"pylibcugraph{cuda_suffix}==23.02.*", + f"rmm{cuda_suffix}==23.2.*", + f"cudf{cuda_suffix}==23.2.*", + f"raft-dask{cuda_suffix}==23.2.*", + f"dask-cudf{cuda_suffix}==23.2.*", + f"pylibcugraph{cuda_suffix}==23.2.*", "cupy-cuda11x", ] diff --git a/python/pylibcugraph/_custom_build/backend.py b/python/pylibcugraph/_custom_build/backend.py index 5bcc0c741d7..f584c386b64 100644 --- a/python/pylibcugraph/_custom_build/backend.py +++ b/python/pylibcugraph/_custom_build/backend.py @@ -21,8 +21,8 @@ def wrapper(config_settings=None): orig_list = getattr(_orig, func.__name__)(config_settings) cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") append_list = [ - f"rmm{cuda_suffix}==23.02.*", - f"pylibraft{cuda_suffix}==23.02.*", + f"rmm{cuda_suffix}==23.2.*", + f"pylibraft{cuda_suffix}==23.2.*", ] return orig_list + append_list diff --git a/python/pylibcugraph/setup.py b/python/pylibcugraph/setup.py index 64cd13e6ad3..901734852d1 100644 --- a/python/pylibcugraph/setup.py +++ b/python/pylibcugraph/setup.py @@ -93,8 +93,8 @@ def get_versions(): package_data={key: ["*.pxd"] for key in find_packages(include=["pylibcugraph*"])}, include_package_data=True, install_requires=[ - f"pylibraft{cuda_suffix}==23.02.*", - f"rmm{cuda_suffix}==23.02.*", + f"pylibraft{cuda_suffix}==23.2.*", + f"rmm{cuda_suffix}==23.2.*", ], extras_require={ "test": [ From 8e01c7ced862375291b33853032e983c1b3e9c75 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 13:12:37 -0500 Subject: [PATCH 37/64] Try pep440 normalization in update-verion.sh --- ci/release/update-version.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 86d58ec3ef8..f01357347dc 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -85,8 +85,11 @@ sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG} # Wheel builds install dask-cuda from source, update its branch sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml +CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}')") +NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}')") + # Wheel builds install intra-RAPIDS dependencies from same release -sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/cugraph/setup.py -sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/cugraph/_custom_build/backend.py -sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/pylibcugraph/setup.py -sed_runner "s/${CURRENT_SHORT_TAG}/${NEXT_SHORT_TAG}/g" python/pylibcugraph/_custom_build/backend.py +sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/cugraph/setup.py +sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/cugraph/_custom_build/backend.py +sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/pylibcugraph/setup.py +sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/pylibcugraph/_custom_build/backend.py From b4ea2a0ea521f493133e4f52f45245547455485e Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 27 Jan 2023 13:31:42 -0500 Subject: [PATCH 38/64] Rebuild --- ci/release/update-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f01357347dc..91448447701 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -85,6 +85,7 @@ sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG} # Wheel builds install dask-cuda from source, update its branch sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml +# Need to distutils-normalize the original version CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}')") NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}')") From 33990d0dca4625d24ea2af12c210d3f6d960bf41 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 28 Jan 2023 12:32:40 -0500 Subject: [PATCH 39/64] Try PIP_FIND_LINKS --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b7e8b87ed56..27756ddea1d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,6 +98,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" #cibw-before-build: "pip install --no-deps ./local-wheelhouse/pylibcugraph*.whl" + cibw-environment: "PIP_FIND_LINKS=/project/local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From 93479d5097494914fdc0e54a7643f6b7275dc2ce Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 30 Jan 2023 14:04:51 -0500 Subject: [PATCH 40/64] Rely on PIP_FIND_LINKS set by shared workflow --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 27756ddea1d..66444e77883 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -97,8 +97,6 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" - #cibw-before-build: "pip install --no-deps ./local-wheelhouse/pylibcugraph*.whl" - cibw-environment: "PIP_FIND_LINKS=/project/local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" wheel-tests-cugraph: needs: wheel-build-cugraph From 9dbcd1ca3f3440bab31aec4af9772688e821bfd4 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Mon, 30 Jan 2023 15:08:48 -0500 Subject: [PATCH 41/64] Avoid using CURRENT_TAG in update-version.sh --- ci/release/update-version.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 91448447701..d16bb09a99b 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -86,11 +86,10 @@ sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG} sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml # Need to distutils-normalize the original version -CURRENT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${CURRENT_SHORT_TAG}')") -NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}')") +NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") # Wheel builds install intra-RAPIDS dependencies from same release -sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/cugraph/setup.py -sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/cugraph/_custom_build/backend.py -sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/pylibcugraph/setup.py -sed_runner "s/${CURRENT_SHORT_TAG_PEP440}/${NEXT_SHORT_TAG_PEP440}/g" python/pylibcugraph/_custom_build/backend.py +sed_runner "s/{cuda_suffix}[^\"].*\",/{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pylibcugraph/setup.py +sed_runner "s/{cuda_suffix}.*\",/{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pylibcugraph/_custom_build/backend.py +sed_runner "s/{cuda_suffix}[^\"].*\",/{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cugraph/setup.py +sed_runner "s/{cuda_suffix}.*\",/{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cugraph/_custom_build/backend.py From 6721aef348783d2ae75dd24c20bdbce740e831a4 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 1 Feb 2023 10:11:55 -0500 Subject: [PATCH 42/64] Disable sccache for cugraph temporarily --- .github/workflows/pr.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 66444e77883..c64dc83d9c2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-sccache-disable with: build_type: pull-request package-name: pylibcugraph @@ -74,6 +74,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + sccache-enabled: "no" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -88,7 +89,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-sccache-disable with: build_type: pull-request package-name: cugraph @@ -98,6 +99,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + sccache-enabled: "no" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From d8397da73bcd22647524cac62cefea2bc15784c1 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 1 Feb 2023 16:29:55 -0500 Subject: [PATCH 43/64] Walk back potential red herring sccache disabling --- .github/workflows/pr.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c64dc83d9c2..66444e77883 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-sccache-disable + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: pylibcugraph @@ -74,7 +74,6 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - sccache-enabled: "no" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -89,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118-sccache-disable + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: cugraph @@ -99,7 +98,6 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - sccache-enabled: "no" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From 3793f95d5b83128203a2b0dffef2f15d3f7f6376 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Wed, 1 Feb 2023 16:30:42 -0500 Subject: [PATCH 44/64] Pass -j8 to skbuild-build-options --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 66444e77883..e2576e00866 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,6 +74,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-build-options: "-j8" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -98,6 +99,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From 24ad67569c0ee95ffb7eb73db3b87d3e184b6335 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 16:52:16 -0800 Subject: [PATCH 45/64] Try disabling parallelism --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e2576e00866..9a1ab5df643 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,7 +74,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" + skbuild-build-options: "-j1" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -99,7 +99,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" + skbuild-build-options: "-j1" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From 70dd04d5a65c5297b7c1597023bfc796a553667b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 19:03:17 -0800 Subject: [PATCH 46/64] Try just using raft headers and not compiling the unnecessary parts. --- cpp/CMakeLists.txt | 1 + cpp/cmake/thirdparty/get_raft.cmake | 5 +++-- python/pylibcugraph/CMakeLists.txt | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fe65da84327..5dd1308637b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -80,6 +80,7 @@ option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc (useful for cud option(BUILD_TESTS "Configure CMake to build tests" ON) option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) option(USE_RAFT_STATIC "Build raft as a static library" OFF) +option(CUGRAPH_COMPILE_RAFT_DIST_LIBS "Compile the raft library instead of using it header-only" ON) option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) option(CUGRAPH_USE_CUGRAPH_OPS_STATIC "Build and statically link the cugraph-ops library" OFF) option(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL "Exclude cugraph-ops targets from cuGraph's 'all' target" OFF) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 43226664e5d..d94827b4e81 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -19,7 +19,7 @@ set(CUGRAPH_BRANCH_VERSION_raft "${CUGRAPH_VERSION_MAJOR}.${CUGRAPH_VERSION_MINO function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC) + set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC COMPILE_RAFT_DIST_LIBS) cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN} ) if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${CUGRAPH_BRANCH_VERSION_raft}") @@ -47,7 +47,7 @@ function(find_and_configure_raft) SOURCE_SUBDIR cpp OPTIONS "RAFT_COMPILE_LIBRARIES OFF" - "RAFT_COMPILE_DIST_LIBRARY ON" + "RAFT_COMPILE_DIST_LIBRARY ${PKG_COMPILE_RAFT_DIST_LIBS}" "BUILD_TESTS OFF" "BUILD_BENCH OFF" "BUILD_SHARED_LIBS ${BUILD_RAFT_SHARED}" @@ -74,4 +74,5 @@ find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft} # even if it's already installed. CLONE_ON_PIN ON USE_RAFT_STATIC ${USE_RAFT_STATIC} + COMPILE_RAFT_DIST_LIBS ${CUGRAPH_COMPILE_RAFT_DIST_LIBS} ) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index eb0ecd42457..c848ad7d5f5 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -65,6 +65,7 @@ if (NOT cugraph_FOUND) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) + set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) From 19c417b6149e9b48fb034c257090d72f4120c5ac Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 19:03:53 -0800 Subject: [PATCH 47/64] Add back some parallelism. --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9a1ab5df643..e2576e00866 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,7 +74,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j1" + skbuild-build-options: "-j8" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -99,7 +99,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j1" + skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From fd9c971186ebd997e1dc9f0d2b37031ca0375b19 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 19:16:12 -0800 Subject: [PATCH 48/64] Fix copyright. --- python/pylibcugraph/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index c848ad7d5f5..1316be28ed3 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at From f5a8b82d10cacdf528c54b25af093606ba2b5247 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 21:01:43 -0800 Subject: [PATCH 49/64] Go back to full parallelism. --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e2576e00866..66444e77883 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,7 +74,6 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -99,7 +98,6 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From 12866d5249c9700f2c2b528af7fc2b9a749f90f2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 1 Feb 2023 21:04:17 -0800 Subject: [PATCH 50/64] Also use headers for cugraph wheels. --- python/cugraph/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 9acce421b9d..ac3b0bc3fd9 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -59,13 +59,13 @@ if(NOT cugraph_FOUND) set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) - set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(_exclude_from_all "") if(CUGRAPH_BUILD_WHEELS) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) + set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) From 9903ddace5cd8abf05ba489636aa77d328c871d0 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Thu, 2 Feb 2023 10:43:08 -0500 Subject: [PATCH 51/64] Revert "Try disabling parallelism" This reverts commit 24ad67569c0ee95ffb7eb73db3b87d3e184b6335. --- .github/workflows/pr.yaml | 2 ++ cpp/CMakeLists.txt | 1 - cpp/cmake/thirdparty/get_raft.cmake | 5 ++--- python/cugraph/CMakeLists.txt | 4 ++-- python/pylibcugraph/CMakeLists.txt | 3 +-- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 66444e77883..e2576e00866 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,6 +74,7 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-build-options: "-j8" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -98,6 +99,7 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5dd1308637b..fe65da84327 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -80,7 +80,6 @@ option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc (useful for cud option(BUILD_TESTS "Configure CMake to build tests" ON) option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) option(USE_RAFT_STATIC "Build raft as a static library" OFF) -option(CUGRAPH_COMPILE_RAFT_DIST_LIBS "Compile the raft library instead of using it header-only" ON) option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) option(CUGRAPH_USE_CUGRAPH_OPS_STATIC "Build and statically link the cugraph-ops library" OFF) option(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL "Exclude cugraph-ops targets from cuGraph's 'all' target" OFF) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index d94827b4e81..43226664e5d 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -19,7 +19,7 @@ set(CUGRAPH_BRANCH_VERSION_raft "${CUGRAPH_VERSION_MAJOR}.${CUGRAPH_VERSION_MINO function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC COMPILE_RAFT_DIST_LIBS) + set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC) cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN} ) if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${CUGRAPH_BRANCH_VERSION_raft}") @@ -47,7 +47,7 @@ function(find_and_configure_raft) SOURCE_SUBDIR cpp OPTIONS "RAFT_COMPILE_LIBRARIES OFF" - "RAFT_COMPILE_DIST_LIBRARY ${PKG_COMPILE_RAFT_DIST_LIBS}" + "RAFT_COMPILE_DIST_LIBRARY ON" "BUILD_TESTS OFF" "BUILD_BENCH OFF" "BUILD_SHARED_LIBS ${BUILD_RAFT_SHARED}" @@ -74,5 +74,4 @@ find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft} # even if it's already installed. CLONE_ON_PIN ON USE_RAFT_STATIC ${USE_RAFT_STATIC} - COMPILE_RAFT_DIST_LIBS ${CUGRAPH_COMPILE_RAFT_DIST_LIBS} ) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index ac3b0bc3fd9..9acce421b9d 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -59,13 +59,13 @@ if(NOT cugraph_FOUND) set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) + set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(_exclude_from_all "") if(CUGRAPH_BUILD_WHEELS) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index 1316be28ed3..eb0ecd42457 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -65,7 +65,6 @@ if (NOT cugraph_FOUND) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) - set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) From 5e76b8c8d85ee38f9bfa553ff731b8fcba0aaacf Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Thu, 2 Feb 2023 10:44:08 -0500 Subject: [PATCH 52/64] Build wheels on new cpu32 instances --- .github/workflows/pr.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e2576e00866..f998db5bc42 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -74,7 +74,8 @@ jobs: extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" + node_type: "cpu32" + #skbuild-build-options: "-j8" wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit @@ -99,7 +100,8 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - skbuild-build-options: "-j8" + node_type: "cpu32" + #skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit From 13d385619534c1d95edcfe444c927a1655134ccd Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 2 Feb 2023 12:31:08 -0800 Subject: [PATCH 53/64] Revert "Revert "Try disabling parallelism"" This reverts commit 9903ddace5cd8abf05ba489636aa77d328c871d0. --- cpp/CMakeLists.txt | 1 + cpp/cmake/thirdparty/get_raft.cmake | 5 +++-- python/cugraph/CMakeLists.txt | 4 ++-- python/pylibcugraph/CMakeLists.txt | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fe65da84327..5dd1308637b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -80,6 +80,7 @@ option(CMAKE_CUDA_LINEINFO "Enable the -lineinfo option for nvcc (useful for cud option(BUILD_TESTS "Configure CMake to build tests" ON) option(USE_CUGRAPH_OPS "Enable all functions that call cugraph-ops" ON) option(USE_RAFT_STATIC "Build raft as a static library" OFF) +option(CUGRAPH_COMPILE_RAFT_DIST_LIBS "Compile the raft library instead of using it header-only" ON) option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) option(CUGRAPH_USE_CUGRAPH_OPS_STATIC "Build and statically link the cugraph-ops library" OFF) option(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL "Exclude cugraph-ops targets from cuGraph's 'all' target" OFF) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 43226664e5d..d94827b4e81 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -19,7 +19,7 @@ set(CUGRAPH_BRANCH_VERSION_raft "${CUGRAPH_VERSION_MAJOR}.${CUGRAPH_VERSION_MINO function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC) + set(oneValueArgs VERSION FORK PINNED_TAG CLONE_ON_PIN USE_RAFT_STATIC COMPILE_RAFT_DIST_LIBS) cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN} ) if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "branch-${CUGRAPH_BRANCH_VERSION_raft}") @@ -47,7 +47,7 @@ function(find_and_configure_raft) SOURCE_SUBDIR cpp OPTIONS "RAFT_COMPILE_LIBRARIES OFF" - "RAFT_COMPILE_DIST_LIBRARY ON" + "RAFT_COMPILE_DIST_LIBRARY ${PKG_COMPILE_RAFT_DIST_LIBS}" "BUILD_TESTS OFF" "BUILD_BENCH OFF" "BUILD_SHARED_LIBS ${BUILD_RAFT_SHARED}" @@ -74,4 +74,5 @@ find_and_configure_raft(VERSION ${CUGRAPH_MIN_VERSION_raft} # even if it's already installed. CLONE_ON_PIN ON USE_RAFT_STATIC ${USE_RAFT_STATIC} + COMPILE_RAFT_DIST_LIBS ${CUGRAPH_COMPILE_RAFT_DIST_LIBS} ) diff --git a/python/cugraph/CMakeLists.txt b/python/cugraph/CMakeLists.txt index 9acce421b9d..ac3b0bc3fd9 100644 --- a/python/cugraph/CMakeLists.txt +++ b/python/cugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -59,13 +59,13 @@ if(NOT cugraph_FOUND) set(BUILD_TESTS OFF) set(BUILD_CUGRAPH_MG_TESTS OFF) set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) - set(BUILD_CUGRAPH_OPS_CPP_TESTS OFF) set(_exclude_from_all "") if(CUGRAPH_BUILD_WHEELS) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) + set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) diff --git a/python/pylibcugraph/CMakeLists.txt b/python/pylibcugraph/CMakeLists.txt index eb0ecd42457..1316be28ed3 100644 --- a/python/pylibcugraph/CMakeLists.txt +++ b/python/pylibcugraph/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -65,6 +65,7 @@ if (NOT cugraph_FOUND) # Statically link dependencies if building wheels set(CUDA_STATIC_RUNTIME ON) set(USE_RAFT_STATIC ON) + set(CUGRAPH_COMPILE_RAFT_DIST_LIBS OFF) set(CUGRAPH_USE_CUGRAPH_OPS_STATIC ON) set(CUGRAPH_EXCLUDE_CUGRAPH_OPS_FROM_ALL ON) set(ALLOW_CLONE_CUGRAPH_OPS ON) From ff02ebc8746332df9bd98543058360834195a785 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 2 Feb 2023 12:37:25 -0800 Subject: [PATCH 54/64] Allow raft template specializations. --- cpp/src/community/legacy/spectral_clustering.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/community/legacy/spectral_clustering.cu b/cpp/src/community/legacy/spectral_clustering.cu index 59067a35927..ea3885a801a 100644 --- a/cpp/src/community/legacy/spectral_clustering.cu +++ b/cpp/src/community/legacy/spectral_clustering.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,9 @@ #include #include +#if defined RAFT_DISTANCE_COMPILED #include +#endif #include #include From a62be6c9ee02e26a1c80f2bbb16f0548704e91b9 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 3 Feb 2023 11:14:05 -0500 Subject: [PATCH 55/64] Address test failures --- .github/workflows/pr.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f998db5bc42..3fa8d79e40e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -110,8 +110,8 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "bash ./datasets/get_test_data.sh && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=mg" + test-before-arm64: "bash ./datasets/get_test_data.sh && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore-glob='*/tests/mg/*'" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From a7db97ca395ec70730890648eca65d3bf19e6ac1 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 3 Feb 2023 15:56:26 -0500 Subject: [PATCH 56/64] Debug cugraph build --- .github/workflows/pr.yaml | 149 +++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 73 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3fa8d79e40e..31dba95ace0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,13 +13,13 @@ jobs: pr-builder: needs: - checks - - conda-cpp-build - - conda-cpp-tests - - conda-notebook-tests - - conda-python-build - - conda-python-tests - - wheel-build-pylibcugraph - - wheel-tests-pylibcugraph + #- conda-cpp-build + #- conda-cpp-tests + #- conda-notebook-tests + #- conda-python-build + #- conda-python-tests + #- wheel-build-pylibcugraph + #- wheel-tests-pylibcugraph - wheel-build-cugraph - wheel-tests-cugraph secrets: inherit @@ -27,68 +27,67 @@ jobs: checks: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-118 - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 - with: - build_type: pull-request - node_type: cpu16 - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 - with: - build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 - with: - build_type: pull-request - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 - with: - build_type: pull-request - node_type: "gpu-latest-1" - arch: "amd64" - container_image: "rapidsai/ci:latest" - run_script: "ci/test_notebooks.sh" - wheel-build-pylibcugraph: - needs: checks - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 - with: - build_type: pull-request - package-name: pylibcugraph - package-dir: python/pylibcugraph - extra-repo: rapidsai/cugraph-ops - extra-repo-sha: branch-23.02 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - node_type: "cpu32" - #skbuild-build-options: "-j8" - wheel-tests-pylibcugraph: - needs: wheel-build-pylibcugraph - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 - with: - build_type: pull-request - package-name: pylibcugraph - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + #conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 + # with: + # build_type: pull-request + # node_type: cpu16 + #conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 + # with: + # build_type: pull-request + #conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 + # with: + # build_type: pull-request + #conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 + # with: + # build_type: pull-request + #conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + # with: + # build_type: pull-request + # node_type: "gpu-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci:latest" + # run_script: "ci/test_notebooks.sh" + #wheel-build-pylibcugraph: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + # with: + # build_type: pull-request + # package-name: pylibcugraph + # package-dir: python/pylibcugraph + # extra-repo: rapidsai/cugraph-ops + # extra-repo-sha: branch-23.02 + # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + # node_type: "cpu32" + #wheel-tests-pylibcugraph: + # needs: wheel-build-pylibcugraph + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + # with: + # build_type: pull-request + # package-name: pylibcugraph + # # On arm also need to install cupy from the specific webpage. + # test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" + # test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + # test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: - needs: wheel-tests-pylibcugraph + #needs: wheel-tests-pylibcugraph secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: @@ -98,10 +97,12 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + #before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + cibw-environment: "SCCACHE_ERROR_LOG=/tmp/sccache_log.txt SCCACHE_LOG=debug" + post-repair-amd64: "rapids-upload-to-s3 cugraph /tmp/sccache_log.txt" + post-repair-arm64: "rapids-upload-to-s3 cugraph /tmp/sccache_log.txt" node_type: "cpu32" - #skbuild-build-options: "-j8" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit @@ -110,8 +111,10 @@ jobs: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "bash ./datasets/get_test_data.sh && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + #test-before-amd64: "cd ./datasets && bash ./datasets/get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "cd ./datasets && bash ./datasets/get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "bash ./datasets/get_test_data.sh && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore-glob='*/tests/mg/*'" + #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 511a09e3d6143d2266959ca9d440ae85107b707c Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 3 Feb 2023 16:21:23 -0500 Subject: [PATCH 57/64] Don't do sccache debugging --- .github/workflows/pr.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 31dba95ace0..dfa387c6c32 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -99,9 +99,6 @@ jobs: extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY #before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - cibw-environment: "SCCACHE_ERROR_LOG=/tmp/sccache_log.txt SCCACHE_LOG=debug" - post-repair-amd64: "rapids-upload-to-s3 cugraph /tmp/sccache_log.txt" - post-repair-arm64: "rapids-upload-to-s3 cugraph /tmp/sccache_log.txt" node_type: "cpu32" wheel-tests-cugraph: needs: wheel-build-cugraph From 7be84454a814df1835a896c37f5ae2d60d9b67df Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 3 Feb 2023 19:20:47 -0500 Subject: [PATCH 58/64] Adjust unittest command --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dfa387c6c32..a6826b037d9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -89,7 +89,7 @@ jobs: wheel-build-cugraph: #needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@no-arm with: build_type: pull-request package-name: cugraph @@ -103,13 +103,13 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@no-arm with: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - #test-before-amd64: "cd ./datasets && bash ./datasets/get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-before-amd64: "cd ./datasets && bash ./datasets/get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + #test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" From 90e2c1c95c384eec18c56954fca744cd0e3978e2 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Fri, 3 Feb 2023 19:40:14 -0500 Subject: [PATCH 59/64] Rebuild --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a6826b037d9..02fae743380 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -110,7 +110,7 @@ jobs: # Always want to test against latest dask/distributed. #test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - # On arm also need to install cupy from the specific webpage. + # On arm also need to install cupy from the specific site. #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" From e35d62d7632a4f3407bc3e092738c5913f622d93 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 4 Feb 2023 09:27:15 -0500 Subject: [PATCH 60/64] Use absolute path for RAPIDS_DATASET_ROOT_DIR --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 02fae743380..0b8ec6b2c44 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,5 +113,5 @@ jobs: # On arm also need to install cupy from the specific site. #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=$(readlink -f ./datasets) pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From d201ac6498ba4d8be0af4ed9f22515a80e58b75f Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 4 Feb 2023 10:18:09 -0500 Subject: [PATCH 61/64] Try a different dataset dir --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0b8ec6b2c44..4ad7aa95928 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,5 +113,5 @@ jobs: # On arm also need to install cupy from the specific site. #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=$(readlink -f ./datasets) pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From e9a16ea353f57f8706b62773165b6dd4d297b276 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 4 Feb 2023 10:57:10 -0500 Subject: [PATCH 62/64] Back to full PR checks --- .github/workflows/pr.yaml | 146 ++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4ad7aa95928..bdb0c1ac679 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,13 +13,13 @@ jobs: pr-builder: needs: - checks - #- conda-cpp-build - #- conda-cpp-tests - #- conda-notebook-tests - #- conda-python-build - #- conda-python-tests - #- wheel-build-pylibcugraph - #- wheel-tests-pylibcugraph + - conda-cpp-build + - conda-cpp-tests + - conda-notebook-tests + - conda-python-build + - conda-python-tests + - wheel-build-pylibcugraph + - wheel-tests-pylibcugraph - wheel-build-cugraph - wheel-tests-cugraph secrets: inherit @@ -27,69 +27,68 @@ jobs: checks: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-118 - #conda-cpp-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 - # with: - # build_type: pull-request - # node_type: cpu16 - #conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 - # with: - # build_type: pull-request - #conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 - # with: - # build_type: pull-request - #conda-python-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 - # with: - # build_type: pull-request - #conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 - # with: - # build_type: pull-request - # node_type: "gpu-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci:latest" - # run_script: "ci/test_notebooks.sh" - #wheel-build-pylibcugraph: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 - # with: - # build_type: pull-request - # package-name: pylibcugraph - # package-dir: python/pylibcugraph - # extra-repo: rapidsai/cugraph-ops - # extra-repo-sha: branch-23.02 - # extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - # skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - # node_type: "cpu32" - #wheel-tests-pylibcugraph: - # needs: wheel-build-pylibcugraph - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 - # with: - # build_type: pull-request - # package-name: pylibcugraph - # # On arm also need to install cupy from the specific webpage. - # test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" - # test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" - # test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 + with: + build_type: pull-request + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/test_notebooks.sh" + wheel-build-pylibcugraph: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + with: + build_type: pull-request + package-name: pylibcugraph + package-dir: python/pylibcugraph + extra-repo: rapidsai/cugraph-ops + extra-repo-sha: branch-23.02 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" + wheel-tests-pylibcugraph: + needs: wheel-build-pylibcugraph + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + with: + build_type: pull-request + package-name: pylibcugraph + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" + test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py" wheel-build-cugraph: - #needs: wheel-tests-pylibcugraph + needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@no-arm + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 with: build_type: pull-request package-name: cugraph @@ -97,21 +96,18 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-23.02 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - #before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=/project/cugraph-ops/" - node_type: "cpu32" wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@no-arm + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 with: build_type: pull-request package-name: cugraph # Always want to test against latest dask/distributed. - #test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific site. - #test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg" test-smoketest: "python ci/wheel_smoke_test_cugraph.py" From 59b04d4bfd9a5d35b1609b5cc68aa6799de17791 Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 4 Feb 2023 11:09:16 -0500 Subject: [PATCH 63/64] Use branch-23.02 wheel workflows --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e29efe7ac85..8a7a0035d66 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: wheel-build-pylibcugraph: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 with: build_type: pull-request package-name: pylibcugraph @@ -77,7 +77,7 @@ jobs: wheel-tests-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 with: build_type: pull-request package-name: pylibcugraph @@ -88,7 +88,7 @@ jobs: wheel-build-cugraph: needs: wheel-tests-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 with: build_type: pull-request package-name: cugraph @@ -101,7 +101,7 @@ jobs: wheel-tests-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 with: build_type: pull-request package-name: cugraph From 0e62fcae8a27f11d67d34c7783920c4e79a619bd Mon Sep 17 00:00:00 2001 From: Sevag Hanssian Date: Sat, 4 Feb 2023 13:16:44 -0500 Subject: [PATCH 64/64] Update build and test wheel workflows to match pr --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/test.yaml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 76298bdc498..26f9cc9c846 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -74,7 +74,7 @@ jobs: wheel-publish-pylibcugraph: needs: wheel-build-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -84,7 +84,7 @@ jobs: wheel-build-cugraph: needs: wheel-publish-pylibcugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -104,7 +104,7 @@ jobs: wheel-publish-cugraph: needs: wheel-build-cugraph secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.02 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f4f589bce61..533a2b5ce29 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibcugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -44,7 +44,7 @@ jobs: test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/pylibcugraph/pylibcugraph/tests" wheel-tests-cugraph: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 with: build_type: nightly branch: ${{ inputs.branch }} @@ -52,7 +52,7 @@ jobs: sha: ${{ inputs.sha }} package-name: cugraph # Always want to test against latest dask/distributed. - test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" - test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets pytest -v ./python/cugraph/cugraph/tests --ignore=tests/mg" + test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.02" + test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v ./python/cugraph/cugraph/tests --ignore=./python/cugraph/cugraph/tests/mg"