From 3bd85750cf825bc85a145faa19c5e4c04ce3d018 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 14:13:07 -0700 Subject: [PATCH 01/12] Update to new workflow to test --- .github/workflows/pr.yaml | 100 +++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c51d5c0a34..079bb08bb4 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,62 +12,62 @@ concurrency: jobs: pr-builder: needs: - - checks - - conda-cpp-build - - conda-cpp-tests - - conda-python-build - - conda-python-tests - - docs-build + #- checks + #- conda-cpp-build + #- conda-cpp-tests + #- conda-python-build + #- conda-python-tests + #- docs-build - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06 - checks: - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 - with: - enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 - with: - build_type: pull-request - node_type: cpu16 - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 - with: - build_type: pull-request - conda-python-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci:latest" - run_script: "ci/build_docs.sh" + #checks: + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 + # with: + # enable_check_generated_files: false + #conda-cpp-build: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + # with: + # build_type: pull-request + # node_type: cpu16 + #conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + # with: + # build_type: pull-request + #conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + # with: + # build_type: pull-request + #conda-python-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + # with: + # build_type: pull-request + #docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci:latest" + # run_script: "ci/build_docs.sh" wheel-build-pylibraft: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: build_type: pull-request package-name: pylibraft @@ -76,7 +76,7 @@ jobs: wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 with: build_type: pull-request package-name: pylibraft @@ -88,7 +88,7 @@ jobs: wheel-build-raft-dask: needs: wheel-tests-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: build_type: pull-request package-name: raft_dask @@ -98,7 +98,7 @@ jobs: wheel-tests-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 with: build_type: pull-request package-name: raft_dask From 15606d666b41e093a865a72649467e654c488940 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 14:24:02 -0700 Subject: [PATCH 02/12] Remove dep on checks --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 079bb08bb4..a3a3a810a5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -65,7 +65,7 @@ jobs: # container_image: "rapidsai/ci:latest" # run_script: "ci/build_docs.sh" wheel-build-pylibraft: - needs: checks + #needs: checks secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: From e424fd7e3b4df5b05a04c7c76fe8de4582a96598 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 14:52:38 -0700 Subject: [PATCH 03/12] bump From d8d8f4d93aec2779ffae4d2ec3eefd8dd6fbc428 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 15:26:36 -0700 Subject: [PATCH 04/12] Update for local wheelhouse being at the root --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a3a3a810a5..3f5ca3d213 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -93,7 +93,7 @@ jobs: build_type: pull-request package-name: raft_dask package-dir: python/raft-dask - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-wheelhouse" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 /local-wheelhouse" skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" wheel-tests-raft-dask: needs: wheel-build-raft-dask From 7a7836a8a9ca71e1bed26c211f9ffd0acc661bc4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 16:27:57 -0700 Subject: [PATCH 05/12] bump From 7d4c568c8d975a04c9e67b84e2206f8bc8e1aa64 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 16:42:47 -0700 Subject: [PATCH 06/12] Skip pylibraft tests for speed --- .github/workflows/pr.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f5ca3d213..1bf3bff175 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -73,20 +73,20 @@ jobs: package-name: pylibraft package-dir: python/pylibraft skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - wheel-tests-pylibraft: - needs: wheel-build-pylibraft - secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 - with: - build_type: pull-request - package-name: pylibraft - test-before-amd64: "pip install cupy-cuda11x" - # On arm also need to install cupy from the specific webpage. - test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test" - test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" + #wheel-tests-pylibraft: + # needs: wheel-build-pylibraft + # secrets: inherit + # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + # with: + # build_type: pull-request + # package-name: pylibraft + # test-before-amd64: "pip install cupy-cuda11x" + # # On arm also need to install cupy from the specific webpage. + # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + # test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test" + # test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" wheel-build-raft-dask: - needs: wheel-tests-pylibraft + needs: wheel-build-pylibraft secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: From 0b49ebe4fc69af8b542f3f653002f81ecc6be7a0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 16:50:32 -0700 Subject: [PATCH 07/12] Remove tests from list too --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1bf3bff175..e2543eb962 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,7 +19,7 @@ jobs: #- conda-python-tests #- docs-build - wheel-build-pylibraft - - wheel-tests-pylibraft + #- wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask secrets: inherit From 22b3445d30aaf8d2d39532da3bb1464563709f51 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 17:13:20 -0700 Subject: [PATCH 08/12] Revert most changes except for new workflow --- .github/workflows/pr.yaml | 122 +++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e2543eb962..0be646409f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,60 +12,60 @@ concurrency: jobs: pr-builder: needs: - #- checks - #- conda-cpp-build - #- conda-cpp-tests - #- conda-python-build - #- conda-python-tests - #- docs-build + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-python-build + - conda-python-tests + - docs-build - wheel-build-pylibraft - #- wheel-tests-pylibraft + - wheel-tests-pylibraft - wheel-build-raft-dask - wheel-tests-raft-dask secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.06 - #checks: - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 - # with: - # enable_check_generated_files: false - #conda-cpp-build: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 - # with: - # build_type: pull-request - # node_type: cpu16 - #conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 - # with: - # build_type: pull-request - #conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 - # with: - # build_type: pull-request - #conda-python-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 - # with: - # build_type: pull-request - #docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci:latest" - # run_script: "ci/build_docs.sh" + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.06 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.06 + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.06 + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.06 + with: + build_type: pull-request + conda-python-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.06 + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.06 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-pylibraft: - #needs: checks + needs: checks secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: @@ -73,21 +73,21 @@ jobs: package-name: pylibraft package-dir: python/pylibraft skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" - #wheel-tests-pylibraft: - # needs: wheel-build-pylibraft - # secrets: inherit - # uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 - # with: - # build_type: pull-request - # package-name: pylibraft - # test-before-amd64: "pip install cupy-cuda11x" - # # On arm also need to install cupy from the specific webpage. - # test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" - # test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test" - # test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" - wheel-build-raft-dask: + wheel-tests-pylibraft: needs: wheel-build-pylibraft secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + with: + build_type: pull-request + package-name: pylibraft + test-before-amd64: "pip install cupy-cuda11x" + # On arm also need to install cupy from the specific webpage. + test-before-arm64: "pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64" + test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test" + test-smoketest: "python ./ci/wheel_smoke_test_pylibraft.py" + wheel-build-raft-dask: + needs: wheel-tests-pylibraft + secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: build_type: pull-request From 419073b81b6c8b7b504bd2c77e3d14d31b38c1d8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Apr 2023 17:23:24 -0700 Subject: [PATCH 09/12] bump From ec8c213a5d313e3a4dd1e4753ccc9169a64d2121 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 29 Apr 2023 16:09:24 -0700 Subject: [PATCH 10/12] Update all workflows --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/test.yaml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0f5f84c158..00a3aac95a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-pylibraft: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -78,7 +78,7 @@ jobs: wheel-publish-pylibraft: needs: wheel-build-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@manylinux_v2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: wheel-build-raft-dask: needs: wheel-publish-pylibraft secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-raft-dask: needs: wheel-build-raft-dask secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@manylinux_v2 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 05e96a6dff..17b2d6f83b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,7 +32,7 @@ jobs: sha: ${{ inputs.sha }} wheel-tests-pylibraft: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 with: build_type: nightly branch: ${{ inputs.branch }} @@ -44,7 +44,7 @@ jobs: test-unittest: "python -m pytest ./python/pylibraft/pylibraft/test" wheel-tests-raft-dask: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.06 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 with: build_type: nightly branch: ${{ inputs.branch }} From 96a58096182b2200ec0064efe3c5d7d0f3143293 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 29 Apr 2023 16:21:24 -0700 Subject: [PATCH 11/12] Remove local wheelhouse logic --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0be646409f..4c4c545a78 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -93,7 +93,7 @@ jobs: build_type: pull-request package-name: raft_dask package-dir: python/raft-dask - before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 /local-wheelhouse" + before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibraft_cu11 rapids-download-wheels-from-s3 ./local-pylibraft && python -m pip install --no-deps ./local-pylibraft/pylibraft*.whl" skbuild-configure-options: "-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" wheel-tests-raft-dask: needs: wheel-build-raft-dask From 4025d63d2f32c62ca1f4d172a1a921e76bc31d75 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 29 Apr 2023 17:44:02 -0700 Subject: [PATCH 12/12] bump