From 1e6fe4f881ad91a7f6e98fc9a862518712ff5f8b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 30 Aug 2024 17:48:15 -0400 Subject: [PATCH 1/7] Use changed-files shared workflow --- .github/workflows/pr.yaml | 121 ++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 70 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0d79568f589..f915bd84471 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -42,66 +42,47 @@ jobs: with: needs: ${{ toJSON(needs) }} changed-files: - runs-on: ubuntu-latest - name: "Check changed files" - outputs: - test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} - test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} - test_notebooks: ${{ steps.changed-files.outputs.notebooks_any_changed == 'true' }} - test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} - steps: - - name: Get PR info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - - name: Checkout code repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - name: Calculate merge base - id: calculate-merge-base - env: - PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} - BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} - run: | - (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") > "$GITHUB_OUTPUT" - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }} - sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} - files_yaml: | - cpp: - - '**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - java: - - '**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - notebooks: - - '**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!java/**' - python: - - '**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!java/**' - - '!notebooks/**' + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@changed-files + with: + files_yaml: | + cpp: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' + java: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + notebooks: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!java/**' + python: + - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!java/**' + - '!notebooks/**' + transform_expr: | + { + "test_cpp": (.cpp_any_changed == "true"), + "test_java": (.java_any_changed == "true"), + "test_notebooks": (.notebooks_any_changed == "true"), + "test_python": (.python_any_changed == "true"), + } checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 @@ -124,7 +105,7 @@ jobs: needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_cpp == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_cpp with: build_type: pull-request conda-python-build: @@ -137,7 +118,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -146,7 +127,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: build_type: pull-request script: "ci/test_python_other.sh" @@ -154,7 +135,7 @@ jobs: needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - if: needs.changed-files.outputs.test_java == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_java with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -175,7 +156,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - if: needs.changed-files.outputs.test_notebooks == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_notebooks with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -218,7 +199,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -235,7 +216,7 @@ jobs: needs: [wheel-build-cudf-polars, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -256,7 +237,7 @@ jobs: needs: [wheel-build-dask-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -276,7 +257,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -287,7 +268,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' + if: fromJSON(needs.changed-files.outputs.transformed_output).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From d7d0607e4adffeec65a1dc21b09f9d97c79f1907 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 30 Aug 2024 18:51:23 -0400 Subject: [PATCH 2/7] Use default transform_expr --- .github/workflows/pr.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f915bd84471..a99d6f12755 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -76,13 +76,6 @@ jobs: - '!img/**' - '!java/**' - '!notebooks/**' - transform_expr: | - { - "test_cpp": (.cpp_any_changed == "true"), - "test_java": (.java_any_changed == "true"), - "test_notebooks": (.notebooks_any_changed == "true"), - "test_python": (.python_any_changed == "true"), - } checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 From 85977fa86bce6b3d9dac01f2ced4b93b76ad8207 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Sat, 31 Aug 2024 11:20:06 -0400 Subject: [PATCH 3/7] Change naming convention --- .github/workflows/pr.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a99d6f12755..7f8ecd6de9b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,7 +46,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@changed-files with: files_yaml: | - cpp: + test_cpp: - '**' - '!CONTRIBUTING.md' - '!README.md' @@ -55,7 +55,7 @@ jobs: - '!java/**' - '!notebooks/**' - '!python/**' - java: + test_java: - '**' - '!CONTRIBUTING.md' - '!README.md' @@ -63,12 +63,12 @@ jobs: - '!img/**' - '!notebooks/**' - '!python/**' - notebooks: + test_notebooks: - '**' - '!CONTRIBUTING.md' - '!README.md' - '!java/**' - python: + test_python: - '**' - '!CONTRIBUTING.md' - '!README.md' From dfd3aa784ad9558a58965422673abaeaa91007aa Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 16 Sep 2024 10:46:39 -0400 Subject: [PATCH 4/7] Change output name --- .github/workflows/pr.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7f8ecd6de9b..01ec12e45ab 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,7 +98,7 @@ jobs: needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_cpp + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp with: build_type: pull-request conda-python-build: @@ -111,7 +111,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -120,7 +120,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: "ci/test_python_other.sh" @@ -128,7 +128,7 @@ jobs: needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_java + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -149,7 +149,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_notebooks + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -192,7 +192,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -209,7 +209,7 @@ jobs: needs: [wheel-build-cudf-polars, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -230,7 +230,7 @@ jobs: needs: [wheel-build-dask-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -250,7 +250,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -261,7 +261,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: fromJSON(needs.changed-files.outputs.transformed_output).test_python + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From 6e5ad64a9d17f901bfe3d9ef0a16c5d428069458 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 16 Sep 2024 17:50:29 -0400 Subject: [PATCH 5/7] Add note --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6159509de65..1ed9d6593e2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,6 +43,8 @@ jobs: needs: ${{ toJSON(needs) }} changed-files: secrets: inherit + # TODO: Change this back to @branch-24.10 once + # https://github.com/rapidsai/shared-workflows/pull/239 is merged uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@changed-files with: files_yaml: | From 7fbc03562f1800ff219a6de3a6e14eaff75450dd Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 17 Sep 2024 14:46:41 -0400 Subject: [PATCH 6/7] Switch to branch-24.10 --- .github/workflows/pr.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1ed9d6593e2..75cae096ff7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -43,9 +43,7 @@ jobs: needs: ${{ toJSON(needs) }} changed-files: secrets: inherit - # TODO: Change this back to @branch-24.10 once - # https://github.com/rapidsai/shared-workflows/pull/239 is merged - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@changed-files + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.10 with: files_yaml: | test_cpp: From bc0076e5d2a927b58cb00cd679a19e3fd8d66df9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 26 Sep 2024 12:43:41 -0400 Subject: [PATCH 7/7] Fix branch --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 42c1d832e98..bc237cc73b0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,7 +44,7 @@ jobs: needs: ${{ toJSON(needs) }} changed-files: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.10 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 with: files_yaml: | test_cpp: