From 7e5ed4628783be47f5c00b32476793eaabb5dad9 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Tue, 15 Oct 2024 14:42:08 +0200 Subject: [PATCH] Update workflows Signed-off-by: Jordan Jacobelli --- .github/workflows/build.yaml | 10 +- .github/workflows/sched.yaml | 10 ++ .github/workflows/test.yaml | 255 +++++++++++++++++++++-------------- 3 files changed, 168 insertions(+), 107 deletions(-) create mode 100644 .github/workflows/sched.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c034752d373..548b309620d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,11 +1,11 @@ name: build on: - push: - branches: - - "branch-*" - tags: - - v[0-9][0-9].[0-9][0-9].[0-9][0-9] +# push: +# branches: +# - "branch-*" +# tags: +# - v[0-9][0-9].[0-9][0-9].[0-9][0-9] workflow_dispatch: inputs: branch: diff --git a/.github/workflows/sched.yaml b/.github/workflows/sched.yaml new file mode 100644 index 00000000000..3d7c69d1b09 --- /dev/null +++ b/.github/workflows/sched.yaml @@ -0,0 +1,10 @@ +name: sched + +on: + schedule: + - cron: '0 */2 * * *' + +jobs: + run: + uses: ./.github/workflows/test.yaml + secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1275aad757c..e2592f1118d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,150 +1,201 @@ name: test on: + workflow_call: + inputs: + branch: + required: false + type: string + default: branch-24.12 + date: + required: false + type: string + default: 2024-10-14 + sha: + required: false + type: string + default: 4dbb8a354a9d4f0b4d82a5bf9747409c6304358f workflow_dispatch: inputs: branch: required: true type: string + default: branch-24.12 date: required: true type: string + default: 2024-10-14 sha: required: true type: string + default: 4dbb8a354a9d4f0b4d82a5bf9747409c6304358f jobs: - conda-cpp-checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - enable_check_symbols: true +# conda-cpp-checks: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# enable_check_symbols: true conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - conda-cpp-memcheck-tests: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" - run_script: "ci/test_cpp_memcheck.sh" - static-configure: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - # Use the wheel container so we can skip conda solves and since our - # primary static consumers (Spark) are not in conda anyway. - container_image: "rapidsai/ci-wheel:latest" - run_script: "ci/configure_cpp_static.sh" - clang-tidy: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - run_script: "ci/clang_tidy.sh" + matrix_filter: '. |= [ + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"l4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtx4090","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"h100","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtxa6000","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"t4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"a100","DRIVER":"latest","DEPENDENCIES":"latest"} + ]' +# conda-cpp-memcheck-tests: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# node_type: "gpu-v100-latest-1" +# arch: "amd64" +# container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" +# run_script: "ci/test_cpp_memcheck.sh" +# static-configure: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# # Use the wheel container so we can skip conda solves and since our +# # primary static consumers (Spark) are not in conda anyway. +# container_image: "rapidsai/ci-wheel:latest" +# run_script: "ci/configure_cpp_static.sh" +# clang-tidy: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# run_script: "ci/clang_tidy.sh" conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_python_cudf.sh" + run_codecov: false + matrix_filter: '. |= [ + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"l4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtx4090","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"h100","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtxa6000","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"t4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"a100","DRIVER":"latest","DEPENDENCIES":"latest"} + ]' conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_python_other.sh" - conda-java-tests: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" - run_script: "ci/test_java.sh" - conda-notebook-tests: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" - run_script: "ci/test_notebooks.sh" + run_codecov: false + matrix_filter: '. |= [ + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"l4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtx4090","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"h100","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtxa6000","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"t4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"a100","DRIVER":"latest","DEPENDENCIES":"latest"} + ]' +# conda-java-tests: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# node_type: "gpu-v100-latest-1" +# arch: "amd64" +# container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" +# run_script: "ci/test_java.sh" +# conda-notebook-tests: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# node_type: "gpu-v100-latest-1" +# arch: "amd64" +# container_image: "rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.11" +# run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@test-nvks-runners with: build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cudf.sh - wheel-tests-dask-cudf: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/test_wheel_dask_cudf.sh - unit-tests-cudf-pandas: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/cudf_pandas_scripts/run_tests.sh - third-party-integration-tests-cudf-pandas: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - node_type: "gpu-v100-latest-1" - container_image: "rapidsai/ci-conda:latest" - run_script: | - ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml + matrix_filter: '. |= [ + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"l4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtx4090","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"h100","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"rtxa6000","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"t4","DRIVER":"latest","DEPENDENCIES":"latest"}, + {"ARCH":"amd64","PY_VER":"3.10","CUDA_VER":"11.8.0","LINUX_VER":"ubuntu22.04","GPU":"a100","DRIVER":"latest","DEPENDENCIES":"latest"} + ]' +# wheel-tests-dask-cudf: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 +# with: +# # This selects "ARCH=amd64 + the latest supported Python + CUDA". +# matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# script: ci/test_wheel_dask_cudf.sh +# unit-tests-cudf-pandas: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# script: ci/cudf_pandas_scripts/run_tests.sh +# third-party-integration-tests-cudf-pandas: +# secrets: inherit +# uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 +# with: +# build_type: nightly +# branch: ${{ inputs.branch }} +# date: ${{ inputs.date }} +# sha: ${{ inputs.sha }} +# node_type: "gpu-v100-latest-1" +# container_image: "rapidsai/ci-conda:latest" +# run_script: | +# ci/cudf_pandas_scripts/third-party-integration/test.sh python/cudf/cudf_pandas_tests/third_party_integration_tests/dependencies.yaml