From 521067c577132c34c97174040df98dc5c7416540 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 23 Jan 2024 21:15:03 +0000 Subject: [PATCH 1/9] Make sure not to trigger extraneous rebuilds --- docs/cudf/source/conf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 01a6c5316bd..986e17aea3e 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -16,10 +16,12 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +import filecmp import glob import os import re import sys +import tempfile import xml.etree.ElementTree as ET from docutils.nodes import Text @@ -126,7 +128,13 @@ def clean_all_xml_files(path): for fn in glob.glob(os.path.join(path, "*.xml")): tree = ET.parse(fn) clean_definitions(tree.getroot()) - tree.write(fn) + with tempfile.NamedTemporaryFile() as tmp_fn: + tree.write(tmp_fn.name) + # Only write files that have actually changed. + if not filecmp.cmp(tmp_fn.name, fn): + tree.write(fn) + + # Breathe Configuration From 0ae18714b3d099ff7b9038a89e001925241f838c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 23 Jan 2024 22:31:55 +0000 Subject: [PATCH 2/9] Remove toctrees from generated pages to speed up the build --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-120_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + docs/cudf/source/conf.py | 3 +++ 4 files changed, 6 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8081d9de8b9..2794125b78a 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -93,6 +93,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables +- sphinx-remove-toctrees - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 1cb8f376f82..91104c55961 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -91,6 +91,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables +- sphinx-remove-toctrees - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 diff --git a/dependencies.yaml b/dependencies.yaml index e62fa86d4d4..09866869a8b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -472,6 +472,7 @@ dependencies: - sphinx-autobuild - sphinx-copybutton - sphinx-markdown-tables + - sphinx-remove-toctrees - sphinxcontrib-websupport notebooks: common: diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 986e17aea3e..5b04335f475 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -64,6 +64,7 @@ class PseudoLexer(RegexLexer): "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx_copybutton", + "sphinx_remove_toctrees", "numpydoc", "IPython.sphinxext.ipython_console_highlighting", "IPython.sphinxext.ipython_directive", @@ -71,6 +72,8 @@ class PseudoLexer(RegexLexer): "myst_nb", ] +remove_from_toctrees = ["user_guide/api_docs/api/*"] + # Preprocess doxygen xml for compatibility with latest Breathe def clean_definitions(root): From e5081a557738860fef2193fdd3564eaa2a7cd461 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 24 Jan 2024 00:51:18 +0000 Subject: [PATCH 3/9] Disable all but the crucial CI jobs --- .github/workflows/pr.yaml | 234 +++++++++++++++++++------------------- 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 41d4e42891c..c80e275632a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,21 +14,21 @@ jobs: needs: - checks - conda-cpp-build - - conda-cpp-checks - - conda-cpp-tests + #- conda-cpp-checks + #- conda-cpp-tests - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests - - conda-notebook-tests + #- conda-python-cudf-tests + #- conda-python-other-tests + #- conda-java-tests + #- conda-notebook-tests - docs-build - - wheel-build-cudf - - wheel-tests-cudf - - wheel-build-dask-cudf - - wheel-tests-dask-cudf - - devcontainer - - unit-tests-cudf-pandas - - pandas-tests + #- wheel-build-cudf + #- wheel-tests-cudf + #- wheel-build-dask-cudf + #- wheel-tests-dask-cudf + #- devcontainer + #- unit-tests-cudf-pandas + #- pandas-tests #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit @@ -44,60 +44,60 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 with: build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 - with: - build_type: pull-request - enable_check_symbols: true - conda-cpp-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 - with: - build_type: pull-request + #conda-cpp-checks: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + # with: + # build_type: pull-request + # enable_check_symbols: true + #conda-cpp-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + # with: + # build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: pull-request - conda-python-cudf-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - with: - build_type: pull-request - test_script: "ci/test_python_cudf.sh" - conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - with: - build_type: pull-request - test_script: "ci/test_python_other.sh" - conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" + #conda-python-cudf-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + # with: + # build_type: pull-request + # test_script: "ci/test_python_cudf.sh" + #conda-python-other-tests: + # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + # with: + # build_type: pull-request + # test_script: "ci/test_python_other.sh" + #conda-java-tests: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_java.sh" + #conda-notebook-tests: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + # with: + # build_type: pull-request + # node_type: "gpu-v100-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/test_notebooks.sh" docs-build: needs: conda-python-build secrets: inherit @@ -108,62 +108,62 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cudf: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 - with: - build_type: pull-request - build-2_28-wheels: "true" - script: "ci/build_wheel_cudf.sh" - wheel-tests-cudf: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - build_type: pull-request - script: ci/test_wheel_cudf.sh - wheel-build-dask-cudf: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - build_type: pull-request - script: "ci/build_wheel_dask_cudf.sh" - wheel-tests-dask-cudf: - needs: wheel-build-dask-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - build_type: pull-request - script: ci/test_wheel_dask_cudf.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 - with: - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; - unit-tests-cudf-pandas: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - build_type: pull-request - script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests using PR branch - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - with: - matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] - build_type: pull-request - script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + #wheel-build-cudf: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + # with: + # build_type: pull-request + # build-2_28-wheels: "true" + # script: "ci/build_wheel_cudf.sh" + #wheel-tests-cudf: + # needs: wheel-build-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # build_type: pull-request + # script: ci/test_wheel_cudf.sh + #wheel-build-dask-cudf: + # needs: wheel-build-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + # build_type: pull-request + # script: "ci/build_wheel_dask_cudf.sh" + #wheel-tests-dask-cudf: + # needs: wheel-build-dask-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + # build_type: pull-request + # script: ci/test_wheel_dask_cudf.sh + #devcontainer: + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + # with: + # build_command: | + # sccache -z; + # build-all -DBUILD_BENCHMARKS=ON --verbose; + # sccache -s; + #unit-tests-cudf-pandas: + # needs: wheel-build-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + # build_type: pull-request + # script: ci/cudf_pandas_scripts/run_tests.sh + #pandas-tests: + # # run the Pandas unit tests using PR branch + # needs: wheel-build-cudf + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # with: + # matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] + # build_type: pull-request + # script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr #pandas-tests-diff: # # diff the results of running the Pandas unit tests and publish a job summary # needs: [pandas-tests-main, pandas-tests-pr] From 0095b7dba0ea4e6651f7f613e10cf8e8146ca032 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 23 Jan 2024 23:14:42 -0800 Subject: [PATCH 4/9] Enable parallel build and disable warnings just to measure perf impact --- ci/build_docs.sh | 8 ++++---- docs/cudf/Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 5a4bf3e0dbc..257a3aa572c 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -40,8 +40,8 @@ popd rapids-logger "Build Python docs" pushd docs/cudf -make dirhtml -make text +make dirhtml O="-j 4" +make text O="-j 4" mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"{html,txt} mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" @@ -49,8 +49,8 @@ popd rapids-logger "Build dask-cuDF Sphinx docs" pushd docs/dask_cudf -make dirhtml -make text +make dirhtml O="-j 4" +make text O="-j 4" mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"{html,txt} mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" diff --git a/docs/cudf/Makefile b/docs/cudf/Makefile index 0b3a6876a5a..e513c89a561 100644 --- a/docs/cudf/Makefile +++ b/docs/cudf/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = -n -v -W --keep-going +SPHINXOPTS = -n -v SPHINXBUILD = sphinx-build SPHINXPROJ = cudf SOURCEDIR = source From cef3c965c79513df68fd02f9bcbe3c8748cf6660 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 24 Jan 2024 06:34:50 -0800 Subject: [PATCH 5/9] Revert "Enable parallel build and disable warnings just to measure perf impact" This reverts commit 0095b7dba0ea4e6651f7f613e10cf8e8146ca032. --- ci/build_docs.sh | 8 ++++---- docs/cudf/Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 257a3aa572c..5a4bf3e0dbc 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -40,8 +40,8 @@ popd rapids-logger "Build Python docs" pushd docs/cudf -make dirhtml O="-j 4" -make text O="-j 4" +make dirhtml +make text mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"{html,txt} mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" @@ -49,8 +49,8 @@ popd rapids-logger "Build dask-cuDF Sphinx docs" pushd docs/dask_cudf -make dirhtml O="-j 4" -make text O="-j 4" +make dirhtml +make text mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"{html,txt} mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" diff --git a/docs/cudf/Makefile b/docs/cudf/Makefile index e513c89a561..0b3a6876a5a 100644 --- a/docs/cudf/Makefile +++ b/docs/cudf/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = -n -v +SPHINXOPTS = -n -v -W --keep-going SPHINXBUILD = sphinx-build SPHINXPROJ = cudf SOURCEDIR = source From 5b5e0788f643af81944d45c5681f87bad0558cc1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 24 Jan 2024 06:43:45 -0800 Subject: [PATCH 6/9] Disable text docs --- ci/build_docs.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 5a4bf3e0dbc..d5e01226df4 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -41,19 +41,21 @@ popd rapids-logger "Build Python docs" pushd docs/cudf make dirhtml -make text -mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"{html,txt} +mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"html mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" -mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" +#make text +#mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"txt +#mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" popd rapids-logger "Build dask-cuDF Sphinx docs" pushd docs/dask_cudf make dirhtml -make text -mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"{html,txt} +mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"html mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" -mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" +#make text +#mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"txt +#mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" popd rapids-upload-docs From 7faafeea08efa17752291096a04fed57ad7f398b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 24 Jan 2024 19:00:33 +0000 Subject: [PATCH 7/9] Use RAPIDS_BUILD_TYPE to determine what to build --- ci/build_docs.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index d5e01226df4..c0a13b362c2 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -43,9 +43,11 @@ pushd docs/cudf make dirhtml mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"html mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" -#make text -#mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"txt -#mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" +if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then + make text + mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"txt + mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" +fi popd rapids-logger "Build dask-cuDF Sphinx docs" @@ -53,9 +55,11 @@ pushd docs/dask_cudf make dirhtml mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"html mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" -#make text -#mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"txt -#mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" +if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then + make text + mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"txt + mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" +fi popd rapids-upload-docs From 1d3d10337fd0e77f3568f6e11ac8c2e47f7a2998 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 24 Jan 2024 21:47:11 +0000 Subject: [PATCH 8/9] Revert "Disable all but the crucial CI jobs" This reverts commit e5081a557738860fef2193fdd3564eaa2a7cd461. --- .github/workflows/pr.yaml | 234 +++++++++++++++++++------------------- 1 file changed, 117 insertions(+), 117 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c80e275632a..41d4e42891c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,21 +14,21 @@ jobs: needs: - checks - conda-cpp-build - #- conda-cpp-checks - #- conda-cpp-tests + - conda-cpp-checks + - conda-cpp-tests - conda-python-build - #- conda-python-cudf-tests - #- conda-python-other-tests - #- conda-java-tests - #- conda-notebook-tests + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - conda-notebook-tests - docs-build - #- wheel-build-cudf - #- wheel-tests-cudf - #- wheel-build-dask-cudf - #- wheel-tests-dask-cudf - #- devcontainer - #- unit-tests-cudf-pandas - #- pandas-tests + - wheel-build-cudf + - wheel-tests-cudf + - wheel-build-dask-cudf + - wheel-tests-dask-cudf + - devcontainer + - unit-tests-cudf-pandas + - pandas-tests #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit @@ -44,60 +44,60 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 with: build_type: pull-request - #conda-cpp-checks: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 - # with: - # build_type: pull-request - # enable_check_symbols: true - #conda-cpp-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 - # with: - # build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.02 + with: + build_type: pull-request + enable_check_symbols: true + conda-cpp-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + with: + build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 with: build_type: pull-request - #conda-python-cudf-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - # with: - # build_type: pull-request - # test_script: "ci/test_python_cudf.sh" - #conda-python-other-tests: - # # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 - # with: - # build_type: pull-request - # test_script: "ci/test_python_other.sh" - #conda-java-tests: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_java.sh" - #conda-notebook-tests: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 - # with: - # build_type: pull-request - # node_type: "gpu-v100-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/test_notebooks.sh" + conda-python-cudf-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_cudf.sh" + conda-python-other-tests: + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + with: + build_type: pull-request + test_script: "ci/test_python_other.sh" + conda-java-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" docs-build: needs: conda-python-build secrets: inherit @@ -108,62 +108,62 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - #wheel-build-cudf: - # needs: checks - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 - # with: - # build_type: pull-request - # build-2_28-wheels: "true" - # script: "ci/build_wheel_cudf.sh" - #wheel-tests-cudf: - # needs: wheel-build-cudf - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - # with: - # build_type: pull-request - # script: ci/test_wheel_cudf.sh - #wheel-build-dask-cudf: - # needs: wheel-build-cudf - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 - # with: - # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - # build_type: pull-request - # script: "ci/build_wheel_dask_cudf.sh" - #wheel-tests-dask-cudf: - # needs: wheel-build-dask-cudf - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - # with: - # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - # build_type: pull-request - # script: ci/test_wheel_dask_cudf.sh - #devcontainer: - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 - # with: - # build_command: | - # sccache -z; - # build-all -DBUILD_BENCHMARKS=ON --verbose; - # sccache -s; - #unit-tests-cudf-pandas: - # needs: wheel-build-cudf - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - # with: - # matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) - # build_type: pull-request - # script: ci/cudf_pandas_scripts/run_tests.sh - #pandas-tests: - # # run the Pandas unit tests using PR branch - # needs: wheel-build-cudf - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 - # with: - # matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] - # build_type: pull-request - # script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + wheel-build-cudf: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + with: + build_type: pull-request + build-2_28-wheels: "true" + script: "ci/build_wheel_cudf.sh" + wheel-tests-cudf: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + with: + build_type: pull-request + script: ci/test_wheel_cudf.sh + wheel-build-dask-cudf: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + build_type: pull-request + script: "ci/build_wheel_dask_cudf.sh" + wheel-tests-dask-cudf: + needs: wheel-build-dask-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + build_type: pull-request + script: ci/test_wheel_dask_cudf.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + with: + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON --verbose; + sccache -s; + unit-tests-cudf-pandas: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + with: + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1"))) + build_type: pull-request + script: ci/cudf_pandas_scripts/run_tests.sh + pandas-tests: + # run the Pandas unit tests using PR branch + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + with: + matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] + build_type: pull-request + script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr #pandas-tests-diff: # # diff the results of running the Pandas unit tests and publish a job summary # needs: [pandas-tests-main, pandas-tests-pr] From 5c8b6281673da58552d9d8f0d3aa762c814f9138 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 25 Jan 2024 11:16:29 -0800 Subject: [PATCH 9/9] Fix quoting Co-authored-by: Bradley Dice --- ci/build_docs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index c0a13b362c2..2b55a9db8af 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -41,11 +41,11 @@ popd rapids-logger "Build Python docs" pushd docs/cudf make dirhtml -mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"html +mkdir -p "${RAPIDS_DOCS_DIR}/cudf/html" mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/cudf/html" if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then make text - mkdir -p "${RAPIDS_DOCS_DIR}/cudf/"txt + mkdir -p "${RAPIDS_DOCS_DIR}/cudf/txt" mv build/text/* "${RAPIDS_DOCS_DIR}/cudf/txt" fi popd @@ -53,11 +53,11 @@ popd rapids-logger "Build dask-cuDF Sphinx docs" pushd docs/dask_cudf make dirhtml -mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"html +mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/html" mv build/dirhtml/* "${RAPIDS_DOCS_DIR}/dask-cudf/html" if [[ "${RAPIDS_BUILD_TYPE}" != "pull-request" ]]; then make text - mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/"txt + mkdir -p "${RAPIDS_DOCS_DIR}/dask-cudf/txt" mv build/text/* "${RAPIDS_DOCS_DIR}/dask-cudf/txt" fi popd