From 1919487b6fe156c5cd9792a8e00b054eab99df06 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Mon, 23 Jan 2023 11:20:01 -0600 Subject: [PATCH 01/17] initial draft --- .github/workflows/build.yaml | 11 +++++++++ .github/workflows/pr.yaml | 11 +++++++++ ci/build_docs.sh | 48 ++++++++++++++++++++++++++++++++++++ dependencies.yaml | 11 +++++++-- 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 ci/build_docs.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 093063c21e7..5cf7c491304 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,17 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + docs-build: + if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} + needs: python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + with: + build_type: branch + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cb83aab31cd..25157c5f2d3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,6 +20,7 @@ jobs: - conda-python-other-tests - conda-java-tests - conda-notebook-tests + - docs-build - wheel-build-cudf - wheel-tests-cudf - wheel-build-dask-cudf @@ -82,6 +83,16 @@ jobs: arch: "amd64" container_image: "rapidsai/ci:latest" run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: needs: checks secrets: inherit diff --git a/ci/build_docs.sh b/ci/build_docs.sh new file mode 100644 index 00000000000..2711ee8bd84 --- /dev/null +++ b/ci/build_docs.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create test conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test +conda activate test + +rapids-print-env + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +VERSION_NUMBER=$(rapids-get-rapids-version-from-git) + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + libcudf \ + cudf + + +# Build Doxygen docs +gpuci_logger "Build Doxygen docs" +pushd cpp/doxygen +doxygen Doxyfile +popd + +# Build Python docs +gpuci_logger "Build Sphinx docs" +pushd docs/cudf +sphinx-build -b dirhtml source _html +sphinx-build -b text source _text +popd + + +if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then + aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" + aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" +fi \ No newline at end of file diff --git a/dependencies.yaml b/dependencies.yaml index 8790853fbb3..e8e4d172fe7 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -40,6 +40,12 @@ files: includes: - develop - py_version + docs: + output: none + includes: + - cudatoolkit + - docs_build + - py_version channels: - rapidsai - rapidsai-nightly @@ -120,10 +126,11 @@ dependencies: - output_types: conda packages: - doxygen=1.8.20 # pre-commit hook needs a specific version. - doc: + docs_build: common: - - output_types: [conda, requirements] + - output_types: [conda] packages: + - doxygen=1.8.20 - myst-nb - nbsphinx - numpydoc From 1627316bc80cdb26782651b14dfc0259093e1310 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 24 Jan 2023 11:18:26 -0600 Subject: [PATCH 02/17] fix typo --- ci/build_docs.sh | 2 +- dependencies.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 2711ee8bd84..63849ad14b4 100644 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -45,4 +45,4 @@ if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" -fi \ No newline at end of file +fi diff --git a/dependencies.yaml b/dependencies.yaml index e8e4d172fe7..cc05f4c5c3f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,7 +9,7 @@ files: - build - cudatoolkit - develop - - doc + - docs_build - notebooks - py_version - run From 32f2cd822e2a784f7a2834fdbb83d7dd74a4861a Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 25 Jan 2023 04:02:24 -0600 Subject: [PATCH 03/17] chmod --- ci/build_docs.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/build_docs.sh diff --git a/ci/build_docs.sh b/ci/build_docs.sh old mode 100644 new mode 100755 From 054cefa199c816f85afd8c9019b229b262f3b5b6 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 25 Jan 2023 09:15:20 -0600 Subject: [PATCH 04/17] add py channel --- ci/build_docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 63849ad14b4..511ed8f14f9 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -18,6 +18,7 @@ rapids-print-env rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) VERSION_NUMBER=$(rapids-get-rapids-version-from-git) rapids-mamba-retry install \ From 9f401af2a327c0d208c3b8f21dd4cf7bcf56bf43 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Sun, 29 Jan 2023 10:06:22 -0600 Subject: [PATCH 05/17] fixes --- ci/build_docs.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 511ed8f14f9..073af59bac9 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -11,8 +11,8 @@ rapids-dependency-file-generator \ --file_key docs \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test -conda activate test +rapids-mamba-retry env create --force -f env.yaml -n docs +conda activate docs rapids-print-env @@ -28,22 +28,22 @@ rapids-mamba-retry install \ cudf -# Build Doxygen docs -gpuci_logger "Build Doxygen docs" +# Build CPP docs +rapids-logger "Build Doxygen docs" pushd cpp/doxygen doxygen Doxyfile popd # Build Python docs -gpuci_logger "Build Sphinx docs" +rapids-logger "Build Sphinx docs" pushd docs/cudf -sphinx-build -b dirhtml source _html -sphinx-build -b text source _text +sphinx-build -b dirhtml source _html -W +sphinx-build -b text source _text -W popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" - aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" fi From 534b6c5197e3b33aa6feb45606dc8ce284d2ff19 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Mon, 6 Feb 2023 13:20:13 -0600 Subject: [PATCH 06/17] rm -W flag --- ci/build_docs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 073af59bac9..5d1ee3f244c 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -37,8 +37,8 @@ popd # Build Python docs rapids-logger "Build Sphinx docs" pushd docs/cudf -sphinx-build -b dirhtml source _html -W -sphinx-build -b text source _text -W +sphinx-build -b dirhtml source _html +sphinx-build -b text source _text popd From c1a20d10f667caa8ffd6b6418e85f62c33976fe4 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 13 Feb 2023 16:43:26 -0500 Subject: [PATCH 07/17] Update ci/build_docs.sh --- ci/build_docs.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 5d1ee3f244c..330b1b458ba 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -24,8 +24,7 @@ VERSION_NUMBER=$(rapids-get-rapids-version-from-git) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcudf \ - cudf + libcudf cudf # Build CPP docs From 52ecf9577d33fcf1fc07d362e7129d8a5ba5af4c Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 13 Feb 2023 16:45:07 -0500 Subject: [PATCH 08/17] update shared workflow branch --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8cbd2c6bb69..731d28b15a2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -57,7 +57,7 @@ jobs: if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: branch node_type: "gpu-latest-1" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index be5de75432f..ec30aa189e0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -86,7 +86,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: pull-request node_type: "gpu-latest-1" From 09f19c142f4fde4ec905306c2487ce24159768c2 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 17 Feb 2023 10:26:22 -0600 Subject: [PATCH 09/17] skip docs nightly --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cd7d99732e3..c276b550648 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -54,7 +54,7 @@ jobs: sha: ${{ inputs.sha }} skip_upload_pkgs: libcudf-example docs-build: - if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} + if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 From 70baa0ffb9a495730068d10d4c3dbecbe8565d76 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Fri, 17 Feb 2023 13:50:08 -0500 Subject: [PATCH 10/17] rm `conf.py` imports According to the [`autodoc` documentation](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html), `autodoc` will automatically import the modules to be documented. The CI docs building process will install the packages that were built in previous stages of the workflow and presumably `autodoc` will import those correctly, which means these lines are not needed. --- docs/cudf/source/conf.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 371a8b4e1c1..3d92d955263 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) 2018-2023, NVIDIA CORPORATION. # # cudf documentation build configuration file, created by @@ -23,11 +22,6 @@ from docutils.nodes import Text from sphinx.addnodes import pending_xref -import cudf - -sys.path.insert(0, os.path.abspath(cudf.__path__[0])) -sys.path.insert(0, os.path.abspath(".")) -sys.path.insert(0, os.path.abspath("../..")) sys.path.append(os.path.abspath("./_ext")) # -- General configuration ------------------------------------------------ From 69ce9e8603691b01cffe82b9645f33d8745fb98f Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Fri, 17 Feb 2023 14:37:14 -0500 Subject: [PATCH 11/17] add generated file dirs to `.gitignore` --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0f81dcb6f2b..da5f0cca84d 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,9 @@ docs/cudf/source/api_docs/generated/* docs/cudf/source/api_docs/api/* docs/cudf/source/user_guide/example_output/* docs/cudf/source/user_guide/cudf.*Dtype.*.rst +_html +_text +jupyter_execute # cibuildwheel /wheelhouse From b0fb54607501afde367d546095062775c164047f Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Fri, 17 Feb 2023 14:37:23 -0500 Subject: [PATCH 12/17] add `dask-cudf` to docs build installs --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 330b1b458ba..87f6671279c 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -24,7 +24,7 @@ VERSION_NUMBER=$(rapids-get-rapids-version-from-git) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcudf cudf + libcudf cudf dask-cudf # Build CPP docs From 9e4ffbb87679f480f8685623daddb46035ccc989 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Fri, 17 Feb 2023 22:09:41 -0500 Subject: [PATCH 13/17] rm unnecessary comments --- ci/build_docs.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 87f6671279c..28b8cd6bd6f 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -27,13 +27,11 @@ rapids-mamba-retry install \ libcudf cudf dask-cudf -# Build CPP docs rapids-logger "Build Doxygen docs" pushd cpp/doxygen doxygen Doxyfile popd -# Build Python docs rapids-logger "Build Sphinx docs" pushd docs/cudf sphinx-build -b dirhtml source _html From 4898bca1919565f7a3214f5485c7b753c91232ef Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Fri, 17 Feb 2023 22:12:13 -0500 Subject: [PATCH 14/17] update dependency list name --- dependencies.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 6778976ea40..92011dd2efd 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,7 +9,7 @@ files: - build - cudatoolkit - develop - - docs_build + - docs - notebooks - py_version - run @@ -45,7 +45,7 @@ files: output: none includes: - cudatoolkit - - docs_build + - docs - py_version channels: - rapidsai @@ -127,7 +127,7 @@ dependencies: - output_types: conda packages: - doxygen=1.8.20 # pre-commit hook needs a specific version. - docs_build: + docs: common: - output_types: [conda] packages: From 740fb29ab7b8422a26e615b6041437fac09ba725 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 21 Feb 2023 16:10:11 -0600 Subject: [PATCH 15/17] add req. rmm tag --- ci/build_docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 28b8cd6bd6f..d37b9db8aaf 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -29,6 +29,7 @@ rapids-mamba-retry install \ rapids-logger "Build Doxygen docs" pushd cpp/doxygen +aws s3 cp s3://rapidsai-docs/librmm/${VERSION_NUMBER}/html/rmm.tag . || echo "Failed to download rmm Doxygen tag" doxygen Doxyfile popd From f1a982e25ee499f20dff2885de019f4e94551431 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:02:56 -0600 Subject: [PATCH 16/17] Update ci/build_docs.sh Co-authored-by: Bradley Dice --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index d37b9db8aaf..5798680c522 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -41,7 +41,7 @@ popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then - aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" + aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" fi From 41edde6c379438749ff960caf940849862d0efd3 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 22 Feb 2023 13:07:30 -0600 Subject: [PATCH 17/17] fix typo --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 5798680c522..9551d98e9fe 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -42,6 +42,6 @@ popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" - aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" fi