From 104997e7f412909ffa0574047f5afe7fe5c570a1 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Mon, 23 Jan 2023 09:27:04 -0600 Subject: [PATCH 01/38] initial draft --- .github/workflows/build.yaml | 11 +++++++++ .github/workflows/pr.yaml | 10 ++++++++ ci/build_docs.sh | 48 ++++++++++++++++++++++++++++++++++++ dependencies.yaml | 12 ++++++++- 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 ci/build_docs.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d0ded9a530..1ba23184ec 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,17 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + docs-build: + if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} + needs: python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + with: + build_type: branch + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-pylibraft: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@main diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c2c64d0003..c5f7d1a0b2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -51,6 +51,16 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-pylibraft: needs: checks secrets: inherit diff --git a/ci/build_docs.sh b/ci/build_docs.sh new file mode 100644 index 0000000000..e065edbe74 --- /dev/null +++ b/ci/build_docs.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create test conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-dependency-file-generator \ + --output requirments \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee requirements.txt + +rapids-mamba-retry env create --force -f env.yaml -n test +conda activate test +pip install -r requirements.txt + +rapids-print-env + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +VERSION_NUMBER=$(rapids-get-rapids-version-from-git) + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + libraft \ + pylibraft \ + raft-dask + + +# Build CPP docs +gpuci_logger "Build CPP docs" +pushd docs +sphinx-build -b dirhtml source build +popd + + + +if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then + aws s3 sync --delete docs/build "s3://rapidsai-docs/raft/${VERSION_NUMBER}/build" +fi \ No newline at end of file diff --git a/dependencies.yaml b/dependencies.yaml index ae900542c0..69c657e430 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -27,6 +27,12 @@ files: includes: - checks - py_version + docs: + output: none + includes: + - cudatoolkit + - docs_build + - py_version channels: - rapidsai - rapidsai-nightly @@ -131,17 +137,21 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 - doc: + docs_build: common: - output_types: [conda, requirements] packages: - breathe - output_types: [requirements] packages: + - pydata_sphinx_theme - sphinx_markdown_tables - output_types: [conda] packages: - doxygen>=1.8.20 + - ipython + - numpydoc + - recommonmark - sphinx-markdown-tables py_version: specific: From 215b54eed0bf6f997bf51bf739f029ccfbb202b4 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 24 Jan 2023 08:50:15 -0600 Subject: [PATCH 02/38] update pr.yaml --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c5f7d1a0b2..ec8ef7d146 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,7 @@ jobs: - conda-cpp-tests - conda-python-build - conda-python-tests + - docs-build - wheel-build-pylibraft - wheel-tests-pylibraft - wheel-build-raft-dask From b25ddea80c50edfbda9160f788bc8e69f87b45af Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 24 Jan 2023 08:51:24 -0600 Subject: [PATCH 03/38] update dependencies.yaml --- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 +++ dependencies.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 87b7075935..d69a83a3a2 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9 +- ipython - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 - libcurand-dev=10.3.0.86 @@ -34,8 +35,10 @@ dependencies: - libcusparse=11.7.5.86 - libfaiss>=1.7.1=cuda* - ninja +- numpydoc - pytest - pytest-cov +- recommonmark - rmm=23.02 - scikit-build>=0.13.1 - scikit-learn diff --git a/dependencies.yaml b/dependencies.yaml index 69c657e430..e304f3e5bc 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,7 +9,7 @@ files: - build - cudatoolkit - develop - - doc + - docs_build - run - test_python test_cpp: From 8f52e857f9d23032bc38fa117fdab82df50433e5 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 24 Jan 2023 08:52:45 -0600 Subject: [PATCH 04/38] update build.sh --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index e065edbe74..dcdf6544c3 100644 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -12,7 +12,7 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml rapids-dependency-file-generator \ - --output requirments \ + --output requirements \ --file_key docs \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee requirements.txt From cb7ebcbcb3396c817a84315173db8f7ae270cb1a Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 24 Jan 2023 10:51:26 -0600 Subject: [PATCH 05/38] chmod --- ci/build_docs.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/build_docs.sh diff --git a/ci/build_docs.sh b/ci/build_docs.sh old mode 100644 new mode 100755 From ec94157dbe0afa915113dd78a7cc62952b7d92aa Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 25 Jan 2023 06:04:24 -0600 Subject: [PATCH 06/38] update deps --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - dependencies.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d69a83a3a2..74dbc9d35c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -43,7 +43,6 @@ dependencies: - scikit-build>=0.13.1 - scikit-learn - scipy -- sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu - ucx-py=0.30 diff --git a/dependencies.yaml b/dependencies.yaml index e304f3e5bc..65d1e85aa0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -145,6 +145,7 @@ dependencies: - output_types: [requirements] packages: - pydata_sphinx_theme + - sphinx_copybutton - sphinx_markdown_tables - output_types: [conda] packages: @@ -152,7 +153,6 @@ dependencies: - ipython - numpydoc - recommonmark - - sphinx-markdown-tables py_version: specific: - output_types: conda From dae6cf1d6cbcb3a810e6d798cf0895717f9e8b3e Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 25 Jan 2023 09:16:59 -0600 Subject: [PATCH 07/38] fix typo --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index dcdf6544c3..a0cbd2a7a6 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -30,7 +30,7 @@ VERSION_NUMBER=$(rapids-get-rapids-version-from-git) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft \ + libraft-split \ pylibraft \ raft-dask From 01540481a18be13b86d9cf8da7abfa3adb8cdb37 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Wed, 25 Jan 2023 11:44:48 -0600 Subject: [PATCH 08/38] install correct packages --- ci/build_docs.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a0cbd2a7a6..fa1957f04b 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -30,7 +30,9 @@ VERSION_NUMBER=$(rapids-get-rapids-version-from-git) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libraft-split \ + libraft-distance \ + libraft-headers \ + libraft-nn \ pylibraft \ raft-dask From 366441c455f17a657455491c5bc8be4c0e92cca2 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Sun, 29 Jan 2023 10:14:40 -0600 Subject: [PATCH 09/38] fixes --- ci/build_docs.sh | 9 +++++---- dependencies.yaml | 15 +++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index fa1957f04b..19721c778f 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -16,8 +16,8 @@ rapids-dependency-file-generator \ --file_key docs \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee requirements.txt -rapids-mamba-retry env create --force -f env.yaml -n test -conda activate test +rapids-mamba-retry env create --force -f env.yaml -n docs +conda activate docs pip install -r requirements.txt rapids-print-env @@ -38,13 +38,14 @@ rapids-mamba-retry install \ # Build CPP docs -gpuci_logger "Build CPP docs" +rapids-logger "Build CPP docs" pushd docs -sphinx-build -b dirhtml source build +sphinx-build -b dirhtml source build -W popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then + rapids-logger "Upload Docs to S3" aws s3 sync --delete docs/build "s3://rapidsai-docs/raft/${VERSION_NUMBER}/build" fi \ No newline at end of file diff --git a/dependencies.yaml b/dependencies.yaml index 65d1e85aa0..f7d54ecaa0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -139,19 +139,18 @@ dependencies: - *libcusparse114 docs_build: common: - - output_types: [conda, requirements] - packages: - - breathe - - output_types: [requirements] - packages: - - pydata_sphinx_theme - - sphinx_copybutton - - sphinx_markdown_tables - output_types: [conda] packages: + - breathe - doxygen>=1.8.20 - ipython - numpydoc + - pip + - pip: + - breathe + - pydata_sphinx_theme + - sphinx_copybutton + - sphinx_markdown_tables - recommonmark py_version: specific: From 9e82d9610d506cd24be3ae8157e32be4d13e8386 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Sun, 29 Jan 2023 11:06:49 -0600 Subject: [PATCH 10/38] run deps gen --- conda/environments/all_cuda-118_arch-x86_64.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 74dbc9d35c..e98db32012 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -36,6 +36,7 @@ dependencies: - libfaiss>=1.7.1=cuda* - ninja - numpydoc +- pip - pytest - pytest-cov - recommonmark @@ -47,4 +48,9 @@ dependencies: - ucx-proc=*=gpu - ucx-py=0.30 - ucx>=1.13.0 +- pip: + - breathe + - pydata_sphinx_theme + - sphinx_copybutton + - sphinx_markdown_tables name: all_cuda-118_arch-x86_64 From f4b18040bcfea9292820fda5395d2bdd62ea043f Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 31 Jan 2023 10:52:46 -0600 Subject: [PATCH 11/38] rm dup breathe --- dependencies.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index f7d54ecaa0..a16bb9cbea 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -147,7 +147,6 @@ dependencies: - numpydoc - pip - pip: - - breathe - pydata_sphinx_theme - sphinx_copybutton - sphinx_markdown_tables From fc83081e72863fafd10ad68e91cc7bcc2b187514 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 31 Jan 2023 11:35:49 -0600 Subject: [PATCH 12/38] update deps --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index e98db32012..7060b59a40 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -49,7 +49,6 @@ dependencies: - ucx-py=0.30 - ucx>=1.13.0 - pip: - - breathe - pydata_sphinx_theme - sphinx_copybutton - sphinx_markdown_tables From 87eca7acf3a7f1cfc7a1c7fdb389bda3ccf39406 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 1 Feb 2023 15:04:24 -0500 Subject: [PATCH 13/38] Fixing dependencies --- conda/environments/all_cuda-118_arch-x86_64.yaml | 7 +++---- dependencies.yaml | 9 ++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7060b59a40..60e35f03f9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -48,8 +48,7 @@ dependencies: - ucx-proc=*=gpu - ucx-py=0.30 - ucx>=1.13.0 -- pip: - - pydata_sphinx_theme - - sphinx_copybutton - - sphinx_markdown_tables +- pydata-sphinx-theme +- sphinx-copybutton +- sphinx-markdown-tables name: all_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index a16bb9cbea..bdaa89456c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -145,12 +145,11 @@ dependencies: - doxygen>=1.8.20 - ipython - numpydoc - - pip - - pip: - - pydata_sphinx_theme - - sphinx_copybutton - - sphinx_markdown_tables + - pydata-sphinx-theme + - sphinxcopybutton + - sphinx-markdown-tables - recommonmark + - pip py_version: specific: - output_types: conda From 11271682b7aa76709de234ed323195d344e2e31d Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 1 Feb 2023 15:13:38 -0500 Subject: [PATCH 14/38] Updating deps --- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 --- dependencies.yaml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 60e35f03f9..2505626ddb 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -48,7 +48,4 @@ dependencies: - ucx-proc=*=gpu - ucx-py=0.30 - ucx>=1.13.0 -- pydata-sphinx-theme -- sphinx-copybutton -- sphinx-markdown-tables name: all_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index bdaa89456c..2472530787 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -146,7 +146,7 @@ dependencies: - ipython - numpydoc - pydata-sphinx-theme - - sphinxcopybutton + - sphinx-copybutton - sphinx-markdown-tables - recommonmark - pip From 74127aa2aa0d884b9a057cdcc9c145144454e4bc Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 1 Feb 2023 15:16:08 -0500 Subject: [PATCH 15/38] Yep --- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2505626ddb..3a224cfab6 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -37,6 +37,7 @@ dependencies: - ninja - numpydoc - pip +- pydata-sphinx-theme - pytest - pytest-cov - recommonmark @@ -44,6 +45,8 @@ dependencies: - scikit-build>=0.13.1 - scikit-learn - scipy +- sphinx-copybutton +- sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu - ucx-py=0.30 From 448dc0abb990faaa25daf5338bd3aa7deb5f69d5 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 1 Feb 2023 16:58:34 -0500 Subject: [PATCH 16/38] Trying to use build.sh for docs --- ci/build_docs.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 19721c778f..ec03965b89 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -39,13 +39,14 @@ rapids-mamba-retry install \ # Build CPP docs rapids-logger "Build CPP docs" -pushd docs -sphinx-build -b dirhtml source build -W -popd +#pushd docs +#sphinx-build -b dirhtml source build -W +#popd +build.sh docs -v -n if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then rapids-logger "Upload Docs to S3" aws s3 sync --delete docs/build "s3://rapidsai-docs/raft/${VERSION_NUMBER}/build" -fi \ No newline at end of file +fi From e4b1a0998d6a6b42c4a6565f1e96d616078c9b06 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 1 Feb 2023 18:38:02 -0500 Subject: [PATCH 17/38] FIxing path --- ci/build_docs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index ec03965b89..74ba67462b 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -42,8 +42,9 @@ rapids-logger "Build CPP docs" #pushd docs #sphinx-build -b dirhtml source build -W #popd +pushd ../ build.sh docs -v -n - +popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then From 5ddccc1dabe9832a65304a696dfaaef6b2164d62 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Thu, 2 Feb 2023 05:04:47 -0600 Subject: [PATCH 18/38] update build_docs.sh --- ci/build_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 74ba67462b..0e9c2bf4af 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -43,7 +43,7 @@ rapids-logger "Build CPP docs" #sphinx-build -b dirhtml source build -W #popd pushd ../ -build.sh docs -v -n +./build.sh docs -v -n popd From ebb5aba9df7755ad891a716754c66386b049ba4e Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Thu, 2 Feb 2023 11:49:23 -0600 Subject: [PATCH 19/38] fixes --- ci/build_docs.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 0e9c2bf4af..15624dd5b3 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -42,9 +42,7 @@ rapids-logger "Build CPP docs" #pushd docs #sphinx-build -b dirhtml source build -W #popd -pushd ../ ./build.sh docs -v -n -popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then From 26c9bf02a0174e8d1944b6398170b1bb8fb0fed3 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 3 Feb 2023 08:21:39 -0600 Subject: [PATCH 20/38] fixes --- dependencies.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 9d0e3b5aae..152aa19fc8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -142,8 +142,11 @@ dependencies: - output_types: [conda] packages: - breathe + - cmake>=3.23.1,!=3.25.0 - doxygen>=1.8.20 + - graphviz - ipython + - ninja - numpydoc - pydata-sphinx-theme - sphinx-copybutton From 7def2ce7158ba38ab2facac0540865be79e3e495 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 3 Feb 2023 08:24:38 -0600 Subject: [PATCH 21/38] update deps --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index f159860592..4712e44367 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9 +- graphviz - ipython - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 From 2a5ceabe3dd1504e897d81421c63435e62c5b5db Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:27:37 -0500 Subject: [PATCH 22/38] rm CMake dependency for building Doxygen docs this commit removes the CMake dependency for building Doxygen docs. it includes these changes: - removes CMake target & associated files for building docs - removed all CMake variables from `Doxyfile.in` - renamed `Doxyfile.in` to `Doxyfile` since it is no longer a CMake template file - updated all `Doxyfile` paths to be relative to its own directory - disabled Doxygen HTML generation since it's not used - only the Doxygen XML content is used by Sphinx via Breathe - add entry to `ci/release/update-version.sh` to keep `Doxyfile` version string up-to-date --- ci/release/update-version.sh | 2 + cpp/CMakeLists.txt | 8 ---- cpp/cmake/doxygen.cmake | 35 -------------- cpp/doxygen/{Doxyfile.in => Doxyfile} | 68 +++++++++++++-------------- 4 files changed, 36 insertions(+), 77 deletions(-) delete mode 100644 cpp/cmake/doxygen.cmake rename cpp/doxygen/{Doxyfile.in => Doxyfile} (97%) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 00f6905032..984ae9ba70 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -69,3 +69,5 @@ sed_runner "s/{cuda_suffix}.*\"\]/{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\"\]/ sed_runner "s/dask-cuda==.*\",/dask-cuda==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/raft-dask/setup.py sed_runner "s/pylibraft{cuda_suffix}.*\",/pylibraft{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/raft-dask/setup.py sed_runner "s/ucx-py{cuda_suffix}.*\",/ucx-py{cuda_suffix}==${NEXT_UCX_PY_SHORT_TAG_PEP440}.*\",/g" python/raft-dask/setup.py + +sed_runner "/^PROJECT_NUMBER/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" cpp/doxygen/Doxyfile diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cf938a5d33..cea37479a4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -710,11 +710,3 @@ endif() if(BUILD_BENCH) include(bench/CMakeLists.txt) endif() - -# ################################################################################################## -# * doxygen targets ---------------------------------------------------------- -include(cmake/doxygen.cmake) -add_doxygen_target( - IN_DOXYFILE doxygen/Doxyfile.in OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile CWD - ${CMAKE_CURRENT_BINARY_DIR} -) diff --git a/cpp/cmake/doxygen.cmake b/cpp/cmake/doxygen.cmake deleted file mode 100644 index a6fddbacd6..0000000000 --- a/cpp/cmake/doxygen.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# - -find_package(Doxygen 1.8.11) - -function(add_doxygen_target) - if(Doxygen_FOUND) - set(options "") - set(oneValueArgs IN_DOXYFILE OUT_DOXYFILE CWD) - set(multiValueArgs "") - cmake_parse_arguments(dox "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) - - message("Command: ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE}") - add_custom_target( - docs_raft - ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} - WORKING_DIRECTORY ${dox_CWD} - VERBATIM - COMMENT "Generate doxygen docs" - ) - else() - message("add_doxygen_target: doxygen exe not found") - endif() -endfunction(add_doxygen_target) diff --git a/cpp/doxygen/Doxyfile.in b/cpp/doxygen/Doxyfile similarity index 97% rename from cpp/doxygen/Doxyfile.in rename to cpp/doxygen/Doxyfile index 07056e503d..a3dc5a1927 100644 --- a/cpp/doxygen/Doxyfile.in +++ b/cpp/doxygen/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = "RAFT C++ API" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @RAFT_VERSION@ +PROJECT_NUMBER = "23.04" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -58,7 +58,7 @@ PROJECT_LOGO = # entered, it will be relative to the location where doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@ +OUTPUT_DIRECTORY = # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- # directories (in 2 levels) under the output directory of each output format and @@ -835,8 +835,8 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/main_page.md \ - @CMAKE_CURRENT_SOURCE_DIR@/include +INPUT = main_page.md \ + ../include # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -879,30 +879,30 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/linalg/symmetrize.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/cache \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/common \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/lap \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/selection \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/csr.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/linalg/lanczos.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/linalg/lanczos.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/cuda_utils.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/cudart_utils.h \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/device_atomics.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/device_utils.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/error.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/handle.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/integer_utils.h \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/interruptible.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/mdarray.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/pow2_utils.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/span.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/vectorized.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/raft.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/cudart_utils.hpp \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/matrix/math.cuh \ - @CMAKE_CURRENT_SOURCE_DIR@/include/raft/matrix/matrix.cuh +EXCLUDE = ../include/raft/sparse/linalg/symmetrize.hpp \ + ../include/raft/cache \ + ../include/raft/common \ + ../include/raft/lap \ + ../include/raft/sparse/selection \ + ../include/raft/sparse/csr.hpp \ + ../include/raft/linalg/lanczos.cuh \ + ../include/raft/linalg/lanczos.hpp \ + ../include/raft/cuda_utils.cuh \ + ../include/raft/cudart_utils.h \ + ../include/raft/util/device_atomics.cuh \ + ../include/raft/device_utils.cuh \ + ../include/raft/error.hpp \ + ../include/raft/handle.hpp \ + ../include/raft/integer_utils.h \ + ../include/raft/interruptible.hpp \ + ../include/raft/mdarray.hpp \ + ../include/raft/pow2_utils.cuh \ + ../include/raft/span.hpp \ + ../include/raft/vectorized.cuh \ + ../include/raft/raft.hpp \ + ../include/raft/core/cudart_utils.hpp \ + ../include/raft/matrix/math.cuh \ + ../include/raft/matrix/matrix.cuh # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -1013,7 +1013,7 @@ FILTER_SOURCE_PATTERNS = # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. -USE_MDFILE_AS_MAINPAGE = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/main_page.md +USE_MDFILE_AS_MAINPAGE = main_page.md #--------------------------------------------------------------------------- # Configuration options related to source browsing @@ -1134,7 +1134,7 @@ IGNORE_PREFIX = # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output # The default value is: YES. -GENERATE_HTML = YES +GENERATE_HTML = NO # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1169,7 +1169,7 @@ HTML_FILE_EXTENSION = .html # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. -HTML_HEADER = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/header.html +HTML_HEADER = header.html # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each # generated HTML page. If the tag is left blank doxygen will generate a standard @@ -2018,7 +2018,7 @@ GENERATE_XML = YES # The default directory is: xml. # This tag requires that the tag GENERATE_XML is set to YES. -XML_OUTPUT = xml +XML_OUTPUT = _xml # If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program # listings (including syntax highlighting and cross-referencing information) to @@ -2270,7 +2270,7 @@ HIDE_UNDOC_RELATIONS = YES # set to NO # The default value is: NO. -HAVE_DOT = @DOXYGEN_DOT_FOUND@ +HAVE_DOT = # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed # to run in parallel. When set to 0 doxygen will base this on the number of @@ -2448,7 +2448,7 @@ INTERACTIVE_SVG = NO # found. If left blank, it is assumed the dot tool can be found in the path. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_PATH = @DOXYGEN_DOT_EXECUTABLE@ +DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the \dotfile From 46e59f099d2d3ec64d94c24769baebb6c5a32d58 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:32:47 -0500 Subject: [PATCH 23/38] update `build.sh` to reflect docs build changes --- build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index b47e1ed862..93f11d11a1 100755 --- a/build.sh +++ b/build.sh @@ -57,6 +57,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= Date: Tue, 7 Feb 2023 11:51:50 -0500 Subject: [PATCH 24/38] rm unused Makefile option this option is spelled incorrectly and therefore not used by `sphinx`. the correct configuration options are specified in `conf.py` --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index b242888a67..d621499d14 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = -Dbreath_projects.RAFT=../cpp/build/xml +SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = RAFT SOURCEDIR = source From f2630fa8347190810a2f7456d8dc53a911c7c088 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:53:40 -0500 Subject: [PATCH 25/38] rm `conf.py` `sys.path` insertions according to the `autodoc` docs below, `autodoc` will automatically import the modules to be documented. this the docs build CI job installs the packages that were previously built in the workflow run, we don't need to modify `sys.path` to include these packages - https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html --- docs/source/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a85dc15b3b..a9a16d700c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,8 +24,6 @@ # is relative to the documentation root, use os.path.abspath to make it # absolute, like shown here. sys.path.insert(0, os.path.abspath("sphinxext")) -sys.path.insert(0, os.path.abspath("../../python/raft-dask")) -sys.path.insert(0, os.path.abspath("../../python/pylibraft")) from github_link import make_linkcode_resolve # noqa From 721007294e28aaf287732be24af22852895de4eb Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:55:23 -0500 Subject: [PATCH 26/38] update `conf.py` to point to Doxygen `_xml` location a previous commit updated the `xml` output location for `Doxygen`, so this commit updates the `sphinx` config accordingly --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a9a16d700c..d8a1b4013d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,7 +54,7 @@ breathe_default_project = "RAFT" breathe_projects = { - "RAFT": "../../cpp/build/xml/", + "RAFT": "../../cpp/doxygen/_xml/", } ipython_mplbackend = "str" From 75044b47d2f98740b9e67c31c25a9b8f1f0c9ecc Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:56:28 -0500 Subject: [PATCH 27/38] update `.gitignore` to ignore doc output paths --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 5d148b836b..80709dbb96 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,10 @@ doxygen_check/ ## cibuildwheel /wheelhouse + +# doxygen +_xml + +# sphinx +_html +_text From 9b15d25d092eabdca002d81967a407581888df57 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 11:59:07 -0500 Subject: [PATCH 28/38] rm `cmake` and `ninja` from `docs` dependencies and re-generate dependency files --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - dependencies.yaml | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4712e44367..638ff13afa 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -37,7 +37,6 @@ dependencies: - libfaiss>=1.7.1=cuda* - ninja - numpydoc -- pip - pydata-sphinx-theme - pytest - pytest-cov diff --git a/dependencies.yaml b/dependencies.yaml index 152aa19fc8..1601340a86 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,7 +9,7 @@ files: - build - cudatoolkit - develop - - docs_build + - docs - run - test_python test_cpp: @@ -31,7 +31,7 @@ files: output: none includes: - cudatoolkit - - docs_build + - docs - py_version channels: - rapidsai @@ -137,22 +137,19 @@ dependencies: - *libcusolver114 - *libcusparse_dev114 - *libcusparse114 - docs_build: + docs: common: - output_types: [conda] packages: - breathe - - cmake>=3.23.1,!=3.25.0 - doxygen>=1.8.20 - graphviz - ipython - - ninja - numpydoc - pydata-sphinx-theme + - recommonmark - sphinx-copybutton - sphinx-markdown-tables - - recommonmark - - pip py_version: specific: - output_types: conda From 9f18bbff4f1bb9512cebaa5df611949f933d658b Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 13:50:44 -0500 Subject: [PATCH 29/38] update `build_docs.sh` --- ci/build_docs.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 15624dd5b3..a2c90027e7 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -11,14 +11,8 @@ rapids-dependency-file-generator \ --file_key docs \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml -rapids-dependency-file-generator \ - --output requirements \ - --file_key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee requirements.txt - rapids-mamba-retry env create --force -f env.yaml -n docs conda activate docs -pip install -r requirements.txt rapids-print-env @@ -37,15 +31,20 @@ rapids-mamba-retry install \ raft-dask -# Build CPP docs -rapids-logger "Build CPP docs" -#pushd docs -#sphinx-build -b dirhtml source build -W -#popd -./build.sh docs -v -n +rapids-logger "Build Doxygen docs" +pushd cpp/doxygen +doxygen Doxyfile +popd + +rapids-logger "Build Sphinx docs" +pushd docs +sphinx-build -b dirhtml source _html +sphinx-build -b text source _text +popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then rapids-logger "Upload Docs to S3" - aws s3 sync --delete docs/build "s3://rapidsai-docs/raft/${VERSION_NUMBER}/build" + aws s3 sync --delete docs/_html "s3://rapidsai-docs/raft/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/_text "s3://rapidsai-docs/raft/${VERSION_NUMBER}/txt" fi From 29272557e671ada2f7ae1b976d4103ebaac8c81d Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 16:40:52 -0500 Subject: [PATCH 30/38] rm dead `.rst` doc files according to the `sphinx` warnings, these `.rst` files aren't used anywhere --- docs/source/cuda_cpp.rst | 11 ----------- docs/source/python.rst | 10 ---------- 2 files changed, 21 deletions(-) delete mode 100644 docs/source/cuda_cpp.rst delete mode 100644 docs/source/python.rst diff --git a/docs/source/cuda_cpp.rst b/docs/source/cuda_cpp.rst deleted file mode 100644 index 7a7cdae086..0000000000 --- a/docs/source/cuda_cpp.rst +++ /dev/null @@ -1,11 +0,0 @@ -CUDA/C++ API -============ - -RAFT is a header-only C++ library with optional pre-compiled shared libraries that can speed up compile times for larger projects. - -.. _api: - -.. toctree:: - :maxdepth: 4 - - cpp_api.rst diff --git a/docs/source/python.rst b/docs/source/python.rst deleted file mode 100644 index 561ec3d029..0000000000 --- a/docs/source/python.rst +++ /dev/null @@ -1,10 +0,0 @@ -RAFT Python APIs -================ - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - pylibraft_api.rst - raft_dask_api.rst - From 06487e753e1bd02b56abbbd283693bd1009fc65d Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 16:46:49 -0500 Subject: [PATCH 31/38] update `developer_guide.md` links these links were invalid and issuing `sphinx` warnings --- docs/source/developer_guide.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/source/developer_guide.md b/docs/source/developer_guide.md index 2f54753cc6..56100b38f7 100644 --- a/docs/source/developer_guide.md +++ b/docs/source/developer_guide.md @@ -140,13 +140,13 @@ RAFT relies on `clang-format` to enforce code style across all C++ and CUDA sour 1. Do not split empty functions/records/namespaces. 2. Two-space indentation everywhere, including the line continuations. 3. Disable reflowing of comments. - The reasons behind these deviations from the Google style guide are given in comments [here](../../cpp/.clang-format). + The reasons behind these deviations from the Google style guide are given in comments [here](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/.clang-format). #### How is the check done? -All formatting checks are done by this python script: [run-clang-format.py](../../cpp/scripts/run-clang-format.py) which is effectively a wrapper over `clang-format`. An error is raised if the code diverges from the format suggested by clang-format. It is expected that the developers run this script to detect and fix formatting violations before creating PR. +All formatting checks are done by this python script: [run-clang-format.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/run-clang-format.py) which is effectively a wrapper over `clang-format`. An error is raised if the code diverges from the format suggested by clang-format. It is expected that the developers run this script to detect and fix formatting violations before creating PR. ##### As part of CI -[run-clang-format.py](../../cpp/scripts/run-clang-format.py) is executed as part of our `ci/checks/style.sh` CI test. If there are any formatting violations, PR author is expected to fix those to get CI passing. Steps needed to fix the formatting violations are described in the subsequent sub-section. +[run-clang-format.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/run-clang-format.py) is executed as part of our `ci/checks/style.sh` CI test. If there are any formatting violations, PR author is expected to fix those to get CI passing. Steps needed to fix the formatting violations are described in the subsequent sub-section. ##### Manually Developers can also manually (or setup this command as part of git pre-commit hook) run this check by executing: @@ -156,10 +156,10 @@ python ./cpp/scripts/run-clang-format.py From the root of the RAFT repository. #### How to know the formatting violations? -When there are formatting errors, [run-clang-format.py](../../cpp/scripts/run-clang-format.py) prints a `diff` command, showing where there are formatting differences. Unfortunately, unlike `flake8`, `clang-format` does NOT print descriptions of the violations, but instead directly formats the code. So, the only way currently to know about formatting differences is to run the diff command as suggested by this script against each violating source file. +When there are formatting errors, [run-clang-format.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/run-clang-format.py) prints a `diff` command, showing where there are formatting differences. Unfortunately, unlike `flake8`, `clang-format` does NOT print descriptions of the violations, but instead directly formats the code. So, the only way currently to know about formatting differences is to run the diff command as suggested by this script against each violating source file. #### How to fix the formatting violations? -When there are formatting violations, [run-clang-format.py](../../cpp/scripts/run-clang-format.py) prints at the end, the exact command that can be run by developers to fix them. This is the easiest way to fix formatting errors. [This screencast](https://asciinema.org/a/287367) shows how developers can check for formatting violations in their branches and also how to fix those, before sending out PRs. +When there are formatting violations, [run-clang-format.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/run-clang-format.py) prints at the end, the exact command that can be run by developers to fix them. This is the easiest way to fix formatting errors. [This screencast](https://asciinema.org/a/287367) shows how developers can check for formatting violations in their branches and also how to fix those, before sending out PRs. In short, to bulk-fix all the formatting violations, execute the following command: ```bash @@ -168,13 +168,13 @@ python ./cpp/scripts/run-clang-format.py -inplace From the root of the RAFT repository. #### clang-format version? -To avoid spurious code style violations we specify the exact clang-format version required, currently `11.1.0`. This is enforced by the [run-clang-format.py](../../cpp/scripts/run-clang-format.py) script itself. Refer [here](../../cpp/README.md#dependencies) for the list of build-time dependencies. +To avoid spurious code style violations we specify the exact clang-format version required, currently `11.1.0`. This is enforced by the [run-clang-format.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/run-clang-format.py) script itself. Refer [here](../build#build-dependencies) for the list of build-time dependencies. #### Additional scripts Along with clang, there are an include checker and copyright checker scripts for checking style, which can be performed as part of CI, as well as manually. ##### #include style -[include_checker.py](../../cpp/scripts/include_checker.py) is used to enforce the include style as follows: +[include_checker.py](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/scripts/include_checker.py) is used to enforce the include style as follows: 1. `#include "..."` should be used for referencing local files only. It is acceptable to be used for referencing files in a sub-folder/parent-folder of the same algorithm, but should never be used to include files in other algorithms or between algorithms and the primitives or other dependencies. 2. `#include <...>` should be used for referencing everything else @@ -184,7 +184,7 @@ python ./cpp/scripts/include_checker.py --inplace [cpp/include cpp/test ... list ``` ##### Copyright header -[copyright.py](../../ci/checks/copyright.py) checks the Copyright header for all git-modified files +[copyright.py](https://github.com/rapidsai/raft/blob/branch-23.04/ci/checks/copyright.py) checks the Copyright header for all git-modified files Manually, you can run the following to bulk-fix the header if only the years need to be updated: ```bash @@ -198,7 +198,7 @@ Call CUDA APIs via the provided helper macros `RAFT_CUDA_TRY`, `RAFT_CUBLAS_TRY` ## Logging ### Introduction -Anything and everything about logging is defined inside [logger.hpp](../../cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. +Anything and everything about logging is defined inside [logger.hpp](https://github.com/rapidsai/raft/blob/branch-23.04/cpp/include/raft/core/logger.hpp). It uses [spdlog](https://github.com/gabime/spdlog) underneath, but this information is transparent to all. ### Usage ```cpp From 4a8852cc2b2c6006e4feef59c8b70cea5033846f Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 7 Feb 2023 17:04:28 -0500 Subject: [PATCH 32/38] add `update-version.sh` entries for all docs files --- ci/release/update-version.sh | 4 ++++ docs/source/build.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 17a545c7c9..83b26bbab2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -75,3 +75,7 @@ for FILE in .github/workflows/*.yaml; do done sed_runner "/^PROJECT_NUMBER/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" cpp/doxygen/Doxyfile + +sed_runner "/^set(RAFT_VERSION/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" docs/source/build.md +sed_runner "/GIT_TAG.*branch-/ s|branch-.*|branch-${NEXT_SHORT_TAG}|g" docs/source/build.md +sed_runner "/rapidsai\/raft/ s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" docs/source/developer_guide.md diff --git a/docs/source/build.md b/docs/source/build.md index 4052e49cf8..bf84e47129 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -266,7 +266,7 @@ When the needed [build dependencies](#build-dependencies) are already satisfied, set(RAFT_GIT_DIR ${CMAKE_CURRENT_BINARY_DIR}/raft CACHE STRING "Path to RAFT repo") ExternalProject_Add(raft GIT_REPOSITORY git@github.com:rapidsai/raft.git - GIT_TAG branch-22.10 + GIT_TAG branch-23.04 PREFIX ${RAFT_GIT_DIR} CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -298,7 +298,7 @@ The following `cmake` snippet enables a flexible configuration of RAFT: ```cmake -set(RAFT_VERSION "22.12") +set(RAFT_VERSION "23.04") set(RAFT_FORK "rapidsai") set(RAFT_PINNED_TAG "branch-${RAFT_VERSION}") From d38b2469f1d7b112a0669caa6fa2f80624003919 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Mon, 13 Feb 2023 18:38:06 -0500 Subject: [PATCH 33/38] updated shared workflows branch --- .github/workflows/build.yaml | 2 +- .github/workflows/pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f580cf3372..5c490cc034 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -56,7 +56,7 @@ jobs: if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} needs: python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: branch node_type: "gpu-latest-1" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 43b3027e93..36816eeb48 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -55,7 +55,7 @@ jobs: docs-build: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@cuda-118 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: pull-request node_type: "gpu-latest-1" From 702396d45f642b11bd22428c068f6f0bc7edcaff Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 14 Feb 2023 20:59:37 -0500 Subject: [PATCH 34/38] document missing arguments --- cpp/include/raft/distance/distance.cuh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/include/raft/distance/distance.cuh b/cpp/include/raft/distance/distance.cuh index 5fd5ebe98d..843a22c593 100644 --- a/cpp/include/raft/distance/distance.cuh +++ b/cpp/include/raft/distance/distance.cuh @@ -42,6 +42,7 @@ namespace distance { * @tparam OutType output type * @tparam FinalLambda user-defined epilogue lamba * @tparam Index_ Index type + * @param handle raft handle for managing expensive resources * @param x first set of points * @param y second set of points * @param dist output distance matrix @@ -89,6 +90,7 @@ void distance(raft::resources const& handle, * @tparam AccType accumulation type * @tparam OutType output type * @tparam Index_ Index type + * @param handle raft handle for managing expensive resources * @param x first set of points * @param y second set of points * @param dist output distance matrix @@ -186,6 +188,7 @@ size_t getWorkspaceSize(const raft::device_matrix_view x, * @tparam AccType accumulation type * @tparam OutType output type * @tparam Index_ Index type + * @param handle raft handle for managing expensive resources * @param x first set of points * @param y second set of points * @param dist output distance matrix From 5a128ed035f3de780b8207c72d90efca8246cdad Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 14 Feb 2023 20:59:48 -0500 Subject: [PATCH 35/38] fix `@file` directives --- cpp/include/raft/stats/adjusted_rand_index.cuh | 2 +- cpp/include/raft/stats/information_criterion.cuh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/raft/stats/adjusted_rand_index.cuh b/cpp/include/raft/stats/adjusted_rand_index.cuh index d2815fe753..87a1b8f686 100644 --- a/cpp/include/raft/stats/adjusted_rand_index.cuh +++ b/cpp/include/raft/stats/adjusted_rand_index.cuh @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * @file adjusted_rand_index.hpp + * @file adjusted_rand_index.cuh * @brief The adjusted Rand index is the corrected-for-chance version of the Rand index. * Such a correction for chance establishes a baseline by using the expected similarity * of all pair-wise comparisons between clusterings specified by a random model. diff --git a/cpp/include/raft/stats/information_criterion.cuh b/cpp/include/raft/stats/information_criterion.cuh index b54f126859..2d865a566d 100644 --- a/cpp/include/raft/stats/information_criterion.cuh +++ b/cpp/include/raft/stats/information_criterion.cuh @@ -14,7 +14,7 @@ * limitations under the License. */ /** - * @file information_criterion.hpp + * @file information_criterion.cuh * @brief These information criteria are used to evaluate the quality of models * by balancing the quality of the fit and the number of parameters. * From 701eeee2c2ffe6823f01aa9a2624511bfe75b6b9 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 14 Feb 2023 21:03:27 -0500 Subject: [PATCH 36/38] rm extraneous header info --- docs/source/conf.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cfa3362d9b..15d41edfbc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,21 +1,5 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +-# Copyright (c) 2018-2023, NVIDIA CORPORATION. -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# import os import sys From d3a20a6b1103c795b562519416b57c727b55cc18 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 14 Feb 2023 21:08:58 -0500 Subject: [PATCH 37/38] fix typo --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 15d41edfbc..2b99d2bfc4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,4 +1,4 @@ --# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import os import sys From 531e370f38f1b61ed2e95b4bdbb4c31ee35c256b Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 17 Feb 2023 10:25:41 -0600 Subject: [PATCH 38/38] skip docs nightly --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5c490cc034..2bcc15a80f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,7 +53,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} docs-build: - if: ${{ startsWith(github.ref, 'refs/heads/branch-') }} + if: github.ref_type == 'branch' && github.event_name == 'push' needs: python-build secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04