Add docs build job (#1168)

The PR adds a `docs_build` process to the PR and Build workflows for this repository. The generated docs are synced to s3 for only the build workflows. Authors: - Jake Awe (https://github.com/AyodeAwe) - Corey J. Nolet (https://github.com/cjnolet) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - AJ Schmidt (https://github.com/ajschmidt8) URL: #1168
rapidsai · Feb 17, 2023 · 013da20 · 013da20
1 parent e14ec63
commit 013da20
Show file tree

Hide file tree

Showing 20 changed files with 162 additions and 141 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -52,6 +52,17 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+  docs-build:
+    if: github.ref_type == 'branch' && github.event_name == 'push'
+    needs: python-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: branch
+      node_type: "gpu-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci:latest"
+      run_script: "ci/build_docs.sh"
   wheel-build-pylibraft:
     secrets: inherit
     uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -17,6 +17,7 @@ jobs:
       - conda-cpp-tests
       - conda-python-build
       - conda-python-tests
+      - docs-build
       - wheel-build-pylibraft
       - wheel-tests-pylibraft
       - wheel-build-raft-dask
@@ -51,6 +52,16 @@ jobs:
     uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
+  docs-build:
+    needs: conda-python-build
+    secrets: inherit
+    uses: rapidsai/shared-action-workflows/.github/workflows/[email protected]
+    with:
+      build_type: pull-request
+      node_type: "gpu-latest-1"
+      arch: "amd64"
+      container_image: "rapidsai/ci:latest"
+      run_script: "ci/build_docs.sh"
   wheel-build-pylibraft:
     needs: checks
     secrets: inherit

diff --git a/.gitignore b/.gitignore
@@ -52,3 +52,10 @@ doxygen_check/
 
 ## cibuildwheel
 /wheelhouse
+
+# doxygen
+_xml
+
+# sphinx
+_html
+_text
diff --git a/build.sh b/build.sh
@@ -57,6 +57,7 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
 "
 LIBRAFT_BUILD_DIR=${LIBRAFT_BUILD_DIR:=${REPODIR}/cpp/build}
 SPHINX_BUILD_DIR=${REPODIR}/docs
+DOXYGEN_BUILD_DIR=${REPODIR}/cpp/doxygen
 RAFT_DASK_BUILD_DIR=${REPODIR}/python/raft-dask/_skbuild
 PYLIBRAFT_BUILD_DIR=${REPODIR}/python/pylibraft/_skbuild
 BUILD_DIRS="${LIBRAFT_BUILD_DIR} ${PYLIBRAFT_BUILD_DIR} ${RAFT_DASK_BUILD_DIR}"
@@ -452,7 +453,8 @@ fi
 
 if hasArg docs; then
     set -x
-    cmake --build ${LIBRAFT_BUILD_DIR} -v --target docs_raft
+    cd ${DOXYGEN_BUILD_DIR}
+    doxygen Doxyfile
     cd ${SPHINX_BUILD_DIR}
-    make html
+    sphinx-build -b dirhtml source _html
 fi
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+rapids-logger "Create test conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key docs \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n docs
+conda activate docs
+
+rapids-print-env
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+VERSION_NUMBER=$(rapids-get-rapids-version-from-git)
+
+rapids-mamba-retry install \
+  --channel "${CPP_CHANNEL}" \
+  --channel "${PYTHON_CHANNEL}" \
+  libraft-distance \
+  libraft-headers \
+  libraft-nn \
+  pylibraft \
+  raft-dask
+
+
+rapids-logger "Build Doxygen docs"
+pushd cpp/doxygen
+doxygen Doxyfile
+popd
+
+rapids-logger "Build Sphinx docs"
+pushd docs
+sphinx-build -b dirhtml source _html
+sphinx-build -b text source _text
+popd
+
+
+if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then
+  rapids-logger "Upload Docs to S3"
+  aws s3 sync --delete docs/_html "s3://rapidsai-docs/raft/${VERSION_NUMBER}/html"
+  aws s3 sync --delete docs/_text "s3://rapidsai-docs/raft/${VERSION_NUMBER}/txt"
+fi
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -72,3 +72,9 @@ sed_runner "s/ucx-py{cuda_suffix}.*\",/ucx-py{cuda_suffix}==${NEXT_UCX_PY_SHORT_
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done
+
+sed_runner "/^PROJECT_NUMBER/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" cpp/doxygen/Doxyfile
+
+sed_runner "/^set(RAFT_VERSION/ s|\".*\"|\"${NEXT_SHORT_TAG}\"|g" docs/source/build.md
+sed_runner "/GIT_TAG.*branch-/ s|branch-.*|branch-${NEXT_SHORT_TAG}|g" docs/source/build.md
+sed_runner "/rapidsai\/raft/ s|branch-[0-9][0-9].[0-9][0-9]|branch-${NEXT_SHORT_TAG}|g" docs/source/developer_guide.md
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -24,6 +24,8 @@ dependencies:
 - doxygen>=1.8.20
 - faiss-proc=*=cuda
 - gcc_linux-64=9
+- graphviz
+- ipython
 - libcublas-dev=11.11.3.6
 - libcublas=11.11.3.6
 - libcurand-dev=10.3.0.86
@@ -34,12 +36,16 @@ dependencies:
 - libcusparse=11.7.5.86
 - libfaiss>=1.7.1=cuda*
 - ninja
+- numpydoc
+- pydata-sphinx-theme
 - pytest
 - pytest-cov
+- recommonmark
 - rmm=23.04
 - scikit-build>=0.13.1
 - scikit-learn
 - scipy
+- sphinx-copybutton
 - sphinx-markdown-tables
 - sysroot_linux-64==2.17
 - ucx-proc=*=gpu

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -724,11 +724,3 @@ endif()
 if(BUILD_BENCH)
   include(bench/CMakeLists.txt)
 endif()
-
-# ##################################################################################################
-# * doxygen targets ----------------------------------------------------------
-include(cmake/doxygen.cmake)
-add_doxygen_target(
-  IN_DOXYFILE doxygen/Doxyfile.in OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile CWD
-  ${CMAKE_CURRENT_BINARY_DIR}
-)
diff --git a/cpp/cmake/doxygen.cmake b/cpp/cmake/doxygen.cmake
diff --git a/cpp/doxygen/Doxyfile.in → cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile.in → cpp/doxygen/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME           = "RAFT C++ API"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = @RAFT_VERSION@
+PROJECT_NUMBER         = "23.04"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -58,7 +58,7 @@ PROJECT_LOGO           =
 # entered, it will be relative to the location where doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = @CMAKE_CURRENT_BINARY_DIR@
+OUTPUT_DIRECTORY       =
 
 # If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
 # directories (in 2 levels) under the output directory of each output format and
@@ -835,8 +835,8 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/main_page.md \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include
+INPUT                  = main_page.md \
+                         ../include
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -879,30 +879,30 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                = @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/linalg/symmetrize.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/cache \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/common \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/lap \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/selection \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/sparse/csr.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/linalg/lanczos.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/linalg/lanczos.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/cuda_utils.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/cudart_utils.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/device_atomics.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/device_utils.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/error.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/handle.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/integer_utils.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/interruptible.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/mdarray.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/pow2_utils.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/span.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/util/vectorized.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/raft.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/core/cudart_utils.hpp \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/matrix/math.cuh \
-                         @CMAKE_CURRENT_SOURCE_DIR@/include/raft/matrix/matrix.cuh
+EXCLUDE                = ../include/raft/sparse/linalg/symmetrize.hpp \
+                         ../include/raft/cache \
+                         ../include/raft/common \
+                         ../include/raft/lap \
+                         ../include/raft/sparse/selection \
+                         ../include/raft/sparse/csr.hpp \
+                         ../include/raft/linalg/lanczos.cuh \
+                         ../include/raft/linalg/lanczos.hpp \
+                         ../include/raft/util/cuda_utils.cuh \
+                         ../include/raft/util/cudart_utils.hpp \
+                         ../include/raft/util/device_atomics.cuh \
+                         ../include/raft/util/device_utils.cuh \
+                         ../include/raft/core/error.hpp \
+                         ../include/raft/core/handle.hpp \
+                         ../include/raft/util/integer_utils.hpp \
+                         ../include/raft/core/interruptible.hpp \
+                         ../include/raft/core/mdarray.hpp \
+                         ../include/raft/util/pow2_utils.cuh \
+                         ../include/raft/core/span.hpp \
+                         ../include/raft/util/vectorized.cuh \
+                         ../include/raft/raft.hpp \
+                         ../include/raft/core/cudart_utils.hpp \
+                         ../include/raft/matrix/math.cuh \
+                         ../include/raft/matrix/matrix.cuh
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -1013,7 +1013,7 @@ FILTER_SOURCE_PATTERNS =
 # (index.html). This can be useful if you have a project on for instance GitHub
 # and want to reuse the introduction page also for the doxygen output.
 
-USE_MDFILE_AS_MAINPAGE = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/main_page.md
+USE_MDFILE_AS_MAINPAGE = main_page.md
 
 #---------------------------------------------------------------------------
 # Configuration options related to source browsing
@@ -1134,7 +1134,7 @@ IGNORE_PREFIX          =
 # If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
 # The default value is: YES.
 
-GENERATE_HTML          = YES
+GENERATE_HTML          = NO
 
 # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -1169,7 +1169,7 @@ HTML_FILE_EXTENSION    = .html
 # of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_HEADER            = @CMAKE_CURRENT_SOURCE_DIR@/doxygen/header.html
+HTML_HEADER            = header.html
 
 # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
 # generated HTML page. If the tag is left blank doxygen will generate a standard
@@ -2018,7 +2018,7 @@ GENERATE_XML           = YES
 # The default directory is: xml.
 # This tag requires that the tag GENERATE_XML is set to YES.
 
-XML_OUTPUT             = xml
+XML_OUTPUT             = _xml
 
 # If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
 # listings (including syntax highlighting and cross-referencing information) to
@@ -2270,7 +2270,7 @@ HIDE_UNDOC_RELATIONS   = YES
 # set to NO
 # The default value is: NO.
 
-HAVE_DOT               = @DOXYGEN_DOT_FOUND@
+HAVE_DOT               =
 
 # The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
 # to run in parallel. When set to 0 doxygen will base this on the number of
@@ -2448,7 +2448,7 @@ INTERACTIVE_SVG        = NO
 # found. If left blank, it is assumed the dot tool can be found in the path.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DOT_PATH               = @DOXYGEN_DOT_EXECUTABLE@
+DOT_PATH               =
 
 # The DOTFILE_DIRS tag can be used to specify one or more directories that
 # contain dot files that are included in the documentation (see the \dotfile

diff --git a/cpp/include/raft/distance/distance.cuh b/cpp/include/raft/distance/distance.cuh
@@ -42,6 +42,7 @@ namespace distance {
  * @tparam OutType output type
  * @tparam FinalLambda user-defined epilogue lamba
  * @tparam Index_ Index type
+ * @param handle raft handle for managing expensive resources
  * @param x first set of points
  * @param y second set of points
  * @param dist output distance matrix
@@ -89,6 +90,7 @@ void distance(raft::resources const& handle,
  * @tparam AccType accumulation type
  * @tparam OutType output type
  * @tparam Index_ Index type
+ * @param handle raft handle for managing expensive resources
  * @param x first set of points
  * @param y second set of points
  * @param dist output distance matrix
@@ -186,6 +188,7 @@ size_t getWorkspaceSize(const raft::device_matrix_view<InType, layout> x,
  * @tparam AccType accumulation type
  * @tparam OutType output type
  * @tparam Index_ Index type
+ * @param handle raft handle for managing expensive resources
  * @param x first set of points
  * @param y second set of points
  * @param dist output distance matrix

diff --git a/cpp/include/raft/stats/adjusted_rand_index.cuh b/cpp/include/raft/stats/adjusted_rand_index.cuh
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 /**
- * @file adjusted_rand_index.hpp
+ * @file adjusted_rand_index.cuh
  * @brief The adjusted Rand index is the corrected-for-chance version of the Rand index.
  * Such a correction for chance establishes a baseline by using the expected similarity
  * of all pair-wise comparisons between clusterings specified by a random model.

diff --git a/cpp/include/raft/stats/information_criterion.cuh b/cpp/include/raft/stats/information_criterion.cuh
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 /**
- * @file information_criterion.hpp
+ * @file information_criterion.cuh
  * @brief These information criteria are used to evaluate the quality of models
  *        by balancing the quality of the fit and the number of parameters.
  *