docs: Improve builds for Envoy and mobile (stamping) (#29887)

Signed-off-by: Ryan Northey <[email protected]>

Signed-off-by: phlax <[email protected]>

api/CONTRIBUTING.md

@@ -23,19 +23,19 @@ documentation.
 The documentation can be built locally in the root of https://github.com/envoyproxy/envoy via:
 
 ```
-docs/build.sh
+ci/do_ci.sh docs
 ```
 
 To skip configuration examples validation:
 
 ```
-SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh
+SPHINX_SKIP_CONFIG_VALIDATION=true ci/do_ci.sh docs
 ```
 
 Or to use a hermetic Docker container:
 
 ```
-./ci/run_envoy_docker.sh './ci/do_ci.sh docs'
+./ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
 ```
 
 This process builds RST documentation directly from the proto files, merges it with the static RST

bazel/BUILD

@@ -41,8 +41,8 @@ genrule(
     outs = ["gnu_build_id.ldscript"],
     cmd = """
       echo --build-id=0x$$(
-          grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\
-        | sed 's/^BUILD_SCM_REVISION //') \\
+          grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \
+        | sed 's/^BUILD_SCM_REVISION //') \
         > $@
     """,
     # Undocumented attr to depend on workspace status files.
@@ -55,8 +55,8 @@ genrule(
     name = "raw_build_id",
     outs = ["raw_build_id.ldscript"],
     cmd = """
-      grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\
-    | sed 's/^BUILD_SCM_REVISION //' \\
+      grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \
+    | sed 's/^BUILD_SCM_REVISION //' \
     | tr -d '\\n' \\
     > $@
     """,

bazel/get_workspace_status

@@ -23,18 +23,21 @@
 if [ -f SOURCE_VERSION ]
 then
     echo "BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
+    echo "ENVOY_BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
     echo "STABLE_BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
     echo "BUILD_SCM_STATUS Distribution"
     exit 0
 fi
 
 if [[ -n "$BAZEL_FAKE_SCM_REVISION" ]]; then
     echo "BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
+    echo "ENVOY_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
     echo "STABLE_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
 else
     # The code below presents an implementation that works for git repository
     git_rev=$(git rev-parse HEAD) || exit 1
     echo "BUILD_SCM_REVISION ${git_rev}"
+    echo "ENVOY_BUILD_SCM_REVISION ${git_rev}"
     echo "STABLE_BUILD_SCM_REVISION ${git_rev}"
 fi
 

ci/do_ci.sh

@@ -650,7 +650,21 @@ case $CI_TARGET in
         setup_clang_toolchain
         echo "generating docs..."
         # Build docs.
-        "${ENVOY_SRCDIR}/docs/build.sh"
+        # We want the binary at the end
+        BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel)
+        [[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
+        rm -rf "${DOCS_OUTPUT_DIR}"
+        mkdir -p "${DOCS_OUTPUT_DIR}"
+        if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then
+            export SPHINX_RUNNER_ARGS="-v warn"
+            BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS")
+        fi
+        if [[ -n "${DOCS_BUILD_RST}" ]]; then
+            bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst
+            cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz
+        fi
+        bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:html
+        tar -xzf bazel-bin/docs/html.tar.gz -C "$DOCS_OUTPUT_DIR"
         ;;
 
     docs-upload)

docs/BUILD

@@ -236,11 +236,15 @@ pkg_tar(
 genrule(
     name = "html_release",
     outs = ["html_release.tar.gz"],
+    # BUILD_SHA must be set in release builds
+    # The Envoy workspace will provide this on stamped builds. For external builds
+    # you must either pass an env var or pass it through the workspace's status.
     cmd = """
     . $(location //bazel:volatile_env) \
+    && _BUILD_SHA=$${BUILD_DOCS_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \
     && $(location //tools/docs:sphinx_runner) \
          $${SPHINX_RUNNER_ARGS:-} \
-         --build_sha="$${BUILD_DOCS_SHA:-$${BUILD_SCM_REVISION}}" \
+         --build_sha="$$_BUILD_SHA" \
          --docs_tag="$${BUILD_DOCS_TAG:-}" \
          --version_file=$(location //:VERSION.txt) \
          --descriptor_path=$(location @envoy_api//:v3_proto_set) \
@@ -280,5 +284,5 @@ genrule(
 
 alias(
     name = "docs",
-    actual = ":html",
+    actual = ":html_release",
 )

docs/README.md

@@ -9,14 +9,14 @@ In both cases, the generated output can be found in `generated/docs`.
 If you have an [existing Envoy development environment](https://github.com/envoyproxy/envoy/tree/main/bazel#quick-start-bazel-build-for-developers), you should have the necessary dependencies and requirements and be able to build the documentation directly.
 
 ```bash
-./docs/build.sh
+./ci/do_ci.sh docs
 ```
 
 By default configuration examples are going to be validated during build. To disable validation,
 set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`:
 
 ```bash
-SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh
+SPHINX_SKIP_CONFIG_VALIDATION=true ./ci/do_ci.sh docs
 ```
 
 ## Using the Docker build container to build the documentation
@@ -27,7 +27,7 @@ image that is used in continuous integration.
 This can be done as follows:
 
 ```
-./ci/run_envoy_docker.sh 'docs/build.sh'
+./ci/run_envoy_docker.sh './ci/do_ci.sh docs'
 ```
 
 To use this method you will need a minimum of 4-5GB of disk space available to accommodate the build image.

docs/build.sh

@@ -1,78 +1,5 @@
 #!/usr/bin/env bash
 
-# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip
-# validation of configuration examples
-
-set -e
-
-
-if [[ ! $(command -v bazel) ]]; then
-    # shellcheck disable=SC2016
-    echo 'ERROR: bazel must be installed and available in "$PATH" to build docs' >&2
-    exit 1
-fi
-
-VERSION="$(cat VERSION.txt)"
-MAIN_BRANCH="refs/heads/main"
-DEV_VERSION_REGEX="-dev$"
-
-# default is to build html only
-BUILD_TYPE=html
-
-if [[ "$VERSION" =~ $DEV_VERSION_REGEX ]]; then
-   if [[ "$CI_BRANCH" == "$MAIN_BRANCH" ]]; then
-       # no need to build html, just rst
-       BUILD_TYPE=rst
-   fi
-else
-    export BUILD_DOCS_TAG="v${VERSION}"
-    echo "BUILD AZP RELEASE BRANCH ${BUILD_DOCS_TAG}"
-    BAZEL_BUILD_OPTIONS+=("--action_env=BUILD_DOCS_TAG")
-fi
-
-# This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files.
-IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}"
-IFS=" " read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}"
-
-# We want the binary at the end
-BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel)
-
-if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then
-    export SPHINX_RUNNER_ARGS="-v warn"
-    BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS")
-fi
-
-# Building html/rst is determined by then needs of CI but can be overridden in dev.
-if [[ "${BUILD_TYPE}" == "html" ]] || [[ -n "${DOCS_BUILD_HTML}" ]]; then
-    BUILD_HTML=1
-    BUILD_HTML_TARGET="//docs"
-    BUILD_HTML_TARBALL="bazel-bin/docs/html.tar.gz"
-    if [[ -n "${CI_BRANCH}" ]] || [[ -n "${DOCS_BUILD_RELEASE}" ]]; then
-        # CI build - use git sha
-        BUILD_HTML_TARGET="//docs:html_release"
-        BUILD_HTML_TARBALL="bazel-bin/docs/html_release.tar.gz"
-    fi
-fi
-if [[ "${BUILD_TYPE}" == "rst" ]] || [[ -n "${DOCS_BUILD_RST}" ]]; then
-    BUILD_RST=1
-fi
-
-# Build html/rst
-if [[ -n "${BUILD_RST}" ]]; then
-    bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst
-fi
-if [[ -n "${BUILD_HTML}" ]]; then
-    bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" "$BUILD_HTML_TARGET"
-fi
-
-[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
-rm -rf "${DOCS_OUTPUT_DIR}"
-mkdir -p "${DOCS_OUTPUT_DIR}"
-
-# Save html/rst to output directory
-if [[ -n "${BUILD_HTML}" ]]; then
-    tar -xzf "$BUILD_HTML_TARBALL" -C "$DOCS_OUTPUT_DIR"
-fi
-if [[ -n "${BUILD_RST}" ]]; then
-    cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz
-fi
+# shellcheck disable=SC2016
+echo 'This script has been removed. Please use `ci/do_ci.sh docs` instead' >&2
+exit 1

mobile/docs/conf.py

@@ -48,11 +48,11 @@ def setup(app):
     app.add_directive('substitution-code-block', SubstitutionCodeBlock)
 
 
-if not os.environ.get('ENVOY_DOCS_RELEASE_LEVEL'):
+if not (release_level := os.environ.get('ENVOY_DOCS_RELEASE_LEVEL')):
     raise Exception("ENVOY_DOCS_RELEASE_LEVEL env var must be defined")
 
-release_level = os.environ['ENVOY_DOCS_RELEASE_LEVEL']
-blob_sha = os.environ['ENVOY_BLOB_SHA']
+if not (blob_sha := os.environ.get("ENVOY_BLOB_SHA")):
+    raise Exception("ENVOY_BLOB_SHA env var must be defined")
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -103,13 +103,12 @@ def setup(app):
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 
-if not os.environ.get('ENVOY_DOCS_VERSION_STRING'):
+# The short X.Y version.
+if not (version := os.environ.get("ENVOY_DOCS_VERSION_STRING")):
     raise Exception("ENVOY_DOCS_VERSION_STRING env var must be defined")
 
-# The short X.Y version.
-version = os.environ['ENVOY_DOCS_VERSION_STRING']
 # The full version, including alpha/beta/rc tags.
-release = os.environ['ENVOY_DOCS_VERSION_STRING']
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tools/base/requirements.in

@@ -15,7 +15,7 @@ envoy.dependency.check>=0.1.7
 envoy.distribution.release>=0.0.9
 envoy.distribution.repo>=0.0.8
 envoy.distribution.verify>=0.0.11
-envoy.docs.sphinx_runner>=0.2.5
+envoy.docs.sphinx_runner>=0.2.9
 envoy.gpg.identity>=0.1.1
 envoy.gpg.sign>=0.2.0
 flake8>=6
@@ -41,6 +41,3 @@ thrift
 verboselogs
 yapf
 yarl>=1.7.2
-
-# This has packaging issues, so use the github tarball until resolved (https://github.com/sphinx-contrib/jquery/issues/15)
-https://github.com/sphinx-contrib/jquery/archive/refs/tags/v3.0.0.zip

tools/base/requirements.txt

@@ -447,6 +447,7 @@ docutils==0.19 \
     # via
     #   envoy-docs-sphinx-runner
     #   sphinx
+    #   sphinx-rtd-theme
 envoy-base-utils==0.4.12 \
     --hash=sha256:2bafcb6be3c1223968c9ee90b7a33d6d93aee16fe99bef37410ea9e4bc1a957f \
     --hash=sha256:b9b409abe83be6911aa03cfe635ed1e2e2b9e44e7c8595b2b7e5c7aae7bf70fc
@@ -488,9 +489,9 @@ envoy-distribution-verify==0.0.11 \
 envoy-docker-utils==0.0.2 \
     --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76
     # via envoy-distribution-distrotest
-envoy-docs-sphinx-runner==0.2.6 \
-    --hash=sha256:045b23f182d9760df693e0c01f6bd0654d0c5690107945e262ce720a976920b7 \
-    --hash=sha256:8166ff4b1b265efaf73db397d7ad0a1a743d71281ec233885c692c24f9349bd1
+envoy-docs-sphinx-runner==0.2.9 \
+    --hash=sha256:1fa789b1d29ea929df67b07e5ca910d62e2057cd229719725030889da53b1a09 \
+    --hash=sha256:4bfa1946104e263471d522b47d683e127124a5ad47334d69de4aea0eac282576
     # via -r requirements.in
 envoy-github-abstract==0.0.22 \
     --hash=sha256:2dd65e2f247a4947d0198b295c82716c13162e30c433b7625c27d59eee7bcf78 \
@@ -1221,12 +1222,18 @@ sphinx==7.1.0 \
     #   -r requirements.in
     #   envoy-docs-sphinx-runner
     #   sphinx-copybutton
+    #   sphinx-rtd-theme
     #   sphinxcontrib-httpdomain
+    #   sphinxcontrib-jquery
     #   sphinxext-rediraffe
 sphinx-copybutton==0.5.2 \
     --hash=sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd \
     --hash=sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e
     # via envoy-docs-sphinx-runner
+sphinx-rtd-theme==2.0.0rc2 \
+    --hash=sha256:d1270effe620df9164b1cd2d617909472a63531e21a716fd22d0fbcedf9d24ff \
+    --hash=sha256:f04df9213acf421c3b42f4f39005c8bc68fc4696c5b4ed4ef13d1678369713f7
+    # via envoy-docs-sphinx-runner
 sphinxcontrib-applehelp==1.0.4 \
     --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
     --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
@@ -1243,11 +1250,12 @@ sphinxcontrib-httpdomain==1.8.1 \
     --hash=sha256:21eefe1270e4d9de8d717cc89ee92cc4871b8736774393bafc5e38a6bb77b1d5 \
     --hash=sha256:6c2dfe6ca282d75f66df333869bb0ce7331c01b475db6809ff9d107b7cdfe04b
     # via envoy-docs-sphinx-runner
-sphinxcontrib.jquery @ https://github.com/sphinx-contrib/jquery/archive/refs/tags/v3.0.0.zip \
-    --hash=sha256:562ad9ac0ac3d8f04a363eb3507ae4b2b856aa04aabab6df7543530fafb849ca
+sphinxcontrib-jquery==4.1 \
+    --hash=sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a \
+    --hash=sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae
     # via
-    #   -r requirements.in
     #   envoy-docs-sphinx-runner
+    #   sphinx-rtd-theme
 sphinxcontrib-jsmath==1.0.1 \
     --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \
     --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8
@@ -1554,6 +1562,4 @@ zstandard==0.21.0 \
 setuptools==68.0.0 \
     --hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \
     --hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235
-    # via
-    #   -r requirements.in
-    #   sphinxcontrib-jquery
+    # via -r requirements.in