From b551561e1b918e943832fa2db1a8b014b0edf74d Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 2 Oct 2023 15:03:21 +0100 Subject: [PATCH] docs: Improve builds for Envoy and mobile (stamping) (#29887) Signed-off-by: Ryan Northey Signed-off-by: phlax --- api/CONTRIBUTING.md | 6 +-- bazel/BUILD | 8 ++-- bazel/get_workspace_status | 3 ++ ci/do_ci.sh | 16 +++++++- docs/BUILD | 8 +++- docs/README.md | 6 +-- docs/build.sh | 79 ++----------------------------------- mobile/docs/build.sh | 52 ------------------------ mobile/docs/conf.py | 13 +++--- tools/base/requirements.in | 5 +-- tools/base/requirements.txt | 24 ++++++----- 11 files changed, 59 insertions(+), 161 deletions(-) delete mode 100755 mobile/docs/build.sh diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index a1e61a7072c4..0ff244623984 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -23,19 +23,19 @@ documentation. The documentation can be built locally in the root of https://github.com/envoyproxy/envoy via: ``` -docs/build.sh +ci/do_ci.sh docs ``` To skip configuration examples validation: ``` -SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +SPHINX_SKIP_CONFIG_VALIDATION=true ci/do_ci.sh docs ``` Or to use a hermetic Docker container: ``` -./ci/run_envoy_docker.sh './ci/do_ci.sh docs' +./ci/run_envoy_docker.sh 'ci/do_ci.sh docs' ``` This process builds RST documentation directly from the proto files, merges it with the static RST diff --git a/bazel/BUILD b/bazel/BUILD index edb5d4a6e1b6..8b15825a9170 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -41,8 +41,8 @@ genrule( outs = ["gnu_build_id.ldscript"], cmd = """ echo --build-id=0x$$( - grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\ - | sed 's/^BUILD_SCM_REVISION //') \\ + grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \ + | sed 's/^BUILD_SCM_REVISION //') \ > $@ """, # Undocumented attr to depend on workspace status files. @@ -55,8 +55,8 @@ genrule( name = "raw_build_id", outs = ["raw_build_id.ldscript"], cmd = """ - grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\ - | sed 's/^BUILD_SCM_REVISION //' \\ + grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \ + | sed 's/^BUILD_SCM_REVISION //' \ | tr -d '\\n' \\ > $@ """, diff --git a/bazel/get_workspace_status b/bazel/get_workspace_status index ca5159e6dea9..bc43475f01ac 100755 --- a/bazel/get_workspace_status +++ b/bazel/get_workspace_status @@ -23,6 +23,7 @@ if [ -f SOURCE_VERSION ] then echo "BUILD_SCM_REVISION $(cat SOURCE_VERSION)" + echo "ENVOY_BUILD_SCM_REVISION $(cat SOURCE_VERSION)" echo "STABLE_BUILD_SCM_REVISION $(cat SOURCE_VERSION)" echo "BUILD_SCM_STATUS Distribution" exit 0 @@ -30,11 +31,13 @@ fi if [[ -n "$BAZEL_FAKE_SCM_REVISION" ]]; then echo "BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" + echo "ENVOY_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" echo "STABLE_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION" else # The code below presents an implementation that works for git repository git_rev=$(git rev-parse HEAD) || exit 1 echo "BUILD_SCM_REVISION ${git_rev}" + echo "ENVOY_BUILD_SCM_REVISION ${git_rev}" echo "STABLE_BUILD_SCM_REVISION ${git_rev}" fi diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 55f49139e33c..187acbc89e35 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -650,7 +650,21 @@ case $CI_TARGET in setup_clang_toolchain echo "generating docs..." # Build docs. - "${ENVOY_SRCDIR}/docs/build.sh" + # We want the binary at the end + BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel) + [[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs + rm -rf "${DOCS_OUTPUT_DIR}" + mkdir -p "${DOCS_OUTPUT_DIR}" + if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then + export SPHINX_RUNNER_ARGS="-v warn" + BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS") + fi + if [[ -n "${DOCS_BUILD_RST}" ]]; then + bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst + cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz + fi + bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:html + tar -xzf bazel-bin/docs/html.tar.gz -C "$DOCS_OUTPUT_DIR" ;; docs-upload) diff --git a/docs/BUILD b/docs/BUILD index 265a8840986f..69548a4aafda 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -236,11 +236,15 @@ pkg_tar( genrule( name = "html_release", outs = ["html_release.tar.gz"], + # BUILD_SHA must be set in release builds + # The Envoy workspace will provide this on stamped builds. For external builds + # you must either pass an env var or pass it through the workspace's status. cmd = """ . $(location //bazel:volatile_env) \ + && _BUILD_SHA=$${BUILD_DOCS_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \ && $(location //tools/docs:sphinx_runner) \ $${SPHINX_RUNNER_ARGS:-} \ - --build_sha="$${BUILD_DOCS_SHA:-$${BUILD_SCM_REVISION}}" \ + --build_sha="$$_BUILD_SHA" \ --docs_tag="$${BUILD_DOCS_TAG:-}" \ --version_file=$(location //:VERSION.txt) \ --descriptor_path=$(location @envoy_api//:v3_proto_set) \ @@ -280,5 +284,5 @@ genrule( alias( name = "docs", - actual = ":html", + actual = ":html_release", ) diff --git a/docs/README.md b/docs/README.md index 923ae33a4bb8..32f9301eaf25 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,14 +9,14 @@ In both cases, the generated output can be found in `generated/docs`. If you have an [existing Envoy development environment](https://github.com/envoyproxy/envoy/tree/main/bazel#quick-start-bazel-build-for-developers), you should have the necessary dependencies and requirements and be able to build the documentation directly. ```bash -./docs/build.sh +./ci/do_ci.sh docs ``` By default configuration examples are going to be validated during build. To disable validation, set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`: ```bash -SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh +SPHINX_SKIP_CONFIG_VALIDATION=true ./ci/do_ci.sh docs ``` ## Using the Docker build container to build the documentation @@ -27,7 +27,7 @@ image that is used in continuous integration. This can be done as follows: ``` -./ci/run_envoy_docker.sh 'docs/build.sh' +./ci/run_envoy_docker.sh './ci/do_ci.sh docs' ``` To use this method you will need a minimum of 4-5GB of disk space available to accommodate the build image. diff --git a/docs/build.sh b/docs/build.sh index 70e1998e6bc7..20089b3a2b6d 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,78 +1,5 @@ #!/usr/bin/env bash -# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip -# validation of configuration examples - -set -e - - -if [[ ! $(command -v bazel) ]]; then - # shellcheck disable=SC2016 - echo 'ERROR: bazel must be installed and available in "$PATH" to build docs' >&2 - exit 1 -fi - -VERSION="$(cat VERSION.txt)" -MAIN_BRANCH="refs/heads/main" -DEV_VERSION_REGEX="-dev$" - -# default is to build html only -BUILD_TYPE=html - -if [[ "$VERSION" =~ $DEV_VERSION_REGEX ]]; then - if [[ "$CI_BRANCH" == "$MAIN_BRANCH" ]]; then - # no need to build html, just rst - BUILD_TYPE=rst - fi -else - export BUILD_DOCS_TAG="v${VERSION}" - echo "BUILD AZP RELEASE BRANCH ${BUILD_DOCS_TAG}" - BAZEL_BUILD_OPTIONS+=("--action_env=BUILD_DOCS_TAG") -fi - -# This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files. -IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}" -IFS=" " read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}" - -# We want the binary at the end -BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel) - -if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then - export SPHINX_RUNNER_ARGS="-v warn" - BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS") -fi - -# Building html/rst is determined by then needs of CI but can be overridden in dev. -if [[ "${BUILD_TYPE}" == "html" ]] || [[ -n "${DOCS_BUILD_HTML}" ]]; then - BUILD_HTML=1 - BUILD_HTML_TARGET="//docs" - BUILD_HTML_TARBALL="bazel-bin/docs/html.tar.gz" - if [[ -n "${CI_BRANCH}" ]] || [[ -n "${DOCS_BUILD_RELEASE}" ]]; then - # CI build - use git sha - BUILD_HTML_TARGET="//docs:html_release" - BUILD_HTML_TARBALL="bazel-bin/docs/html_release.tar.gz" - fi -fi -if [[ "${BUILD_TYPE}" == "rst" ]] || [[ -n "${DOCS_BUILD_RST}" ]]; then - BUILD_RST=1 -fi - -# Build html/rst -if [[ -n "${BUILD_RST}" ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst -fi -if [[ -n "${BUILD_HTML}" ]]; then - bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" "$BUILD_HTML_TARGET" -fi - -[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs -rm -rf "${DOCS_OUTPUT_DIR}" -mkdir -p "${DOCS_OUTPUT_DIR}" - -# Save html/rst to output directory -if [[ -n "${BUILD_HTML}" ]]; then - tar -xzf "$BUILD_HTML_TARBALL" -C "$DOCS_OUTPUT_DIR" -fi -if [[ -n "${BUILD_RST}" ]]; then - cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz -fi +# shellcheck disable=SC2016 +echo 'This script has been removed. Please use `ci/do_ci.sh docs` instead' >&2 +exit 1 diff --git a/mobile/docs/build.sh b/mobile/docs/build.sh deleted file mode 100755 index bec4b177bf55..000000000000 --- a/mobile/docs/build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -e - -# shellcheck disable=SC1091 -. tools/shell_utils.sh - -# We need to set ENVOY_DOCS_VERSION_STRING and ENVOY_DOCS_RELEASE_LEVEL for Sphinx. -# We also validate that the tag and version match at this point if needed. - -# Docs for release tags are reserved for vX.Y.Z versions. -# vX.Y.Z.ddmmyy do not publish tagged docs. -VERSION_NUMBER=$(cat mobile/VERSION) -if [[ "$GITHUB_REF_TYPE" == "tag" ]] && [[ "${VERSION_NUMBER}" =~ ^[0-9]+\.[0-9]+\.[0-9]$ ]] -then - # Check the git tag matches the version number in the VERSION file. - if [ "v${VERSION_NUMBER}" != "${GITHUB_REF_NAME}" ]; then - echo "Given git tag does not match the VERSION file content:" - echo "${GITHUB_REF_NAME} vs $(cat mobile/VERSION)" - exit 1 - fi - # Check the version_history.rst contains current release version. - grep --fixed-strings "$VERSION_NUMBER" docs/root/intro/version_history.rst \ - || (echo "Git tag not found in version_history.rst" && exit 1) - - # Now that we now there is a match, we can use the tag. - export ENVOY_DOCS_VERSION_STRING="tag-$GITHUB_REF_NAME" - export ENVOY_DOCS_RELEASE_LEVEL=tagged - export ENVOY_BLOB_SHA="$GITHUB_REF_NAME" -else - BUILD_SHA=$(git rev-parse HEAD) - export ENVOY_DOCS_VERSION_STRING="${VERSION_NUMBER}"-"${BUILD_SHA:0:6}" - export ENVOY_DOCS_RELEASE_LEVEL=pre-release - export ENVOY_BLOB_SHA="$BUILD_SHA" -fi - -SCRIPT_DIR=$(dirname "$0") -BUILD_DIR=build_docs -[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs -[[ -z "${GENERATED_RST_DIR}" ]] && GENERATED_RST_DIR=generated/rst - -rm -rf "${DOCS_OUTPUT_DIR}" -mkdir -p "${DOCS_OUTPUT_DIR}" - -rm -rf "${GENERATED_RST_DIR}" -mkdir -p "${GENERATED_RST_DIR}" - -source_venv "$BUILD_DIR" -pip install -r "${SCRIPT_DIR}"/requirements.txt --no-deps --require-hashes - -rsync -av "${SCRIPT_DIR}"/root/ "${SCRIPT_DIR}"/conf.py "${GENERATED_RST_DIR}" -sphinx-build -W --keep-going -b html "${GENERATED_RST_DIR}" "${DOCS_OUTPUT_DIR}" diff --git a/mobile/docs/conf.py b/mobile/docs/conf.py index 80270f95c7af..9618124b2704 100644 --- a/mobile/docs/conf.py +++ b/mobile/docs/conf.py @@ -48,11 +48,11 @@ def setup(app): app.add_directive('substitution-code-block', SubstitutionCodeBlock) -if not os.environ.get('ENVOY_DOCS_RELEASE_LEVEL'): +if not (release_level := os.environ.get('ENVOY_DOCS_RELEASE_LEVEL')): raise Exception("ENVOY_DOCS_RELEASE_LEVEL env var must be defined") -release_level = os.environ['ENVOY_DOCS_RELEASE_LEVEL'] -blob_sha = os.environ['ENVOY_BLOB_SHA'] +if not (blob_sha := os.environ.get("ENVOY_BLOB_SHA")): + raise Exception("ENVOY_BLOB_SHA env var must be defined") # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -103,13 +103,12 @@ def setup(app): # |version| and |release|, also used in various other places throughout the # built documents. -if not os.environ.get('ENVOY_DOCS_VERSION_STRING'): +# The short X.Y version. +if not (version := os.environ.get("ENVOY_DOCS_VERSION_STRING")): raise Exception("ENVOY_DOCS_VERSION_STRING env var must be defined") -# The short X.Y version. -version = os.environ['ENVOY_DOCS_VERSION_STRING'] # The full version, including alpha/beta/rc tags. -release = os.environ['ENVOY_DOCS_VERSION_STRING'] +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/tools/base/requirements.in b/tools/base/requirements.in index c0c77a086bf5..00dd34986e8d 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -15,7 +15,7 @@ envoy.dependency.check>=0.1.7 envoy.distribution.release>=0.0.9 envoy.distribution.repo>=0.0.8 envoy.distribution.verify>=0.0.11 -envoy.docs.sphinx_runner>=0.2.5 +envoy.docs.sphinx_runner>=0.2.9 envoy.gpg.identity>=0.1.1 envoy.gpg.sign>=0.2.0 flake8>=6 @@ -41,6 +41,3 @@ thrift verboselogs yapf yarl>=1.7.2 - -# This has packaging issues, so use the github tarball until resolved (https://github.com/sphinx-contrib/jquery/issues/15) -https://github.com/sphinx-contrib/jquery/archive/refs/tags/v3.0.0.zip diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 397a48db90cd..09c30e31d763 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -447,6 +447,7 @@ docutils==0.19 \ # via # envoy-docs-sphinx-runner # sphinx + # sphinx-rtd-theme envoy-base-utils==0.4.12 \ --hash=sha256:2bafcb6be3c1223968c9ee90b7a33d6d93aee16fe99bef37410ea9e4bc1a957f \ --hash=sha256:b9b409abe83be6911aa03cfe635ed1e2e2b9e44e7c8595b2b7e5c7aae7bf70fc @@ -488,9 +489,9 @@ envoy-distribution-verify==0.0.11 \ envoy-docker-utils==0.0.2 \ --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76 # via envoy-distribution-distrotest -envoy-docs-sphinx-runner==0.2.6 \ - --hash=sha256:045b23f182d9760df693e0c01f6bd0654d0c5690107945e262ce720a976920b7 \ - --hash=sha256:8166ff4b1b265efaf73db397d7ad0a1a743d71281ec233885c692c24f9349bd1 +envoy-docs-sphinx-runner==0.2.9 \ + --hash=sha256:1fa789b1d29ea929df67b07e5ca910d62e2057cd229719725030889da53b1a09 \ + --hash=sha256:4bfa1946104e263471d522b47d683e127124a5ad47334d69de4aea0eac282576 # via -r requirements.in envoy-github-abstract==0.0.22 \ --hash=sha256:2dd65e2f247a4947d0198b295c82716c13162e30c433b7625c27d59eee7bcf78 \ @@ -1221,12 +1222,18 @@ sphinx==7.1.0 \ # -r requirements.in # envoy-docs-sphinx-runner # sphinx-copybutton + # sphinx-rtd-theme # sphinxcontrib-httpdomain + # sphinxcontrib-jquery # sphinxext-rediraffe sphinx-copybutton==0.5.2 \ --hash=sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd \ --hash=sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e # via envoy-docs-sphinx-runner +sphinx-rtd-theme==2.0.0rc2 \ + --hash=sha256:d1270effe620df9164b1cd2d617909472a63531e21a716fd22d0fbcedf9d24ff \ + --hash=sha256:f04df9213acf421c3b42f4f39005c8bc68fc4696c5b4ed4ef13d1678369713f7 + # via envoy-docs-sphinx-runner sphinxcontrib-applehelp==1.0.4 \ --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \ --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e @@ -1243,11 +1250,12 @@ sphinxcontrib-httpdomain==1.8.1 \ --hash=sha256:21eefe1270e4d9de8d717cc89ee92cc4871b8736774393bafc5e38a6bb77b1d5 \ --hash=sha256:6c2dfe6ca282d75f66df333869bb0ce7331c01b475db6809ff9d107b7cdfe04b # via envoy-docs-sphinx-runner -sphinxcontrib.jquery @ https://github.com/sphinx-contrib/jquery/archive/refs/tags/v3.0.0.zip \ - --hash=sha256:562ad9ac0ac3d8f04a363eb3507ae4b2b856aa04aabab6df7543530fafb849ca +sphinxcontrib-jquery==4.1 \ + --hash=sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a \ + --hash=sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae # via - # -r requirements.in # envoy-docs-sphinx-runner + # sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 \ --hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \ --hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8 @@ -1554,6 +1562,4 @@ zstandard==0.21.0 \ setuptools==68.0.0 \ --hash=sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f \ --hash=sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235 - # via - # -r requirements.in - # sphinxcontrib-jquery + # via -r requirements.in