CI: Documentation test gen, links, images, etc.

Introduces both building the documentation sites with `mdbook` and
`cargo doc` as well as placing them in the same directory structure as
they exist in on GitHub pages.

Utilising `htmltest` to test the links, images, scripts etc in the
`mdbook` pages (there are lots of invalid anchors in the rustdoc, so not
 testing the links for now - but we could later).

To use locally: `make test-docs` will run the tests for you.

Extra notes:
* Had to use a fork of `htmltest`, as awaiting the merging of a critical
  bugfix.
* Split up each of the test targets in `cloudbuild.yaml`. This requires
  a bit more work when writing new test targets in the Makefile, but
  will be easier to determine why things go wrong in CI.
* Broke up the Dockerfile into sections, to make it faster for local
  rebuilding.

Fix googleforgames#367

build/Makefile

@@ -47,7 +47,7 @@ version:
 	@echo $(package_version)
 
 # Run all tests
-test: ensure-build-image test-quilkin test-examples
+test: ensure-build-image test-quilkin test-examples test-docs
 
 # test only the quilkin crate
 test-quilkin: ensure-build-image
@@ -67,6 +67,17 @@ test-examples: ensure-build-image
 	docker run --rm $(common_rust_args) -w /workspace/examples/quilkin-filter-example \
 		--entrypoint=cargo $(BUILD_IMAGE_TAG) fmt -- --check
 
+# Run tests against documentation
+test-docs: ensure-build-image
+	docker run --rm $(common_rust_args) \
+		--entrypoint=bash $(BUILD_IMAGE_TAG) -c \
+		 	'export RUSTDOCFLAGS="-Dwarnings" && mkdir /tmp/docs && \
+			mkdir -p ./target/doc; \
+			cargo doc --workspace --no-deps && cd docs && mdbook build --dest-dir /tmp/docs/book && \
+			cp -r /workspace/target/doc /tmp/docs/api && \
+			rm /tmp/docs/book/print.html && \
+			htmltest -c /workspace/docs/htmltest.yaml /tmp/docs'
+
 # Build all binaries, images and related artifacts
 build: binary-archive build-image
 

build/build-image/Dockerfile

@@ -23,11 +23,28 @@ ENV RUSTUP_HOME=/usr/local/rustup \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 
+# Install packages
 RUN set -eux && \
     apt-get update && \
     apt-get install -y wget zip build-essential libssl-dev pkg-config python3-pip && \
-    pip3 install live-server && \
-    wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init && \
+    pip3 install live-server
+
+# Install Go
+WORKDIR /usr/local
+ENV GO_VERSION=1.17.2
+RUN wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \
+    tar -xzf go${GO_VERSION}.linux-amd64.tar.gz && rm go${GO_VERSION}.linux-amd64.tar.gz
+ENV PATH /usr/local/go/bin:$PATH
+
+# Install htmltest
+WORKDIR /tmp
+# TODO (markmandel): replace with official release once https://github.com/wjdp/htmltest/pull/181 is merged and released.
+RUN git clone --branch debug/panic --depth=5 https://github.com/markmandel/htmltest.git && \
+    cd htmltest && ./build.sh && cp ./bin/htmltest /usr/local/bin/ && \
+    rm -r /tmp/htmltest
+
+# Install Rust
+RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init && \
     chmod +x rustup-init && \
     ./rustup-init -y --no-modify-path --default-toolchain $RUST_TOOLCHAIN && \
     rm rustup-init && \

cloudbuild.yaml

@@ -31,8 +31,22 @@ steps:
     args:
       - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
       - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
-      - test
-    id: test
+      - test-quilkin
+    id: test-quilkin
+  - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
+    dir: ./build
+    args:
+      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
+      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
+      - test-examples
+    id: test-examples
+  - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
+    dir: ./build
+    args:
+      - BUILD_IMAGE_TAG=${_BUILD_IMAGE_TAG}
+      - BUILD_IMAGE_ARG=--cache-from ${_BUILD_IMAGE_TAG}
+      - test-docs
+    id: test-docs
   - name: us-docker.pkg.dev/$PROJECT_ID/ci/make-docker
     dir: ./build
     args:

docs/htmltest.yaml

@@ -0,0 +1,30 @@
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TestFilesConcurrently: false
+IgnoreInternalEmptyHash: true
+IgnoreEmptyHref: true
+CheckExternal: true
+ExternalTimeout: 60
+IgnoreURLs:
+  - "http://localhost:9091/"
+IgnoreInternalURLs:
+  - "book/print.html"
+  - "print.html"
+  - "../print.html"
+IgnoreDirs:
+  - "api"
+HTTPHeaders:
+  User-Agent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"
+  Accept: "text/html"