Release and Debug image release pipeline (#214)

This appends to the release `cloudbuild.yaml` to create a release and
debug tagged image when run based on the `distroless` image.

This also includes several steps that are needed for ongoing internal
compliance - such as maintaining a list of dependencies and their
licences, zipping MPL licenced code, etc.

Fixes #201

.dockerignore

@@ -0,0 +1,20 @@
+#
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.*
+target/*/deps
+target/*/incremental
+target/*/build

.gcloudignore

@@ -15,4 +15,5 @@
 #
 
 .*
+!.dockerignore
 target

.gitignore

@@ -17,9 +17,14 @@
 .*
 !.gitignore
 !.gcloudignore
+!.dockerignore
 
 *.iml
 
+license.html
+license.csv
+dependencies-src.zip
+
 ### Rust template
 # Generated by Cargo
 # will have compiled files and executables

about.toml

@@ -0,0 +1,24 @@
+#
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+accepted = [
+    "Apache-2.0",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "CC0-1.0",
+    "MIT",
+    "MPL-2.0",
+]

build/release/Dockerfile.builder

@@ -30,6 +30,7 @@ RUN set -eux; \
     rm rustup-init; \
     chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
     cargo install cross; \
+    cargo install cargo-about; \
     rustup --version; \
     cargo --version; \
     rustc --version; \

build/release/Dockerfile.release

@@ -0,0 +1,31 @@
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG PROFILE
+
+FROM gcr.io/distroless/cc:nonroot as base
+WORKDIR /
+COPY ./license.html .
+COPY ./dependencies-src.zip .
+COPY --chown=nonroot:nonroot ./build/release/quilkin.yaml /etc/quilkin/quilkin.yaml
+
+FROM base as release
+COPY ./target/x86_64-unknown-linux-gnu/release/quilkin .
+
+FROM base as debug
+COPY ./target/x86_64-unknown-linux-gnu/debug/quilkin .
+
+FROM $PROFILE
+USER nonroot:nonroot
+ENTRYPOINT ["/quilkin", "--filename", "/etc/quilkin/quilkin.yaml"]

build/release/README.md

@@ -5,9 +5,19 @@ release:
 
 * Linux executable for both release and debug.
 * Windows executable for both release and debug.
+* Docker images for both release and debug.
 
 The executables are stored under `gs://$PROJECT_ID-quilkin-releases` in a zip file named quilkin-${version}.zip, 
 where the version is the version stored in Cargo.toml.
 
+This `cloudbuild.yaml` assumes there is a docker container repository called `release` already created in your 
+project, and will create the Docker images therein. The images will be tagged with both ${version} and ${version}-debug 
+where the version is the version stored in Cargo.toml.
+
+The ${version}-debug tagged version runs the debug binary of Quilkin, where the ${version} tag runs the production 
+release binary.
+
+To create this, run: `gcloud artifacts repositories create release --repository-format=docker --location=us`.
+
 If you need to pass extra arguments to the `make` target, the target comes with an `$(ARGS)` parameter than can be
-used.
+used.

build/release/archive_dependencies.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+#
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+set -eo pipefail
+
+CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+
+# Need to grabs source for MPL, GPL, LGPL, and CDDL licenced dependencies
+# and include it in the Docker image
+
+# This should be reviewed before each release to make sure we're capturing all
+# the dependencies we need.
+
+rm dependencies-src.zip || true
+
+dependencies=("slog-json")
+
+zip="$(pwd)/dependencies-src.zip"
+pushd "$CARGO_HOME/registry/src"
+for d in "${dependencies[@]}"; do
+  path=$(find . -type d -name "$d-*")
+  echo "Archiving $d:$path"
+  zip -rv "$zip" "$path"
+done
+popd

build/release/cloudbuild.yaml

@@ -35,6 +35,18 @@ steps:
     id: cross-version
     entrypoint: cross
     args: ['--version']
+  - name: builder
+    id: cargo-about
+    entrypoint: bash
+    args:
+      - '-c'
+      - 'cargo about generate license.html.hbs > license.html'
+  - name: builder
+    id: dependencies-src.zip
+    entrypoint: bash
+    args:
+      - '-c'
+      - './build/release/archive_dependencies.sh'
   - name: builder
     id: x86_64-unknown-linux-gnu-debug
     entrypoint: cross
@@ -43,6 +55,18 @@ steps:
     id: x86_64-unknown-linux-gnu-release
     entrypoint: cross
     args: ['build', '--target', 'x86_64-unknown-linux-gnu', '--release']
+  - name: builder
+    id: docker-linux-debug
+    entrypoint: bash
+    args:
+      - '-c'
+      - 'docker build -t $_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY/quilkin:$(cat ./version)-debug --build-arg PROFILE=debug -f ./build/release/Dockerfile.release .'
+  - name: builder
+    id: docker-linux-release
+    entrypoint: bash
+    args:
+      - '-c'
+      - 'docker build -t $_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY/quilkin:$(cat ./version) --build-arg PROFILE=release -f ./build/release/Dockerfile.release .'
   - name: builder
     id: x86_64-pc-windows-gnu-debug
     entrypoint: cross
@@ -58,15 +82,20 @@ steps:
       - 'zip quilkin-$(cat ./version).zip target/x86_64-unknown-linux-gnu/debug/quilkin ./target/x86_64-unknown-linux-gnu/release/quilkin ./target/x86_64-pc-windows-gnu/debug/quilkin.exe ./target/x86_64-pc-windows-gnu/release/quilkin.exe'
 artifacts:
   objects:
-    location: 'gs://$PROJECT_ID-quilkin-releases/'
+    location: '$_STORAGE'
     paths:
       - '*.zip'
+      - 'license.html'
+  images:
+    - $_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY/quilkin
 options:
   env:
     - "CARGO_HOME=/workspace/.cargo"
     - "RUST_BACKTRACE=1"
   machineType: N1_HIGHCPU_8
+  dynamic_substitutions: true
 substitutions:
-  _STORAGE: gs://$PROJECT_ID-quilkin-releases
-
+  _STORAGE: gs://${PROJECT_ID}-quilkin-releases
+  _LOCATION: us
+  _REPOSITORY: release
 timeout: 1800s

build/release/quilkin.yaml

@@ -0,0 +1,21 @@
+#
+# Copyright 2021 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: v1alpha1
+static:
+  endpoints:
+    - name: noop
+      address: 127.0.0.1:0

license.csv.hbs

@@ -0,0 +1,17 @@
+{{!
+Copyright 2021 Google LLC All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+}}"External Library Name","Link to License","License Name"
+{{#each licenses}}{{#each used_by}}"{{crate.name}}","{{crate.repository}}","{{../name}}"
+{{/each}}{{/each}}

license.html.hbs

@@ -0,0 +1,86 @@
+<!--
+Copyright 2021 Google LLC All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<html>
+
+<head>
+    <style>
+        @media (prefers-color-scheme: dark) {
+            body {
+                background: #333;
+                color: white;
+            }
+            a {
+                color: skyblue;
+            }
+        }
+        .container {
+            font-family: sans-serif;
+            max-width: 800px;
+            margin: 0 auto;
+        }
+        .intro {
+            text-align: center;
+        }
+        .licenses-list {
+            list-style-type: none;
+            margin: 0;
+            padding: 0;
+        }
+        .license-used-by {
+            margin-top: -10px;
+        }
+        .license-text {
+            max-height: 200px;
+            overflow-y: scroll;
+            white-space: pre-wrap;
+        }
+    </style>
+</head>
+
+<body>
+    <main class="container">
+        <div class="intro">
+            <h1>Third Party Licenses</h1>
+            <p>This page lists the licenses of the projects used in cargo-about.</p>
+        </div>
+
+        <h2>Overview of licenses:</h2>
+        <ul class="licenses-overview">
+            {{#each overview}}
+            <li><a href="#{{id}}">{{name}}</a> ({{count}})</li>
+            {{/each}}
+        </ul>
+
+        <h2>All license text:</h2>
+        <ul class="licenses-list">
+            {{#each licenses}}
+            <li class="license">
+                <h3 id="{{id}}">{{name}}</h3>
+                <h4>Used by:</h4>
+                <ul class="license-used-by">
+                    {{#each used_by}}
+                    <li><a href="{{#if crate.repository}} {{crate.repository}} {{else}} https://crates.io/crates/{{crate.name}} {{/if}}">{{crate.name}} {{crate.version}}</a></li>
+                    {{/each}}
+                </ul>
+                <pre class="license-text">{{text}}</pre>
+            </li>
+            {{/each}}
+        </ul>
+    </main>
+</body>
+
+</html>