diff --git a/integrations/cloudbuild/build-coverage-remote.yaml b/integrations/cloudbuild/build-coverage-remote.yaml new file mode 100644 index 00000000000000..1d78608cfe8a4d --- /dev/null +++ b/integrations/cloudbuild/build-coverage-remote.yaml @@ -0,0 +1,18 @@ +steps: + - name: gcr.io/$PROJECT_ID/remote-builder + env: + - GIT_CLONE_CMD=git clone --recurse-submodules https://github.com/project-chip/connectedhomeip.git; + - RUN_COVERAGE_CMD=connectedhomeip/scripts/build_coverage.sh; + - CLEANUP_CMD=rm -rf connectedhomeip; + - PROJECT_ID=$PROJECT_ID + +logsBucket: matter-build-automation-coverage-logs + +# Global timeout for all steps +timeout: 21600s +queueTtl: 21600s + +artifacts: + objects: + location: "gs://matter-build-automation-coverage-logs/$PROJECT_ID/$COMMIT_SHA/" + paths: ["/workspace/coverage_html.tar.gz"] \ No newline at end of file diff --git a/integrations/docker/images/build-all.sh b/integrations/docker/images/build-all.sh index 4aff2747ed571a..e2b942b0887a9f 100755 --- a/integrations/docker/images/build-all.sh +++ b/integrations/docker/images/build-all.sh @@ -21,7 +21,7 @@ # https://github.com/project-chip/connectedhomeip/issues/710 # set -e -find "$(git rev-parse --show-toplevel)"/integrations/docker/images/ -name Dockerfile ! -path "*chip-cert-bins/*" | while read -r dockerfile; do +find "$(git rev-parse --show-toplevel)"/integrations/docker/images/ -name Dockerfile ! -path "*chip-cert-bins/*" ! -path "*chip-build-remote-builder/*" | while read -r dockerfile; do pushd "$(dirname "$dockerfile")" >/dev/null ./build.sh "$@" popd >/dev/null diff --git a/integrations/docker/images/chip-build-remote-builder/Dockerfile b/integrations/docker/images/chip-build-remote-builder/Dockerfile new file mode 100644 index 00000000000000..ebb6072113a3e6 --- /dev/null +++ b/integrations/docker/images/chip-build-remote-builder/Dockerfile @@ -0,0 +1,4 @@ +FROM google/cloud-sdk + +COPY run.sh /bin +CMD ["bash", "-xe", "/bin/run.sh"] diff --git a/integrations/docker/images/chip-build-remote-builder/README.md b/integrations/docker/images/chip-build-remote-builder/README.md new file mode 100644 index 00000000000000..6c6e72ccd8221b --- /dev/null +++ b/integrations/docker/images/chip-build-remote-builder/README.md @@ -0,0 +1,24 @@ +# Docker `gcr.io:\$PROJECT_ID:remote-builder` + +`gcr.io:\$PROJECT_ID:remote-builder` is the name of the Docker image used by +Matter for continuous integration and coverage statistics using Google Cloud +Platform. + +Contents of this directory: + +- build.sh - utility for building (and optionally) tagging and pushing the + remote-builder Docker image +- cloudbuild.yaml - build config file contains instructions for Cloud Build to + build, package, and push the remote-builder Docker image. +- Dockerfile - description of the image + +In order to use remote-builder, you need to first build the builder: + +`./build.sh` + +When using the remote-builder image, the following will happen: + +1. A temporary SSH key will be created in your Container Builder workspace +2. SSH into a virtual machine instance with your configured flags +3. Your command will be run inside that compute engine instance's workspace +4. The workspace will be copied back to your Container Builder workspace diff --git a/integrations/docker/images/chip-build-remote-builder/build.sh b/integrations/docker/images/chip-build-remote-builder/build.sh new file mode 100755 index 00000000000000..7e33a953862ce8 --- /dev/null +++ b/integrations/docker/images/chip-build-remote-builder/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# build.sh - utility for building (and optionally) tagging and pushing +# the a Docker image +# + +gcloud builds submit --config=cloudbuild.yaml . diff --git a/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml b/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml new file mode 100644 index 00000000000000..23ca7f79627862 --- /dev/null +++ b/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml @@ -0,0 +1,6 @@ +steps: + - name: "gcr.io/cloud-builders/docker" + args: ["build", "-t", "gcr.io/$PROJECT_ID/remote-builder", "."] +images: + - "gcr.io/$PROJECT_ID/remote-builder" +tags: ["cloud-builders-community"] diff --git a/integrations/docker/images/chip-build-remote-builder/run.sh b/integrations/docker/images/chip-build-remote-builder/run.sh new file mode 100755 index 00000000000000..dd1a173d4e389e --- /dev/null +++ b/integrations/docker/images/chip-build-remote-builder/run.sh @@ -0,0 +1,70 @@ +#!/bin/bash -xe + +# +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# run.sh - utility for running a Docker image +# +# This script expects to live in a directory named after the image +# with a version file next to it. So: use symlinks +# +USERNAME=${USERNAME:-ubuntu} +PROJECT=$(gcloud info --format='value(config.project)') +INSTANCE_NAME=${INSTANCE_NAME:-matter-remote-builder} +ZONE=${ZONE:-us-central1-c} +GCLOUD=${GCLOUD:-gcloud} +RETRIES=${RETRIES:-3} + +# Run command on the instance via ssh +function ssh() { + "$GCLOUD" compute ssh --project="$PROJECT" --zone="$ZONE" "$USERNAME@$INSTANCE_NAME" -- "$1" +} + +# Always delete workspace after attempting build +function cleanup() { + ssh "$CLEANUP_CMD" +} + +"$GCLOUD" config set project "$PROJECT_ID" +"$GCLOUD" config set compute/zone "$ZONE" + +trap cleanup EXIT + +RETRY_COUNT=1 +while [ "$(ssh 'printf pass')" != "pass" ]; do + echo "[Try $RETRY_COUNT of $RETRIES] Waiting for instance to start accepting SSH connections..." + if [ "$RETRY_COUNT" == "$RETRIES" ]; then + echo "Retry limit reached, giving up!" + exit 1 + fi + sleep 10 + RETRY_COUNT=$(($RETRY_COUNT + 1)) +done + +# Cleanup workspace if there is leftover +ssh "$CLEANUP_CMD" + +# Setup workspace with connectedhomeip +ssh "$GIT_CLONE_CMD" + +# Run coverage tests +ssh "$RUN_COVERAGE_CMD" + +"$GCLOUD" compute scp --project="$PROJECT" --zone="$ZONE" \ + "$USERNAME@$INSTANCE_NAME:/home/ubuntu/connectedhomeip/out/coverage/coverage/coverage_html.tar.gz" "$PWD" + +# Always delete workspace after build +ssh "$CLEANUP_CMD"