feat: allow tracing build system with [debug ci] (#2389)

With this PR you are now able to put [debug ci] into a commit message
and have all BASH scripts set -x, which prints all commands ran.

Going forward, I encourage checking `if [ -n "${BUILD_SYSTEM_DEBUG:-}" ]
; then ... fi` with verbose debug dumps of data.

Changes:
- setup_env exports this flag
- all other bash scripts in build-system or in e.g. scripts/run_tests
module locations use this flag

barretenberg/cpp/scripts/run_tests

@@ -3,7 +3,8 @@
 # 1. The number of ignition transcripts to download.
 # 2. The set of gtest binary names to run.
 # 3-n. The arguments to pass to the gtest binaries.
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 NUM_TRANSCRIPTS=$1
 TESTS=$2

build-system/scripts/build

@@ -22,6 +22,7 @@
 # - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers.
 # - Push the image tagged with the commit hash to the cache.
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -euo pipefail
 
 REPOSITORY=$1

build-system/scripts/build_local

@@ -6,6 +6,7 @@
 # If DOCKERFILE is excluded it tries to default to ./Dockerfile then ./<PROJECT_DIR_NAME>/Dockerfile
 # If REPO is excluded it defaults to PROJECT_DIR_NAME.
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 TARGET_PROJECT=${1:-}

build-system/scripts/calculate_content_hash

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/calculate_image_uri

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/check_npm_version

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 readonly LOCAL_VERSION=$(node -pe "require('./package.json').version")

build-system/scripts/check_rebuild

@@ -1,5 +1,6 @@
 #!/bin/bash
 # If this script fails (nonzero exit), then the caller should rebuild.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -euo pipefail
 
 TAG=$1

build-system/scripts/clean_image_tags

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -e
 
 REPOSITORY=$1

build-system/scripts/cond_run_script

@@ -4,6 +4,7 @@
 # Arguments are:
 # 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success.
 # 2... ARGS: Script and arguments to run.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/cond_spot_run_build

@@ -1,7 +1,8 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1
 CPUS=$2
 
-cond_spot_run_script $REPOSITORY $CPUS build $REPOSITORY
+cond_spot_run_script $REPOSITORY $CPUS build $REPOSITORY

build-system/scripts/cond_spot_run_script

@@ -11,6 +11,7 @@
 #
 # Env vars are:
 # TAG_POSTFIX: Optional. If provided we check for the image tag with this postfix to determine if re-run is required.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/cond_spot_run_test

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -euo pipefail
 
 REPOSITORY=$1
@@ -12,4 +13,5 @@ SCRIPT=$(query_manifest relativeProjectDir $REPOSITORY)/$SCRIPT
 
 # Specify a TAG_POSTFIX as the JOB_NAME
 mkdir -p /tmp/test-logs
-TAG_POSTFIX=$JOB_NAME cond_spot_run_script $REPOSITORY $CPUS $SCRIPT $@ | tee "/tmp/test-logs/$JOB_NAME.log"
+export TAG_POSTFIX=$JOB_NAME 
+cond_spot_run_script $REPOSITORY $CPUS $SCRIPT $@ | tee "/tmp/test-logs/$JOB_NAME.log"

build-system/scripts/create_dockerhub_manifest

@@ -5,6 +5,7 @@
 # 3. Adds the arch specific tagged image to each list
 # 4. Pushes the 2 lists
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 if [ -z "$COMMIT_TAG" ]; then

build-system/scripts/create_ecr_manifest

@@ -5,6 +5,7 @@
 # 3. Creates a manifest list using a platform agnositc image uri, adds each image to it
 # 4. Pushes the manifest list 
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -e
 
 REPOSITORY=$1
@@ -35,4 +36,4 @@ done
 IFS=$OLD_IFS
 unset OLD_IFS
 
-docker manifest push --purge $FINAL
+docker manifest push --purge $FINAL

build-system/scripts/deploy

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/deploy_dockerhub

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 if [ -z "$COMMIT_TAG" ]; then

build-system/scripts/deploy_ecr

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/deploy_global

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Deployment script for global service (e.g. company website and metrics).
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/deploy_npm

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 readonly REPOSITORY=$1

build-system/scripts/deploy_s3

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/deploy_service

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 # Redeploy service with latest image.

build-system/scripts/deploy_terraform

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/ensure_apt_package

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 if dpkg -s $1 &> /dev/null; then

build-system/scripts/ensure_repo

@@ -1,6 +1,7 @@
 #!/bin/bash
 # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it
 # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 LIFECYCLE_POLICY='{

build-system/scripts/ensure_terraform

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Downloads and installs `terraform` if it's not installed.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 [ ! -f /usr/local/bin/terraform ] || exit 0

build-system/scripts/erase_image_tags

@@ -1,6 +1,7 @@
 #!/bin/bash
 # Erase the image tag associated with the last commit for the given repository.
 # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/extract_repo

@@ -1,5 +1,6 @@
 #!/bin/bash
 # Given a repository, extracts the builds entire /usr/src dir to the given path.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/extract_tag_version

@@ -3,6 +3,7 @@
 # then checks if the commit tag variable (if any)
 # is a valid semver & echoes that valid semver.
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY="$1"

build-system/scripts/force_deploy_build

@@ -4,6 +4,7 @@
 #
 # usage: ./deploy_force_build <repository> <force-build>
 # example: ./deploy_force_build aztec-sandbox true
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -e
 
 REPOSITORY=$1

build-system/scripts/init_submodules

@@ -1,5 +1,6 @@
 #!/bin/bash
 # For a given repository, init any required submodules.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -euo pipefail
 
 REPOSITORY=$1

build-system/scripts/query_manifest

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 CMD=$1
 REPO=$2

build-system/scripts/remote_run_script

@@ -6,6 +6,7 @@
 # 2... ARGS: Script and arguments to run.
 #
 # e.g. remote_run_script 1.2.3.4 build my_repo
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 IP=$1

build-system/scripts/remote_runner

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts

build-system/scripts/request_spot

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 NAME=$1

build-system/scripts/setup_env

@@ -13,6 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-}
 JOB_NAME=$3
 GIT_REPOSITORY_URL=${4:-}
 BRANCH=${5:-}
+COMMIT_MESSAGE=${6:-}
 
 BASH_ENV=${BASH_ENV:-}
 BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
@@ -99,6 +100,12 @@ if [ -n "$COMMIT_HASH" ]; then
   mkdir -p ~/.ssh
   echo ${BUILD_INSTANCE_KEY:-} | base64 -d > ~/.ssh/build_instance_key
   chmod 600 ~/.ssh/build_instance_key
+  COMMIT_MESSAGE=`git log -n 1 --pretty=format:"%s" HEAD`
+  # if our commit messages has [debug ci] anywhere in it, we set -x in all build system BASH files
+  # this can also be used for other verbose logging
+  if [[ "$COMMIT_MESSAGE" == *"[debug ci]"* ]]; then
+    echo export BUILD_SYSTEM_DEBUG=1 >> $BASH_ENV
+  fi
 fi
 
 set +e

build-system/scripts/spot_run_script

@@ -6,6 +6,7 @@
 #
 # Env vars:
 # JOB_NAME: Set within setup-env. The job name as per CI.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 CONTENT_HASH=$1
 CPUS=$2

build-system/scripts/spot_run_test_script

@@ -9,6 +9,7 @@ cd $(query_manifest projectDir $REPOSITORY)
 
 mkdir -p /tmp/test-logs
 
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -o pipefail
 
 CONTENT_HASH=$(calculate_content_hash $REPOSITORY)

build-system/scripts/store_test_benchmark_logs

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

build-system/scripts/tag_remote_image

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 REPOSITORY=$1

circuits/cpp/scripts/build_run_tests_docker_local

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 # To be called only LOCALLY for testing WITH docker.

circuits/cpp/scripts/run_coverage

@@ -1,4 +1,5 @@
 #!bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
 # To be called LOCALLY for testing WITHOUT docker.

circuits/cpp/scripts/run_tests

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 # To be called from CI for testing with docker and AWS.
 # Can't be called locally unless AWS credentials are set up.

circuits/cpp/scripts/run_tests_local

@@ -1,5 +1,6 @@
 #!/bin/bash
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 DIR="$(dirname "$0")"
 

scripts/ci/store_test_benchmark_logs

@@ -1,4 +1,5 @@
 #!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -e
 
 AZTEC_GITHUB_TOKEN=$1

yarn-project/canary/scripts/cond_run_script

@@ -14,8 +14,8 @@
 # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a
 #    successful run. The script will only run if there were relevant code changes since the last successful commit.
 # 3... ARGS: Script to run and args.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
-set -x
 
 REPOSITORY=$1
 shift

yarn-project/canary/scripts/run_tests

@@ -1,7 +1,8 @@
 #!/bin/bash
 # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests).
 # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose.
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 export TEST=$1
 export IMAGE=${2:-canary}

yarn-project/end-to-end/scripts/run_tests

@@ -1,7 +1,8 @@
 #!/bin/bash
 # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests).
 # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose.
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 export TEST=$1
 export COMPOSE_FILE=${2:-docker-compose.yml}

yarn-project/end-to-end/scripts/run_tests_local

@@ -1,7 +1,8 @@
 #!/bin/bash
 # This script is used to run an e2e test in CI (see config.yml and cond_run_script).
 # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose.
-set -exu
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
 
 export TEST=$1
 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml}