From 27bd884f9be8fd2c1ee5ca4872edc934c292cd74 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 21:29:06 -0400 Subject: [PATCH 01/65] ci: use content hash in build system --- build-system/scripts/build | 10 +++---- build-system/scripts/calculate_content_hash | 21 ++++++++++++++ build-system/scripts/check_rebuild | 32 ++++++--------------- build-system/scripts/cond_spot_run_build | 16 ++++------- build-system/scripts/cond_spot_run_script | 13 ++++----- build-system/scripts/last_successful_commit | 18 ------------ build_manifest.json | 2 +- 7 files changed, 46 insertions(+), 66 deletions(-) create mode 100644 build-system/scripts/calculate_content_hash delete mode 100755 build-system/scripts/last_successful_commit diff --git a/build-system/scripts/build b/build-system/scripts/build index d830434eb30..81aae10ca1f 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -49,19 +49,19 @@ function fetch_image() { # Ensure ECR repository exists. ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" cd $BUILD_DIR # If we have previously successful commit, we can early out if nothing relevant has changed since. -if [[ $FORCE_BUILD == 'false' ]] && check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +if [[ $FORCE_BUILD == 'false' ]] && check_rebuild "$CONTENT_HASH" $REPOSITORY; then echo "No rebuild necessary. Retagging..." STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true + tag_remote_image $REPOSITORY cache-$CONTENT_HASH-$STAGE cache-$COMMIT_HASH-$STAGE || true done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$COMMIT_HASH untag_remote_image $REPOSITORY tainted exit 0 fi diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash new file mode 100644 index 00000000000..7274e089645 --- /dev/null +++ b/build-system/scripts/calculate_content_hash @@ -0,0 +1,21 @@ +#!/bin/bash + +REPOSITORY=$1 + +# Compute REBUILD_PATTERNS from the build manifest, also printing it to the terminal (/dev/tty). +echo "Rebuild patterns:" +REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY | tee /dev/tty) + +git ls-files -s ${COMMIT_HASH} | while read -r line; do + # an example line is + # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json + # this format is beneficial as it grabs the hashes from git efficiently + # we will next filter by our rebuild patterns + # then we pipe the hash portion of each file to git hash-object to produce our content hash + for pattern in $REBUILD_PATTERNS; do + if [[ "$filepath" =~ $pattern ]]; then + echo "$line" + break + fi + done +done | git hash-object --stdin diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 5093e796b0b..f72515b5027 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -4,36 +4,20 @@ # The rebuild patterns are taken from the build manifest (computed from set of dependencies). set -euo pipefail -BASE_COMMIT=$1 +TAG=$1 REPOSITORY=$2 # If given nothing, then exit with failure to rebuild -[ -n "$BASE_COMMIT" ] || exit 1 +[ -n "$CONTENT_HASH" ] || exit 1 -# If a tainted tag exists, remove it exit with failure to rebuild. -if image_exists $REPOSITORY tainted; then - echo "$REPOSITORY has been tainted. Will rebuild." - exit 1 -fi - -# Compute .rebuild_patterns from the build manifest. -query_manifest rebuildPatterns $REPOSITORY > .rebuild_patterns - -echo "Rebuild patterns:" -cat .rebuild_patterns - -git config diff.renameLimit 999999 - -# Get list of files that differ -different_files=$(list_file_diff ${BASE_COMMIT} ${COMMIT_HASH}) || { - echo "list_file_diff failed. Rebuild required."; - exit 1; -} - -if grep -f .rebuild_patterns <<< "$different_files" &> /dev/null; then +if ! image_exists $REPOSITORY $TAG; then echo "Rebuild required." exit 1 +elif image_exists $REPOSITORY tainted; then + # If a tainted tag exists, remove it exit with failure to rebuild. + echo "$REPOSITORY has been tainted. Will rebuild." + exit 1 else echo "No rebuild required." exit 0 -fi \ No newline at end of file +fi diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 4e9b38ff751..e5793eef4d4 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -10,18 +10,12 @@ DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) init_submodules $REPOSITORY -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY) -echo "Last successful commit: $LAST_SUCCESSFUL_COMMIT" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +if ! check_rebuild "$CONTENT_HASH" $REPOSITORY; then spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ -else - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$LAST_SUCCESSFUL_COMMIT cache-$COMMIT_HASH -fi \ No newline at end of file + tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG +fi diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index fa12e39493e..8e8c7159910 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -11,7 +11,7 @@ # # Arguments are: # 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a +# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. set -e @@ -21,11 +21,10 @@ shift SUCCESS_TAG=$1 shift -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG) +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" -echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT" - -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +if ! check_rebuild "$CONTENT_HASH" $REPOSITORY; then spot_run_script $@ - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG -fi \ No newline at end of file + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG +fi diff --git a/build-system/scripts/last_successful_commit b/build-system/scripts/last_successful_commit deleted file mode 100755 index 03b0fe625ad..00000000000 --- a/build-system/scripts/last_successful_commit +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -REPOSITORY=$1 - -if [ -n "$2" ]; then - TAG_POSTFIX=-$2 -fi - -# We are assuming that there has been a successful build of the given repo -# within the last 50 commits. If not return nothing. -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - >&2 echo "Checking if image exists for commit $COMMIT_HASH..." - if image_exists $REPOSITORY cache-${COMMIT_HASH}$TAG_POSTFIX; then - >&2 echo "Found." - echo $COMMIT_HASH - exit 0 - fi -done \ No newline at end of file diff --git a/build_manifest.json b/build_manifest.json index 7ae4ef62ae6..052ddd0ce7e 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -517,4 +517,4 @@ "types" ] } -} \ No newline at end of file +} From 26b81e6f389b296bcf2226fb56e732e6eb711c77 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 21:37:52 -0400 Subject: [PATCH 02/65] ci: use content hash in build system --- build-system/scripts/calculate_content_hash | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-system/scripts/calculate_content_hash diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash old mode 100644 new mode 100755 From 2381e37eb31bfa8f97d20345ea98ac53ff6a0611 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 21:41:56 -0400 Subject: [PATCH 03/65] fix: no tty in ci --- build-system/scripts/calculate_content_hash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 7274e089645..bf76d222d31 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -2,9 +2,10 @@ REPOSITORY=$1 -# Compute REBUILD_PATTERNS from the build manifest, also printing it to the terminal (/dev/tty). +# Compute REBUILD_PATTERNS from the build manifest +REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) echo "Rebuild patterns:" -REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY | tee /dev/tty) +echo $REBUILD_PATTERNS git ls-files -s ${COMMIT_HASH} | while read -r line; do # an example line is From 14f06473f584080f34a1789981615e25408b6ace Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 22:03:58 -0400 Subject: [PATCH 04/65] fix: align to use content hash everywhere --- build-system/scripts/build | 31 ++++++++----------- build-system/scripts/changed | 8 ++--- build-system/scripts/cond_spot_run_build | 4 +-- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/deploy | 8 ++--- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 4 +-- build-system/scripts/deploy_global | 8 ++--- build-system/scripts/erase_image_tags | 21 +++++++------ .../end-to-end/scripts/cond_run_script | 6 ++-- 10 files changed, 45 insertions(+), 49 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 81aae10ca1f..dc05bcb6b16 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -55,13 +55,8 @@ echo "Content hash: $CONTENT_HASH" cd $BUILD_DIR # If we have previously successful commit, we can early out if nothing relevant has changed since. -if [[ $FORCE_BUILD == 'false' ]] && check_rebuild "$CONTENT_HASH" $REPOSITORY; then - echo "No rebuild necessary. Retagging..." - STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') - for STAGE in $STAGES; do - tag_remote_image $REPOSITORY cache-$CONTENT_HASH-$STAGE cache-$COMMIT_HASH-$STAGE || true - done - tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$COMMIT_HASH +if [[ $FORCE_BUILD == 'false' ]] && check_rebuild cache-"$CONTENT_HASH" $REPOSITORY; then + echo "No rebuild necessary." untag_remote_image $REPOSITORY tainted exit 0 fi @@ -94,13 +89,13 @@ PARENTS=$(cat $DOCKERFILE | sed -n -e "s/^FROM $ECR_DEPLOY_URL\/\([^[:space:]]\+ for PARENT in $PARENTS; do # Extract repository name (i.e. discard tag). PARENT_REPO=${PARENT%:*} - PARENT_COMMIT_HASH=$(last_successful_commit $PARENT_REPO) + PARENT_CONTENT_HASH=$(calculate_content_hash $PARENT_REPO) # There must be a parent image to continue. - if [ -z "$PARENT_COMMIT_HASH" ]; then + if [ -z "$PARENT_CONTENT_HASH" ]; then echo "No parent image found for $PARENT_REPO" exit 1 fi - PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_COMMIT_HASH + PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:cache-$PARENT_CONTENT_HASH echo "Pulling dependency $PARENT_REPO..." fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. @@ -113,18 +108,18 @@ CACHE_FROM="" STAGES=$(cat $DOCKERFILE | sed -n -e 's/^FROM .* AS \(.*\)/\1/p') for STAGE in $STAGES; do # Get the last build of this stage to leverage layer caching. - if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then + if [ -n "$CONTENT_HASH" ]; then echo "Pulling stage: $STAGE" - STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT-$STAGE + STAGE_IMAGE_LAST_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE if fetch_image $STAGE_IMAGE_LAST_URI; then STAGE_CACHE_FROM="--cache-from $STAGE_IMAGE_LAST_URI" fi fi echo "Building stage: $STAGE" - STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH-$STAGE + STAGE_IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH-$STAGE # Build our dockerfile, add timing information - docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \ + docker build --target $STAGE $STAGE_CACHE_FROM -t $STAGE_IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \ | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done # We don't want to have redo this stages work when building the final image. Use it as a layer cache. @@ -136,8 +131,8 @@ for STAGE in $STAGES; do done # Pull previous image to use it as a layer cache if it exists. -if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then - LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$LAST_SUCCESSFUL_COMMIT +if [ -n "$CONTENT_HASH" ]; then + LAST_SUCCESSFUL_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH echo "Pulling previous build of $REPOSITORY..." fetch_image $LAST_SUCCESSFUL_URI || true CACHE_FROM="--cache-from $LAST_SUCCESSFUL_URI $CACHE_FROM" @@ -145,10 +140,10 @@ if [ -n "$LAST_SUCCESSFUL_COMMIT" ]; then fi # Build the actual image and give it a commit tag. -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH +IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH echo "Building image: $IMAGE_COMMIT_URI" # Build our dockerfile, add timing information -docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_COMMIT_HASH=$COMMIT_HASH . \ +docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE $CACHE_FROM --build-arg COMMIT_TAG=$COMMIT_TAG --build-arg ARG_CONTENT_HASH=$CONTENT_HASH . \ | while read line ; do echo "$(date "+%H:%M:%S")| $line"; done echo "Pushing image: $IMAGE_COMMIT_URI" docker push $IMAGE_COMMIT_URI > /dev/null 2>&1 diff --git a/build-system/scripts/changed b/build-system/scripts/changed index fb2358ff119..19f477fa505 100755 --- a/build-system/scripts/changed +++ b/build-system/scripts/changed @@ -1,11 +1,11 @@ #!/bin/bash -# Returns true if the file at path has changed since the LAST_SUCCESSFUL_COMMIT. -LAST_SUCCESSFUL_COMMIT=$1 +# Returns true if the file at path has changed since the CONTENT_HASH. +CONTENT_HASH=$1 [ -n "$COMMIT_HASH" ] || exit 1 -if git diff --name-only $LAST_SUCCESSFUL_COMMIT $COMMIT_HASH | grep -q $2; then +if git diff --name-only $CONTENT_HASH $COMMIT_HASH | grep -q $2; then exit 0 else exit 1 -fi \ No newline at end of file +fi diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index e5793eef4d4..7db25ca0dc8 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -15,7 +15,7 @@ echo "Content hash: $CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) -if ! check_rebuild "$CONTENT_HASH" $REPOSITORY; then +if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 8e8c7159910..879c864b0f8 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -24,7 +24,7 @@ shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" -if ! check_rebuild "$CONTENT_HASH" $REPOSITORY; then +if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then spot_run_script $@ tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index d8974f3a3e9..99c420b9a27 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -11,9 +11,9 @@ cd $(query_manifest projectDir $REPOSITORY) deploy_ecr $REPOSITORY # Bail out if nothing changed. -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $DEPLOY_TAG-deployed) -echo "Last successfully deployed commit: $LAST_SUCCESSFUL_COMMIT" -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Last successfully deployed commit: $CONTENT_HASH" +if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then echo "No changes detected, skipping deployment." exit 0 fi @@ -26,4 +26,4 @@ for SERVICE in $SERVICES; do done # Tag the image as deployed. -tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$DEPLOY_TAG-deployed \ No newline at end of file +tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 1f1db34287f..128593b272b 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -56,4 +56,4 @@ docker tag $IMAGE_COMMIT_URI $IMAGE_LATEST_URI # Push tagged image to dockerhub. docker push $IMAGE_DEPLOY_URI # Push :latest image to dockerhub -docker push $IMAGE_LATEST_URI \ No newline at end of file +docker push $IMAGE_LATEST_URI diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index e149ec09ae7..0368b0b4c80 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -2,7 +2,7 @@ set -e REPOSITORY=$1 -IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$COMMIT_HASH +IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH # Login to build region and pull the build. ensure_repo $REPOSITORY $ECR_REGION @@ -22,4 +22,4 @@ if [ -n "$COMMIT_TAG" ]; then tag_remote_image $REPOSITORY $COMMIT_HASH $PROJECT-$COMMIT_TAG $ECR_DEPLOY_REGION fi -tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION \ No newline at end of file +tag_remote_image $REPOSITORY $COMMIT_HASH $DEPLOY_TAG $ECR_DEPLOY_REGION diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index 31cdef01613..dbe3054cc3e 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -10,9 +10,9 @@ cd $(query_manifest projectDir $REPOSITORY) deploy_ecr $REPOSITORY # Bail out if nothing changed. -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $DEPLOY_TAG-deployed) -echo "Last successfully deployed commit: $LAST_SUCCESSFUL_COMMIT" -if check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Last successfully deployed commit: $CONTENT_HASH" +if check_rebuild cache-$CONTENT_HASH-$DEPLOY_TAG-deployed $REPOSITORY; then echo "No changes detected, skipping deployment." exit 0 fi @@ -22,4 +22,4 @@ deploy_terraform "" ./terraform deploy_service $REPOSITORY # Tag the image as deployed. -tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$DEPLOY_TAG-deployed \ No newline at end of file +tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$DEPLOY_TAG-deployed diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 930f4d4d789..7fc7894db51 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -10,13 +10,14 @@ if [ -n "$3" ]; then TAG_POSTFIX=-$3 fi -for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do - TAG=cache-${COMMIT_HASH}$TAG_POSTFIX - if image_exists $REPOSITORY $TAG; then - echo "Erasing image tag: $REPOSITORY:$TAG" - aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null - if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then - break - fi - fi -done \ No newline at end of file +# TODO adapt to call calculate_content_hash and +# for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do +# TAG=cache-${COMMIT_HASH}$TAG_POSTFIX +# if image_exists $REPOSITORY $TAG; then +# echo "Erasing image tag: $REPOSITORY:$TAG" +# aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null +# if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then +# break +# fi +# fi +# done diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 96adcc9e8e2..c1eae55ff32 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -23,12 +23,12 @@ shift SCRIPT_TO_RUN=$1 shift -LAST_SUCCESSFUL_COMMIT=$(last_successful_commit $REPOSITORY $SUCCESS_TAG) +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Last successful commit for $SUCCESS_TAG: $LAST_SUCCESSFUL_COMMIT" +echo "Last successful commit for $SUCCESS_TAG: $CONTENT_HASH" echo "Script to run is $SCRIPT_TO_RUN $@" -if ! check_rebuild "$LAST_SUCCESSFUL_COMMIT" $REPOSITORY; then +if ! check_rebuild cache-$COMMIT_HASH-$SUCCESS_TAG $REPOSITORY; then "$SCRIPT_TO_RUN" "$@" tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG fi From e10704e30d54f6a3bd5b2bd0baeb224b1184645f Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 22:10:05 -0400 Subject: [PATCH 05/65] fix: align to use content hash everywhere --- build-system/scripts/build | 1 + build-system/scripts/cond_spot_run_build | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index dc05bcb6b16..79847298779 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -1,4 +1,5 @@ #!/bin/bash +set -x # # Builds a docker image and pushes it to it's repository. Leverages caches where possible. # Cached images include previous successfully built images (including multi-stages) built on this branch. diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 7db25ca0dc8..168616ac10a 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -ex set -o pipefail REPOSITORY=$1 From daa820112ffcd254fa9a0063dc358e511abcc268 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 22:33:45 -0400 Subject: [PATCH 06/65] fix: don't print extra info in content hash --- build-system/scripts/calculate_content_hash | 2 -- build-system/scripts/tag_remote_image | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index bf76d222d31..34387e24b33 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -4,8 +4,6 @@ REPOSITORY=$1 # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) -echo "Rebuild patterns:" -echo $REBUILD_PATTERNS git ls-files -s ${COMMIT_HASH} | while read -r line; do # an example line is diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index ee42f32cb87..2e10084df42 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -32,4 +32,4 @@ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ --image-manifest "$EXISTING_TAG_MANIFEST" > /dev/null 2>&1 -fi \ No newline at end of file +fi From 8e320e64acf5196589babd29f74718bbb89ff824 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 22:37:40 -0400 Subject: [PATCH 07/65] Remove debug flag --- build-system/scripts/build | 1 - build-system/scripts/cond_spot_run_build | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 79847298779..dc05bcb6b16 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -1,5 +1,4 @@ #!/bin/bash -set -x # # Builds a docker image and pushes it to it's repository. Leverages caches where possible. # Cached images include previous successfully built images (including multi-stages) built on this branch. diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 168616ac10a..7db25ca0dc8 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,5 +1,5 @@ #!/bin/bash -set -ex +set -e set -o pipefail REPOSITORY=$1 From 987206d01356ca27029e198cae63cc3ec04badcd Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 22:51:19 -0400 Subject: [PATCH 08/65] fix build script --- build-system/scripts/check_rebuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index f72515b5027..d80e9493684 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -8,7 +8,7 @@ TAG=$1 REPOSITORY=$2 # If given nothing, then exit with failure to rebuild -[ -n "$CONTENT_HASH" ] || exit 1 +[ -n "$TAG" ] || exit 1 if ! image_exists $REPOSITORY $TAG; then echo "Rebuild required." From 5e22d278b46e10a36b70be848548bcbfcc05aacd Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 23:06:00 -0400 Subject: [PATCH 09/65] fix build script --- .circleci/config.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd239494034..2ad9b3eaa66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,21 +25,12 @@ parameters: default: "system" # This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout. -# Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits. -# We need historical commit hashes to calculate diffs between previous and current commits. +# Initially it just fetches the repo metadata for the current commit hash to a depth of 1 commit. # It then checks out the fetched head to actually download the data. checkout: &checkout run: name: "Checkout code" command: | - function retry_10() { - # Retries up to 10 times with 10 second intervals - for i in $(seq 1 10); do - "$@" && return || sleep 10 - done - echo "$@ failed after 10 attempts" - exit 1 - } cd $HOME mkdir -p .ssh chmod 0700 .ssh @@ -57,10 +48,15 @@ checkout: &checkout git remote add origin $CIRCLE_REPOSITORY_URL # Only download metadata when fetching. - retry_10 git fetch --depth 50 --filter=blob:none origin $CIRCLE_SHA1 + retry_10 git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD - # Initialize submodules recursively (retry 10 times on failure) - retry_10 git submodule update --init --recursive + +# checkout_submodules: &checkout_submodules +# run: +# name: "Checkout code" +# command: | +# # Initialize submodules recursively (retry 10 times on failure) +# retry_10 git submodule update --init --recursive # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of @@ -1445,5 +1441,3 @@ workflows: requires: - build-deployment-canary <<: *deploy_defaults - - From 21f5f915f4b5d58a012a94eca1d7ef3d41467c24 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 23:43:37 -0400 Subject: [PATCH 10/65] Continue to use content hash throughout --- build-system/remote_build/remote_build | 4 ++-- build-system/scripts/calculate_content_hash | 1 + circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 4 +++- yarn-project/end-to-end/scripts/run_tests_local | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index 60f8a206847..c2cdd90f79f 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -14,7 +14,7 @@ git remote add origin $GIT_REPOSITORY_URL # Only download metadata when fetching. git config remote.origin.promisor true git config remote.origin.partialclonefilter blob:none -git fetch --depth 50 origin $COMMIT_HASH +git fetch --depth 1 origin $COMMIT_HASH git checkout FETCH_HEAD # Checkout barretenberg submodule only. git submodule update --init build-system @@ -25,4 +25,4 @@ BASH_ENV=/tmp/bash_env echo "Calling setup env..." source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" echo "Calling build..." -build $@ \ No newline at end of file +build $@ diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 34387e24b33..37f080e4c74 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,6 +1,7 @@ #!/bin/bash REPOSITORY=$1 +COMMIT_HASH=${2:-$COMMIT_HASH} # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index b7dd5c27b8b..d52ffd4c3c0 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -4,8 +4,10 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH +CONTENT_HASH=$(calculate_content_hash barretenberg-x86_64-linux-clang-assert) +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$CONTENT_HASH docker pull $IMAGE_URI diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 98e2b67ffba..64f0a9f0fcd 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -6,6 +6,7 @@ set -e export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} + if [ -n "$COMMIT_HASH" ]; then aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com @@ -16,4 +17,4 @@ if [ -n "$COMMIT_HASH" ]; then fi docker-compose -f $COMPOSE_FILE rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end From edd9dea90f8793c03efa19931c10acead9db66ef Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 2 Sep 2023 23:55:31 -0400 Subject: [PATCH 11/65] Bring back retry_10 for now --- .circleci/config.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ad9b3eaa66..d899c582ecc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,12 +25,21 @@ parameters: default: "system" # This build step checks out the code from the repository. It has a hardcoded readonly key to allow the checkout. -# Initially it just fetches the repo metadata for the current commit hash to a depth of 1 commit. +# Initially it just fetches the repo metadata for the current commit hash to a depth of 50 commits. +# We need historical commit hashes to calculate diffs between previous and current commits. # It then checks out the fetched head to actually download the data. checkout: &checkout run: name: "Checkout code" command: | + function retry_10() { + # Retries up to 10 times with 10 second intervals + for i in $(seq 1 10); do + "$@" && return || sleep 10 + done + echo "$@ failed after 10 attempts" + exit 1 + } cd $HOME mkdir -p .ssh chmod 0700 .ssh @@ -51,13 +60,6 @@ checkout: &checkout retry_10 git fetch --depth 1 --filter=blob:none origin $CIRCLE_SHA1 git checkout FETCH_HEAD -# checkout_submodules: &checkout_submodules -# run: -# name: "Checkout code" -# command: | -# # Initialize submodules recursively (retry 10 times on failure) -# retry_10 git submodule update --init --recursive - # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. # It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds. From d3f8ff86e84c642b55812644b771ce5128eafd40 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 00:19:53 -0400 Subject: [PATCH 12/65] fix: more content hash --- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 5 ++++- .../barretenberg/cpp/scripts/run_aztec_circuits_tests | 4 +++- circuits/cpp/barretenberg/cpp/scripts/run_tests | 4 +++- .../canary/scripts/docker-compose-e2e-sandbox.yml | 2 +- yarn-project/canary/scripts/run_tests | 10 +++++++--- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index b344ea40025..39255f86a0e 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -4,7 +4,10 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH +export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" +REPOSITORY=barretenberg-x86_64-linux-clang-assert +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH docker pull $IMAGE_URI diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index 0c3358300cd..c18a6c65513 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -19,7 +19,9 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-circuits-${ARCH}-linux-clang-builder-runner:cache-$COMMIT_HASH +REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 409379f4e01..486ae8a50b8 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -12,7 +12,9 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$COMMIT_HASH +REPOSITORY="barretenberg-x86_64-linux-clang-assert" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH docker pull $IMAGE_URI diff --git a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml index d677792416c..a37fd7880fc 100644 --- a/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml +++ b/yarn-project/canary/scripts/docker-compose-e2e-sandbox.yml @@ -13,7 +13,7 @@ services: - '8545:8545' sandbox: - image: $ECR_URL/aztec-sandbox:cache-$COMMIT_HASH + image: $ECR_URL/aztec-sandbox:cache-$CONTENT_HASH environment: DEBUG: 'aztec:*' ETHEREUM_HOST: http://fork:8545 diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 9a99a2018e6..a18409b903f 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -7,6 +7,10 @@ export TEST=$1 export IMAGE=${2:-canary} export COMPOSE_FILE=${3:-docker-compose.yml} +export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" +CONTENT_HASH=$(calculate_content_hash $IMAGE) +echo "Content hash: $CONTENT_HASH" + if [ "$TEST" = "uniswap_trade_on_l1_from_l2.test.ts" ]; then export FORK_URL=https://mainnet.infura.io/v3/9928b52099854248b3a096be07a6b23c export FORK_BLOCK_NUMBER=17514288 @@ -15,9 +19,9 @@ fi if [ -n "$COMMIT_HASH" ]; then $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$COMMIT_HASH aztecprotocol/canary:latest + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH aztecprotocol/canary:latest fi docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from canary \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from canary From 1543d964aa30d1c77dee5829852fc91b153af943 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 11:30:04 -0400 Subject: [PATCH 13/65] More content hash usage --- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 7 ++----- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 7 ++----- .../cpp/scripts/run_aztec_circuits_tests | 4 ++-- circuits/cpp/barretenberg/cpp/scripts/run_tests | 6 ++---- circuits/cpp/scripts/run_tests | 4 +++- yarn-project/canary/scripts/run_tests | 14 +++++--------- yarn-project/end-to-end/scripts/run_tests_local | 10 +++++++--- 7 files changed, 23 insertions(+), 29 deletions(-) diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index d52ffd4c3c0..5c8632142f0 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -4,11 +4,8 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -CONTENT_HASH=$(calculate_content_hash barretenberg-x86_64-linux-clang-assert) -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert:cache-$CONTENT_HASH - +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) docker pull $IMAGE_URI TESTS=( diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index 39255f86a0e..f4a1ecd2316 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -4,11 +4,8 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" -REPOSITORY=barretenberg-x86_64-linux-clang-assert -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH - +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index c18a6c65513..39b54ddb5e6 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -20,8 +20,8 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 486ae8a50b8..571f77c4204 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -12,10 +12,8 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -REPOSITORY="barretenberg-x86_64-linux-clang-assert" -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH - +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) docker pull $IMAGE_URI if [ -f "$TESTS" ]; then diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index 13f679a3ab6..e41a08c8e9c 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -25,7 +25,9 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/circuits-$ARCH-linux-clang-assert:cache-$COMMIT_HASH +REPOSITORY="circuits-$ARCH-linux-clang-assert" +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI # run tests in docker image diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index a18409b903f..aa9ddf4fd28 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -7,21 +7,17 @@ export TEST=$1 export IMAGE=${2:-canary} export COMPOSE_FILE=${3:-docker-compose.yml} -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" -CONTENT_HASH=$(calculate_content_hash $IMAGE) -echo "Content hash: $CONTENT_HASH" - if [ "$TEST" = "uniswap_trade_on_l1_from_l2.test.ts" ]; then export FORK_URL=https://mainnet.infura.io/v3/9928b52099854248b3a096be07a6b23c export FORK_BLOCK_NUMBER=17514288 fi -if [ -n "$COMMIT_HASH" ]; then - $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH aztecprotocol/canary:latest -fi +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) +docker pull $IMAGE_URI +docker tag $IMAGE_URI aztecprotocol/canary:latest docker-compose rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from canary diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 64f0a9f0fcd..9a6fcfcf750 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -5,14 +5,18 @@ set -e export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} - +REPOSITORY=barretenberg-x86_64-linux-clang-assert +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +# use the image rebuild patterns to compute a content hash, use this to get a URI +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) +docker pull $IMAGE_URI if [ -n "$COMMIT_HASH" ]; then aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com for REPO in end-to-end; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH aztecprotocol/$REPO:latest + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest done fi From 4338f9b9a6e0765f2935403fb25cb8352cb71a42 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 11:47:08 -0400 Subject: [PATCH 14/65] fix: more content hash --- build-system/scripts/remote_run_script | 8 ++++---- build-system/scripts/spot_run_script | 10 ++++++---- yarn-project/end-to-end/scripts/cond_run_script | 6 +++--- yarn-project/end-to-end/scripts/run_tests | 15 +++++++-------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index f393e2649da..7971e10f05f 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -2,9 +2,9 @@ set -e IP=$1 -FULL_PATH=$2 -shift -shift +CONTENT_HASH=$2 +FULL_PATH=$3 +shift 3 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) @@ -14,4 +14,4 @@ SCRIPT_NAME=$(basename $FULL_PATH) scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. # Run script on remote instance. -ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 1f3ee311fed..9e35ac53996 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -4,10 +4,12 @@ # 2... ARGS: Arguments to pass to remote_run_script. set -e +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) + # On any sort of exit (error or not), kill spot request so it doesn't count against quota. function on_exit { - if [ -f "sir-$COMMIT_HASH:$JOB_NAME.txt" ]; then - SIR=$(cat sir-$COMMIT_HASH:$JOB_NAME.txt) + if [ -f "sir-$CONTENT_HASH:$JOB_NAME.txt" ]; then + SIR=$(cat sir-$CONTENT_HASH:$JOB_NAME.txt) echo "Cancelling spot instance request $SIR (silently)" aws ec2 cancel-spot-instance-requests --spot-instance-request-ids $SIR >/dev/null 2>&1 || true fi @@ -18,11 +20,11 @@ SPEC=$1 shift # Get spot instance. -IP=$(request_spot $COMMIT_HASH:$JOB_NAME $SPEC) +IP=$(request_spot $CONTENT_HASH:$JOB_NAME $SPEC) # Run script remotely on spot instance, capturing success or failure. set +e -remote_run_script $IP $@ +remote_run_script $IP $CONTENT_HASH $@ CODE=$? # Shutdown spot. diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index c1eae55ff32..a833fa1efa8 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -25,10 +25,10 @@ shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Last successful commit for $SUCCESS_TAG: $CONTENT_HASH" +echo "Content: $CONTENT_HASH" echo "Script to run is $SCRIPT_TO_RUN $@" -if ! check_rebuild cache-$COMMIT_HASH-$SUCCESS_TAG $REPOSITORY; then +if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then "$SCRIPT_TO_RUN" "$@" - tag_remote_image $REPOSITORY cache-$COMMIT_HASH cache-$COMMIT_HASH-$SUCCESS_TAG + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index ad3f295edf4..da2f161501d 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -6,14 +6,13 @@ set -e export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} -if [ -n "$COMMIT_HASH" ]; then - $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +CONTENT_HASH=$(calculate_content_hash $REPO) +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - for REPO in aztec.js end-to-end aztec-sandbox; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$COMMIT_HASH aztecprotocol/$REPO:latest - done -fi +for REPO in aztec.js end-to-end aztec-sandbox; do + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest +done docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end \ No newline at end of file +docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end From f0a0ba1d62537a6fdcd49ed168d19747047be57a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 11:56:08 -0400 Subject: [PATCH 15/65] More updates --- build-system/scripts/changed | 11 ------- build-system/scripts/list_file_diff | 30 ------------------- .../end-to-end/scripts/run_tests_local | 15 ++++------ yarn-project/types/Dockerfile | 2 +- 4 files changed, 7 insertions(+), 51 deletions(-) delete mode 100755 build-system/scripts/changed delete mode 100755 build-system/scripts/list_file_diff diff --git a/build-system/scripts/changed b/build-system/scripts/changed deleted file mode 100755 index 19f477fa505..00000000000 --- a/build-system/scripts/changed +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Returns true if the file at path has changed since the CONTENT_HASH. -CONTENT_HASH=$1 - -[ -n "$COMMIT_HASH" ] || exit 1 - -if git diff --name-only $CONTENT_HASH $COMMIT_HASH | grep -q $2; then - exit 0 -else - exit 1 -fi diff --git a/build-system/scripts/list_file_diff b/build-system/scripts/list_file_diff deleted file mode 100755 index d0721fd7351..00000000000 --- a/build-system/scripts/list_file_diff +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Lists files that have changed between two commits, including files in submodules. -# Needed because `git diff --submodule=diff --name-only` doesn't work as one would hope. -set -euo pipefail - -BASE_COMMIT=$1 -COMMIT_HASH=${2:-HEAD} - -if [ -z "$BASE_COMMIT" ]; then - echo "Usage $0 from_commit [to_commit]" - exit 1 -fi - -cd $ROOT_PATH - -git --no-pager diff --name-only ${BASE_COMMIT} ${COMMIT_HASH} - -for SUBMODULE_PATH in $(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'); do - [ -f $SUBMODULE_PATH/.git ] || continue - if [ "$(git ls-tree $BASE_COMMIT $SUBMODULE_PATH | awk '{print $2}')" = "tree" ]; then - # In the base commit, we were not a submodule, and now we are. Output all files as having changed. - (cd $SUBMODULE_PATH && - git ls-files | awk "\$0=\"$SUBMODULE_PATH/\"\$0") - continue - fi - SM_PREV_HASH=$(git ls-tree $BASE_COMMIT $SUBMODULE_PATH | awk '{print $3}') - SM_CURR_HASH=$(git ls-tree $COMMIT_HASH $SUBMODULE_PATH | awk '{print $3}') - (cd $SUBMODULE_PATH && - git --no-pager diff --name-only ${SM_PREV_HASH} ${SM_CURR_HASH} | awk "\$0=\"$SUBMODULE_PATH/\"\$0") -done diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 9a6fcfcf750..fde67bc4f16 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -6,19 +6,16 @@ set -e export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} REPOSITORY=barretenberg-x86_64-linux-clang-assert -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) +IMAGE_URI=$(calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI -if [ -n "$COMMIT_HASH" ]; then - aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com +aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com - for REPO in end-to-end; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest - done -fi +for REPO in end-to-end; do + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest +done docker-compose -f $COMPOSE_FILE rm -f docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/types/Dockerfile b/yarn-project/types/Dockerfile index 6031d076e0c..3f97c9648b0 100644 --- a/yarn-project/types/Dockerfile +++ b/yarn-project/types/Dockerfile @@ -11,4 +11,4 @@ RUN yarn workspaces focus --production > /dev/null FROM node:18-alpine COPY --from=builder /usr/src/yarn-project/types /usr/src/yarn-project/types WORKDIR /usr/src/yarn-project/types -ENTRYPOINT ["yarn"] \ No newline at end of file +ENTRYPOINT ["yarn"] From 0568bf80795b1c6c217609dcd01e3cf53665e899 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:10:50 -0400 Subject: [PATCH 16/65] small refactor --- yarn-project/end-to-end/scripts/run_tests_local | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index fde67bc4f16..bc6a4ed2fca 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -13,8 +13,8 @@ docker pull $IMAGE_URI aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com for REPO in end-to-end; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest + docker pull $IMAGE_URI + docker tag $IMAGE_URI aztecprotocol/$REPO:latest done docker-compose -f $COMPOSE_FILE rm -f From 653c7e162b84b02c454762f9d0a5dad5c77fdcbb Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:20:25 -0400 Subject: [PATCH 17/65] fix run_tests --- build-system/scripts/remote_run_script | 5 +++-- circuits/cpp/scripts/run_tests | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 7971e10f05f..cf7741cb695 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -9,9 +9,10 @@ shift 3 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) SCRIPT_NAME=$(basename $FULL_PATH) +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. -# Run script on remote instance. -ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +# Run script on remote instance, passing environment variables. +ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH IMAGE_URI=\"$IMAGE_URI\" COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index e41a08c8e9c..c97e4b24a69 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -26,8 +26,6 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="circuits-$ARCH-linux-clang-assert" -# use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI # run tests in docker image From c7cfd9fc932dcb1a64d7e34c3912a4a14c70b860 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:22:28 -0400 Subject: [PATCH 18/65] fix run_tests --- circuits/cpp/scripts/run_tests | 1 + 1 file changed, 1 insertion(+) diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index c97e4b24a69..edb48cc7baf 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -26,6 +26,7 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="circuits-$ARCH-linux-clang-assert" +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI # run tests in docker image From b9ca48750b2db9aefc2474390f85044fb278ccb9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:24:22 -0400 Subject: [PATCH 19/65] fix run_tests --- build-system/scripts/build | 4 ++-- build-system/scripts/cond_spot_run_build | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index dc05bcb6b16..4add19d1f1a 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -35,8 +35,6 @@ echo "Working directory: $PWD" echo "Dockerfile: $DOCKERFILE" echo "Build directory: $BUILD_DIR" -init_submodules $REPOSITORY - function fetch_image() { echo "Pulling: $1" if ! docker pull $1 > /dev/null 2>&1; then @@ -61,6 +59,8 @@ if [[ $FORCE_BUILD == 'false' ]] && check_rebuild cache-"$CONTENT_HASH" $REPOSIT exit 0 fi +init_submodules $REPOSITORY + # Validate any terraform if it exists. if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then ensure_terraform diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 7db25ca0dc8..f003d145a15 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -8,14 +8,13 @@ SPEC=$1 shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) -init_submodules $REPOSITORY - CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then + init_submodules $REPOSITORY spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi From a7285bea657121dbbbf7706298c2423918c125fc Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:30:09 -0400 Subject: [PATCH 20/65] fix run_tests --- build-system/scripts/remote_run_script | 3 +-- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 3 ++- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 5 ++--- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 3 +-- circuits/cpp/barretenberg/cpp/scripts/run_tests | 3 ++- yarn-project/canary/scripts/run_tests | 3 +-- yarn-project/end-to-end/scripts/run_tests_local | 2 +- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index cf7741cb695..c1d99adff0f 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -9,10 +9,9 @@ shift 3 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) SCRIPT_NAME=$(basename $FULL_PATH) -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. # Run script on remote instance, passing environment variables. -ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH IMAGE_URI=\"$IMAGE_URI\" COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 5c8632142f0..3a036c69fa7 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -4,8 +4,9 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI TESTS=( diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index f4a1ecd2316..5d1fae52c77 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -3,9 +3,8 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -# use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) +REPOSITORY=barretenberg-x86_64-linux-clang-assert +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index 39b54ddb5e6..0846513a166 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -20,8 +20,7 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" -# use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 571f77c4204..9d9d1260abc 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -13,7 +13,8 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri barretenberg-x86_64-linux-clang-assert) +REPOSITORY=barretenberg-x86_64-linux-clang-assert +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI if [ -f "$TESTS" ]; then diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index aa9ddf4fd28..2310babe82f 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -14,8 +14,7 @@ fi $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -# use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH" docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index bc6a4ed2fca..2c6d8079ccc 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -7,7 +7,7 @@ export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI=$(calculate_image_uri $REPOSITORY) +IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" docker pull $IMAGE_URI aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com From 0b7145c0a8fccbaba13f1e5be2c8566932fb706a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:35:02 -0400 Subject: [PATCH 21/65] fix run_tests --- build-system/scripts/query_manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index f968f8fb57c..04750820623 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,6 +1,6 @@ #!/bin/bash set -e - +set -x CMD=$1 REPO=$2 From 0a85ac1665f49cefb571939cb1a8fd4147d25a2e Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:37:10 -0400 Subject: [PATCH 22/65] faster cond_spot_run_script --- build-system/scripts/cond_spot_run_script | 1 + build-system/scripts/cond_spot_run_test_script | 2 -- build-system/scripts/spot_run_test_script | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 879c864b0f8..e788388dd4d 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -25,6 +25,7 @@ CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then + init_submodules $REPOSITORY spot_run_script $@ tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index d4bccd922cd..792d314d8a5 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -5,8 +5,6 @@ REPOSITORY=$2 shift shift -init_submodules $REPOSITORY - cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 0fb7e510434..71dcc90ed85 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -5,8 +5,6 @@ REPOSITORY=$2 shift shift -init_submodules $REPOSITORY - cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs From eca925fd491682ccabe2290bec5a210e4a442095 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:44:19 -0400 Subject: [PATCH 23/65] fix: more content hash --- circuits/cpp/barretenberg/ts/scripts/run_tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circuits/cpp/barretenberg/ts/scripts/run_tests b/circuits/cpp/barretenberg/ts/scripts/run_tests index aad403925c4..cd417d0724a 100755 --- a/circuits/cpp/barretenberg/ts/scripts/run_tests +++ b/circuits/cpp/barretenberg/ts/scripts/run_tests @@ -2,5 +2,5 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$COMMIT_HASH -docker run --rm $IMAGE_URI \ No newline at end of file +IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$CONTENT_HASH +docker run --rm $IMAGE_URI From f026034ac15ec73bdb2cd4d82f3d25d0f71d5685 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:51:33 -0400 Subject: [PATCH 24/65] fix build manifest --- build_manifest.json | 1 - 1 file changed, 1 deletion(-) diff --git a/build_manifest.json b/build_manifest.json index 052ddd0ce7e..2c03c68a20c 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -274,7 +274,6 @@ ], "dependencies": [ "aztec.js", - "cli", "foundation", "l1-artifacts", "noir-contracts" From 82852319556d5a56020b7ee2924d422a562f9e6b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 13:54:39 -0400 Subject: [PATCH 25/65] fix build manifest --- build_manifest.json | 1 - 1 file changed, 1 deletion(-) diff --git a/build_manifest.json b/build_manifest.json index 2c03c68a20c..351983cf8c8 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -305,7 +305,6 @@ "aztec-sandbox", "aztec.js", "circuits.js", - "cli", "ethereum", "foundation", "l1-artifacts", From 976caf92cb8740772d13f58bdc0dc7a7db8cf086 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:03:57 -0400 Subject: [PATCH 26/65] debugging --- yarn-project/end-to-end/scripts/cond_run_script | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index a833fa1efa8..ecbb1a7c241 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -15,6 +15,7 @@ # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. set -e +set -x REPOSITORY=$1 shift From 4f5d9ea4500664309ce4852740ec44054279b2b7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:09:46 -0400 Subject: [PATCH 27/65] debugging --- yarn-project/end-to-end/scripts/cond_run_script | 4 +--- yarn-project/end-to-end/scripts/run_tests | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index ecbb1a7c241..29430cc365a 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -24,9 +24,7 @@ shift SCRIPT_TO_RUN=$1 shift -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) - -echo "Content: $CONTENT_HASH" +echo "Content hash: $CONTENT_HASH" echo "Script to run is $SCRIPT_TO_RUN $@" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index da2f161501d..540940b8d0d 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -6,7 +6,6 @@ set -e export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} -CONTENT_HASH=$(calculate_content_hash $REPO) $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null for REPO in aztec.js end-to-end aztec-sandbox; do From 98d18843070b53010b9fbbf5968962af2c76c3f5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:14:35 -0400 Subject: [PATCH 28/65] debugging --- build-system/remote_build/remote_build | 2 +- build-system/scripts/build | 2 +- build-system/scripts/build_local | 2 +- build-system/scripts/check_npm_version | 4 ++-- build-system/scripts/check_rebuild | 2 +- build-system/scripts/cond_spot_run_build | 4 ++-- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/cond_spot_run_test_script | 4 ++-- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 2 +- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 2 +- build-system/scripts/deploy_global | 2 +- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 4 ++-- build-system/scripts/deploy_service | 4 ++-- build-system/scripts/deploy_terraform | 4 ++-- build-system/scripts/ensure_apt_package | 4 ++-- build-system/scripts/ensure_repo | 4 ++-- build-system/scripts/ensure_terraform | 4 ++-- build-system/scripts/erase_image_tags | 2 +- build-system/scripts/extract_repo | 4 ++-- build-system/scripts/init_submodules | 2 +- build-system/scripts/query_manifest | 4 ++-- build-system/scripts/remote_run_script | 2 +- build-system/scripts/request_spot | 4 ++-- build-system/scripts/retry_10 | 2 +- build-system/scripts/setup_env | 2 +- build-system/scripts/spot_run_script | 2 +- build-system/scripts/spot_run_test_script | 4 ++-- build-system/scripts/spot_run_tests | 2 +- build-system/scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 2 +- 33 files changed, 46 insertions(+), 46 deletions(-) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index c2cdd90f79f..e51b9a55327 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/build-system/scripts/build b/build-system/scripts/build index 4add19d1f1a..24fd93587b2 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -22,7 +22,7 @@ # - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers. # - Push the image tagged with the commit hash to the cache. -set -euo pipefail +set -xeuo pipefail REPOSITORY=$1 FORCE_BUILD=${2:-"false"} diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 9df4d1d0e0a..0c96f4bd8b3 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -e +set -xe TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index ee61fd5bccf..0f5a07a6105 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} @@ -14,4 +14,4 @@ elif [ "$LOCAL_VERSION" != "$PUBLISHED_VERSION" ] && [ "$LOCAL_VERSION" == "$HIG else echo "Expect npm version number to be higher than '$PUBLISHED_VERSION'. Current local version is '$LOCAL_VERSION'." exit 1 -fi \ No newline at end of file +fi diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index d80e9493684..e58eabd6ff2 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -2,7 +2,7 @@ # Fails if any files matching the rebuild patterns, have changed since the base commit. # If this script fails (nonzero exit), then the caller should rebuild. # The rebuild patterns are taken from the build manifest (computed from set of dependencies). -set -euo pipefail +set -xeuo pipefail TAG=$1 REPOSITORY=$2 diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index f003d145a15..101e3922ae7 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,6 +1,6 @@ #!/bin/bash -set -e -set -o pipefail +set -xe +set -xo pipefail REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index e788388dd4d..d95b5d2e267 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -e +set -xe REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index 792d314d8a5..764976eebc7 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,5 +9,5 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -o pipefail +set -xo pipefail cond_spot_run_script $REPOSITORY $JOB_NAME 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index 07734f637a8..18583ef238c 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -e +set -xe cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index 99c420b9a27..bafc24b26e0 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 128593b272b..f2e2c5aafd0 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index 0368b0b4c80..e3674d803cd 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index dbe3054cc3e..3b3b09ff1d3 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -e +set -xe REPOSITORY=$1 diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 71185633370..9880e49b910 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index 9b000dfa58e..32b98746130 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) @@ -46,4 +46,4 @@ aws s3 cp \ if [ -n "$AWS_CLOUDFRONT_DISTRIBUTION" ]; then aws cloudfront create-invalidation --distribution-id $AWS_CLOUDFRONT_DISTRIBUTION --paths "/*" -fi \ No newline at end of file +fi diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 37d9e728884..3794a369d54 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,8 +1,8 @@ #!/bin/bash -set -e +set -xe # Redeploy service with latest image. SERVICE_NAME=$1 if aws ecs list-services --region $ECR_DEPLOY_REGION --cluster setup | grep "/$SERVICE_NAME\"" > /dev/null; then aws ecs update-service --region $ECR_DEPLOY_REGION --cluster setup --service $SERVICE_NAME --force-new-deployment -fi \ No newline at end of file +fi diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index 00e0ad22aeb..ed29165cdc2 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 TF_DIR=$2 @@ -38,4 +38,4 @@ for RESOURCE in $TO_TAINT; do terraform taint $RESOURCE || true done -terraform apply -input=false -auto-approve \ No newline at end of file +terraform apply -input=false -auto-approve diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index e546752c23a..91101021701 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,9 +1,9 @@ #!/bin/bash -set -e +set -xe if dpkg -s $1 &> /dev/null; then exit 0 fi >&2 echo "Installing $1..." -sudo apt update &> /dev/null && sudo apt install --force-yes $1 &> /dev/null \ No newline at end of file +sudo apt update &> /dev/null && sudo apt install --force-yes $1 &> /dev/null diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 31089f8f64c..8b1bf4a90bd 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -e +set -xe LIFECYCLE_POLICY='{ "rules": [ @@ -37,4 +37,4 @@ elif [ -n "$REFRESH_LIFECYCLE" ]; then echo "Refreshing lifecycle rules for repo: $REPOSITORY" aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 fi -echo \ No newline at end of file +echo diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index c1686f38f64..0de6ed96b20 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -e +set -xe [ ! -f /usr/local/bin/terraform ] || exit 0 @@ -10,4 +10,4 @@ curl -sSL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terrafor sudo apt install -y unzip unzip terraform.zip sudo mv terraform /usr/local/bin/ -rm terraform.zip \ No newline at end of file +rm terraform.zip diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 7fc7894db51..696351d8646 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -e +set -xe REPOSITORY=$1 TILL_COMMIT_HASH=$2 diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 64576acae4c..8946aed1a36 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -e +set -xe REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} @@ -17,4 +17,4 @@ docker cp $TEMP_CONTAINER:$EXTRACT_FROM $EXTRACT_TO docker rm -v $TEMP_CONTAINER > /dev/null echo "Extracted contents:" -ls -al $EXTRACT_TO \ No newline at end of file +ls -al $EXTRACT_TO diff --git a/build-system/scripts/init_submodules b/build-system/scripts/init_submodules index c456aa1a995..1efa2db83f3 100755 --- a/build-system/scripts/init_submodules +++ b/build-system/scripts/init_submodules @@ -1,6 +1,6 @@ #!/bin/bash # For a given repository, init any required submodules. -set -euo pipefail +set -xeuo pipefail REPOSITORY=$1 diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index 04750820623..b0876e644ae 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,6 +1,6 @@ #!/bin/bash -set -e -set -x +set -xe +set -xx CMD=$1 REPO=$2 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index c1d99adff0f..607c57d6e5a 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe IP=$1 CONTENT_HASH=$2 diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 42f2faaf90d..34ec26b1287 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe NAME=$1 CPUS=${2:-32} @@ -63,4 +63,4 @@ done >&2 echo "Waiting for SSH at $IP..." while ! nc -z $IP 22; do sleep 1; done; -echo $IP \ No newline at end of file +echo $IP diff --git a/build-system/scripts/retry_10 b/build-system/scripts/retry_10 index 2d111c6e3dc..e0e499ba624 100755 --- a/build-system/scripts/retry_10 +++ b/build-system/scripts/retry_10 @@ -1,4 +1,4 @@ -set -eu +set -xeu # Retries up to 10 times with 10 second intervals for i in $(seq 1 10); do "$@" && exit || sleep 10 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 0cb8e22f8d9..2558d345a72 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,7 +6,7 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -e +set -xe COMMIT_HASH=$1 COMMIT_TAG=$2 diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 9e35ac53996..2fff25475cb 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -2,7 +2,7 @@ # Runs a test script on a remote spot instance. Arguments are: # 1. SPEC: Instance specification filename. # 2... ARGS: Arguments to pass to remote_run_script. -set -e +set -xe CONTENT_HASH=$(calculate_content_hash $REPOSITORY) diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 71dcc90ed85..9bf31569d0a 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,5 +9,5 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -o pipefail +set -xo pipefail spot_run_script 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index df2f738473b..36f9494ae3f 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -e +set -xe spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index 424a9ff3188..08ea86826cf 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 2e10084df42..ffeae1693ac 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe REPOSITORY=$1 EXISTING_TAG=$2 From 23107f0d950b2f2d81bee23b653b6c2580b2f9e9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:19:14 -0400 Subject: [PATCH 29/65] debugging --- bootstrap.sh | 2 +- bootstrap_docker.sh | 4 ++-- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 4 ++-- circuits/cpp/barretenberg/bootstrap_docker.sh | 4 ++-- circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh | 4 ++-- circuits/cpp/barretenberg/cpp/bootstrap.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 4 ++-- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 4 ++-- circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh | 2 +- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 4 ++-- circuits/cpp/barretenberg/cpp/scripts/run_tests | 4 ++-- circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh | 4 ++-- circuits/cpp/barretenberg/scripts/bindgen.sh | 2 +- circuits/cpp/barretenberg/sol/scripts/install_foundry.sh | 4 ++-- circuits/cpp/bootstrap.sh | 2 +- circuits/cpp/format.sh | 2 +- circuits/cpp/scripts/build_run_tests_docker_local | 4 ++-- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 4 ++-- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- l1-contracts/bootstrap.sh | 4 ++-- l1-contracts/scripts/install_foundry.sh | 4 ++-- scripts/ci/store_test_benchmark_logs | 2 +- scripts/fix_subrepo_edge_case.sh | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- scripts/git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/ext/bashplus/test/setup | 2 +- .../git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/test-more-bash/ext/bashplus/lib/bash+.bash | 2 +- .../git-subrepo/ext/test-more-bash/ext/bashplus/test/setup | 2 +- .../ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/test/setup | 2 +- scripts/git-subrepo/lib/git-subrepo | 4 ++-- scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/init-test | 2 +- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 4 ++-- .../git-subrepo/note/subtree-rebase-fail-example/test.bash | 2 +- scripts/git-subrepo/note/test-subrepo-push.sh | 2 +- scripts/git-subrepo/pkg/bin/generate-help-functions.pl | 2 +- scripts/git_subrepo.sh | 2 +- scripts/migrate_barretenberg_branch.sh | 2 +- scripts/update.sh | 4 ++-- yarn-project/bootstrap.sh | 2 +- yarn-project/canary/scripts/run_tests | 2 +- yarn-project/canary/scripts/update_packages.sh | 4 ++-- yarn-project/end-to-end/scripts/cond_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests_local | 2 +- yarn-project/end-to-end/scripts/start_e2e.sh | 4 ++-- yarn-project/end-to-end/scripts/start_p2p_e2e.sh | 4 ++-- yarn-project/end-to-end/src/guides/up_quick_start.sh | 2 +- yarn-project/l1-artifacts/scripts/generate-artifacts.sh | 2 +- yarn-project/noir-contracts/scripts/install_noir.sh | 4 ++-- yarn-project/noir-contracts/scripts/install_noirup.sh | 4 ++-- yarn-project/noir-contracts/scripts/types.sh | 4 ++-- yarn-project/noir-contracts/src/scripts/compile.sh | 6 +++--- yarn-project/p2p-bootstrap/scripts/start_bootnode.sh | 4 ++-- 60 files changed, 86 insertions(+), 86 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 62b93deb023..72729dd117f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu export CLEAN=${1:-} diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index d4df940ff47..abdd58d453e 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -e +set -xe TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) @@ -39,4 +39,4 @@ if [ -z "$TARGET_PROJECT" ]; then echo echo "Success! You could now run e.g.:" echo " docker run -ti --rm aztecprotocol/end-to-end:latest e2e_private_token_contract.test" -fi \ No newline at end of file +fi diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 1deabdaf3e9..d9e920d87f5 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -e +set -xe BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs @@ -54,7 +54,7 @@ function test() { $BB prove_and_verify -c $CRS_PATH -b ./target/$dir_name.bytecode > /dev/null 2>&1 fi result=$? - set -e + set -xe if [ $result -eq 0 ]; then echo -e "\033[32mPASSED\033[0m" diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index 737f9d07563..85889e5cad9 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,8 +1,8 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -e +set -xe COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) -build_local \ No newline at end of file +build_local diff --git a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh index 756c130ee8c..ff4692b4758 100755 --- a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/sh # Script is assumed to be run from -set -eu +set -xeu bb() { ../build/bin/bb "$@" -v @@ -9,4 +9,4 @@ bb() { bb gates bb prove -o proof bb write_vk -o vk -bb verify -k vk -p proof \ No newline at end of file +bb verify -k vk -p proof diff --git a/circuits/cpp/barretenberg/cpp/bootstrap.sh b/circuits/cpp/barretenberg/cpp/bootstrap.sh index 869bbc9c885..ef389491f40 100755 --- a/circuits/cpp/barretenberg/cpp/bootstrap.sh +++ b/circuits/cpp/barretenberg/cpp/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Clean. rm -rf ./build diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index 20c99bf8993..2eb2bb23b11 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 3a036c69fa7..dca51c90fbb 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -e +set -xe $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert @@ -30,7 +30,7 @@ TESTS=( TESTS_STR="${TESTS[@]}" docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp; \ (cd srs_db && ./download_ignition.sh 1); \ cd build; \ diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index 5d1fae52c77..c25434289ee 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/bash # Executes the bb binary test script. -set -eu +set -xeu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert @@ -8,6 +8,6 @@ IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTE docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp/bin-test; \ ./bin-test.sh" diff --git a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh index 3a8eecf2438..3dd76aa9c46 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Clean. rm -rf ./src/wasi-sdk-* diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index 0846513a166..f5ca2db2902 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. @@ -37,7 +37,7 @@ echo "*** Running Aztec circuits tests on commit: $AZTEC_COMMIT" # run tests in docker RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/; \ git clone https://github.com/AztecProtocol/aztec3-packages.git; \ cd /usr/src/aztec3-packages/circuits/cpp; \ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 9d9d1260abc..4c340fd948b 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -e +set -xe NUM_TRANSCRIPTS=$1 TESTS=$2 @@ -22,7 +22,7 @@ if [ -f "$TESTS" ]; then fi docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/barretenberg/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ cd /usr/src/barretenberg/cpp/build; \ diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index 1632c035781..801b52d9652 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -e +set -xe mkdir -p ignition cd ignition @@ -45,4 +45,4 @@ for TRANSCRIPT in $(seq 0 $NUM); do else download $NUM fi -done \ No newline at end of file +done diff --git a/circuits/cpp/barretenberg/scripts/bindgen.sh b/circuits/cpp/barretenberg/scripts/bindgen.sh index 1a2034a213a..43740d746e1 100755 --- a/circuits/cpp/barretenberg/scripts/bindgen.sh +++ b/circuits/cpp/barretenberg/scripts/bindgen.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json diff --git a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh index 3842f6a4318..02bcbf8db3b 100755 --- a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh +++ b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -eu +set -xeu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" @@ -18,4 +18,4 @@ chmod +x $BIN_PATH export PATH=$FOUNDRY_BIN_DIR:$PATH # Use version. -foundryup \ No newline at end of file +foundryup diff --git a/circuits/cpp/bootstrap.sh b/circuits/cpp/bootstrap.sh index 0462e0085f6..b9f487e5678 100755 --- a/circuits/cpp/bootstrap.sh +++ b/circuits/cpp/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash # Takes a list of targets from commandline # Takes CLEAN as an environment variable. If passed, cleans build artifacts -set -eu +set -xeu export WASI_VERSION=20 diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index 20c99bf8993..2eb2bb23b11 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index 3bf05cf8363..711e77bd6ab 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. @@ -31,7 +31,7 @@ time docker build -f $DOCKERFILE -t $IMAGE_URI . # run tests in docker image RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run time docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/circuits/cpp/barretenberg/cpp/srs_db; \ ln -sf /usr/src/circuits/cpp/barretenberg/cpp/srs_db /usr/src/circuits/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index 174d11ac634..bca464157e8 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -e +set -xe # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index edb48cc7baf..4979a550fce 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. @@ -32,7 +32,7 @@ docker pull $IMAGE_URI # run tests in docker image RUN_ARGS="$@" # helper var necessary for some reason to pass all args to docker run time docker run --rm -t $IMAGE_URI /bin/sh -c "\ - set -e; \ + set -xe; \ cd /usr/src/circuits/cpp/barretenberg/cpp/srs_db; \ ln -sf /usr/src/circuits/cpp/barretenberg/cpp/srs_db /usr/src/circuits/cpp/srs_db; \ ./download_ignition.sh $NUM_TRANSCRIPTS; \ diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index e209cda65cf..3c73ab7db69 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 984ad38ec3b..6364a2f564c 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe # Run clang-tidy on all C++ source files # diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 3cb814fc4d5..cfe00c96988 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu cd "$(dirname "$0")" @@ -16,4 +16,4 @@ forge install --no-commit git submodule update --init --recursive ./lib # Compile contracts -forge build \ No newline at end of file +forge build diff --git a/l1-contracts/scripts/install_foundry.sh b/l1-contracts/scripts/install_foundry.sh index 3842f6a4318..02bcbf8db3b 100755 --- a/l1-contracts/scripts/install_foundry.sh +++ b/l1-contracts/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -eu +set -xeu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" @@ -18,4 +18,4 @@ chmod +x $BIN_PATH export PATH=$FOUNDRY_BIN_DIR:$PATH # Use version. -foundryup \ No newline at end of file +foundryup diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 057d64b4bc3..1008451bf31 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe AZTEC_GITHUB_TOKEN=$1 diff --git a/scripts/fix_subrepo_edge_case.sh b/scripts/fix_subrepo_edge_case.sh index 6888875eaff..7a236435868 100755 --- a/scripts/fix_subrepo_edge_case.sh +++ b/scripts/fix_subrepo_edge_case.sh @@ -6,7 +6,7 @@ # in the face of e.g. a .gitrepo whitespace change, but it's a fallback, # we only have this issue in master, and the file should only be edited # generally by subrepo commands. -set -eu +set -xeu SUBREPO_PATH="${1:-}" echo "Auto-fixing squashed parent in $SUBREPO_PATH/.gitrepo." diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index a691460242d..47dd76f0936 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -e +set -xe shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index dccdac08376..01c6b30e76f 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -e +set -xe [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/bashplus/test/setup b/scripts/git-subrepo/ext/bashplus/test/setup index c8b138d7e32..a3535659a5a 100644 --- a/scripts/git-subrepo/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -e -o pipefail +set -xe -o pipefail PATH=$PWD/bin:$PATH diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index a691460242d..47dd76f0936 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -e +set -xe shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index 2e1a70c2079..e6635ef9766 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -e +set -xe [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup index 35124d0f69d..176f989ef32 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -e -u -o pipefail +set -xe -u -o pipefail [[ $BASH_VERSION == 4.0* ]] && set +u run=0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash index cc34dcb9a2a..a4f0d192449 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash @@ -6,7 +6,7 @@ Test::Tap:die() { echo "$@" >&2; trap EXIT; exit 1; } #------------------------------------------------------------------------------ -set -e -u -o pipefail +set -xe -u -o pipefail [[ ${BASH_VERSION-} == 4.0* ]] && set +u # shellcheck disable=2034 diff --git a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash index 6fa9f56d93f..815dd498f95 100644 --- a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash +++ b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020. Ingy döt Net. -set -e -u -o pipefail +set -xe -u -o pipefail # shellcheck disable=2034 Test__More_VERSION=0.0.5 diff --git a/scripts/git-subrepo/ext/test-more-bash/test/setup b/scripts/git-subrepo/ext/test-more-bash/test/setup index 99df35d614f..1d910f3ceb1 100644 --- a/scripts/git-subrepo/ext/test-more-bash/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/test/setup @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e -u -o pipefail +set -xe -u -o pipefail BASHLIB=$(find "$PWD" -type d | grep -E '/(bin|lib)$' | xargs -n1 printf "%s:") PATH=$BASHLIB:$PATH diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index a6d5d96a1f6..15db6c7dcfa 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -e +set -xe export FILTER_BRANCH_SQUELCH_WARNING=1 @@ -1902,7 +1902,7 @@ RUN() { fi fi rc=$? - set -e + set -xe if [[ $rc -ne 0 ]]; then OK=false diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 123bb54cbb5..7f865d2ff1e 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -e +set -xe help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index a20854de88a..4d4ef58690b 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex cat $0 # Show this script in the output REPO=${1:-ingydotnet/djson-pm} diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 21bf8365b75..7fbf6fcbdb4 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -xe set -x # Make a directory to work in: @@ -43,7 +43,7 @@ set -x # commit: git rebase master subrepo-fake || true - # Command fails on conflict but `|| true` prevents `set -e` from stopping + # Command fails on conflict but `|| true` prevents `set -xe` from stopping # here. # Show the current branch status: diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index df8b818cee6..ac77fe7361a 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex rm -fr repo{1,2,3} mkdir repo{1,2} diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index afceb5efa92..40160368c9e 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex # Make a directory to work in: { diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index 40b6af8f2b3..1157f52c6f0 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -e +set -xe ... } diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index df9743a2bd5..0651c7c7da5 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu SCRIPT_DIR=$(dirname "$(realpath "$0")") diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh index 9f3001c7421..d78935bf4fb 100755 --- a/scripts/migrate_barretenberg_branch.sh +++ b/scripts/migrate_barretenberg_branch.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Usage: ./this.sh # Script for migrating PRs from barretenberg repo to aztec-packages. diff --git a/scripts/update.sh b/scripts/update.sh index 76446d014a4..b374d765a14 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Script for running after updating the working copy with remote changes. # Similar to bootstrap, but more lightweight and oriented towards working on end-to-end. @@ -14,4 +14,4 @@ echo -e '\n\033[1mRebuild Noir contracts...\033[0m' (cd yarn-project/noir-contracts && yarn noir:build:all 2> /dev/null) echo -e '\n\033[1mRebuild circuits wasm...\033[0m' -(cd circuits/cpp && cmake --build --preset wasm -j --target aztec3-circuits.wasm) \ No newline at end of file +(cd circuits/cpp && cmake --build --preset wasm -j --target aztec3-circuits.wasm) diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 39036119ed7..a2879ddef87 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Navigate to script folder cd "$(dirname "$0")" diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 2310babe82f..7b87da78fdc 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -xe export TEST=$1 export IMAGE=${2:-canary} diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index 7fa52399a23..cbd8e7e2f76 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -e +set -xe COMMIT_TAG=$1 @@ -20,4 +20,4 @@ for PKG in $(jq --raw-output ".dependencies | keys[] | select(contains(\"@aztec/ jq --arg v $VERSION ".dependencies[\"$PKG\"] = \$v" package.json > $TMP && mv $TMP package.json done -jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json \ No newline at end of file +jq ".references = []" tsconfig.json > $TMP && mv $TMP tsconfig.json diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 29430cc365a..b6f3a7d295b 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -e +set -xe set -x REPOSITORY=$1 diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index 540940b8d0d..4d6f2c8e5bc 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -xe export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 2c6d8079ccc..9187887d638 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -xe export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/start_e2e.sh b/yarn-project/end-to-end/scripts/start_e2e.sh index 617982ba7e8..45f6d13800a 100755 --- a/yarn-project/end-to-end/scripts/start_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -xeu export NODE_NO_WARNINGS=1 -node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ \ No newline at end of file +node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ diff --git a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh index 144fda44211..ed24c853826 100755 --- a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -xeu export DEBUG='aztec:*' export ARCHIVER_POLLING_INTERVAL=500 export P2P_CHECK_INTERVAL=50 @@ -14,4 +14,4 @@ export P2P_SERVER='false' export P2P_ENABLED='true' export DEBUG='aztec:*,libp2p:*' -yarn test e2e_p2p_network.test.ts \ No newline at end of file +yarn test e2e_p2p_network.test.ts diff --git a/yarn-project/end-to-end/src/guides/up_quick_start.sh b/yarn-project/end-to-end/src/guides/up_quick_start.sh index c27ff3454f3..85012b10027 100755 --- a/yarn-project/end-to-end/src/guides/up_quick_start.sh +++ b/yarn-project/end-to-end/src/guides/up_quick_start.sh @@ -1,7 +1,7 @@ # Run locally from end-to-end folder while running anvil and sandbox with: # PATH=$PATH:../node_modules/.bin ./src/guides/up_quick_start.sh -set -eux +set -xeux # docs:start:declare-accounts ALICE="0x2e13f0201905944184fc2c09d29fcf0cac07647be171656a275f63d99b819360" diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index 1c4bd5e2ba9..86b8bb95708 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euo pipefail; +set -xeuo pipefail; target_dir=./generated diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index 126cd93c1e8..8021c0685ca 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -1,8 +1,8 @@ #!/bin/bash # Script to install noirup and the latest aztec nargo -set -eu +set -xeu VERSION="aztec" # Install nargo -noirup -v $VERSION \ No newline at end of file +noirup -v $VERSION diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index 86d056bc185..efa83d5b668 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest nargo -set -eu +set -xeu SPECIFIED_HOME=${1:-$HOME} @@ -19,4 +19,4 @@ mkdir -p $NARGO_MAN_DIR curl -# -Ls $BIN_URL -o $BIN_PATH chmod +x $BIN_PATH -export PATH=$NARGO_BIN_DIR:$PATH \ No newline at end of file +export PATH=$NARGO_BIN_DIR:$PATH diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 65fb9ccffce..9f499a4e0d7 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -7,8 +7,8 @@ # - you can run `yarn noir:types:all` to create all noir artifacts and types consumed by aztec packages. # Enable strict mode: -# Exit on error (set -e), treat unset variables as an error (set -u), -set -eu; +# Exit on error (set -xe), treat unset variables as an error (set -u), +set -xeu; artifacts_dir="src/artifacts" types_dir="src/types" diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 99500bb4de9..384c1d0d537 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -7,10 +7,10 @@ # yarn noir:build private_token ecdsa_account # Enable strict mode: -# Exit on error (set -e), treat unset variables as an error (set -u), +# Exit on error (set -xe), treat unset variables as an error (set -u), # and propagate the exit status of the first failing command in a pipeline (set -o pipefail). -set -euo pipefail; +set -xeuo pipefail; # Run build scripts ./scripts/compile.sh "$@" -./scripts/types.sh "$@" \ No newline at end of file +./scripts/types.sh "$@" diff --git a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh index 613d1125971..7621979ab44 100755 --- a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh +++ b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh @@ -1,8 +1,8 @@ #! /bin/bash -set -eu +set -xeu cd .. export P2P_TCP_LISTEN_PORT=40400 export PEER_ID='0a260024080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1224080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1a44080112402df8b977f356c6e34fa021c9647973234dff4df706c185794405aafb556723cf5ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e' yarn build -yarn start \ No newline at end of file +yarn start From 0403e6ec15cc9db2498e4fb836d5dc042b066061 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:23:25 -0400 Subject: [PATCH 30/65] script hardening --- yarn-project/canary/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests_local | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index 7b87da78fdc..fc39a781c95 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xe +set -xeu export TEST=$1 export IMAGE=${2:-canary} diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index 4d6f2c8e5bc..ba874d060cc 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xe +set -xeu export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 9187887d638..01b9ac4c10b 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xe +set -xeu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} From 18eea4f7afb704edea75bddb1f0de679eab0c3f2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:34:41 -0400 Subject: [PATCH 31/65] Add content hash to cond_run_script --- yarn-project/end-to-end/scripts/cond_run_script | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index b6f3a7d295b..96f9463f8cb 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -xe +set -xeu set -x REPOSITORY=$1 @@ -24,10 +24,11 @@ shift SCRIPT_TO_RUN=$1 shift +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" echo "Script to run is $SCRIPT_TO_RUN $@" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - "$SCRIPT_TO_RUN" "$@" + CONTENT_HASH="$CONTENT_HASH" "$SCRIPT_TO_RUN" "$@" tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi From d797d3e8be6fc2284a7941124329126096e78e10 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:36:34 -0400 Subject: [PATCH 32/65] try 100% ubuntu --- .circleci/config.yml | 211 ++++++++++++++--------------- build-system/scripts/check_rebuild | 2 +- 2 files changed, 106 insertions(+), 107 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d899c582ecc..c043260f679 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,9 +77,9 @@ save_logs: &save_logs jobs: barretenberg-wasm-linux-clang: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -88,9 +88,9 @@ jobs: command: cond_spot_run_build barretenberg-wasm-linux-clang 64 barretenberg-x86_64-linux-gcc: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -99,9 +99,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-gcc 64 barretenberg-x86_64-linux-clang: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -110,9 +110,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang 64 barretenberg-x86_64-linux-clang-fuzzing: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -121,9 +121,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 64 barretenberg-x86_64-linux-clang-assert: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -132,9 +132,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 64 barretenberg-stdlib-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -144,9 +144,9 @@ jobs: - *save_logs barretenberg-dsl-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -156,9 +156,9 @@ jobs: - *save_logs barretenberg-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -168,9 +168,9 @@ jobs: - *save_logs barretenberg-honk-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -180,9 +180,9 @@ jobs: - *save_logs barretenberg-proof-system-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -192,9 +192,9 @@ jobs: - *save_logs barretenberg-stdlib-recursion-turbo-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -204,9 +204,9 @@ jobs: - *save_logs barretenberg-stdlib-recursion-ultra-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -216,9 +216,9 @@ jobs: - *save_logs barretenberg-join-split-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -228,9 +228,9 @@ jobs: - *save_logs barretenberg-benchmark-aggregator: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - attach_workspace: at: /tmp/test-logs @@ -252,9 +252,9 @@ jobs: command: build bb.js bb-js-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -263,9 +263,9 @@ jobs: command: cond_spot_run_tests bb.js barretenberg-acir-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -274,9 +274,9 @@ jobs: command: cond_spot_run_build barretenberg-acir-tests 32 circuits-wasm-linux-clang: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -285,9 +285,9 @@ jobs: command: cond_spot_run_build circuits-wasm-linux-clang 64 circuits-wasm-linux-clang-assert: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -296,9 +296,9 @@ jobs: command: cond_spot_run_build circuits-wasm-linux-clang-assert 64 circuits-x86_64-linux-clang-tidy: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -307,9 +307,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang-tidy 64 circuits-x86_64-linux-clang: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -318,9 +318,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang 64 circuits-x86_64-linux-clang-assert: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -329,9 +329,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang-assert 64 circuits-wasm-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -341,9 +341,9 @@ jobs: - *save_logs circuits-x86_64-tests: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -355,7 +355,7 @@ jobs: circuits-end: docker: - image: cimg/base:current - resource_class: small + steps: - run: name: "Noop" @@ -467,9 +467,8 @@ jobs: command: build key-store noir-contracts-build: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 steps: - *checkout - *setup_env @@ -714,9 +713,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-sandbox-example: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -725,9 +724,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_sandbox_example.test.ts docker-compose-e2e-sandbox.yml e2e-multi-transfer-contract: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -820,9 +819,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-escrow-contract: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -915,9 +914,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-p2p: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -926,9 +925,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_p2p_network.test.ts e2e-browser-sandbox: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -937,9 +936,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_aztec_js_browser.test.ts docker-compose-e2e-sandbox.yml aztec-rpc-sandbox: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -960,9 +959,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-canary-test: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -973,7 +972,7 @@ jobs: e2e-join: docker: - image: cimg/base:current - resource_class: small + steps: - run: name: "Noop" @@ -982,7 +981,7 @@ jobs: e2e-end: docker: - image: cimg/base:current - resource_class: small + steps: - run: name: "Noop" @@ -1129,7 +1128,7 @@ jobs: deploy-end: docker: - image: cimg/base:current - resource_class: small + steps: - run: name: "Noop" @@ -1147,9 +1146,9 @@ jobs: command: build canary true run-deployment-canary-uniswap: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -1158,9 +1157,9 @@ jobs: command: spot_run_test_script ./scripts/run_tests canary uniswap_trade_on_l1_from_l2.test.ts canary docker-compose.yml run-deployment-canary-browser: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env @@ -1169,9 +1168,9 @@ jobs: command: spot_run_test_script ./scripts/run_tests canary aztec_js_browser.test.ts canary docker-compose.yml run-deployment-canary-cli: - docker: - - image: aztecprotocol/alpine-build-image - resource_class: small + machine: + image: ubuntu-2004:202010-01 + steps: - *checkout - *setup_env diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index e58eabd6ff2..51eea5112a6 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -19,5 +19,5 @@ elif image_exists $REPOSITORY tainted; then exit 1 else echo "No rebuild required." - exit 0 + exit 1 # TODO TOGGLE fi From 88054d1f3a41e7989fe928a3b488c86849d9b9b5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:38:16 -0400 Subject: [PATCH 33/65] Hardening --- build-system/scripts/query_manifest | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index b0876e644ae..ad2b23b5668 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,6 +1,5 @@ #!/bin/bash -set -xe -set -xx +set -xeu CMD=$1 REPO=$2 From ef1db55e12bf41bedbdfe23fbd9e7e529b989b27 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:44:32 -0400 Subject: [PATCH 34/65] Hardening --- bootstrap_docker.sh | 2 +- build-system/remote_build/remote_build | 2 +- build-system/scripts/build_local | 2 +- build-system/scripts/check_npm_version | 2 +- build-system/scripts/cond_spot_run_build | 2 +- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/cond_spot_run_test_script | 2 +- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 2 +- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 2 +- build-system/scripts/deploy_global | 2 +- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 2 +- build-system/scripts/deploy_service | 2 +- build-system/scripts/deploy_terraform | 2 +- build-system/scripts/ensure_apt_package | 2 +- build-system/scripts/ensure_repo | 2 +- build-system/scripts/ensure_terraform | 2 +- build-system/scripts/erase_image_tags | 2 +- build-system/scripts/extract_repo | 2 +- build-system/scripts/remote_run_script | 2 +- build-system/scripts/request_spot | 2 +- build-system/scripts/setup_env | 2 +- build-system/scripts/spot_run_script | 2 +- build-system/scripts/spot_run_test_script | 2 +- build-system/scripts/spot_run_tests | 2 +- build-system/scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 2 +- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 4 ++-- circuits/cpp/barretenberg/bootstrap_docker.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 2 +- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 2 +- circuits/cpp/barretenberg/cpp/scripts/run_tests | 2 +- circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh | 2 +- circuits/cpp/format.sh | 2 +- circuits/cpp/scripts/build_run_tests_docker_local | 2 +- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 2 +- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- scripts/ci/store_test_benchmark_logs | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- scripts/git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/test-more-bash/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/lib/git-subrepo | 4 ++-- scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 2 +- scripts/git-subrepo/pkg/bin/generate-help-functions.pl | 2 +- yarn-project/canary/scripts/update_packages.sh | 2 +- 52 files changed, 54 insertions(+), 54 deletions(-) diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index abdd58d453e..3282f954d9c 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -xe +set -xeu TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index e51b9a55327..83e33bf356c 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 0c96f4bd8b3..2215c8c45a9 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -xe +set -xeu TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index 0f5a07a6105..91bd924b9e5 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 101e3922ae7..699d61108fa 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu set -xo pipefail REPOSITORY=$1 diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index d95b5d2e267..143cc9d4ab4 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -xe +set -xeu REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index 764976eebc7..575f839465a 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu SCRIPT_PATH=$1 REPOSITORY=$2 shift diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index 18583ef238c..c898c0700d2 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xe +set -xeu cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index bafc24b26e0..0eacf7847da 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index f2e2c5aafd0..09783f59ee1 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index e3674d803cd..7c8f8dbe96b 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index 3b3b09ff1d3..e54afb05133 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -xe +set -xeu REPOSITORY=$1 diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 9880e49b910..9df0e10d1a5 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index 32b98746130..cd076e59077 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 3794a369d54..34b5df87f1e 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu # Redeploy service with latest image. SERVICE_NAME=$1 diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index ed29165cdc2..1a813cc4591 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 TF_DIR=$2 diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index 91101021701..fe5d3d39f9f 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu if dpkg -s $1 &> /dev/null; then exit 0 diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 8b1bf4a90bd..d0f62dc3700 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -xe +set -xeu LIFECYCLE_POLICY='{ "rules": [ diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index 0de6ed96b20..d70f1d9cf84 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -xe +set -xeu [ ! -f /usr/local/bin/terraform ] || exit 0 diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 696351d8646..a2bd8acb257 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -xe +set -xeu REPOSITORY=$1 TILL_COMMIT_HASH=$2 diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 8946aed1a36..1756b361811 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -xe +set -xeu REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 607c57d6e5a..a4efda7cf6a 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu IP=$1 CONTENT_HASH=$2 diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 34ec26b1287..82a01f56937 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu NAME=$1 CPUS=${2:-32} diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 2558d345a72..3de75a1aaf5 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,7 +6,7 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -xe +set -xeu COMMIT_HASH=$1 COMMIT_TAG=$2 diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 2fff25475cb..4fda0e93134 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -2,7 +2,7 @@ # Runs a test script on a remote spot instance. Arguments are: # 1. SPEC: Instance specification filename. # 2... ARGS: Arguments to pass to remote_run_script. -set -xe +set -xeu CONTENT_HASH=$(calculate_content_hash $REPOSITORY) diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 9bf31569d0a..97c93946ac9 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu SCRIPT_PATH=$1 REPOSITORY=$2 shift diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index 36f9494ae3f..e9b14c1fd88 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xe +set -xeu spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index 08ea86826cf..9ac9ff71c04 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index ffeae1693ac..0bf67437019 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu REPOSITORY=$1 EXISTING_TAG=$2 diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index d9e920d87f5..92b68e6580c 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -xe +set -xeu BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs @@ -54,7 +54,7 @@ function test() { $BB prove_and_verify -c $CRS_PATH -b ./target/$dir_name.bytecode > /dev/null 2>&1 fi result=$? - set -xe + set -xeu if [ $result -eq 0 ]; then echo -e "\033[32mPASSED\033[0m" diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index 85889e5cad9..3db07de7bc7 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -xe +set -xeu COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index 2eb2bb23b11..f866bea9c15 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index dca51c90fbb..66117d7dbf7 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -xe +set -xeu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index f5ca2db2902..a40e8b3c97e 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 4c340fd948b..24c68e7467b 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -xe +set -xeu NUM_TRANSCRIPTS=$1 TESTS=$2 diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index 801b52d9652..d9e1bcd7fe5 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -xe +set -xeu mkdir -p ignition cd ignition diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index 2eb2bb23b11..f866bea9c15 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index 711e77bd6ab..fa64b29e330 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index bca464157e8..7979f7d1c6c 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -xe +set -xeu # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index 4979a550fce..eb8a2a28a6c 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index 3c73ab7db69..a67649e471d 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 6364a2f564c..13e46f5469c 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu # Run clang-tidy on all C++ source files # diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 1008451bf31..2b36f14a4ea 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xe +set -xeu AZTEC_GITHUB_TOKEN=$1 diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index 47dd76f0936..57d3e19655c 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xe +set -xeu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index 01c6b30e76f..16ea238245f 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xe +set -xeu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index 47dd76f0936..57d3e19655c 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xe +set -xeu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index e6635ef9766..28d8df1c78f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xe +set -xeu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index 15db6c7dcfa..78ed42e0bc3 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -xe +set -xeu export FILTER_BRANCH_SQUELCH_WARNING=1 @@ -1902,7 +1902,7 @@ RUN() { fi fi rc=$? - set -xe + set -xeu if [[ $rc -ne 0 ]]; then OK=false diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 7f865d2ff1e..1933b6eb280 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xe +set -xeu help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 7fbf6fcbdb4..5e9694a0999 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -xe +set -xeu set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index 1157f52c6f0..b23a011ef5f 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xe +set -xeu ... } diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index cbd8e7e2f76..608bcdb3035 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -xe +set -xeu COMMIT_TAG=$1 From 28dffcc6580da9426027d972071a3a988aaec2fe Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:48:12 -0400 Subject: [PATCH 35/65] Fixes after hardening --- build-system/scripts/cond_spot_run_build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 699d61108fa..ace078d5841 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -13,8 +13,8 @@ echo "Content hash: $CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) -if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then +if ! check_rebuild cache-$CONTENT_HASH $REPOSITORY; then init_submodules $REPOSITORY spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ - tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG + tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH fi From cc329e8d2e6f11017e35c65348ffead8038dcb42 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:51:51 -0400 Subject: [PATCH 36/65] revert --- .circleci/config.yml | 211 ++++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 105 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c043260f679..d899c582ecc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,9 +77,9 @@ save_logs: &save_logs jobs: barretenberg-wasm-linux-clang: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -88,9 +88,9 @@ jobs: command: cond_spot_run_build barretenberg-wasm-linux-clang 64 barretenberg-x86_64-linux-gcc: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -99,9 +99,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-gcc 64 barretenberg-x86_64-linux-clang: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -110,9 +110,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang 64 barretenberg-x86_64-linux-clang-fuzzing: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -121,9 +121,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 64 barretenberg-x86_64-linux-clang-assert: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -132,9 +132,9 @@ jobs: command: cond_spot_run_build barretenberg-x86_64-linux-clang-assert 64 barretenberg-stdlib-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -144,9 +144,9 @@ jobs: - *save_logs barretenberg-dsl-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -156,9 +156,9 @@ jobs: - *save_logs barretenberg-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -168,9 +168,9 @@ jobs: - *save_logs barretenberg-honk-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -180,9 +180,9 @@ jobs: - *save_logs barretenberg-proof-system-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -192,9 +192,9 @@ jobs: - *save_logs barretenberg-stdlib-recursion-turbo-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -204,9 +204,9 @@ jobs: - *save_logs barretenberg-stdlib-recursion-ultra-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -216,9 +216,9 @@ jobs: - *save_logs barretenberg-join-split-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -228,9 +228,9 @@ jobs: - *save_logs barretenberg-benchmark-aggregator: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - attach_workspace: at: /tmp/test-logs @@ -252,9 +252,9 @@ jobs: command: build bb.js bb-js-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -263,9 +263,9 @@ jobs: command: cond_spot_run_tests bb.js barretenberg-acir-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -274,9 +274,9 @@ jobs: command: cond_spot_run_build barretenberg-acir-tests 32 circuits-wasm-linux-clang: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -285,9 +285,9 @@ jobs: command: cond_spot_run_build circuits-wasm-linux-clang 64 circuits-wasm-linux-clang-assert: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -296,9 +296,9 @@ jobs: command: cond_spot_run_build circuits-wasm-linux-clang-assert 64 circuits-x86_64-linux-clang-tidy: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -307,9 +307,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang-tidy 64 circuits-x86_64-linux-clang: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -318,9 +318,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang 64 circuits-x86_64-linux-clang-assert: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -329,9 +329,9 @@ jobs: command: cond_spot_run_build circuits-x86_64-linux-clang-assert 64 circuits-wasm-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -341,9 +341,9 @@ jobs: - *save_logs circuits-x86_64-tests: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -355,7 +355,7 @@ jobs: circuits-end: docker: - image: cimg/base:current - + resource_class: small steps: - run: name: "Noop" @@ -467,8 +467,9 @@ jobs: command: build key-store noir-contracts-build: - machine: - image: ubuntu-2004:202010-01 + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -713,9 +714,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-sandbox-example: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -724,9 +725,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_sandbox_example.test.ts docker-compose-e2e-sandbox.yml e2e-multi-transfer-contract: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -819,9 +820,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-escrow-contract: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -914,9 +915,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-p2p: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -925,9 +926,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_p2p_network.test.ts e2e-browser-sandbox: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -936,9 +937,9 @@ jobs: command: cond_spot_run_tests end-to-end e2e_aztec_js_browser.test.ts docker-compose-e2e-sandbox.yml aztec-rpc-sandbox: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -959,9 +960,9 @@ jobs: working_directory: yarn-project/end-to-end e2e-canary-test: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -972,7 +973,7 @@ jobs: e2e-join: docker: - image: cimg/base:current - + resource_class: small steps: - run: name: "Noop" @@ -981,7 +982,7 @@ jobs: e2e-end: docker: - image: cimg/base:current - + resource_class: small steps: - run: name: "Noop" @@ -1128,7 +1129,7 @@ jobs: deploy-end: docker: - image: cimg/base:current - + resource_class: small steps: - run: name: "Noop" @@ -1146,9 +1147,9 @@ jobs: command: build canary true run-deployment-canary-uniswap: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -1157,9 +1158,9 @@ jobs: command: spot_run_test_script ./scripts/run_tests canary uniswap_trade_on_l1_from_l2.test.ts canary docker-compose.yml run-deployment-canary-browser: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env @@ -1168,9 +1169,9 @@ jobs: command: spot_run_test_script ./scripts/run_tests canary aztec_js_browser.test.ts canary docker-compose.yml run-deployment-canary-cli: - machine: - image: ubuntu-2004:202010-01 - + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env From d471bcc5a59c4467f9ab15abc8ffa5506d77a5b2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:57:00 -0400 Subject: [PATCH 37/65] Pass content hash to spot_run_script --- build-system/scripts/cond_spot_run_build | 2 +- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/spot_run_script | 13 ++++++------- build-system/scripts/spot_run_test_script | 5 ++++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index ace078d5841..7b4dc1552aa 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -15,6 +15,6 @@ cd $(query_manifest buildDir $REPOSITORY) if ! check_rebuild cache-$CONTENT_HASH $REPOSITORY; then init_submodules $REPOSITORY - spot_run_script $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ + spot_run_script $CONTENT_HASH $SPEC $BUILD_SYSTEM_PATH/remote_build/remote_build $REPOSITORY $@ tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH fi diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 143cc9d4ab4..8b17520078a 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -26,6 +26,6 @@ echo "Content hash: $CONTENT_HASH" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY - spot_run_script $@ + spot_run_script $CONTENT_HASH $@ tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 4fda0e93134..094d03cedb7 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -1,10 +1,12 @@ #!/bin/bash # Runs a test script on a remote spot instance. Arguments are: -# 1. SPEC: Instance specification filename. -# 2... ARGS: Arguments to pass to remote_run_script. +# 1. Content hash for our repository contents. Used for identify spot jobs and image tags. +# 2. SPEC: Instance specification filename. +# 3... ARGS: Arguments to pass to remote_run_script. set -xeu - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +CONTENT_HASH=$1 +SPEC=$2 +shift 2 # On any sort of exit (error or not), kill spot request so it doesn't count against quota. function on_exit { @@ -16,9 +18,6 @@ function on_exit { } trap on_exit EXIT -SPEC=$1 -shift - # Get spot instance. IP=$(request_spot $CONTENT_HASH:$JOB_NAME $SPEC) diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 97c93946ac9..b121bd65cbc 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -10,4 +10,7 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs set -xo pipefail -spot_run_script 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "Content hash: $CONTENT_HASH" +spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" From 03b0e01dea7765066bc5226f4f4761c2b93d5e2c Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 14:58:23 -0400 Subject: [PATCH 38/65] Hardening safety --- build-system/scripts/request_spot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 82a01f56937..d756ad64898 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -51,7 +51,7 @@ fi aws ec2 create-tags --resources $IID --tags "Key=Name,Value=$NAME" aws ec2 create-tags --resources $IID --tags "Key=Group,Value=build-instance" -while [ -z "$IP" ]; do +while [ -z "${IP:-}" ]; do sleep 1 IP=$(aws ec2 describe-instances \ --filter "Name=instance-id,Values=$IID" \ From c4ce73d8c74a209008381a10b0faeab0ee133a19 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:04:40 -0400 Subject: [PATCH 39/65] Hardening compatibility --- bootstrap_docker.sh | 4 ++-- build-system/scripts/setup_env | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index 3282f954d9c..b51bdf9a16e 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -xeu +set -xe TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) @@ -32,7 +32,7 @@ if [ -z "$TARGET_PROJECT" ]; then fi fi -source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) +source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER build_local $TARGET_PROJECT $ONLY_TARGET if [ -z "$TARGET_PROJECT" ]; then diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 3de75a1aaf5..3284b4778fc 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -11,8 +11,8 @@ set -xeu COMMIT_HASH=$1 COMMIT_TAG=$2 JOB_NAME=$3 -GIT_REPOSITORY_URL=$4 -BRANCH=$5 +GIT_REPOSITORY_URL=${4:-} +BRANCH=${5:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat PROJECT) From 216f5e7453fca82219f06cde31fcaa1489ed9cb3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:15:45 -0400 Subject: [PATCH 40/65] Update yarn prepare to not add non-existing deps --- .../yarn-project-base/scripts/update_build_manifest.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs b/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs index 36adc602a58..b915dd8733d 100644 --- a/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs +++ b/yarn-project/yarn-project-base/scripts/update_build_manifest.mjs @@ -21,7 +21,10 @@ function updateBuildManifest(buildManifestFile, allDependencies, projectKey, opt // Update the "dependencies" key in the corresponding section of the buildManifestData // Take just the folder name component - const updatedDependencies = aztecDependencies.map(packageName => packageName.split('/')[1]); + // Filter out dependencies that are not themselves in the manifest: + const updatedDependencies = aztecDependencies + .map(packageName => packageName.split('/')[1]) + .filter(depProjectKey => !!buildManifestData[depProjectKey]); // If we are just checking, throw if dependencies don't match if (options.checkOnly) { From c123b7e57535e574e57734f54e7f3b0552a1b589 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:17:53 -0400 Subject: [PATCH 41/65] Update yarn prepare to not add non-existing deps --- build_manifest.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_manifest.json b/build_manifest.json index 351983cf8c8..f7634ffbae7 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -305,6 +305,7 @@ "aztec-sandbox", "aztec.js", "circuits.js", + "cli", "ethereum", "foundation", "l1-artifacts", @@ -515,4 +516,4 @@ "types" ] } -} +} \ No newline at end of file From 364dfa5c774bcc608e37803789cda331a27ebbde Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:18:28 -0400 Subject: [PATCH 42/65] Retoggle no build --- build-system/scripts/check_rebuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 51eea5112a6..e58eabd6ff2 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -19,5 +19,5 @@ elif image_exists $REPOSITORY tainted; then exit 1 else echo "No rebuild required." - exit 1 # TODO TOGGLE + exit 0 fi From 83bb5a9c1a0c32aeb94b132840971949c021e6a7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:21:38 -0400 Subject: [PATCH 43/65] Remove debugging --- bootstrap.sh | 2 +- bootstrap_docker.sh | 2 +- build-system/remote_build/remote_build | 2 +- build-system/scripts/build | 2 +- build-system/scripts/build_local | 2 +- build-system/scripts/check_npm_version | 2 +- build-system/scripts/check_rebuild | 2 +- build-system/scripts/cond_spot_run_build | 4 ++-- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/cond_spot_run_test_script | 4 ++-- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 2 +- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 2 +- build-system/scripts/deploy_global | 2 +- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 2 +- build-system/scripts/deploy_service | 2 +- build-system/scripts/deploy_terraform | 2 +- build-system/scripts/ensure_apt_package | 2 +- build-system/scripts/ensure_repo | 2 +- build-system/scripts/ensure_terraform | 2 +- build-system/scripts/erase_image_tags | 2 +- build-system/scripts/extract_repo | 2 +- build-system/scripts/init_submodules | 2 +- build-system/scripts/query_manifest | 2 +- build-system/scripts/remote_run_script | 2 +- build-system/scripts/request_spot | 2 +- build-system/scripts/retry_10 | 2 +- build-system/scripts/setup_env | 2 +- build-system/scripts/spot_run_script | 2 +- build-system/scripts/spot_run_test_script | 4 ++-- build-system/scripts/spot_run_tests | 2 +- build-system/scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 2 +- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- circuits/cpp/barretenberg/bootstrap_docker.sh | 2 +- circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/bootstrap.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh | 2 +- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 2 +- circuits/cpp/barretenberg/cpp/scripts/run_tests | 2 +- circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh | 2 +- circuits/cpp/barretenberg/scripts/bindgen.sh | 2 +- circuits/cpp/barretenberg/sol/scripts/install_foundry.sh | 2 +- circuits/cpp/bootstrap.sh | 2 +- circuits/cpp/format.sh | 2 +- circuits/cpp/scripts/build_run_tests_docker_local | 2 +- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 2 +- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- l1-contracts/bootstrap.sh | 2 +- l1-contracts/scripts/install_foundry.sh | 2 +- scripts/ci/store_test_benchmark_logs | 2 +- scripts/fix_subrepo_edge_case.sh | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- scripts/git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/ext/bashplus/test/setup | 2 +- scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/test-more-bash/ext/bashplus/lib/bash+.bash | 2 +- .../git-subrepo/ext/test-more-bash/ext/bashplus/test/setup | 2 +- .../ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/test/setup | 2 +- scripts/git-subrepo/lib/git-subrepo | 2 +- scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/init-test | 2 +- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 2 +- .../git-subrepo/note/subtree-rebase-fail-example/test.bash | 2 +- scripts/git-subrepo/note/test-subrepo-push.sh | 2 +- scripts/git-subrepo/pkg/bin/generate-help-functions.pl | 2 +- scripts/git_subrepo.sh | 2 +- scripts/migrate_barretenberg_branch.sh | 2 +- scripts/update.sh | 2 +- yarn-project/bootstrap.sh | 2 +- yarn-project/canary/scripts/run_tests | 2 +- yarn-project/canary/scripts/update_packages.sh | 2 +- yarn-project/end-to-end/scripts/cond_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests_local | 2 +- yarn-project/end-to-end/scripts/start_e2e.sh | 2 +- yarn-project/end-to-end/scripts/start_p2p_e2e.sh | 2 +- yarn-project/end-to-end/src/guides/up_quick_start.sh | 2 +- yarn-project/l1-artifacts/scripts/generate-artifacts.sh | 2 +- yarn-project/noir-contracts/scripts/install_noir.sh | 2 +- yarn-project/noir-contracts/scripts/install_noirup.sh | 2 +- yarn-project/noir-contracts/scripts/types.sh | 2 +- yarn-project/noir-contracts/src/scripts/compile.sh | 2 +- yarn-project/p2p-bootstrap/scripts/start_bootnode.sh | 2 +- 93 files changed, 96 insertions(+), 96 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 72729dd117f..62b93deb023 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu export CLEAN=${1:-} diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index b51bdf9a16e..a8025f08aa1 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -xe +set -e TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index 83e33bf356c..8eab556ee57 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/build-system/scripts/build b/build-system/scripts/build index 24fd93587b2..4add19d1f1a 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -22,7 +22,7 @@ # - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers. # - Push the image tagged with the commit hash to the cache. -set -xeuo pipefail +set -euo pipefail REPOSITORY=$1 FORCE_BUILD=${2:-"false"} diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 2215c8c45a9..bb1d6340844 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -xeu +set -eu TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index 91bd924b9e5..0d7de47828e 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index e58eabd6ff2..d80e9493684 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -2,7 +2,7 @@ # Fails if any files matching the rebuild patterns, have changed since the base commit. # If this script fails (nonzero exit), then the caller should rebuild. # The rebuild patterns are taken from the build manifest (computed from set of dependencies). -set -xeuo pipefail +set -euo pipefail TAG=$1 REPOSITORY=$2 diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 7b4dc1552aa..61f262307e5 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,6 +1,6 @@ #!/bin/bash -set -xeu -set -xo pipefail +set -eu +set -o pipefail REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 8b17520078a..154b70c8b68 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -xeu +set -eu REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index 575f839465a..c760e69d00f 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,5 +9,5 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -xo pipefail +set -o pipefail cond_spot_run_script $REPOSITORY $JOB_NAME 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index c898c0700d2..78097379f1f 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xeu +set -eu cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index 0eacf7847da..ae6d92a69ba 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 09783f59ee1..d7e0442d30b 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index 7c8f8dbe96b..aa512cc1b4b 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index e54afb05133..5114a067a74 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -xeu +set -eu REPOSITORY=$1 diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 9df0e10d1a5..1847e2463d1 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index cd076e59077..947062da206 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 34b5df87f1e..c126becdf56 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Redeploy service with latest image. SERVICE_NAME=$1 diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index 1a813cc4591..637ef5c8be1 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 TF_DIR=$2 diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index fe5d3d39f9f..33326c7f1fd 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if dpkg -s $1 &> /dev/null; then exit 0 diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index d0f62dc3700..186b204cd4e 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -xeu +set -eu LIFECYCLE_POLICY='{ "rules": [ diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index d70f1d9cf84..e04231e79f1 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -xeu +set -eu [ ! -f /usr/local/bin/terraform ] || exit 0 diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index a2bd8acb257..8f99c3f1569 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -xeu +set -eu REPOSITORY=$1 TILL_COMMIT_HASH=$2 diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 1756b361811..8bd7ef2c606 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -xeu +set -eu REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} diff --git a/build-system/scripts/init_submodules b/build-system/scripts/init_submodules index 1efa2db83f3..c456aa1a995 100755 --- a/build-system/scripts/init_submodules +++ b/build-system/scripts/init_submodules @@ -1,6 +1,6 @@ #!/bin/bash # For a given repository, init any required submodules. -set -xeuo pipefail +set -euo pipefail REPOSITORY=$1 diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index ad2b23b5668..ba7b8c1fe33 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu CMD=$1 REPO=$2 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index a4efda7cf6a..c33d386f1f9 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu IP=$1 CONTENT_HASH=$2 diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index d756ad64898..d87fd829eee 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu NAME=$1 CPUS=${2:-32} diff --git a/build-system/scripts/retry_10 b/build-system/scripts/retry_10 index e0e499ba624..2d111c6e3dc 100755 --- a/build-system/scripts/retry_10 +++ b/build-system/scripts/retry_10 @@ -1,4 +1,4 @@ -set -xeu +set -eu # Retries up to 10 times with 10 second intervals for i in $(seq 1 10); do "$@" && exit || sleep 10 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 3284b4778fc..2df97fbfd5a 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,7 +6,7 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -xeu +set -eu COMMIT_HASH=$1 COMMIT_TAG=$2 diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 094d03cedb7..63f16928c85 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -3,7 +3,7 @@ # 1. Content hash for our repository contents. Used for identify spot jobs and image tags. # 2. SPEC: Instance specification filename. # 3... ARGS: Arguments to pass to remote_run_script. -set -xeu +set -eu CONTENT_HASH=$1 SPEC=$2 shift 2 diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index b121bd65cbc..06d4d455380 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,7 +9,7 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -xo pipefail +set -o pipefail CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index e9b14c1fd88..e7d8183c048 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xeu +set -eu spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index 9ac9ff71c04..d44f3c2cd9f 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 0bf67437019..741009ff125 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 EXISTING_TAG=$2 diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 92b68e6580c..b04b43fcd9b 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -xeu +set -eu BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index 3db07de7bc7..e2f88e22525 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -xeu +set -eu COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) diff --git a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh index ff4692b4758..eb634c96065 100755 --- a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/sh # Script is assumed to be run from -set -xeu +set -eu bb() { ../build/bin/bb "$@" -v diff --git a/circuits/cpp/barretenberg/cpp/bootstrap.sh b/circuits/cpp/barretenberg/cpp/bootstrap.sh index ef389491f40..869bbc9c885 100755 --- a/circuits/cpp/barretenberg/cpp/bootstrap.sh +++ b/circuits/cpp/barretenberg/cpp/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Clean. rm -rf ./build diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index f866bea9c15..48315e39621 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 66117d7dbf7..235aa111ff4 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -xeu +set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index c25434289ee..b06c26036ba 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/bash # Executes the bb binary test script. -set -xeu +set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh index 3dd76aa9c46..3a8eecf2438 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Clean. rm -rf ./src/wasi-sdk-* diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index a40e8b3c97e..d67176f098f 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 24c68e7467b..965307ff5a5 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -xeu +set -eu NUM_TRANSCRIPTS=$1 TESTS=$2 diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index d9e1bcd7fe5..478f442f9af 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -xeu +set -eu mkdir -p ignition cd ignition diff --git a/circuits/cpp/barretenberg/scripts/bindgen.sh b/circuits/cpp/barretenberg/scripts/bindgen.sh index 43740d746e1..1a2034a213a 100755 --- a/circuits/cpp/barretenberg/scripts/bindgen.sh +++ b/circuits/cpp/barretenberg/scripts/bindgen.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json diff --git a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh index 02bcbf8db3b..e085beb76d5 100755 --- a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh +++ b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -xeu +set -eu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/circuits/cpp/bootstrap.sh b/circuits/cpp/bootstrap.sh index b9f487e5678..0462e0085f6 100755 --- a/circuits/cpp/bootstrap.sh +++ b/circuits/cpp/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash # Takes a list of targets from commandline # Takes CLEAN as an environment variable. If passed, cleans build artifacts -set -xeu +set -eu export WASI_VERSION=20 diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index f866bea9c15..48315e39621 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index fa64b29e330..a075cbba172 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index 7979f7d1c6c..d0e4bb097f4 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -xeu +set -eu # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index eb8a2a28a6c..b3fbc43f459 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index a67649e471d..4c942a0f13a 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 13e46f5469c..e239fc3b1a6 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Run clang-tidy on all C++ source files # diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index cfe00c96988..58aba5449eb 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu cd "$(dirname "$0")" diff --git a/l1-contracts/scripts/install_foundry.sh b/l1-contracts/scripts/install_foundry.sh index 02bcbf8db3b..e085beb76d5 100755 --- a/l1-contracts/scripts/install_foundry.sh +++ b/l1-contracts/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -xeu +set -eu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 2b36f14a4ea..8f87ba04462 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu AZTEC_GITHUB_TOKEN=$1 diff --git a/scripts/fix_subrepo_edge_case.sh b/scripts/fix_subrepo_edge_case.sh index 7a236435868..6888875eaff 100755 --- a/scripts/fix_subrepo_edge_case.sh +++ b/scripts/fix_subrepo_edge_case.sh @@ -6,7 +6,7 @@ # in the face of e.g. a .gitrepo whitespace change, but it's a fallback, # we only have this issue in master, and the file should only be edited # generally by subrepo commands. -set -xeu +set -eu SUBREPO_PATH="${1:-}" echo "Auto-fixing squashed parent in $SUBREPO_PATH/.gitrepo." diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index 57d3e19655c..e0bc1f73913 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xeu +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index 16ea238245f..b52e17b55a6 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xeu +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/bashplus/test/setup b/scripts/git-subrepo/ext/bashplus/test/setup index a3535659a5a..c8b138d7e32 100644 --- a/scripts/git-subrepo/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -xe -o pipefail +set -e -o pipefail PATH=$PWD/bin:$PATH diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index 57d3e19655c..e0bc1f73913 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xeu +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index 28d8df1c78f..e0e15cfc49d 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xeu +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup index 176f989ef32..35124d0f69d 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -xe -u -o pipefail +set -e -u -o pipefail [[ $BASH_VERSION == 4.0* ]] && set +u run=0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash index a4f0d192449..cc34dcb9a2a 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash @@ -6,7 +6,7 @@ Test::Tap:die() { echo "$@" >&2; trap EXIT; exit 1; } #------------------------------------------------------------------------------ -set -xe -u -o pipefail +set -e -u -o pipefail [[ ${BASH_VERSION-} == 4.0* ]] && set +u # shellcheck disable=2034 diff --git a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash index 815dd498f95..6fa9f56d93f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash +++ b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020. Ingy döt Net. -set -xe -u -o pipefail +set -e -u -o pipefail # shellcheck disable=2034 Test__More_VERSION=0.0.5 diff --git a/scripts/git-subrepo/ext/test-more-bash/test/setup b/scripts/git-subrepo/ext/test-more-bash/test/setup index 1d910f3ceb1..99df35d614f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/test/setup @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xe -u -o pipefail +set -e -u -o pipefail BASHLIB=$(find "$PWD" -type d | grep -E '/(bin|lib)$' | xargs -n1 printf "%s:") PATH=$BASHLIB:$PATH diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index 78ed42e0bc3..9f4a3384cd1 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -xeu +set -eu export FILTER_BRANCH_SQUELCH_WARNING=1 diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 1933b6eb280..98e34dd272b 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xeu +set -eu help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index 4d4ef58690b..a20854de88a 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex cat $0 # Show this script in the output REPO=${1:-ingydotnet/djson-pm} diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 5e9694a0999..62bbd9617c1 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -xeu +set -eu set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index ac77fe7361a..df8b818cee6 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex rm -fr repo{1,2,3} mkdir repo{1,2} diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index 40160368c9e..afceb5efa92 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex # Make a directory to work in: { diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index b23a011ef5f..5dd949e6ed9 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xeu +set -eu ... } diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index 0651c7c7da5..df9743a2bd5 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_DIR=$(dirname "$(realpath "$0")") diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh index d78935bf4fb..9f3001c7421 100755 --- a/scripts/migrate_barretenberg_branch.sh +++ b/scripts/migrate_barretenberg_branch.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Usage: ./this.sh # Script for migrating PRs from barretenberg repo to aztec-packages. diff --git a/scripts/update.sh b/scripts/update.sh index b374d765a14..ab78b6827dc 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Script for running after updating the working copy with remote changes. # Similar to bootstrap, but more lightweight and oriented towards working on end-to-end. diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index a2879ddef87..39036119ed7 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Navigate to script folder cd "$(dirname "$0")" diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index fc39a781c95..dac0f0bd921 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -eu export TEST=$1 export IMAGE=${2:-canary} diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index 608bcdb3035..0b5a3c59b43 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -xeu +set -eu COMMIT_TAG=$1 diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 96f9463f8cb..e8159655bf2 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -xeu +set -eu set -x REPOSITORY=$1 diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index ba874d060cc..fad024b1eb2 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -eu export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 01b9ac4c10b..846d54173c4 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -eu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/start_e2e.sh b/yarn-project/end-to-end/scripts/start_e2e.sh index 45f6d13800a..018c3d79603 100755 --- a/yarn-project/end-to-end/scripts/start_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu export NODE_NO_WARNINGS=1 node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ diff --git a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh index ed24c853826..9b1b661a7fe 100755 --- a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu export DEBUG='aztec:*' export ARCHIVER_POLLING_INTERVAL=500 export P2P_CHECK_INTERVAL=50 diff --git a/yarn-project/end-to-end/src/guides/up_quick_start.sh b/yarn-project/end-to-end/src/guides/up_quick_start.sh index 85012b10027..c27ff3454f3 100755 --- a/yarn-project/end-to-end/src/guides/up_quick_start.sh +++ b/yarn-project/end-to-end/src/guides/up_quick_start.sh @@ -1,7 +1,7 @@ # Run locally from end-to-end folder while running anvil and sandbox with: # PATH=$PATH:../node_modules/.bin ./src/guides/up_quick_start.sh -set -xeux +set -eux # docs:start:declare-accounts ALICE="0x2e13f0201905944184fc2c09d29fcf0cac07647be171656a275f63d99b819360" diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index 86b8bb95708..1c4bd5e2ba9 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeuo pipefail; +set -euo pipefail; target_dir=./generated diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index 8021c0685ca..c0c9d70d729 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest aztec nargo -set -xeu +set -eu VERSION="aztec" diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index efa83d5b668..11ba9b15d31 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest nargo -set -xeu +set -eu SPECIFIED_HOME=${1:-$HOME} diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 9f499a4e0d7..a9d249c437f 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -8,7 +8,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), -set -xeu; +set -eu; artifacts_dir="src/artifacts" types_dir="src/types" diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 384c1d0d537..74719291c65 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -9,7 +9,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), # and propagate the exit status of the first failing command in a pipeline (set -o pipefail). -set -xeuo pipefail; +set -euo pipefail; # Run build scripts ./scripts/compile.sh "$@" diff --git a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh index 7621979ab44..2abace6942d 100755 --- a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh +++ b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu cd .. export P2P_TCP_LISTEN_PORT=40400 From ac55fb013595a988d71ee5ca79d2ef6df2ca5785 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:24:31 -0400 Subject: [PATCH 44/65] Remove cli dep --- build_manifest.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build_manifest.json b/build_manifest.json index f7634ffbae7..351983cf8c8 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -305,7 +305,6 @@ "aztec-sandbox", "aztec.js", "circuits.js", - "cli", "ethereum", "foundation", "l1-artifacts", @@ -516,4 +515,4 @@ "types" ] } -} \ No newline at end of file +} From 6aabc6d26f9251415cd2aeb71f178adba733958d Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:40:49 -0400 Subject: [PATCH 45/65] Revert "Remove debugging" This reverts commit 83bb5a9c1a0c32aeb94b132840971949c021e6a7. --- bootstrap.sh | 2 +- bootstrap_docker.sh | 2 +- build-system/remote_build/remote_build | 2 +- build-system/scripts/build | 2 +- build-system/scripts/build_local | 2 +- build-system/scripts/check_npm_version | 2 +- build-system/scripts/check_rebuild | 2 +- build-system/scripts/cond_spot_run_build | 4 ++-- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/cond_spot_run_test_script | 4 ++-- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 2 +- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 2 +- build-system/scripts/deploy_global | 2 +- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 2 +- build-system/scripts/deploy_service | 2 +- build-system/scripts/deploy_terraform | 2 +- build-system/scripts/ensure_apt_package | 2 +- build-system/scripts/ensure_repo | 2 +- build-system/scripts/ensure_terraform | 2 +- build-system/scripts/erase_image_tags | 2 +- build-system/scripts/extract_repo | 2 +- build-system/scripts/init_submodules | 2 +- build-system/scripts/query_manifest | 2 +- build-system/scripts/remote_run_script | 2 +- build-system/scripts/request_spot | 2 +- build-system/scripts/retry_10 | 2 +- build-system/scripts/setup_env | 2 +- build-system/scripts/spot_run_script | 2 +- build-system/scripts/spot_run_test_script | 4 ++-- build-system/scripts/spot_run_tests | 2 +- build-system/scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 2 +- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- circuits/cpp/barretenberg/bootstrap_docker.sh | 2 +- circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/bootstrap.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh | 2 +- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 2 +- circuits/cpp/barretenberg/cpp/scripts/run_tests | 2 +- circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh | 2 +- circuits/cpp/barretenberg/scripts/bindgen.sh | 2 +- circuits/cpp/barretenberg/sol/scripts/install_foundry.sh | 2 +- circuits/cpp/bootstrap.sh | 2 +- circuits/cpp/format.sh | 2 +- circuits/cpp/scripts/build_run_tests_docker_local | 2 +- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 2 +- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- l1-contracts/bootstrap.sh | 2 +- l1-contracts/scripts/install_foundry.sh | 2 +- scripts/ci/store_test_benchmark_logs | 2 +- scripts/fix_subrepo_edge_case.sh | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- scripts/git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/ext/bashplus/test/setup | 2 +- scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/test-more-bash/ext/bashplus/lib/bash+.bash | 2 +- .../git-subrepo/ext/test-more-bash/ext/bashplus/test/setup | 2 +- .../ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/test/setup | 2 +- scripts/git-subrepo/lib/git-subrepo | 2 +- scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/init-test | 2 +- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 2 +- .../git-subrepo/note/subtree-rebase-fail-example/test.bash | 2 +- scripts/git-subrepo/note/test-subrepo-push.sh | 2 +- scripts/git-subrepo/pkg/bin/generate-help-functions.pl | 2 +- scripts/git_subrepo.sh | 2 +- scripts/migrate_barretenberg_branch.sh | 2 +- scripts/update.sh | 2 +- yarn-project/bootstrap.sh | 2 +- yarn-project/canary/scripts/run_tests | 2 +- yarn-project/canary/scripts/update_packages.sh | 2 +- yarn-project/end-to-end/scripts/cond_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests_local | 2 +- yarn-project/end-to-end/scripts/start_e2e.sh | 2 +- yarn-project/end-to-end/scripts/start_p2p_e2e.sh | 2 +- yarn-project/end-to-end/src/guides/up_quick_start.sh | 2 +- yarn-project/l1-artifacts/scripts/generate-artifacts.sh | 2 +- yarn-project/noir-contracts/scripts/install_noir.sh | 2 +- yarn-project/noir-contracts/scripts/install_noirup.sh | 2 +- yarn-project/noir-contracts/scripts/types.sh | 2 +- yarn-project/noir-contracts/src/scripts/compile.sh | 2 +- yarn-project/p2p-bootstrap/scripts/start_bootnode.sh | 2 +- 93 files changed, 96 insertions(+), 96 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 62b93deb023..72729dd117f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu export CLEAN=${1:-} diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index a8025f08aa1..b51bdf9a16e 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -e +set -xe TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index 8eab556ee57..83e33bf356c 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/build-system/scripts/build b/build-system/scripts/build index 4add19d1f1a..24fd93587b2 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -22,7 +22,7 @@ # - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers. # - Push the image tagged with the commit hash to the cache. -set -euo pipefail +set -xeuo pipefail REPOSITORY=$1 FORCE_BUILD=${2:-"false"} diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index bb1d6340844..2215c8c45a9 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -eu +set -xeu TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index 0d7de47828e..91bd924b9e5 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index d80e9493684..e58eabd6ff2 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -2,7 +2,7 @@ # Fails if any files matching the rebuild patterns, have changed since the base commit. # If this script fails (nonzero exit), then the caller should rebuild. # The rebuild patterns are taken from the build manifest (computed from set of dependencies). -set -euo pipefail +set -xeuo pipefail TAG=$1 REPOSITORY=$2 diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 61f262307e5..7b4dc1552aa 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,6 +1,6 @@ #!/bin/bash -set -eu -set -o pipefail +set -xeu +set -xo pipefail REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 154b70c8b68..8b17520078a 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -eu +set -xeu REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index c760e69d00f..575f839465a 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,5 +9,5 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -o pipefail +set -xo pipefail cond_spot_run_script $REPOSITORY $JOB_NAME 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index 78097379f1f..c898c0700d2 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -eu +set -xeu cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index ae6d92a69ba..0eacf7847da 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index d7e0442d30b..09783f59ee1 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index aa512cc1b4b..7c8f8dbe96b 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index 5114a067a74..e54afb05133 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -eu +set -xeu REPOSITORY=$1 diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 1847e2463d1..9df0e10d1a5 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index 947062da206..cd076e59077 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index c126becdf56..34b5df87f1e 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Redeploy service with latest image. SERVICE_NAME=$1 diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index 637ef5c8be1..1a813cc4591 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 TF_DIR=$2 diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index 33326c7f1fd..fe5d3d39f9f 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu if dpkg -s $1 &> /dev/null; then exit 0 diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 186b204cd4e..d0f62dc3700 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -eu +set -xeu LIFECYCLE_POLICY='{ "rules": [ diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index e04231e79f1..d70f1d9cf84 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -eu +set -xeu [ ! -f /usr/local/bin/terraform ] || exit 0 diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 8f99c3f1569..a2bd8acb257 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -eu +set -xeu REPOSITORY=$1 TILL_COMMIT_HASH=$2 diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 8bd7ef2c606..1756b361811 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -eu +set -xeu REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} diff --git a/build-system/scripts/init_submodules b/build-system/scripts/init_submodules index c456aa1a995..1efa2db83f3 100755 --- a/build-system/scripts/init_submodules +++ b/build-system/scripts/init_submodules @@ -1,6 +1,6 @@ #!/bin/bash # For a given repository, init any required submodules. -set -euo pipefail +set -xeuo pipefail REPOSITORY=$1 diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index ba7b8c1fe33..ad2b23b5668 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu CMD=$1 REPO=$2 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index c33d386f1f9..a4efda7cf6a 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu IP=$1 CONTENT_HASH=$2 diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index d87fd829eee..d756ad64898 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu NAME=$1 CPUS=${2:-32} diff --git a/build-system/scripts/retry_10 b/build-system/scripts/retry_10 index 2d111c6e3dc..e0e499ba624 100755 --- a/build-system/scripts/retry_10 +++ b/build-system/scripts/retry_10 @@ -1,4 +1,4 @@ -set -eu +set -xeu # Retries up to 10 times with 10 second intervals for i in $(seq 1 10); do "$@" && exit || sleep 10 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 2df97fbfd5a..3284b4778fc 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,7 +6,7 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -eu +set -xeu COMMIT_HASH=$1 COMMIT_TAG=$2 diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 63f16928c85..094d03cedb7 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -3,7 +3,7 @@ # 1. Content hash for our repository contents. Used for identify spot jobs and image tags. # 2. SPEC: Instance specification filename. # 3... ARGS: Arguments to pass to remote_run_script. -set -eu +set -xeu CONTENT_HASH=$1 SPEC=$2 shift 2 diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 06d4d455380..b121bd65cbc 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,7 +9,7 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -o pipefail +set -xo pipefail CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index e7d8183c048..e9b14c1fd88 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -eu +set -xeu spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index d44f3c2cd9f..9ac9ff71c04 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 741009ff125..0bf67437019 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu REPOSITORY=$1 EXISTING_TAG=$2 diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index b04b43fcd9b..92b68e6580c 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -eu +set -xeu BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index e2f88e22525..3db07de7bc7 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -eu +set -xeu COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) diff --git a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh index eb634c96065..ff4692b4758 100755 --- a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/sh # Script is assumed to be run from -set -eu +set -xeu bb() { ../build/bin/bb "$@" -v diff --git a/circuits/cpp/barretenberg/cpp/bootstrap.sh b/circuits/cpp/barretenberg/cpp/bootstrap.sh index 869bbc9c885..ef389491f40 100755 --- a/circuits/cpp/barretenberg/cpp/bootstrap.sh +++ b/circuits/cpp/barretenberg/cpp/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Clean. rm -rf ./build diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index 48315e39621..f866bea9c15 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 235aa111ff4..66117d7dbf7 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -eu +set -xeu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index b06c26036ba..c25434289ee 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/bash # Executes the bb binary test script. -set -eu +set -xeu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh index 3a8eecf2438..3dd76aa9c46 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Clean. rm -rf ./src/wasi-sdk-* diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index d67176f098f..a40e8b3c97e 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 965307ff5a5..24c68e7467b 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -eu +set -xeu NUM_TRANSCRIPTS=$1 TESTS=$2 diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index 478f442f9af..d9e1bcd7fe5 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -eu +set -xeu mkdir -p ignition cd ignition diff --git a/circuits/cpp/barretenberg/scripts/bindgen.sh b/circuits/cpp/barretenberg/scripts/bindgen.sh index 1a2034a213a..43740d746e1 100755 --- a/circuits/cpp/barretenberg/scripts/bindgen.sh +++ b/circuits/cpp/barretenberg/scripts/bindgen.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json diff --git a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh index e085beb76d5..02bcbf8db3b 100755 --- a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh +++ b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -eu +set -xeu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/circuits/cpp/bootstrap.sh b/circuits/cpp/bootstrap.sh index 0462e0085f6..b9f487e5678 100755 --- a/circuits/cpp/bootstrap.sh +++ b/circuits/cpp/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash # Takes a list of targets from commandline # Takes CLEAN as an environment variable. If passed, cleans build artifacts -set -eu +set -xeu export WASI_VERSION=20 diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index 48315e39621..f866bea9c15 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index a075cbba172..fa64b29e330 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index d0e4bb097f4..7979f7d1c6c 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -eu +set -xeu # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index b3fbc43f459..eb8a2a28a6c 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index 4c942a0f13a..a67649e471d 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index e239fc3b1a6..13e46f5469c 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Run clang-tidy on all C++ source files # diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 58aba5449eb..cfe00c96988 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu cd "$(dirname "$0")" diff --git a/l1-contracts/scripts/install_foundry.sh b/l1-contracts/scripts/install_foundry.sh index e085beb76d5..02bcbf8db3b 100755 --- a/l1-contracts/scripts/install_foundry.sh +++ b/l1-contracts/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -eu +set -xeu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 8f87ba04462..2b36f14a4ea 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu AZTEC_GITHUB_TOKEN=$1 diff --git a/scripts/fix_subrepo_edge_case.sh b/scripts/fix_subrepo_edge_case.sh index 6888875eaff..7a236435868 100755 --- a/scripts/fix_subrepo_edge_case.sh +++ b/scripts/fix_subrepo_edge_case.sh @@ -6,7 +6,7 @@ # in the face of e.g. a .gitrepo whitespace change, but it's a fallback, # we only have this issue in master, and the file should only be edited # generally by subrepo commands. -set -eu +set -xeu SUBREPO_PATH="${1:-}" echo "Auto-fixing squashed parent in $SUBREPO_PATH/.gitrepo." diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index e0bc1f73913..57d3e19655c 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -eu +set -xeu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index b52e17b55a6..16ea238245f 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -eu +set -xeu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/bashplus/test/setup b/scripts/git-subrepo/ext/bashplus/test/setup index c8b138d7e32..a3535659a5a 100644 --- a/scripts/git-subrepo/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -e -o pipefail +set -xe -o pipefail PATH=$PWD/bin:$PATH diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index e0bc1f73913..57d3e19655c 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -eu +set -xeu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index e0e15cfc49d..28d8df1c78f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -eu +set -xeu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup index 35124d0f69d..176f989ef32 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -e -u -o pipefail +set -xe -u -o pipefail [[ $BASH_VERSION == 4.0* ]] && set +u run=0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash index cc34dcb9a2a..a4f0d192449 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash @@ -6,7 +6,7 @@ Test::Tap:die() { echo "$@" >&2; trap EXIT; exit 1; } #------------------------------------------------------------------------------ -set -e -u -o pipefail +set -xe -u -o pipefail [[ ${BASH_VERSION-} == 4.0* ]] && set +u # shellcheck disable=2034 diff --git a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash index 6fa9f56d93f..815dd498f95 100644 --- a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash +++ b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020. Ingy döt Net. -set -e -u -o pipefail +set -xe -u -o pipefail # shellcheck disable=2034 Test__More_VERSION=0.0.5 diff --git a/scripts/git-subrepo/ext/test-more-bash/test/setup b/scripts/git-subrepo/ext/test-more-bash/test/setup index 99df35d614f..1d910f3ceb1 100644 --- a/scripts/git-subrepo/ext/test-more-bash/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/test/setup @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e -u -o pipefail +set -xe -u -o pipefail BASHLIB=$(find "$PWD" -type d | grep -E '/(bin|lib)$' | xargs -n1 printf "%s:") PATH=$BASHLIB:$PATH diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index 9f4a3384cd1..78ed42e0bc3 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -eu +set -xeu export FILTER_BRANCH_SQUELCH_WARNING=1 diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 98e34dd272b..1933b6eb280 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -eu +set -xeu help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index a20854de88a..4d4ef58690b 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex cat $0 # Show this script in the output REPO=${1:-ingydotnet/djson-pm} diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 62bbd9617c1..5e9694a0999 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -eu +set -xeu set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index df8b818cee6..ac77fe7361a 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex rm -fr repo{1,2,3} mkdir repo{1,2} diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index afceb5efa92..40160368c9e 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -xex # Make a directory to work in: { diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index 5dd949e6ed9..b23a011ef5f 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -eu +set -xeu ... } diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index df9743a2bd5..0651c7c7da5 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu SCRIPT_DIR=$(dirname "$(realpath "$0")") diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh index 9f3001c7421..d78935bf4fb 100755 --- a/scripts/migrate_barretenberg_branch.sh +++ b/scripts/migrate_barretenberg_branch.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Usage: ./this.sh # Script for migrating PRs from barretenberg repo to aztec-packages. diff --git a/scripts/update.sh b/scripts/update.sh index ab78b6827dc..b374d765a14 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Script for running after updating the working copy with remote changes. # Similar to bootstrap, but more lightweight and oriented towards working on end-to-end. diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 39036119ed7..a2879ddef87 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu +set -xeu # Navigate to script folder cd "$(dirname "$0")" diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index dac0f0bd921..fc39a781c95 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -eu +set -xeu export TEST=$1 export IMAGE=${2:-canary} diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index 0b5a3c59b43..608bcdb3035 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -eu +set -xeu COMMIT_TAG=$1 diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index e8159655bf2..96f9463f8cb 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -eu +set -xeu set -x REPOSITORY=$1 diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index fad024b1eb2..ba874d060cc 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -eu +set -xeu export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 846d54173c4..01b9ac4c10b 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -eu +set -xeu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/start_e2e.sh b/yarn-project/end-to-end/scripts/start_e2e.sh index 018c3d79603..45f6d13800a 100755 --- a/yarn-project/end-to-end/scripts/start_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -xeu export NODE_NO_WARNINGS=1 node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ diff --git a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh index 9b1b661a7fe..ed24c853826 100755 --- a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -xeu export DEBUG='aztec:*' export ARCHIVER_POLLING_INTERVAL=500 export P2P_CHECK_INTERVAL=50 diff --git a/yarn-project/end-to-end/src/guides/up_quick_start.sh b/yarn-project/end-to-end/src/guides/up_quick_start.sh index c27ff3454f3..85012b10027 100755 --- a/yarn-project/end-to-end/src/guides/up_quick_start.sh +++ b/yarn-project/end-to-end/src/guides/up_quick_start.sh @@ -1,7 +1,7 @@ # Run locally from end-to-end folder while running anvil and sandbox with: # PATH=$PATH:../node_modules/.bin ./src/guides/up_quick_start.sh -set -eux +set -xeux # docs:start:declare-accounts ALICE="0x2e13f0201905944184fc2c09d29fcf0cac07647be171656a275f63d99b819360" diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index 1c4bd5e2ba9..86b8bb95708 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -euo pipefail; +set -xeuo pipefail; target_dir=./generated diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index c0c9d70d729..8021c0685ca 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest aztec nargo -set -eu +set -xeu VERSION="aztec" diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index 11ba9b15d31..efa83d5b668 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest nargo -set -eu +set -xeu SPECIFIED_HOME=${1:-$HOME} diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index a9d249c437f..9f499a4e0d7 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -8,7 +8,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), -set -eu; +set -xeu; artifacts_dir="src/artifacts" types_dir="src/types" diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 74719291c65..384c1d0d537 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -9,7 +9,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), # and propagate the exit status of the first failing command in a pipeline (set -o pipefail). -set -euo pipefail; +set -xeuo pipefail; # Run build scripts ./scripts/compile.sh "$@" diff --git a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh index 2abace6942d..7621979ab44 100755 --- a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh +++ b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -eu +set -xeu cd .. export P2P_TCP_LISTEN_PORT=40400 From a7c57cd43ac8de7e83011a80b478474bfd273ef1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:43:26 -0400 Subject: [PATCH 46/65] Fix docker ordering --- yarn-project/end-to-end/scripts/run_tests_local | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 01b9ac4c10b..f4685d86a6f 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -8,9 +8,8 @@ export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" -docker pull $IMAGE_URI - aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com +docker pull $IMAGE_URI for REPO in end-to-end; do docker pull $IMAGE_URI From c92149812d37f3f0be4f1a795bb7d861c9d0577b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:54:11 -0400 Subject: [PATCH 47/65] Fix e2e --- yarn-project/end-to-end/scripts/run_tests_local | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index f4685d86a6f..918f6241505 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,19 +1,16 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -e export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} -REPOSITORY=barretenberg-x86_64-linux-clang-assert -# use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" + aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 278380418400.dkr.ecr.us-east-2.amazonaws.com -docker pull $IMAGE_URI for REPO in end-to-end; do - docker pull $IMAGE_URI - docker tag $IMAGE_URI aztecprotocol/$REPO:latest + docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH + docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest done docker-compose -f $COMPOSE_FILE rm -f From 833b90e4f3729b115187f79ad7616145aa842a5a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:57:04 -0400 Subject: [PATCH 48/65] Fix debugging --- build-system/scripts/calculate_content_hash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 37f080e4c74..551e0d4f819 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,5 +1,7 @@ #!/bin/bash +set -xeu + REPOSITORY=$1 COMMIT_HASH=${2:-$COMMIT_HASH} From 87d7c9f4687ff58bdf8fc1fdc287339d6b3fd09a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 15:59:21 -0400 Subject: [PATCH 49/65] Fix content hashing --- build-system/scripts/calculate_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 551e0d4f819..305cb5f7606 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -8,7 +8,7 @@ COMMIT_HASH=${2:-$COMMIT_HASH} # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) -git ls-files -s ${COMMIT_HASH} | while read -r line; do +git ls-tree -r ${COMMIT_HASH} | while read -r line; do # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json # this format is beneficial as it grabs the hashes from git efficiently From 76108cc770935e5c1f4348088ad60facee8a59c7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:00:09 -0400 Subject: [PATCH 50/65] Fix content hashing --- build-system/scripts/calculate_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 305cb5f7606..79729781d46 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -15,7 +15,7 @@ git ls-tree -r ${COMMIT_HASH} | while read -r line; do # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash for pattern in $REBUILD_PATTERNS; do - if [[ "$filepath" =~ $pattern ]]; then + if [[ "$line" =~ $pattern ]]; then echo "$line" break fi From 715dffe172e867977e9cf06bd75cb3d67bfa89cc Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:00:54 -0400 Subject: [PATCH 51/65] Fix content hashing --- build-system/scripts/calculate_content_hash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 79729781d46..581f72793a1 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -14,8 +14,9 @@ git ls-tree -r ${COMMIT_HASH} | while read -r line; do # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash + filepath=$(echo "$line" | awk '{print $4}') for pattern in $REBUILD_PATTERNS; do - if [[ "$line" =~ $pattern ]]; then + if [[ "$filepath" =~ $pattern ]]; then echo "$line" break fi From fb31ac943693ca860419328518351ca7c815ef12 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:02:30 -0400 Subject: [PATCH 52/65] Remove debugging from fixed content hashing --- build-system/scripts/calculate_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 581f72793a1..1e7f19dd5a6 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -1,6 +1,6 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 COMMIT_HASH=${2:-$COMMIT_HASH} From a88ec8f8a24c4039e6eceebb980b2d5ae6c240cd Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:17:59 -0400 Subject: [PATCH 53/65] air test fix --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 92b68e6580c..68a7c2f4c40 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -11,7 +11,7 @@ BRANCH=kw/acvm-0-24 # Pull down the test vectors from the noir repo, if we don't have the folder already. if [ ! -d acir_tests ]; then - if [ -n "$TEST_SRC" ]; then + if [ -n "${TEST_SRC:-}" ]; then cp -R $TEST_SRC acir_tests else rm -rf noir From de85aa6ed7057e5f4443a8ffd16cc1713f5db27c Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:24:05 -0400 Subject: [PATCH 54/65] air test fix --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 68a7c2f4c40..c11b1dd10bf 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -68,7 +68,7 @@ function test() { cd .. } -if [ -n "$1" ]; then +if [ -n "{$1:-}" ]; then test $1 else for DIR in $(find -maxdepth 1 -type d -not -path '.'); do From fb3396510a90f22c8b69062056bfcca9e4266c30 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:27:01 -0400 Subject: [PATCH 55/65] air test fix --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index c11b1dd10bf..e21fe6af9c6 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -68,7 +68,7 @@ function test() { cd .. } -if [ -n "{$1:-}" ]; then +if [ -n "${1:-}" ]; then test $1 else for DIR in $(find -maxdepth 1 -type d -not -path '.'); do From da039c3cae46f083bf048591b5b21ab347ffe11b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:38:10 -0400 Subject: [PATCH 56/65] air test fix --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index e21fe6af9c6..5465cbcb8bd 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -48,7 +48,7 @@ function test() { cd $1 set +e - if [ -n "$VERBOSE" ]; then + if [ -n "${VERBOSE:-}" ]; then $BB prove_and_verify -v -c $CRS_PATH -b ./target/$dir_name.bytecode else $BB prove_and_verify -c $CRS_PATH -b ./target/$dir_name.bytecode > /dev/null 2>&1 From 33eb2f6abb8c71fa1e61ed814d0a5e2600373255 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:52:19 -0400 Subject: [PATCH 57/65] Try to speed up content hashing --- .circleci/config.yml | 9 ++++----- build-system/scripts/calculate_content_hash | 21 +++++++-------------- scripts/ci/store_test_benchmark_logs | 6 +++--- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47e9909b82b..3506e409adb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1265,10 +1265,10 @@ workflows: - barretenberg-stdlib-recursion-turbo-tests - barretenberg-stdlib-recursion-ultra-tests - barretenberg-join-split-tests - filters: - branches: - only: - - master + # filters: + # branches: + # only: + # - master <<: *defaults - bb-js: requires: @@ -1458,4 +1458,3 @@ workflows: requires: - build-deployment-canary <<: *deploy_defaults - diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 1e7f19dd5a6..0ecc960521e 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -8,17 +8,10 @@ COMMIT_HASH=${2:-$COMMIT_HASH} # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) -git ls-tree -r ${COMMIT_HASH} | while read -r line; do - # an example line is - # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json - # this format is beneficial as it grabs the hashes from git efficiently - # we will next filter by our rebuild patterns - # then we pipe the hash portion of each file to git hash-object to produce our content hash - filepath=$(echo "$line" | awk '{print $4}') - for pattern in $REBUILD_PATTERNS; do - if [[ "$filepath" =~ $pattern ]]; then - echo "$line" - break - fi - done -done | git hash-object --stdin +AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') +# an example line is +# 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json +# this format is beneficial as it grabs the hashes from git efficiently +# we will next filter by our rebuild patterns +# then we pipe the hash portion of each file to git hash-object to produce our content hash +git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 2b36f14a4ea..44e1c9dbbeb 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -33,11 +33,11 @@ cat /tmp/csv/new.csv \ if [ -s /tmp/csv/trimmed.csv ]; then cd /tmp - git clone --depth 1 https://$AZTEC_GITHUB_TOKEN:@github.com/AztecProtocol/benchmark-archive + git clone --depth 1 git@github.aaakk.us.kg-logs:AztecProtocol/benchmark-archive.git cd benchmark-archive - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com + git config user.email "circleci@bot" + git config user.name "CircleCi Bot" cat /tmp/csv/trimmed.csv >>benchmarks.csv git add benchmarks.csv git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH" From 51c5e4bc39108da3aa390f1e0423b2fd540a0948 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:53:03 -0400 Subject: [PATCH 58/65] Revert "Revert "Remove debugging"" This reverts commit 6aabc6d26f9251415cd2aeb71f178adba733958d. --- bootstrap.sh | 2 +- bootstrap_docker.sh | 2 +- build-system/remote_build/remote_build | 2 +- build-system/scripts/build | 2 +- build-system/scripts/build_local | 2 +- build-system/scripts/check_npm_version | 2 +- build-system/scripts/check_rebuild | 2 +- build-system/scripts/cond_spot_run_build | 4 ++-- build-system/scripts/cond_spot_run_script | 2 +- build-system/scripts/cond_spot_run_test_script | 4 ++-- build-system/scripts/cond_spot_run_tests | 2 +- build-system/scripts/deploy | 2 +- build-system/scripts/deploy_dockerhub | 2 +- build-system/scripts/deploy_ecr | 2 +- build-system/scripts/deploy_global | 2 +- build-system/scripts/deploy_npm | 2 +- build-system/scripts/deploy_s3 | 2 +- build-system/scripts/deploy_service | 2 +- build-system/scripts/deploy_terraform | 2 +- build-system/scripts/ensure_apt_package | 2 +- build-system/scripts/ensure_repo | 2 +- build-system/scripts/ensure_terraform | 2 +- build-system/scripts/erase_image_tags | 2 +- build-system/scripts/extract_repo | 2 +- build-system/scripts/init_submodules | 2 +- build-system/scripts/query_manifest | 2 +- build-system/scripts/remote_run_script | 2 +- build-system/scripts/request_spot | 2 +- build-system/scripts/retry_10 | 2 +- build-system/scripts/setup_env | 2 +- build-system/scripts/spot_run_script | 2 +- build-system/scripts/spot_run_test_script | 4 ++-- build-system/scripts/spot_run_tests | 2 +- build-system/scripts/store_test_benchmark_logs | 2 +- build-system/scripts/tag_remote_image | 2 +- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 2 +- circuits/cpp/barretenberg/bootstrap_docker.sh | 2 +- circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/bootstrap.sh | 2 +- circuits/cpp/barretenberg/cpp/format.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh | 2 +- .../cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 2 +- circuits/cpp/barretenberg/cpp/scripts/run_tests | 2 +- circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh | 2 +- circuits/cpp/barretenberg/scripts/bindgen.sh | 2 +- circuits/cpp/barretenberg/sol/scripts/install_foundry.sh | 2 +- circuits/cpp/bootstrap.sh | 2 +- circuits/cpp/format.sh | 2 +- circuits/cpp/scripts/build_run_tests_docker_local | 2 +- circuits/cpp/scripts/run_coverage | 2 +- circuits/cpp/scripts/run_tests | 2 +- circuits/cpp/scripts/run_tests_local | 2 +- circuits/cpp/scripts/tidy.sh | 2 +- l1-contracts/bootstrap.sh | 2 +- l1-contracts/scripts/install_foundry.sh | 2 +- scripts/ci/store_test_benchmark_logs | 2 +- scripts/fix_subrepo_edge_case.sh | 2 +- scripts/git-subrepo/ext/bashplus/bin/bash+ | 2 +- scripts/git-subrepo/ext/bashplus/lib/bash+.bash | 2 +- scripts/git-subrepo/ext/bashplus/test/setup | 2 +- scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ | 2 +- .../ext/test-more-bash/ext/bashplus/lib/bash+.bash | 2 +- .../git-subrepo/ext/test-more-bash/ext/bashplus/test/setup | 2 +- .../ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash | 2 +- scripts/git-subrepo/ext/test-more-bash/test/setup | 2 +- scripts/git-subrepo/lib/git-subrepo | 2 +- scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash | 2 +- scripts/git-subrepo/note/init-test | 2 +- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 2 +- .../git-subrepo/note/subtree-rebase-fail-example/test.bash | 2 +- scripts/git-subrepo/note/test-subrepo-push.sh | 2 +- scripts/git-subrepo/pkg/bin/generate-help-functions.pl | 2 +- scripts/git_subrepo.sh | 2 +- scripts/migrate_barretenberg_branch.sh | 2 +- scripts/update.sh | 2 +- yarn-project/bootstrap.sh | 2 +- yarn-project/canary/scripts/run_tests | 2 +- yarn-project/canary/scripts/update_packages.sh | 2 +- yarn-project/end-to-end/scripts/cond_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 2 +- yarn-project/end-to-end/scripts/run_tests_local | 2 +- yarn-project/end-to-end/scripts/start_e2e.sh | 2 +- yarn-project/end-to-end/scripts/start_p2p_e2e.sh | 2 +- yarn-project/end-to-end/src/guides/up_quick_start.sh | 2 +- yarn-project/l1-artifacts/scripts/generate-artifacts.sh | 2 +- yarn-project/noir-contracts/scripts/install_noir.sh | 2 +- yarn-project/noir-contracts/scripts/install_noirup.sh | 2 +- yarn-project/noir-contracts/scripts/types.sh | 2 +- yarn-project/noir-contracts/src/scripts/compile.sh | 2 +- yarn-project/p2p-bootstrap/scripts/start_bootnode.sh | 2 +- 93 files changed, 96 insertions(+), 96 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 72729dd117f..62b93deb023 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu export CLEAN=${1:-} diff --git a/bootstrap_docker.sh b/bootstrap_docker.sh index b51bdf9a16e..a8025f08aa1 100755 --- a/bootstrap_docker.sh +++ b/bootstrap_docker.sh @@ -16,7 +16,7 @@ # cd yarn-project/end-to-end # ONLY_TARGET=false ../../bootstrap_docker.sh -set -xe +set -e TARGET_PROJECT=$1 COMMIT_HASH=$(git rev-parse HEAD) diff --git a/build-system/remote_build/remote_build b/build-system/remote_build/remote_build index 83e33bf356c..8eab556ee57 100755 --- a/build-system/remote_build/remote_build +++ b/build-system/remote_build/remote_build @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts diff --git a/build-system/scripts/build b/build-system/scripts/build index 24fd93587b2..4add19d1f1a 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -22,7 +22,7 @@ # - Perform the build of the image itself. With the cache primed we should only have to rebuild the necessary layers. # - Push the image tagged with the commit hash to the cache. -set -xeuo pipefail +set -euo pipefail REPOSITORY=$1 FORCE_BUILD=${2:-"false"} diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index 2215c8c45a9..bb1d6340844 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -6,7 +6,7 @@ # If DOCKERFILE is excluded it tries to default to ./Dockerfile then .//Dockerfile # If REPO is excluded it defaults to PROJECT_DIR_NAME. -set -xeu +set -eu TARGET_PROJECT=$1 ONLY_TARGET=$2 diff --git a/build-system/scripts/check_npm_version b/build-system/scripts/check_npm_version index 91bd924b9e5..0d7de47828e 100755 --- a/build-system/scripts/check_npm_version +++ b/build-system/scripts/check_npm_version @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu readonly LOCAL_VERSION=$(node -pe "require('./package.json').version") readonly PACKAGE_NAME=${1:-./} diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index e58eabd6ff2..d80e9493684 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -2,7 +2,7 @@ # Fails if any files matching the rebuild patterns, have changed since the base commit. # If this script fails (nonzero exit), then the caller should rebuild. # The rebuild patterns are taken from the build manifest (computed from set of dependencies). -set -xeuo pipefail +set -euo pipefail TAG=$1 REPOSITORY=$2 diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 7b4dc1552aa..61f262307e5 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -1,6 +1,6 @@ #!/bin/bash -set -xeu -set -xo pipefail +set -eu +set -o pipefail REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 8b17520078a..154b70c8b68 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a prefix after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Arguments to pass to spot_run_script. -set -xeu +set -eu REPOSITORY=$1 shift diff --git a/build-system/scripts/cond_spot_run_test_script b/build-system/scripts/cond_spot_run_test_script index 575f839465a..c760e69d00f 100755 --- a/build-system/scripts/cond_spot_run_test_script +++ b/build-system/scripts/cond_spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,5 +9,5 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -xo pipefail +set -o pipefail cond_spot_run_script $REPOSITORY $JOB_NAME 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/build-system/scripts/cond_spot_run_tests b/build-system/scripts/cond_spot_run_tests index c898c0700d2..78097379f1f 100755 --- a/build-system/scripts/cond_spot_run_tests +++ b/build-system/scripts/cond_spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xeu +set -eu cond_spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/deploy b/build-system/scripts/deploy index 0eacf7847da..ae6d92a69ba 100755 --- a/build-system/scripts/deploy +++ b/build-system/scripts/deploy @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 SERVICES=${2:-$REPOSITORY} diff --git a/build-system/scripts/deploy_dockerhub b/build-system/scripts/deploy_dockerhub index 09783f59ee1..d7e0442d30b 100755 --- a/build-system/scripts/deploy_dockerhub +++ b/build-system/scripts/deploy_dockerhub @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ -z "$COMMIT_TAG" ]; then echo "Will only push tagged builds to dockerhub. Skipping." diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr index 7c8f8dbe96b..aa512cc1b4b 100755 --- a/build-system/scripts/deploy_ecr +++ b/build-system/scripts/deploy_ecr @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 IMAGE_COMMIT_URI=$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH diff --git a/build-system/scripts/deploy_global b/build-system/scripts/deploy_global index e54afb05133..5114a067a74 100755 --- a/build-system/scripts/deploy_global +++ b/build-system/scripts/deploy_global @@ -1,6 +1,6 @@ #!/bin/bash # Deployment script for global service (e.g. company website and metrics). -set -xeu +set -eu REPOSITORY=$1 diff --git a/build-system/scripts/deploy_npm b/build-system/scripts/deploy_npm index 9df0e10d1a5..1847e2463d1 100755 --- a/build-system/scripts/deploy_npm +++ b/build-system/scripts/deploy_npm @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu readonly REPOSITORY=$1 readonly STANDALONE=$2 diff --git a/build-system/scripts/deploy_s3 b/build-system/scripts/deploy_s3 index cd076e59077..947062da206 100755 --- a/build-system/scripts/deploy_s3 +++ b/build-system/scripts/deploy_s3 @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 # A global deploy, doesn't namespace under a deploy tag (e.g. company website, metrics) diff --git a/build-system/scripts/deploy_service b/build-system/scripts/deploy_service index 34b5df87f1e..c126becdf56 100755 --- a/build-system/scripts/deploy_service +++ b/build-system/scripts/deploy_service @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Redeploy service with latest image. SERVICE_NAME=$1 diff --git a/build-system/scripts/deploy_terraform b/build-system/scripts/deploy_terraform index 1a813cc4591..637ef5c8be1 100755 --- a/build-system/scripts/deploy_terraform +++ b/build-system/scripts/deploy_terraform @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 TF_DIR=$2 diff --git a/build-system/scripts/ensure_apt_package b/build-system/scripts/ensure_apt_package index fe5d3d39f9f..33326c7f1fd 100755 --- a/build-system/scripts/ensure_apt_package +++ b/build-system/scripts/ensure_apt_package @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if dpkg -s $1 &> /dev/null; then exit 0 diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index d0f62dc3700..186b204cd4e 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -1,7 +1,7 @@ #!/bin/bash # Logs the shell into the ECR instance at the given region, establishes if the given repository exists, creates it if it # doesn't, and re-applies thie lifecycle policy (determines when images should be automatically deleted) if it does. -set -xeu +set -eu LIFECYCLE_POLICY='{ "rules": [ diff --git a/build-system/scripts/ensure_terraform b/build-system/scripts/ensure_terraform index d70f1d9cf84..e04231e79f1 100755 --- a/build-system/scripts/ensure_terraform +++ b/build-system/scripts/ensure_terraform @@ -1,6 +1,6 @@ #!/bin/bash # Downloads and installs `terraform` if it's not installed. -set -xeu +set -eu [ ! -f /usr/local/bin/terraform ] || exit 0 diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index a2bd8acb257..8f99c3f1569 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -1,7 +1,7 @@ #!/bin/bash # Erase the image tag associated with the last commit for the given repository. # If TILL_COMMIT_HASH is given, erase tags going back in time until we reach TILL_COMMIT_HASH. -set -xeu +set -eu REPOSITORY=$1 TILL_COMMIT_HASH=$2 diff --git a/build-system/scripts/extract_repo b/build-system/scripts/extract_repo index 1756b361811..8bd7ef2c606 100755 --- a/build-system/scripts/extract_repo +++ b/build-system/scripts/extract_repo @@ -1,6 +1,6 @@ #!/bin/bash # Given a repository, extracts the builds entire /usr/src dir to the given path. -set -xeu +set -eu REPOSITORY=$1 EXTRACT_FROM=${2:-/usr/src} diff --git a/build-system/scripts/init_submodules b/build-system/scripts/init_submodules index 1efa2db83f3..c456aa1a995 100755 --- a/build-system/scripts/init_submodules +++ b/build-system/scripts/init_submodules @@ -1,6 +1,6 @@ #!/bin/bash # For a given repository, init any required submodules. -set -xeuo pipefail +set -euo pipefail REPOSITORY=$1 diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index ad2b23b5668..ba7b8c1fe33 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu CMD=$1 REPO=$2 diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index a4efda7cf6a..c33d386f1f9 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu IP=$1 CONTENT_HASH=$2 diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index d756ad64898..d87fd829eee 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu NAME=$1 CPUS=${2:-32} diff --git a/build-system/scripts/retry_10 b/build-system/scripts/retry_10 index e0e499ba624..2d111c6e3dc 100755 --- a/build-system/scripts/retry_10 +++ b/build-system/scripts/retry_10 @@ -1,4 +1,4 @@ -set -xeu +set -eu # Retries up to 10 times with 10 second intervals for i in $(seq 1 10); do "$@" && exit || sleep 10 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 3284b4778fc..2df97fbfd5a 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,7 +6,7 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. -set -xeu +set -eu COMMIT_HASH=$1 COMMIT_TAG=$2 diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 094d03cedb7..63f16928c85 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -3,7 +3,7 @@ # 1. Content hash for our repository contents. Used for identify spot jobs and image tags. # 2. SPEC: Instance specification filename. # 3... ARGS: Arguments to pass to remote_run_script. -set -xeu +set -eu CONTENT_HASH=$1 SPEC=$2 shift 2 diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index b121bd65cbc..06d4d455380 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_PATH=$1 REPOSITORY=$2 shift @@ -9,7 +9,7 @@ cd $(query_manifest projectDir $REPOSITORY) mkdir -p /tmp/test-logs -set -xo pipefail +set -o pipefail CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" diff --git a/build-system/scripts/spot_run_tests b/build-system/scripts/spot_run_tests index e9b14c1fd88..e7d8183c048 100755 --- a/build-system/scripts/spot_run_tests +++ b/build-system/scripts/spot_run_tests @@ -1,4 +1,4 @@ #!/bin/bash -set -xeu +set -eu spot_run_test_script ./scripts/run_tests $@ diff --git a/build-system/scripts/store_test_benchmark_logs b/build-system/scripts/store_test_benchmark_logs index 9ac9ff71c04..d44f3c2cd9f 100755 --- a/build-system/scripts/store_test_benchmark_logs +++ b/build-system/scripts/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 shift diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 0bf67437019..741009ff125 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu REPOSITORY=$1 EXISTING_TAG=$2 diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index 5465cbcb8bd..67367756220 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -3,7 +3,7 @@ # BB: to specify a different binary to test with (e.g. bb.js or bb.js-dev). # VERBOSE: to enable logging for each test. -set -xeu +set -eu BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs diff --git a/circuits/cpp/barretenberg/bootstrap_docker.sh b/circuits/cpp/barretenberg/bootstrap_docker.sh index 3db07de7bc7..e2f88e22525 100755 --- a/circuits/cpp/barretenberg/bootstrap_docker.sh +++ b/circuits/cpp/barretenberg/bootstrap_docker.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script builds the projects listed in build_mainifest.sh. -set -xeu +set -eu COMMIT_HASH=$(git rev-parse HEAD) source ./build-system/scripts/setup_env $COMMIT_HASH '' mainframe_$USER $(git rev-parse --show-toplevel) diff --git a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh index ff4692b4758..eb634c96065 100755 --- a/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/bin-test/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/sh # Script is assumed to be run from -set -xeu +set -eu bb() { ../build/bin/bb "$@" -v diff --git a/circuits/cpp/barretenberg/cpp/bootstrap.sh b/circuits/cpp/barretenberg/cpp/bootstrap.sh index ef389491f40..869bbc9c885 100755 --- a/circuits/cpp/barretenberg/cpp/bootstrap.sh +++ b/circuits/cpp/barretenberg/cpp/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Clean. rm -rf ./build diff --git a/circuits/cpp/barretenberg/cpp/format.sh b/circuits/cpp/barretenberg/cpp/format.sh index f866bea9c15..48315e39621 100755 --- a/circuits/cpp/barretenberg/cpp/format.sh +++ b/circuits/cpp/barretenberg/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 66117d7dbf7..235aa111ff4 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # This script runs all test suites that have not been broken out into their own jobs for parallelisation. # Might be better to list exclusions here rather than inclusions as risky to maintain. -set -xeu +set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index c25434289ee..b06c26036ba 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -1,6 +1,6 @@ #!/bin/bash # Executes the bb binary test script. -set -xeu +set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh index 3dd76aa9c46..3a8eecf2438 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/install-wasi-sdk.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Clean. rm -rf ./src/wasi-sdk-* diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index a40e8b3c97e..d67176f098f 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 24c68e7467b..965307ff5a5 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -3,7 +3,7 @@ # 1. The number of ignition transcripts to download. # 2. The set of gtest binary names to run. # 3-n. The arguments to pass to the gtest binaries. -set -xeu +set -eu NUM_TRANSCRIPTS=$1 TESTS=$2 diff --git a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh index d9e1bcd7fe5..478f442f9af 100755 --- a/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh +++ b/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh @@ -13,7 +13,7 @@ # If a checksums file is available, it will be used to validate if a download is required # and also check the validity of the downloaded transcripts. If not the script downloads # whatever is requested but does not check the validity of the downloads. -set -xeu +set -eu mkdir -p ignition cd ignition diff --git a/circuits/cpp/barretenberg/scripts/bindgen.sh b/circuits/cpp/barretenberg/scripts/bindgen.sh index 43740d746e1..1a2034a213a 100755 --- a/circuits/cpp/barretenberg/scripts/bindgen.sh +++ b/circuits/cpp/barretenberg/scripts/bindgen.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu #find ./cpp/src -type f -name "c_bind*.hpp" | ./scripts/decls_json.py > exports.json cat ./scripts/c_bind_files.txt | ./scripts/decls_json.py > exports.json diff --git a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh index 02bcbf8db3b..e085beb76d5 100755 --- a/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh +++ b/circuits/cpp/barretenberg/sol/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -xeu +set -eu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/circuits/cpp/bootstrap.sh b/circuits/cpp/bootstrap.sh index b9f487e5678..0462e0085f6 100755 --- a/circuits/cpp/bootstrap.sh +++ b/circuits/cpp/bootstrap.sh @@ -1,7 +1,7 @@ #!/bin/bash # Takes a list of targets from commandline # Takes CLEAN as an environment variable. If passed, cleans build artifacts -set -xeu +set -eu export WASI_VERSION=20 diff --git a/circuits/cpp/format.sh b/circuits/cpp/format.sh index f866bea9c15..48315e39621 100755 --- a/circuits/cpp/format.sh +++ b/circuits/cpp/format.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu if [ "$1" == "staged" ]; then echo Formatting staged files... diff --git a/circuits/cpp/scripts/build_run_tests_docker_local b/circuits/cpp/scripts/build_run_tests_docker_local index fa64b29e330..a075cbba172 100755 --- a/circuits/cpp/scripts/build_run_tests_docker_local +++ b/circuits/cpp/scripts/build_run_tests_docker_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called only LOCALLY for testing WITH docker. # Builds a docker image and runs tests in it. diff --git a/circuits/cpp/scripts/run_coverage b/circuits/cpp/scripts/run_coverage index 7979f7d1c6c..d0e4bb097f4 100755 --- a/circuits/cpp/scripts/run_coverage +++ b/circuits/cpp/scripts/run_coverage @@ -1,5 +1,5 @@ #!bin/bash -set -xeu +set -eu # To be called LOCALLY for testing WITHOUT docker. # diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index eb8a2a28a6c..b3fbc43f459 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # To be called from CI for testing with docker and AWS. # Can't be called locally unless AWS credentials are set up. diff --git a/circuits/cpp/scripts/run_tests_local b/circuits/cpp/scripts/run_tests_local index a67649e471d..4c942a0f13a 100755 --- a/circuits/cpp/scripts/run_tests_local +++ b/circuits/cpp/scripts/run_tests_local @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu DIR="$(dirname "$0")" diff --git a/circuits/cpp/scripts/tidy.sh b/circuits/cpp/scripts/tidy.sh index 13e46f5469c..e239fc3b1a6 100755 --- a/circuits/cpp/scripts/tidy.sh +++ b/circuits/cpp/scripts/tidy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Run clang-tidy on all C++ source files # diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index cfe00c96988..58aba5449eb 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu cd "$(dirname "$0")" diff --git a/l1-contracts/scripts/install_foundry.sh b/l1-contracts/scripts/install_foundry.sh index 02bcbf8db3b..e085beb76d5 100755 --- a/l1-contracts/scripts/install_foundry.sh +++ b/l1-contracts/scripts/install_foundry.sh @@ -1,5 +1,5 @@ #!/bin/sh -set -xeu +set -eu export FOUNDRY_DIR="$PWD/.foundry" FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs index 44e1c9dbbeb..1d9350e6c98 100755 --- a/scripts/ci/store_test_benchmark_logs +++ b/scripts/ci/store_test_benchmark_logs @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu AZTEC_GITHUB_TOKEN=$1 diff --git a/scripts/fix_subrepo_edge_case.sh b/scripts/fix_subrepo_edge_case.sh index 7a236435868..6888875eaff 100755 --- a/scripts/fix_subrepo_edge_case.sh +++ b/scripts/fix_subrepo_edge_case.sh @@ -6,7 +6,7 @@ # in the face of e.g. a .gitrepo whitespace change, but it's a fallback, # we only have this issue in master, and the file should only be edited # generally by subrepo commands. -set -xeu +set -eu SUBREPO_PATH="${1:-}" echo "Auto-fixing squashed parent in $SUBREPO_PATH/.gitrepo." diff --git a/scripts/git-subrepo/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/bashplus/bin/bash+ index 57d3e19655c..e0bc1f73913 100755 --- a/scripts/git-subrepo/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xeu +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash index 16ea238245f..b52e17b55a6 100644 --- a/scripts/git-subrepo/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xeu +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/bashplus/test/setup b/scripts/git-subrepo/ext/bashplus/test/setup index a3535659a5a..c8b138d7e32 100644 --- a/scripts/git-subrepo/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -xe -o pipefail +set -e -o pipefail PATH=$PWD/bin:$PATH diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ index 57d3e19655c..e0bc1f73913 100755 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/bin/bash+ @@ -5,7 +5,7 @@ # Copyright (c) 2013-2020 Ingy döt Net #------------------------------------------------------------------------------ -set -xeu +set -eu shopt -s compat31 &>/dev/null || true #------------------------------------------------------------------------------ diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash index 28d8df1c78f..e0e15cfc49d 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/lib/bash+.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020 Ingy döt Net -set -xeu +set -eu [[ ${BASHPLUS_VERSION-} ]] && return 0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup index 176f989ef32..35124d0f69d 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/ext/bashplus/test/setup @@ -7,7 +7,7 @@ # how nice Bash can be. #------------------------------------------------------------------------------ -set -xe -u -o pipefail +set -e -u -o pipefail [[ $BASH_VERSION == 4.0* ]] && set +u run=0 diff --git a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash index a4f0d192449..cc34dcb9a2a 100644 --- a/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash +++ b/scripts/git-subrepo/ext/test-more-bash/ext/test-tap-bash/lib/test/tap.bash @@ -6,7 +6,7 @@ Test::Tap:die() { echo "$@" >&2; trap EXIT; exit 1; } #------------------------------------------------------------------------------ -set -xe -u -o pipefail +set -e -u -o pipefail [[ ${BASH_VERSION-} == 4.0* ]] && set +u # shellcheck disable=2034 diff --git a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash index 815dd498f95..6fa9f56d93f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash +++ b/scripts/git-subrepo/ext/test-more-bash/lib/test/more.bash @@ -2,7 +2,7 @@ # # Copyright (c) 2013-2020. Ingy döt Net. -set -xe -u -o pipefail +set -e -u -o pipefail # shellcheck disable=2034 Test__More_VERSION=0.0.5 diff --git a/scripts/git-subrepo/ext/test-more-bash/test/setup b/scripts/git-subrepo/ext/test-more-bash/test/setup index 1d910f3ceb1..99df35d614f 100644 --- a/scripts/git-subrepo/ext/test-more-bash/test/setup +++ b/scripts/git-subrepo/ext/test-more-bash/test/setup @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xe -u -o pipefail +set -e -u -o pipefail BASHLIB=$(find "$PWD" -type d | grep -E '/(bin|lib)$' | xargs -n1 printf "%s:") PATH=$BASHLIB:$PATH diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index 78ed42e0bc3..9f4a3384cd1 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -7,7 +7,7 @@ # shellcheck disable=1090,1091,2034 # Exit on any errors: -set -xeu +set -eu export FILTER_BRANCH_SQUELCH_WARNING=1 diff --git a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash index 1933b6eb280..98e34dd272b 100644 --- a/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash +++ b/scripts/git-subrepo/lib/git-subrepo.d/help-functions.bash @@ -2,7 +2,7 @@ # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xeu +set -eu help:all() { cat <<'...' diff --git a/scripts/git-subrepo/note/init-test b/scripts/git-subrepo/note/init-test index 4d4ef58690b..a20854de88a 100755 --- a/scripts/git-subrepo/note/init-test +++ b/scripts/git-subrepo/note/init-test @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex cat $0 # Show this script in the output REPO=${1:-ingydotnet/djson-pm} diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 5e9694a0999..62bbd9617c1 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -xeu +set -eu set -x # Make a directory to work in: diff --git a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash index ac77fe7361a..df8b818cee6 100755 --- a/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash +++ b/scripts/git-subrepo/note/subtree-rebase-fail-example/test.bash @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex rm -fr repo{1,2,3} mkdir repo{1,2} diff --git a/scripts/git-subrepo/note/test-subrepo-push.sh b/scripts/git-subrepo/note/test-subrepo-push.sh index 40160368c9e..afceb5efa92 100644 --- a/scripts/git-subrepo/note/test-subrepo-push.sh +++ b/scripts/git-subrepo/note/test-subrepo-push.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -xex +set -ex # Make a directory to work in: { diff --git a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl index b23a011ef5f..5dd949e6ed9 100644 --- a/scripts/git-subrepo/pkg/bin/generate-help-functions.pl +++ b/scripts/git-subrepo/pkg/bin/generate-help-functions.pl @@ -33,7 +33,7 @@ sub write_start { # DO NOT EDIT. This file generated by pkg/bin/generate-help-functions.pl. -set -xeu +set -eu ... } diff --git a/scripts/git_subrepo.sh b/scripts/git_subrepo.sh index 0651c7c7da5..df9743a2bd5 100755 --- a/scripts/git_subrepo.sh +++ b/scripts/git_subrepo.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu SCRIPT_DIR=$(dirname "$(realpath "$0")") diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh index d78935bf4fb..9f3001c7421 100755 --- a/scripts/migrate_barretenberg_branch.sh +++ b/scripts/migrate_barretenberg_branch.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Usage: ./this.sh # Script for migrating PRs from barretenberg repo to aztec-packages. diff --git a/scripts/update.sh b/scripts/update.sh index b374d765a14..ab78b6827dc 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Script for running after updating the working copy with remote changes. # Similar to bootstrap, but more lightweight and oriented towards working on end-to-end. diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index a2879ddef87..39036119ed7 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeu +set -eu # Navigate to script folder cd "$(dirname "$0")" diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index fc39a781c95..dac0f0bd921 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -eu export TEST=$1 export IMAGE=${2:-canary} diff --git a/yarn-project/canary/scripts/update_packages.sh b/yarn-project/canary/scripts/update_packages.sh index 608bcdb3035..0b5a3c59b43 100755 --- a/yarn-project/canary/scripts/update_packages.sh +++ b/yarn-project/canary/scripts/update_packages.sh @@ -1,4 +1,4 @@ -set -xeu +set -eu COMMIT_TAG=$1 diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 96f9463f8cb..e8159655bf2 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -14,7 +14,7 @@ # 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a # successful run. The script will only run if there were relevant code changes since the last successful commit. # 3... ARGS: Script to run and args. -set -xeu +set -eu set -x REPOSITORY=$1 diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index ba874d060cc..fad024b1eb2 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -xeu +set -eu export TEST=$1 export COMPOSE_FILE=${2:-docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local index 918f6241505..e7736622417 100755 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ b/yarn-project/end-to-end/scripts/run_tests_local @@ -1,7 +1,7 @@ #!/bin/bash # This script is used to run an e2e test in CI (see config.yml and cond_run_script). # It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -set -e +set -eu export TEST=$1 export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} diff --git a/yarn-project/end-to-end/scripts/start_e2e.sh b/yarn-project/end-to-end/scripts/start_e2e.sh index 45f6d13800a..018c3d79603 100755 --- a/yarn-project/end-to-end/scripts/start_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu export NODE_NO_WARNINGS=1 node ${NODE_ARGS-} --openssl-legacy-provider --experimental-vm-modules $(yarn bin jest) --no-cache --runInBand --passWithNoTests $@ diff --git a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh index ed24c853826..9b1b661a7fe 100755 --- a/yarn-project/end-to-end/scripts/start_p2p_e2e.sh +++ b/yarn-project/end-to-end/scripts/start_p2p_e2e.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu export DEBUG='aztec:*' export ARCHIVER_POLLING_INTERVAL=500 export P2P_CHECK_INTERVAL=50 diff --git a/yarn-project/end-to-end/src/guides/up_quick_start.sh b/yarn-project/end-to-end/src/guides/up_quick_start.sh index 85012b10027..c27ff3454f3 100755 --- a/yarn-project/end-to-end/src/guides/up_quick_start.sh +++ b/yarn-project/end-to-end/src/guides/up_quick_start.sh @@ -1,7 +1,7 @@ # Run locally from end-to-end folder while running anvil and sandbox with: # PATH=$PATH:../node_modules/.bin ./src/guides/up_quick_start.sh -set -xeux +set -eux # docs:start:declare-accounts ALICE="0x2e13f0201905944184fc2c09d29fcf0cac07647be171656a275f63d99b819360" diff --git a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh index 86b8bb95708..1c4bd5e2ba9 100755 --- a/yarn-project/l1-artifacts/scripts/generate-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/generate-artifacts.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeuo pipefail; +set -euo pipefail; target_dir=./generated diff --git a/yarn-project/noir-contracts/scripts/install_noir.sh b/yarn-project/noir-contracts/scripts/install_noir.sh index 8021c0685ca..c0c9d70d729 100755 --- a/yarn-project/noir-contracts/scripts/install_noir.sh +++ b/yarn-project/noir-contracts/scripts/install_noir.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest aztec nargo -set -xeu +set -eu VERSION="aztec" diff --git a/yarn-project/noir-contracts/scripts/install_noirup.sh b/yarn-project/noir-contracts/scripts/install_noirup.sh index efa83d5b668..11ba9b15d31 100755 --- a/yarn-project/noir-contracts/scripts/install_noirup.sh +++ b/yarn-project/noir-contracts/scripts/install_noirup.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to install noirup and the latest nargo -set -xeu +set -eu SPECIFIED_HOME=${1:-$HOME} diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 9f499a4e0d7..a9d249c437f 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -8,7 +8,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), -set -xeu; +set -eu; artifacts_dir="src/artifacts" types_dir="src/types" diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 384c1d0d537..74719291c65 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -9,7 +9,7 @@ # Enable strict mode: # Exit on error (set -xe), treat unset variables as an error (set -u), # and propagate the exit status of the first failing command in a pipeline (set -o pipefail). -set -xeuo pipefail; +set -euo pipefail; # Run build scripts ./scripts/compile.sh "$@" diff --git a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh index 7621979ab44..2abace6942d 100755 --- a/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh +++ b/yarn-project/p2p-bootstrap/scripts/start_bootnode.sh @@ -1,5 +1,5 @@ #! /bin/bash -set -xeu +set -eu cd .. export P2P_TCP_LISTEN_PORT=40400 From 1e3ac2092753f68645a2063d34acb69d834d86ea Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 16:57:30 -0400 Subject: [PATCH 59/65] Use content hash in erase_image_tags --- build-system/scripts/erase_image_tags | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build-system/scripts/erase_image_tags b/build-system/scripts/erase_image_tags index 8f99c3f1569..4c61679f835 100755 --- a/build-system/scripts/erase_image_tags +++ b/build-system/scripts/erase_image_tags @@ -10,14 +10,14 @@ if [ -n "$3" ]; then TAG_POSTFIX=-$3 fi -# TODO adapt to call calculate_content_hash and -# for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do -# TAG=cache-${COMMIT_HASH}$TAG_POSTFIX -# if image_exists $REPOSITORY $TAG; then -# echo "Erasing image tag: $REPOSITORY:$TAG" -# aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null -# if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then -# break -# fi -# fi -# done +for COMMIT_HASH in $(git log -n 50 --pretty=format:"%H"); do + CONTENT_HASH=$(calculate_content_hash $REPOSITORY $COMMIT_HASH) + TAG=cache-${CONTENT_HASH}$TAG_POSTFIX + if image_exists $REPOSITORY $TAG; then + echo "Erasing image tag: $REPOSITORY:$TAG" + aws ecr batch-delete-image --region=$ECR_REGION --repository-name $1 --image-ids imageTag=$TAG > /dev/null + if [ -z "$TILL_COMMIT_HASH" -o "$COMMIT_HASH" = "$TILL_COMMIT_HASH" ]; then + break + fi + fi +done From 6326c5fe309a786b8a11ebc41ac511d7cb0277cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:31:01 -0400 Subject: [PATCH 60/65] Reinstate master for aggregator --- .circleci/config.yml | 8 ++++---- build_manifest.json | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3506e409adb..700f39f1f37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1265,10 +1265,10 @@ workflows: - barretenberg-stdlib-recursion-turbo-tests - barretenberg-stdlib-recursion-ultra-tests - barretenberg-join-split-tests - # filters: - # branches: - # only: - # - master + filters: + branches: + only: + - master <<: *defaults - bb-js: requires: diff --git a/build_manifest.json b/build_manifest.json index 84a51991e9d..c765b5a68d8 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -126,7 +126,9 @@ "buildDir": ".", "dockerfile": "docs/Dockerfile", "rebuildPatterns": [ - "^." + "^docs/", + "^.*\\.cpp", + "^.*\\.ts" ], "dependencies": [] }, From 9d2250a88f6be2ebd34dc433564eea9ecfc7a23c Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:35:07 -0400 Subject: [PATCH 61/65] self-review --- scripts/git-subrepo/note/recreate-rebase-conflict.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git-subrepo/note/recreate-rebase-conflict.sh b/scripts/git-subrepo/note/recreate-rebase-conflict.sh index 62bbd9617c1..8e1103e575d 100644 --- a/scripts/git-subrepo/note/recreate-rebase-conflict.sh +++ b/scripts/git-subrepo/note/recreate-rebase-conflict.sh @@ -43,7 +43,7 @@ set -x # commit: git rebase master subrepo-fake || true - # Command fails on conflict but `|| true` prevents `set -xe` from stopping + # Command fails on conflict but `|| true` prevents `set -e` from stopping # here. # Show the current branch status: From 6b09e024796c239dcdfdf2b7c58d31cb7b3b1e6d Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:35:44 -0400 Subject: [PATCH 62/65] self-review --- yarn-project/noir-contracts/scripts/types.sh | 2 +- yarn-project/noir-contracts/src/scripts/compile.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index a9d249c437f..65fb9ccffce 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -7,7 +7,7 @@ # - you can run `yarn noir:types:all` to create all noir artifacts and types consumed by aztec packages. # Enable strict mode: -# Exit on error (set -xe), treat unset variables as an error (set -u), +# Exit on error (set -e), treat unset variables as an error (set -u), set -eu; artifacts_dir="src/artifacts" diff --git a/yarn-project/noir-contracts/src/scripts/compile.sh b/yarn-project/noir-contracts/src/scripts/compile.sh index 74719291c65..3edb7482cad 100755 --- a/yarn-project/noir-contracts/src/scripts/compile.sh +++ b/yarn-project/noir-contracts/src/scripts/compile.sh @@ -7,7 +7,7 @@ # yarn noir:build private_token ecdsa_account # Enable strict mode: -# Exit on error (set -xe), treat unset variables as an error (set -u), +# Exit on error (set -e), treat unset variables as an error (set -u), # and propagate the exit status of the first failing command in a pipeline (set -o pipefail). set -euo pipefail; From 6607e004e18ad14d31eccbbb7cad874dc37376c1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:36:51 -0400 Subject: [PATCH 63/65] fix build patterns in docs --- build_manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_manifest.json b/build_manifest.json index c765b5a68d8..b26d813c3e6 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -127,8 +127,8 @@ "dockerfile": "docs/Dockerfile", "rebuildPatterns": [ "^docs/", - "^.*\\.cpp", - "^.*\\.ts" + "^.*.cpp^", + "^.*.ts^" ], "dependencies": [] }, From 2e8f98946ccf76fd90be178a948aba789826ab98 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:38:28 -0400 Subject: [PATCH 64/65] fix build patterns in docs --- build_manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_manifest.json b/build_manifest.json index b26d813c3e6..c4b3b24cdac 100644 --- a/build_manifest.json +++ b/build_manifest.json @@ -127,8 +127,8 @@ "dockerfile": "docs/Dockerfile", "rebuildPatterns": [ "^docs/", - "^.*.cpp^", - "^.*.ts^" + "^.*.cpp$", + "^.*.ts$" ], "dependencies": [] }, From d53ae4527c2c8fa377df01244f1c303926656a45 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sun, 3 Sep 2023 17:39:51 -0400 Subject: [PATCH 65/65] Remove debugging from fixed content hashing --- scripts/git-subrepo/lib/git-subrepo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git-subrepo/lib/git-subrepo b/scripts/git-subrepo/lib/git-subrepo index 9f4a3384cd1..8ee1621915c 100755 --- a/scripts/git-subrepo/lib/git-subrepo +++ b/scripts/git-subrepo/lib/git-subrepo @@ -1902,7 +1902,7 @@ RUN() { fi fi rc=$? - set -xeu + set -eu if [[ $rc -ne 0 ]]; then OK=false