From a63423cc0c0a3c854316eca03efe2e78dd9308fc Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 3 Oct 2023 15:21:00 +0000 Subject: [PATCH 1/4] Remove sandbox base image and force_deploy_build. Generalise in check_rebuild. --- .circleci/config.yml | 108 +++++++++----------- build-system/scripts/check_rebuild | 14 ++- build-system/scripts/force_deploy_build | 29 ------ build_manifest.yml | 11 +- yarn-project/aztec-sandbox/Dockerfile.final | 10 -- 5 files changed, 64 insertions(+), 108 deletions(-) delete mode 100755 build-system/scripts/force_deploy_build delete mode 100644 yarn-project/aztec-sandbox/Dockerfile.final diff --git a/.circleci/config.yml b/.circleci/config.yml index 96343056735..8e9c9099cd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -417,17 +417,6 @@ jobs: name: Test command: cond_spot_run_container yarn-project 64 test | add_timestamps - aztec-sandbox-base: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large - steps: - - *checkout - - *setup_env - - run: - name: "Build and test" - command: force_deploy_build aztec-sandbox-base false - aztec-sandbox-x86_64: machine: image: ubuntu-2204:2023.07.2 @@ -437,7 +426,7 @@ jobs: - *setup_env - run: name: "Build and test" - command: force_deploy_build aztec-sandbox false x86_64 + command: build aztec-sandbox false x86_64 aztec-sandbox-arm64: machine: @@ -452,7 +441,7 @@ jobs: # out to eu-west2 despite the image being locally tagged, resulting in unauthorised 401. Weird docker bug? command: | echo "export DOCKER_BUILDKIT=" > $BASH_ENV - force_deploy_build aztec-sandbox false arm64 + build aztec-sandbox false arm64 aztec-sandbox-ecr-manifest: machine: @@ -530,7 +519,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_2_pxes.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-multiple-accounts-1-enc-key: machine: @@ -542,7 +531,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-deploy-contract: machine: @@ -554,7 +543,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_deploy_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-lending-contract: machine: @@ -566,7 +555,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_lending_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-token-contract: machine: @@ -578,7 +567,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_token_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-private-airdrop: machine: @@ -590,7 +579,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_private_airdrop.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-private-token-contract: machine: @@ -602,7 +591,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_private_token_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-sandbox-example: machine: @@ -614,7 +603,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_sandbox_example.test.ts ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-multi-transfer-contract: machine: @@ -626,7 +615,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multi_transfer.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-block-building: machine: @@ -638,7 +627,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_block_building.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-nested-contract: machine: @@ -650,7 +639,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_nested_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-non-contract-account: machine: @@ -662,7 +651,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_non_contract_account.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-cross-chain-messaging: machine: @@ -674,7 +663,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-public-cross-chain-messaging: machine: @@ -686,7 +675,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-public-to-private-messaging: machine: @@ -698,7 +687,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-account-contracts: machine: @@ -710,7 +699,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_account_contracts.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-escrow-contract: machine: @@ -722,7 +711,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_escrow_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-pending-commitments-contract: machine: @@ -734,7 +723,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-ordering: machine: @@ -746,7 +735,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_ordering.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } uniswap-trade-on-l1-from-l2: machine: @@ -758,7 +747,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } integration-archiver-l1-to-l2: machine: @@ -770,7 +759,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } integration-l1-publisher: machine: @@ -782,7 +771,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local integration_l1_publisher.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-public-token-contract: machine: @@ -794,7 +783,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_token_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-cli: machine: @@ -806,7 +795,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cli.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-p2p: machine: @@ -818,7 +807,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_p2p_network.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-browser-sandbox: machine: @@ -830,7 +819,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-e2e-sandbox-browser.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-card-game: machine: @@ -842,7 +831,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local e2e_card_game.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } pxe-sandbox: machine: @@ -854,7 +843,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local pxe_sandbox.test.ts ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } cli-docs-sandbox: machine: @@ -866,7 +855,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local cli_docs_sandbox.test.ts ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } guides-writing-an-account-contract: machine: @@ -878,7 +867,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local guides/writing_an_account_contract.test.ts - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } guides-dapp-testing: machine: @@ -890,7 +879,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local guides/dapp_testing.test.ts ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } guides-sample-dapp: machine: @@ -902,7 +891,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local sample-dapp ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } guides-up-quick-start: machine: @@ -914,7 +903,7 @@ jobs: - run: name: "Test" command: cond_run_script end-to-end ./scripts/run_tests_local guides/up_quick_start.test.ts ./scripts/docker-compose-e2e-sandbox.yml - environment: { DEBUG: 'aztec:*' } + environment: { DEBUG: "aztec:*" } e2e-canary-test: machine: @@ -937,7 +926,7 @@ jobs: - run: name: "Benchmark" command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts - environment: { DEBUG: 'aztec:benchmarks:*,aztec:sequencer' } + environment: { DEBUG: "aztec:benchmarks:*,aztec:sequencer" } build-docs: machine: @@ -1087,10 +1076,6 @@ deploy_defaults: &deploy_defaults - slack/notify: event: fail branch_pattern: "master" -yarn_project: &yarn_project - requires: - - yarn-project - <<: *defaults e2e_test: &e2e_test requires: - e2e-join @@ -1199,20 +1184,27 @@ workflows: - yarn-project <<: *defaults - - end-to-end: *yarn_project - - aztec-sandbox-base: *yarn_project - - canary: *yarn_project - - build-docs: *yarn_project + - end-to-end: + requires: + - yarn-project + <<: *defaults + - canary: + requires: + - yarn-project + <<: *defaults + - build-docs: + requires: + - yarn-project + <<: *defaults - aztec-sandbox-x86_64: requires: - - aztec-sandbox-base + - yarn-project <<: *defaults - aztec-sandbox-arm64: requires: - - aztec-sandbox-base + - yarn-project <<: *defaults - - aztec-sandbox-ecr-manifest: requires: - aztec-sandbox-x86_64 diff --git a/build-system/scripts/check_rebuild b/build-system/scripts/check_rebuild index 22d351b1575..6e4e4b3384c 100755 --- a/build-system/scripts/check_rebuild +++ b/build-system/scripts/check_rebuild @@ -13,8 +13,18 @@ REPOSITORY=$2 [[ "$COMMIT_MESSAGE" == *"[ci rebuild $REPOSITORY]"* ]] && exit 1 [[ "$COMMIT_MESSAGE" == *"[ci rebuild]"* ]] && exit 1 +COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY) +DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) + +# If we are deploying, and our Dockerfile uses the COMMIT_TAG arg, we need to rebuild to ensure the latest version +# number is baked into the image. +# TODO: This should probably be properly handled by e.g. release-please bumping the version number as part of its PR +# which would invalidate the content hash, rather than us using Dockerfile ARGS. +if [ -n "$COMMIT_TAG_VERSION" ] && grep -qE "^ARG COMMIT_TAG=" $DOCKERFILE; then + exit 1 +fi + +# If the image doesn't exist, we need to rebuild. if ! image_exists $REPOSITORY $TAG; then exit 1 -else - exit 0 fi diff --git a/build-system/scripts/force_deploy_build b/build-system/scripts/force_deploy_build deleted file mode 100755 index 894efd48acb..00000000000 --- a/build-system/scripts/force_deploy_build +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# -# This script forces a build if we're in a deployment run, otherwise runs `build` as usual -# -# usage: ./deploy_force_build -# example: ./deploy_force_build aztec-sandbox true -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -e - -REPOSITORY=$1 -shift -FORCE_BUILD=$1 -shift - -# if FORCE_BUILD is already set, just continue with it -if [[ $FORCE_BUILD == 'true' ]]; then - build $REPOSITORY true $@ - exit 0 -fi - -COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) - -if [ -z "$COMMIT_TAG_VERSION" ]; then - # Not a version tag, build normally - build $REPOSITORY false $@ -else - # Force build - build $REPOSITORY true $@ -fi diff --git a/build_manifest.yml b/build_manifest.yml index bdb1f1361e9..d2f0286cb72 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -107,18 +107,11 @@ yarn-project: - yarn-project-base - noir-contracts-build -aztec-sandbox-base: - buildDir: yarn-project - projectDir: yarn-project/aztec-sandbox - dependencies: - - yarn-project - aztec-sandbox: buildDir: yarn-project projectDir: yarn-project/aztec-sandbox - dockerfile: Dockerfile.final dependencies: - - aztec-sandbox-base + - yarn-project boxes-blank-react: buildDir: yarn-project @@ -168,4 +161,4 @@ docs: - ^.*/noir-version.json$ - ^.*.nr$ dependencies: - - yarn-project \ No newline at end of file + - yarn-project diff --git a/yarn-project/aztec-sandbox/Dockerfile.final b/yarn-project/aztec-sandbox/Dockerfile.final deleted file mode 100644 index a9cc84909f1..00000000000 --- a/yarn-project/aztec-sandbox/Dockerfile.final +++ /dev/null @@ -1,10 +0,0 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec-sandbox-base AS builder - -FROM node:18-alpine - -COPY --from=builder /usr/src/ /usr/src/ -WORKDIR /usr/src/yarn-project/aztec-sandbox - -ENTRYPOINT ["yarn"] -CMD [ "start" ] -EXPOSE 8080 \ No newline at end of file From 802f82b68175411db57d52d5649ae14694bb11ed Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 3 Oct 2023 19:00:48 +0000 Subject: [PATCH 2/4] I see why we had a base now. This should work though. --- yarn-project/aztec-sandbox/Dockerfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/yarn-project/aztec-sandbox/Dockerfile b/yarn-project/aztec-sandbox/Dockerfile index 3f3a2e6cb3d..2a9c82d0869 100644 --- a/yarn-project/aztec-sandbox/Dockerfile +++ b/yarn-project/aztec-sandbox/Dockerfile @@ -1,13 +1,19 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project ARG COMMIT_TAG="" -# Update pxe version if COMMIT_TAG has been used +# Start a new image as we need arch specific. +FROM node:18-alpine as builder +RUN apk add jq +COPY --from=yarn-project /usr/src/ /usr/src/ + +# Update pxe version if COMMIT_TAG has been used. WORKDIR /usr/src/yarn-project/pxe RUN if [[ -n "${COMMIT_TAG}" ]]; then \ jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ fi +# Update sandbox version if COMMIT_TAG has been used. WORKDIR /usr/src/yarn-project/aztec-sandbox RUN if [[ -n "${COMMIT_TAG}" ]]; then \ jq --arg v ${COMMIT_TAG} '.version = $v' package.json > _temp && mv _temp package.json; \ From b402dd4a8e0881a6cc1ce59882b17a1965bf06fc Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 3 Oct 2023 19:48:17 +0000 Subject: [PATCH 3/4] Fix. --- .circleci/config.yml | 2 +- build-system/scripts/calculate_image_uri | 5 +---- build-system/scripts/create_ecr_manifest | 20 ++++++++------------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e9c9099cd5..0a71f85677f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -452,7 +452,7 @@ jobs: - *setup_env - run: name: "Create ECR manifest" - command: create_ecr_manifest aztec-sandbox-base aztec-sandbox x86_64,arm64 + command: create_ecr_manifest aztec-sandbox x86_64,arm64 boxes-blank-react: machine: diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index 38038a88319..bb276ed6187 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -3,8 +3,5 @@ set -eu REPOSITORY=$1 -# Assume we might be calling this directly without build system in path -export ROOT_PATH=$(git rev-parse --show-toplevel) -export PATH="$PATH:$ROOT_PATH/build-system/scripts:$ROOT_PATH/build-system/bin" CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +echo "$ECR_URL/$REPOSITORY:cache-$CONTENT_HASH" diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index a020ddcfd2a..52711f24afe 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -3,24 +3,21 @@ # 1. Logs into ECR and ensures we have the given repository # 2. Computes the image uri of the cached images for the given repository given the list of architectures # 3. Creates a manifest list using a platform agnositc image uri, adds each image to it -# 4. Pushes the manifest list +# 4. Pushes the manifest list [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -e REPOSITORY=$1 -FINAL_IMAGE_NAME=$2 -ARCH_LIST=$3 +ARCH_LIST=$2 # Ensure ECR repository exists. retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +IMAGE_URI=$(calculate_image_uri $REPOSITORY) +echo "Image URI: $IMAGE_URI" -FINAL=$ECR_URL/$FINAL_IMAGE_NAME:cache-$CONTENT_HASH - -echo "Creating manifest list $FINAL..." +echo "Creating manifest list..." export DOCKER_CLI_EXPERIMENTAL=enabled @@ -28,10 +25,9 @@ OLD_IFS=$IFS IFS=',' for A in $ARCH_LIST do - IMAGE=$ECR_URL/$FINAL_IMAGE_NAME:cache-$CONTENT_HASH-$A - echo "Adding image $IMAGE to manifest list" - docker manifest create $FINAL \ - --amend $IMAGE + ARCH_IMAGE=$IMAGE_URI-$A + echo "Adding image $ARCH_IMAGE to manifest list." + docker manifest create $IMAGE_URI --amend $ARCH_IMAGE done IFS=$OLD_IFS unset OLD_IFS From dc6bb7c42696b0c786ee93ed465fb3b5b73726e1 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 3 Oct 2023 19:52:14 +0000 Subject: [PATCH 4/4] Fix. --- build-system/scripts/create_ecr_manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index 52711f24afe..44d887a190f 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -6,7 +6,7 @@ # 4. Pushes the manifest list [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -e +set -eu REPOSITORY=$1 ARCH_LIST=$2 @@ -32,4 +32,4 @@ done IFS=$OLD_IFS unset OLD_IFS -docker manifest push --purge $FINAL +docker manifest push --purge $IMAGE_URI