From 1e3812591c7bbad0a1af7bb7237ddea145a55b83 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 17:22:34 +0000 Subject: [PATCH 01/29] Build nargo against Ubuntu 20 for better compatability --- noir/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noir/Dockerfile b/noir/Dockerfile index 000292e0a47..bd509450287 100644 --- a/noir/Dockerfile +++ b/noir/Dockerfile @@ -1,10 +1,10 @@ -FROM rust:bookworm +FROM rust:bullseye WORKDIR /usr/src/noir COPY . . RUN ./scripts/bootstrap_native.sh # When running the container, mount the users home directory to same location. -FROM ubuntu:lunar +FROM ubuntu:focal # Install Tini as nargo doesn't handle signals properly. # Install git as nargo needs it to clone. RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean From 31cf1a9a2abca62156fac7e9396c17050ad0e558 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 17:58:01 +0000 Subject: [PATCH 02/29] retry a docker create thing. --- build-system/scripts/create_ecr_manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index 0495c3c165b..4e258f59df3 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -24,7 +24,7 @@ IFS=',' for A in $ARCH_LIST; do IMAGE_URI=$(calculate_image_uri $REPOSITORY $A) echo "Adding image $IMAGE_URI to manifest list $MULTIARCH_IMAGE_URI..." - docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI + retry docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI done retry docker manifest push --purge $MULTIARCH_IMAGE_URI From a99b05d4aa73af9b1e287a889b90902a050c1b12 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 18:38:54 +0000 Subject: [PATCH 03/29] We don't deploy to ecr anymore, only dockerhub. Use public ecr for build images. --- barretenberg/acir_tests/Dockerfile.bb | 6 +-- barretenberg/acir_tests/Dockerfile.bb.js | 4 +- barretenberg/acir_tests/Dockerfile.bb.sol | 6 +-- .../acir_tests/Dockerfile.noir_acir_tests | 2 +- barretenberg/cpp/dockerfiles/Dockerfile.bench | 2 +- barretenberg/ts/Dockerfile | 2 +- boxes/Dockerfile | 6 +-- build-system/scripts/build | 2 +- build-system/scripts/build_local | 6 +-- build-system/scripts/cond_run_compose | 1 - build-system/scripts/deploy_ecr | 39 ------------------- build-system/scripts/ecr_login | 2 +- build-system/scripts/remote_run_script | 3 +- build-system/scripts/setup_env | 3 +- docs/Dockerfile | 2 +- noir-projects/Dockerfile | 4 +- yarn-project/Dockerfile | 2 +- yarn-project/Dockerfile.prod | 2 +- yarn-project/aztec-faucet/Dockerfile | 2 +- yarn-project/aztec/Dockerfile | 2 +- yarn-project/cli/Dockerfile | 2 +- yarn-project/end-to-end/Dockerfile | 2 +- .../scripts/docker-compose-bootstrap.yml | 2 +- yarn-project/yarn-project-base/Dockerfile | 10 ++--- .../Dockerfile.dockerignore.v24 | 2 +- 25 files changed, 35 insertions(+), 81 deletions(-) delete mode 100755 build-system/scripts/deploy_ecr diff --git a/barretenberg/acir_tests/Dockerfile.bb b/barretenberg/acir_tests/Dockerfile.bb index 006db0e5333..4369c5f1033 100644 --- a/barretenberg/acir_tests/Dockerfile.bb +++ b/barretenberg/acir_tests/Dockerfile.bb @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0-alpine RUN apk update && apk add git bash curl jq coreutils @@ -10,7 +10,7 @@ COPY . . # Run every acir test through native bb build prove_then_verify flow for UltraPlonk. # This ensures we test independent pk construction through real/garbage witness data paths. RUN FLOW=prove_then_verify ./run_acir_tests.sh -# This flow is essentially the GoblinUltraHonk equivalent to the UltraPlonk "prove and verify". (This functionality is +# This flow is essentially the GoblinUltraHonk equivalent to the UltraPlonk "prove and verify". (This functionality is # accessed via the goblin "accumulate" mechanism). RUN FLOW=accumulate_and_verify_goblin ./run_acir_tests.sh # This is a "full" Goblin flow. It constructs and verifies four proofs: GoblinUltraHonk, ECCVM, Translator, and merge diff --git a/barretenberg/acir_tests/Dockerfile.bb.js b/barretenberg/acir_tests/Dockerfile.bb.js index b894826a5c2..ec6bac811da 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.js +++ b/barretenberg/acir_tests/Dockerfile.bb.js @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/bb.js +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0 COPY --from=0 /usr/src/barretenberg/ts-build /usr/src/barretenberg/ts diff --git a/barretenberg/acir_tests/Dockerfile.bb.sol b/barretenberg/acir_tests/Dockerfile.bb.sol index fe87fffc51e..dddb7e122f9 100644 --- a/barretenberg/acir_tests/Dockerfile.bb.sol +++ b/barretenberg/acir_tests/Dockerfile.bb.sol @@ -1,6 +1,6 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-sol -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-compile-acir-tests as noir-acir-tests +FROM aztecprotocol/barretenberg-x86_64-linux-clang-assert +FROM aztecprotocol/barretenberg-x86_64-linux-clang-sol +FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests FROM node:18.19.0-alpine RUN apk update && apk add git bash curl jq diff --git a/barretenberg/acir_tests/Dockerfile.noir_acir_tests b/barretenberg/acir_tests/Dockerfile.noir_acir_tests index 5131e94e279..8fd78de30dc 100644 --- a/barretenberg/acir_tests/Dockerfile.noir_acir_tests +++ b/barretenberg/acir_tests/Dockerfile.noir_acir_tests @@ -2,7 +2,7 @@ # However, the noir subrepo has no concept of the aztec build pipeline, so the Dockerfile wouldn't make sense there. # So, it lives here. # This chains off the nargo build, and creates a container with a compiled set of acir tests. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir +FROM aztecprotocol/noir RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean ENV PATH="/usr/src/noir/target/release:${PATH}" WORKDIR /usr/src/noir/test_programs diff --git a/barretenberg/cpp/dockerfiles/Dockerfile.bench b/barretenberg/cpp/dockerfiles/Dockerfile.bench index 0535d2fb3a9..42226685814 100644 --- a/barretenberg/cpp/dockerfiles/Dockerfile.bench +++ b/barretenberg/cpp/dockerfiles/Dockerfile.bench @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang +FROM aztecprotocol/barretenberg-x86_64-linux-clang WORKDIR /usr/src/barretenberg/cpp RUN apk update && apk add curl libstdc++ jq RUN ./scripts/ci/ultra_honk_bench.sh diff --git a/barretenberg/ts/Dockerfile b/barretenberg/ts/Dockerfile index d28b5661b2d..855dc17be37 100644 --- a/barretenberg/ts/Dockerfile +++ b/barretenberg/ts/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-wasm-linux-clang +FROM aztecprotocol/barretenberg-wasm-linux-clang FROM node:18.19.0-alpine COPY --from=0 /usr/src/barretenberg /usr/src/barretenberg diff --git a/boxes/Dockerfile b/boxes/Dockerfile index 9241170b544..573511a4181 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -1,8 +1,8 @@ # Builds the boxes (they were copied into yarn-project-base so the cli can unbox). # Produces a container that can be run to test a specific box. See docker-compose.yml. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec AS aztec -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects +FROM aztecprotocol/aztec AS aztec +FROM aztecprotocol/noir as noir +FROM aztecprotocol/noir-projects as noir-projects # We need yarn. Start fresh container. FROM node:18.19.0 diff --git a/build-system/scripts/build b/build-system/scripts/build index c82e6bde0a6..c3adc4a3f1b 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -109,7 +109,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do fi # Substitute references to parent repo, with the relevent built image uri. - DEPLOY_URI=$ECR_DEPLOY_URL/$PARENT_REPO + DEPLOY_URI=aztecprotocol/$PARENT_REPO PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE done diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index d32d70eb349..51315797390 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -64,8 +64,7 @@ for E in "${PROJECTS[@]}"; do fi fi - # TODO: Move everything to aztecprotocol/* - DEPLOY_IMAGE_URI=$ECR_DEPLOY_URL/$REPO:latest + DEPLOY_IMAGE_URI=aztecprotocol/$REPO:latest CACHE_IMAGE_URI=$(calculate_image_uri $REPO) ARR=(${CACHE_IMAGE_URI//:/ }) REPO_URI=${ARR[0]} @@ -111,9 +110,6 @@ for E in "${PROJECTS[@]}"; do fi fi - # Retag for aztecprotocol dockerhub. - docker tag $DEPLOY_IMAGE_URI aztecprotocol/$REPO:latest - echo -e "${BOLD}Tagged${RESET}: aztecprotocol/$REPO:latest" echo -e "${BOLD}SHA256${RESET}: $(docker inspect --format='{{.Id}}' $DEPLOY_IMAGE_URI)" diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose index 89c8749690b..b67fa075e06 100755 --- a/build-system/scripts/cond_run_compose +++ b/build-system/scripts/cond_run_compose @@ -27,7 +27,6 @@ if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then echo "Pulling $REPO..." REPO_IMAGE_URI=$(calculate_image_uri $REPO) retry docker pull $REPO_IMAGE_URI - docker tag $REPO_IMAGE_URI $ECR_DEPLOY_URL/$REPO docker tag $REPO_IMAGE_URI aztecprotocol/$REPO done diff --git a/build-system/scripts/deploy_ecr b/build-system/scripts/deploy_ecr deleted file mode 100755 index c156f99e71e..00000000000 --- a/build-system/scripts/deploy_ecr +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -REPOSITORY=$1 -ARCH=${2:-""} -IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) -if [[ -n "$ARCH" ]]; then - IMAGE_COMMIT_URI=$IMAGE_COMMIT_URI-$ARCH -fi - -# Login to build region and pull the build. -retry ensure_repo $REPOSITORY $ECR_REGION -retry docker pull $IMAGE_COMMIT_URI > /dev/null 2>&1 - -# Ensure ECR repository exists in deployment region. -retry ensure_repo $REPOSITORY $ECR_DEPLOY_REGION - -# Push image to deployment repo with commit hash tag and optional architecture e.g: -# falafel:deadbeefcafebabe1337c0de-x86_64 -IMAGE_DEPLOY_COMMIT_URI=$ECR_DEPLOY_URL/$REPOSITORY:$COMMIT_HASH -CURRENT_TAG=$COMMIT_HASH -NEW_TAG_COMMIT_TAG=$PROJECT-$COMMIT_TAG -NEW_TAG_DEPLOY_TAG=$DEPLOY_TAG -if [[ -n "$ARCH" ]]; then - IMAGE_DEPLOY_COMMIT_URI=$IMAGE_DEPLOY_COMMIT_URI-$ARCH - CURRENT_TAG=$CURRENT_TAG-$ARCH - NEW_TAG_COMMIT_TAG=$NEW_TAG_COMMIT_TAG-$ARCH - NEW_TAG_DEPLOY_TAG=$NEW_TAG_DEPLOY_TAG-$ARCH -fi -retry docker tag $IMAGE_COMMIT_URI $IMAGE_DEPLOY_COMMIT_URI -retry docker push $IMAGE_DEPLOY_COMMIT_URI - -# Tag image with full version if we have one. Allows deployment of precise image version if rollback needed. -if [ -n "${COMMIT_TAG:-}" ]; then - retry tag_remote_image $REPOSITORY $CURRENT_TAG $NEW_TAG_COMMIT_TAG $ECR_DEPLOY_REGION -fi - -retry tag_remote_image $REPOSITORY $CURRENT_TAG $NEW_TAG_DEPLOY_TAG $ECR_DEPLOY_REGION diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 5d9e6671e59..b44d06edb9a 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,5 +1,5 @@ #!/usr/bin/env bash set -euo pipefail REGION=${1:-$ECR_REGION} -aws ecr get-login-password --region $REGION \ +aws ecr-public get-login-password --region $REGION \ | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 059417cced0..855b3f14610 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -28,11 +28,10 @@ ssh -A -F $SSH_CONFIG_PATH $IP " export BRANCH=${BRANCH:-} export PULL_REQUEST=${PULL_REQUEST:-} export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD - export ECR_DEPLOY_URL=$ECR_DEPLOY_URL export ECR_URL=$ECR_URL export BUILD_SYSTEM_DEBUG=${BUILD_SYSTEM_DEBUG:-} - # temp while we transitioning to avm + # temp while we transitioning to avm export AVM_ENABLED=${AVM_ENABLED:-} ./remote_runner $@ " diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index dc6ad03d0df..672133f6f7a 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -81,9 +81,8 @@ echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV echo export ECR_REGION=us-east-2 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV -echo export ECR_URL=278380418400.dkr.ecr.us-east-2.amazonaws.com >> $BASH_ENV +echo export ECR_URL=public.ecr.aws/aztec >> $BASH_ENV echo export ECR_DEPLOY_REGION=eu-west-2 >> $BASH_ENV -echo export ECR_DEPLOY_URL=278380418400.dkr.ecr.eu-west-2.amazonaws.com >> $BASH_ENV echo export DOCKERHUB_ACCOUNT=aztecprotocol >> $BASH_ENV echo export DOCKERHUB_USERNAME=aztecprotocolci >> $BASH_ENV echo export PROJECT=$PROJECT >> $BASH_ENV diff --git a/docs/Dockerfile b/docs/Dockerfile index 847379f1ccf..d963bf86ec7 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project +FROM aztecprotocol/yarn-project WORKDIR /usr/src COPY . . WORKDIR /usr/src/docs diff --git a/noir-projects/Dockerfile b/noir-projects/Dockerfile index d4fa7dc97d8..24a5c6da24f 100644 --- a/noir-projects/Dockerfile +++ b/noir-projects/Dockerfile @@ -1,5 +1,5 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/avm-transpiler as transpiler +FROM aztecprotocol/noir as noir +FROM aztecprotocol/avm-transpiler as transpiler FROM ubuntu:lunar # Copy in nargo diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 9de7d8776b8..3732722687e 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -4,7 +4,7 @@ # - Run the tests. # - Run the formatter checks. # Any subsequent build steps needed to support downstream containers should be done in those containers build files. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder +FROM aztecprotocol/yarn-project-base as builder # Copy in the entire workspace. COPY . . diff --git a/yarn-project/Dockerfile.prod b/yarn-project/Dockerfile.prod index b935d818fd6..c355f7b6a1f 100644 --- a/yarn-project/Dockerfile.prod +++ b/yarn-project/Dockerfile.prod @@ -1,6 +1,6 @@ # This productionifies the workspace, removing all developer dependencies and producing a final slim image from which # we then generate downstream multiarch containers to execute the specific projects. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS yarn-project +FROM aztecprotocol/yarn-project AS yarn-project # Need new arch specific image. FROM node:18.19.0 AS builder diff --git a/yarn-project/aztec-faucet/Dockerfile b/yarn-project/aztec-faucet/Dockerfile index f4567db7482..047f7cd0cac 100644 --- a/yarn-project/aztec-faucet/Dockerfile +++ b/yarn-project/aztec-faucet/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod +FROM aztecprotocol/yarn-project-prod WORKDIR /usr/src/yarn-project/aztec-faucet ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec-faucet/dest/bin/index.js"] EXPOSE 8080 diff --git a/yarn-project/aztec/Dockerfile b/yarn-project/aztec/Dockerfile index e8a8e8d45d1..13b2000744d 100644 --- a/yarn-project/aztec/Dockerfile +++ b/yarn-project/aztec/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/aztec/dest/bin/index.js"] EXPOSE 8080 diff --git a/yarn-project/cli/Dockerfile b/yarn-project/cli/Dockerfile index 21dfcc65e97..b782d46e801 100644 --- a/yarn-project/cli/Dockerfile +++ b/yarn-project/cli/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-prod AS yarn-project-prod +FROM aztecprotocol/yarn-project-prod AS yarn-project-prod ENTRYPOINT ["node", "--no-warnings", "/usr/src/yarn-project/cli/dest/bin/index.js"] # The version has been updated in yarn-project-prod. diff --git a/yarn-project/end-to-end/Dockerfile b/yarn-project/end-to-end/Dockerfile index 3e81baa6f2c..49f5a14711d 100644 --- a/yarn-project/end-to-end/Dockerfile +++ b/yarn-project/end-to-end/Dockerfile @@ -1,4 +1,4 @@ -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project AS builder +FROM aztecprotocol/yarn-project AS builder # Build web bundle for browser tests WORKDIR /usr/src/yarn-project/end-to-end diff --git a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml index 9ea2293cb24..d9788104ec6 100644 --- a/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml +++ b/yarn-project/p2p-bootstrap/scripts/docker-compose-bootstrap.yml @@ -1,7 +1,7 @@ version: '3' services: p2p-bootstrap: - image: 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec:latest + image: aztecprotocol/aztec:latest command: 'start --p2p-bootstrap' ports: - '40400:40400' diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 933ec230be2..e51a024b4fa 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -42,11 +42,11 @@ # RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src # - Create final slim image by copying needed dirs into a fresh image. # -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files +FROM aztecprotocol/l1-contracts as contracts +FROM aztecprotocol/bb.js as bb.js +FROM aztecprotocol/noir-projects as noir-projects +FROM aztecprotocol/noir-packages as noir-packages +FROM aztecprotocol/boxes-files as boxes-files FROM node:18.19.0 RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean diff --git a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 b/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 index 421235079e3..0627db619ac 100644 --- a/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 +++ b/yarn-project/yarn-project-base/Dockerfile.dockerignore.v24 @@ -5,7 +5,7 @@ # # This took a fair bit of trial and error to get the right result. If you need to meddle, try: # - Commenting out everything after the COPY . . in the dockerfile. -# - ONLY_TARGET=1 ../bootstrap_docker.sh yarn-project-base && docker run -ti --rm 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base:latest sh -c 'du -ha .' +# - ONLY_TARGET=1 ../bootstrap_docker.sh yarn-project-base && docker run -ti --rm aztecprotocol/yarn-project-base:latest sh -c 'du -ha .' # Exclude everything to start. * From 60a38d32a317e99981f49cbaa03f156895bb2980 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 19:10:02 +0000 Subject: [PATCH 04/29] bootstrap scripts try to use cache by default. l1-contracts will attempt to use ci cache. --- avm-transpiler/bootstrap.sh | 3 +++ barretenberg/bootstrap.sh | 3 +++ bootstrap.sh | 18 +++++------------- l1-contracts/Dockerfile | 3 +++ l1-contracts/bootstrap.sh | 3 +++ l1-contracts/bootstrap_cache.sh | 8 ++++++++ noir/bootstrap.sh | 3 +++ noir/bootstrap_cache.sh | 1 - 8 files changed, 28 insertions(+), 14 deletions(-) create mode 100755 l1-contracts/bootstrap_cache.sh diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 916d8107876..2c726cb4e43 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -16,4 +16,7 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + ./scripts/bootstrap_native.sh \ No newline at end of file diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index c16fd294b6c..9d398f7105b 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -eu +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + cd "$(dirname "$0")" (cd cpp && ./bootstrap.sh $@) diff --git a/bootstrap.sh b/bootstrap.sh index 818d5795637..fb1e5c8b2db 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -61,19 +61,11 @@ PROJECTS=( # Build projects locally for P in "${PROJECTS[@]}"; do - if [ -n "${BOOTSTRAP_USE_REMOTE_CACHE:-}" ] && [ -f "$P/bootstrap_cache.sh" ]; then - echo "**************************************" - echo -e "\033[1mBootstrapping $P from remote cache...\033[0m" - echo "**************************************" - echo - $P/bootstrap_cache.sh - else - echo "**************************************" - echo -e "\033[1mBootstrapping $P...\033[0m" - echo "**************************************" - echo - $P/bootstrap.sh - fi + echo "**************************************" + echo -e "\033[1mBootstrapping $P...\033[0m" + echo "**************************************" + echo + $P/bootstrap.sh echo echo done diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index 1ced5c8672e..bb0511f6657 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -13,3 +13,6 @@ RUN forge clean && forge fmt --check && forge build && forge test RUN yarn && yarn lint RUN git add . && yarn slither && yarn slither-has-diff RUN forge build + +FROM scratch +COPY --from=0 /usr/src/l1-contracts/out /usr/src/l1-contracts/out \ No newline at end of file diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index f776b6072dc..c11b7d56383 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -15,6 +15,9 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + # Clean rm -rf broadcast cache out serve diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh new file mode 100755 index 00000000000..5d9d9c7931e --- /dev/null +++ b/l1-contracts/bootstrap_cache.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu + +cd "$(dirname "$0")" +source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" +extract_repo l1-contracts /usr/src/l1-contracts/out . diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 5ebe7ade090..1f9506904a4 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -15,5 +15,8 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh \ No newline at end of file diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 672702416bd..ac919f3ca65 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -8,4 +8,3 @@ echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" extract_repo noir-packages /usr/src/noir/packages ./ echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" extract_repo noir /usr/src/noir/target/release ./target/ - From 97822b101f5bb7660ed37a2a752b488810ecfdb7 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:04:32 +0000 Subject: [PATCH 05/29] Remove old cache images. --- avm-transpiler/bootstrap_cache.sh | 2 ++ barretenberg/bootstrap.sh | 3 --- barretenberg/cpp/bootstrap.sh | 3 +++ barretenberg/{ => cpp}/bootstrap_cache.sh | 7 +++---- barretenberg/ts/bootstrap.sh | 5 ++++- barretenberg/ts/bootstrap_cache.sh | 12 ++++++++++++ bootstrap.sh | 2 +- build-system/scripts/remove_old_images | 13 +++++++++++++ l1-contracts/bootstrap_cache.sh | 2 ++ l1-contracts/src/core/libraries/ConstantsGen.sol | 12 ++++-------- noir/bootstrap_cache.sh | 3 +++ yarn-project/circuits.js/package.json | 2 +- 12 files changed, 48 insertions(+), 18 deletions(-) rename barretenberg/{ => cpp}/bootstrap_cache.sh (54%) create mode 100755 barretenberg/ts/bootstrap_cache.sh create mode 100755 build-system/scripts/remove_old_images diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index a618d231593..080136e2be8 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -7,3 +7,5 @@ source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" extract_repo avm-transpiler \ /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/ + +remove_old_images avm-transpiler diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index 9d398f7105b..c16fd294b6c 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash set -eu -# Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit - cd "$(dirname "$0")" (cd cpp && ./bootstrap.sh $@) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 0b9295bafbd..32de0856a4d 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -31,6 +31,9 @@ fi # Download ignition transcripts. (cd ./srs_db && ./download_ignition.sh 0) +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + # Pick native toolchain file. ARCH=$(uname -m) if [ "$OS" == "macos" ]; then diff --git a/barretenberg/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh similarity index 54% rename from barretenberg/bootstrap_cache.sh rename to barretenberg/cpp/bootstrap_cache.sh index 71e081f77e8..a739a6cab66 100755 --- a/barretenberg/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -2,12 +2,11 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null +source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m" -extract_repo bb.js \ +extract_repo barretenberg-wasm-linux-clang \ /usr/src/barretenberg/cpp/build-wasm/bin ./cpp/build-wasm \ /usr/src/barretenberg/cpp/build-wasm-threads/bin ./cpp/build-wasm-threads -echo -e "\033[1mBuilding ESM bb.ts...\033[0m" -(cd ts && SKIP_CPP_BUILD=1 ./scripts/build_wasm.sh && ./bootstrap.sh esm) +remove_old_images barretenberg-wasm-linux-clang diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index 1c0464dc212..f8f4dfd56b8 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -18,10 +18,13 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + yarn install --immutable echo "Building with command 'yarn $BUILD_CMD'..." yarn $BUILD_CMD # Make bin globally available. npm link -echo "Barretenberg ts build successful" \ No newline at end of file +echo "Barretenberg ts build successful" diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh new file mode 100755 index 00000000000..c5e39052643 --- /dev/null +++ b/barretenberg/ts/bootstrap_cache.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eu + +cd "$(dirname "$0")" +source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m" +extract_repo bb.js /usr/src/barretenberg/ts/dest . +# Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. +yarn install + +remove_old_images bb.js diff --git a/bootstrap.sh b/bootstrap.sh index fb1e5c8b2db..8cd728f1d02 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -65,7 +65,7 @@ for P in "${PROJECTS[@]}"; do echo -e "\033[1mBootstrapping $P...\033[0m" echo "**************************************" echo - $P/bootstrap.sh + (cd $P && ./bootstrap.sh) echo echo done diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images new file mode 100755 index 00000000000..128fbbdf444 --- /dev/null +++ b/build-system/scripts/remove_old_images @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Removes all cache-* docker images for the given repository that are not the current content hash. +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +shift + +IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) +for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do + echo "Removing $IMAGE..." + docker rmi $IMAGE +done diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 5d9d9c7931e..911796ce9b5 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -6,3 +6,5 @@ source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" extract_repo l1-contracts /usr/src/l1-contracts/out . + +remove_old_images l1-contracts diff --git a/l1-contracts/src/core/libraries/ConstantsGen.sol b/l1-contracts/src/core/libraries/ConstantsGen.sol index f8a7ae2576e..db3f98aacbe 100644 --- a/l1-contracts/src/core/libraries/ConstantsGen.sol +++ b/l1-contracts/src/core/libraries/ConstantsGen.sol @@ -76,14 +76,10 @@ library Constants { uint256 internal constant MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS = 1000; uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_PRIVATE_FUNCTION_IN_FIELDS = 500; uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_UNCONSTRAINED_FUNCTION_IN_FIELDS = 500; - uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = - 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; - uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = - 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; - uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = - 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; - uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = - 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; + uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; + uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; + uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; + uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; uint256 internal constant L1_TO_L2_MESSAGE_ORACLE_CALL_LENGTH = 25; uint256 internal constant MAX_NOTE_FIELDS_LENGTH = 20; uint256 internal constant GET_NOTE_ORACLE_RETURN_LENGTH = 23; diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index ac919f3ca65..1cec6c81d8e 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -8,3 +8,6 @@ echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" extract_repo noir-packages /usr/src/noir/packages ./ echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" extract_repo noir /usr/src/noir/target/release ./target/ + +remove_old_images noir-packages +remove_old_images noir diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index 715e844ad06..a3888d384bb 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -26,7 +26,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts && cd ../../l1-contracts && ./.foundry/bin/forge fmt", + "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "inherits": [ From f4d59db4ddb773144345a4786c151f0e99aa8e5b Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:40:46 +0000 Subject: [PATCH 06/29] Better retry. --- build-system/scripts/retry | 10 ++++++++-- build-system/scripts/setup_env | 7 ++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build-system/scripts/retry b/build-system/scripts/retry index 0489aa226fd..2172e7dcfd9 100755 --- a/build-system/scripts/retry +++ b/build-system/scripts/retry @@ -1,7 +1,13 @@ +if [ -n "$RETRY_DISABLED" ]; then + "$@" && exit || exit 1 +fi + ATTEMPTS=3 -# Retries up to 3 times with 10 second intervals +# Retries up to 3 times with 5 second intervals for i in $(seq 1 $ATTEMPTS); do - "$@" && exit || sleep 10 + "$@" && exit + [ "$i" != "$ATTEMPTS" ] && sleep 5 done + >&2 echo "$@ failed after $ATTEMPTS attempts" exit 1 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index dc6ad03d0df..14482b99fb0 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,6 +6,10 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. + +# Save current options as we might be sourcing. +CURRENT_SHELL_OPTS=$(set +o) +trap 'eval "$CURRENT_SHELL_OPTS"' EXIT [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu @@ -139,6 +143,3 @@ cat $BASH_ENV # Having written the variables to $BASH_ENV, we now want to set them in this shell context. source $BASH_ENV - -# This script is often "sourced", we don't want calling shell to exit on a subsequent error! -set +e From 6ac200099db87684ec74093699ddd83eb661aca3 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:48:27 +0000 Subject: [PATCH 07/29] Early out if can't use cache. --- avm-transpiler/bootstrap_cache.sh | 2 ++ barretenberg/cpp/bootstrap_cache.sh | 2 ++ barretenberg/ts/bootstrap_cache.sh | 2 ++ l1-contracts/bootstrap_cache.sh | 2 ++ noir/bootstrap_cache.sh | 2 ++ 5 files changed, 10 insertions(+) diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 080136e2be8..96f68ab0e76 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index a739a6cab66..9f420deda2e 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index c5e39052643..ceb9e561db4 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 911796ce9b5..a4fc1b05a3f 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 1cec6c81d8e..7a3523ab93b 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null From bd3a1bdac12c07d4aa1b555f43b074f034d0548c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:49:56 +0000 Subject: [PATCH 08/29] fix unbound --- build-system/scripts/retry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/retry b/build-system/scripts/retry index 2172e7dcfd9..6cbfdc14649 100755 --- a/build-system/scripts/retry +++ b/build-system/scripts/retry @@ -1,4 +1,4 @@ -if [ -n "$RETRY_DISABLED" ]; then +if [ -n "${RETRY_DISABLED:-}" ]; then "$@" && exit || exit 1 fi From c52fec804e1157e0416935eeabd5da0b2788229c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:52:28 +0000 Subject: [PATCH 09/29] Remove codegen sol file. --- .../src/core/libraries/ConstantsGen.sol | 120 ------------------ 1 file changed, 120 deletions(-) delete mode 100644 l1-contracts/src/core/libraries/ConstantsGen.sol diff --git a/l1-contracts/src/core/libraries/ConstantsGen.sol b/l1-contracts/src/core/libraries/ConstantsGen.sol deleted file mode 100644 index db3f98aacbe..00000000000 --- a/l1-contracts/src/core/libraries/ConstantsGen.sol +++ /dev/null @@ -1,120 +0,0 @@ -// GENERATED FILE - DO NOT EDIT, RUN yarn remake-constants in circuits.js -// SPDX-License-Identifier: Apache-2.0 -// Copyright 2023 Aztec Labs. -pragma solidity >=0.8.18; - -/** - * @title Constants Library - * @author Aztec Labs - * @notice Library that contains constants used throughout the Aztec protocol - */ -library Constants { - // Prime field modulus - uint256 internal constant P = - 21888242871839275222246405745257275088548364400416034343698204186575808495617; - uint256 internal constant MAX_FIELD_VALUE = P - 1; - - uint256 internal constant ARGS_LENGTH = 16; - uint256 internal constant RETURN_VALUES_LENGTH = 4; - uint256 internal constant MAX_NEW_COMMITMENTS_PER_CALL = 16; - uint256 internal constant MAX_NEW_NULLIFIERS_PER_CALL = 16; - uint256 internal constant MAX_PRIVATE_CALL_STACK_LENGTH_PER_CALL = 4; - uint256 internal constant MAX_PUBLIC_CALL_STACK_LENGTH_PER_CALL = 4; - uint256 internal constant MAX_NEW_L2_TO_L1_MSGS_PER_CALL = 2; - uint256 internal constant MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL = 16; - uint256 internal constant MAX_PUBLIC_DATA_READS_PER_CALL = 16; - uint256 internal constant MAX_READ_REQUESTS_PER_CALL = 32; - uint256 internal constant MAX_NULLIFIER_KEY_VALIDATION_REQUESTS_PER_CALL = 1; - uint256 internal constant MAX_NEW_COMMITMENTS_PER_TX = 64; - uint256 internal constant MAX_NON_REVERTIBLE_COMMITMENTS_PER_TX = 8; - uint256 internal constant MAX_REVERTIBLE_COMMITMENTS_PER_TX = 56; - uint256 internal constant MAX_NEW_NULLIFIERS_PER_TX = 64; - uint256 internal constant MAX_NON_REVERTIBLE_NULLIFIERS_PER_TX = 8; - uint256 internal constant MAX_REVERTIBLE_NULLIFIERS_PER_TX = 56; - uint256 internal constant MAX_PRIVATE_CALL_STACK_LENGTH_PER_TX = 8; - uint256 internal constant MAX_PUBLIC_CALL_STACK_LENGTH_PER_TX = 8; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX = 3; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX = 5; - uint256 internal constant MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 32; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 16; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 16; - uint256 internal constant MAX_PUBLIC_DATA_READS_PER_TX = 32; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_DATA_READS_PER_TX = 16; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_DATA_READS_PER_TX = 16; - uint256 internal constant MAX_NEW_L2_TO_L1_MSGS_PER_TX = 2; - uint256 internal constant MAX_NEW_CONTRACTS_PER_TX = 1; - uint256 internal constant MAX_READ_REQUESTS_PER_TX = 128; - uint256 internal constant MAX_NULLIFIER_KEY_VALIDATION_REQUESTS_PER_TX = 4; - uint256 internal constant NUM_ENCRYPTED_LOGS_HASHES_PER_TX = 1; - uint256 internal constant NUM_UNENCRYPTED_LOGS_HASHES_PER_TX = 1; - uint256 internal constant NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP = 16; - uint256 internal constant VK_TREE_HEIGHT = 3; - uint256 internal constant FUNCTION_TREE_HEIGHT = 5; - uint256 internal constant CONTRACT_TREE_HEIGHT = 16; - uint256 internal constant NOTE_HASH_TREE_HEIGHT = 32; - uint256 internal constant PUBLIC_DATA_TREE_HEIGHT = 40; - uint256 internal constant NULLIFIER_TREE_HEIGHT = 20; - uint256 internal constant L1_TO_L2_MSG_TREE_HEIGHT = 16; - uint256 internal constant ROLLUP_VK_TREE_HEIGHT = 8; - uint256 internal constant ARTIFACT_FUNCTION_TREE_MAX_HEIGHT = 5; - uint256 internal constant CONTRACT_SUBTREE_HEIGHT = 0; - uint256 internal constant CONTRACT_SUBTREE_SIBLING_PATH_LENGTH = 16; - uint256 internal constant NOTE_HASH_SUBTREE_HEIGHT = 6; - uint256 internal constant NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH = 26; - uint256 internal constant NULLIFIER_SUBTREE_HEIGHT = 6; - uint256 internal constant PUBLIC_DATA_SUBTREE_HEIGHT = 5; - uint256 internal constant ARCHIVE_HEIGHT = 16; - uint256 internal constant NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH = 14; - uint256 internal constant PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH = 35; - uint256 internal constant L1_TO_L2_MSG_SUBTREE_HEIGHT = 4; - uint256 internal constant L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH = 12; - uint256 internal constant FUNCTION_SELECTOR_NUM_BYTES = 4; - uint256 internal constant MAPPING_SLOT_PEDERSEN_SEPARATOR = 4; - uint256 internal constant NUM_FIELDS_PER_SHA256 = 2; - uint256 internal constant ARGS_HASH_CHUNK_LENGTH = 32; - uint256 internal constant ARGS_HASH_CHUNK_COUNT = 32; - uint256 internal constant MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS = 1000; - uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_PRIVATE_FUNCTION_IN_FIELDS = 500; - uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_UNCONSTRAINED_FUNCTION_IN_FIELDS = 500; - uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; - uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; - uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; - uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; - uint256 internal constant L1_TO_L2_MESSAGE_ORACLE_CALL_LENGTH = 25; - uint256 internal constant MAX_NOTE_FIELDS_LENGTH = 20; - uint256 internal constant GET_NOTE_ORACLE_RETURN_LENGTH = 23; - uint256 internal constant MAX_NOTES_PER_PAGE = 10; - uint256 internal constant VIEW_NOTE_ORACLE_RETURN_LENGTH = 212; - uint256 internal constant AZTEC_ADDRESS_LENGTH = 1; - uint256 internal constant CALL_CONTEXT_LENGTH = 8; - uint256 internal constant CONTENT_COMMITMENT_LENGTH = 7; - uint256 internal constant CONTRACT_DEPLOYMENT_DATA_LENGTH = 6; - uint256 internal constant CONTRACT_STORAGE_READ_LENGTH = 2; - uint256 internal constant CONTRACT_STORAGE_UPDATE_REQUEST_LENGTH = 2; - uint256 internal constant ETH_ADDRESS_LENGTH = 1; - uint256 internal constant FUNCTION_DATA_LENGTH = 4; - uint256 internal constant FUNCTION_LEAF_PREIMAGE_LENGTH = 5; - uint256 internal constant GLOBAL_VARIABLES_LENGTH = 6; - uint256 internal constant HEADER_LENGTH = 25; - uint256 internal constant L1_TO_L2_MESSAGE_LENGTH = 8; - uint256 internal constant L2_TO_L1_MESSAGE_LENGTH = 2; - uint256 internal constant NEW_CONTRACT_DATA_LENGTH = 3; - uint256 internal constant NULLIFIER_KEY_VALIDATION_REQUEST_LENGTH = 4; - uint256 internal constant NULLIFIER_KEY_VALIDATION_REQUEST_CONTEXT_LENGTH = 5; - uint256 internal constant PARTIAL_STATE_REFERENCE_LENGTH = 8; - uint256 internal constant PRIVATE_CALL_STACK_ITEM_LENGTH = 219; - uint256 internal constant PRIVATE_CIRCUIT_PUBLIC_INPUTS_LENGTH = 214; - uint256 internal constant PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH = 194; - uint256 internal constant STATE_REFERENCE_LENGTH = 10; - uint256 internal constant TX_CONTEXT_DATA_LENGTH = 11; - uint256 internal constant TX_REQUEST_LENGTH = 17; - uint256 internal constant GET_NOTES_ORACLE_RETURN_LENGTH = 674; - uint256 internal constant COMMITMENTS_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant NULLIFIERS_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant PUBLIC_DATA_WRITES_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant CONTRACTS_NUM_BYTES_PER_BASE_ROLLUP = 32; - uint256 internal constant CONTRACT_DATA_NUM_BYTES_PER_BASE_ROLLUP = 64; - uint256 internal constant CONTRACT_DATA_NUM_BYTES_PER_BASE_ROLLUP_UNPADDED = 52; - uint256 internal constant L2_TO_L1_MSGS_NUM_BYTES_PER_BASE_ROLLUP = 64; - uint256 internal constant LOGS_HASHES_NUM_BYTES_PER_BASE_ROLLUP = 64; -} From 332e1a39d5b5af3f103b2b65ff644ad059fd884c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 13:48:42 +0000 Subject: [PATCH 10/29] fix --- build_manifest.yml | 3 ++- l1-contracts/bootstrap.sh | 6 +++--- yarn-project/circuits.js/package.json | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index bac4b7bc86e..9517efe5c8b 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -226,4 +226,5 @@ avm-transpiler: dockerfile: avm-transpiler/Dockerfile rebuildPatterns: - ^avm-transpiler/ - - ^noir/ + dependencies: + - noir diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index c11b7d56383..489ee2a80eb 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -15,15 +15,15 @@ if [ -n "$CMD" ]; then fi fi +# Install foundry. +. ./scripts/install_foundry.sh + # Attempt to just pull artefacts from CI and exit on success. ./bootstrap_cache.sh && exit # Clean rm -rf broadcast cache out serve -# Install foundry. -. ./scripts/install_foundry.sh - # Install forge install --no-commit diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index a3888d384bb..715e844ad06 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -26,7 +26,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts", + "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts && cd ../../l1-contracts && ./.foundry/bin/forge fmt", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "inherits": [ From e40e51d6b9e12aa710afc8648831fe7b9befc2b8 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:08:07 +0000 Subject: [PATCH 11/29] avm-transpiler to target ubuntu 20 --- avm-transpiler/Dockerfile | 6 +++--- build-system/scripts/remove_old_images | 2 +- noir/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/avm-transpiler/Dockerfile b/avm-transpiler/Dockerfile index 1f55f8ba40d..95a4ead89ee 100644 --- a/avm-transpiler/Dockerfile +++ b/avm-transpiler/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:bookworm +FROM rust:bullseye WORKDIR /usr/src COPY ./avm-transpiler ./avm-transpiler @@ -8,6 +8,6 @@ WORKDIR /usr/src/avm-transpiler RUN apt-get update && apt-get install -y git RUN ./scripts/bootstrap_native.sh -FROM ubuntu:lunar +FROM ubuntu:focal COPY --from=0 /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler -ENTRYPOINT ["sh", "-c"] \ No newline at end of file +ENTRYPOINT ["sh", "-c"] diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images index 128fbbdf444..7c1c927e6b6 100755 --- a/build-system/scripts/remove_old_images +++ b/build-system/scripts/remove_old_images @@ -9,5 +9,5 @@ shift IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do echo "Removing $IMAGE..." - docker rmi $IMAGE + docker rmi --force $IMAGE done diff --git a/noir/Dockerfile b/noir/Dockerfile index bd509450287..3a478c3f95a 100644 --- a/noir/Dockerfile +++ b/noir/Dockerfile @@ -9,4 +9,4 @@ FROM ubuntu:focal # Install git as nargo needs it to clone. RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean COPY --from=0 /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo -ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"] From 6ece897a4fccf05872d0e64aedd8b0971cb64950 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:14:08 +0000 Subject: [PATCH 12/29] fix --- build_manifest.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index 9517efe5c8b..7a144219eff 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -146,7 +146,7 @@ yarn-project-base: yarn-project: buildDir: yarn-project rebuildPatterns: - - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ - ^yarn-project/Dockerfile$ - ^yarn-project/cli/aztec-cli dependencies: @@ -226,5 +226,4 @@ avm-transpiler: dockerfile: avm-transpiler/Dockerfile rebuildPatterns: - ^avm-transpiler/ - dependencies: - - noir + - ^noir/ From 2465a8d0751be71776e5fc27e9bebd4c7a1e7aab Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:20:27 +0000 Subject: [PATCH 13/29] fix --- build_manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_manifest.yml b/build_manifest.yml index 7a144219eff..bac4b7bc86e 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -146,7 +146,7 @@ yarn-project-base: yarn-project: buildDir: yarn-project rebuildPatterns: - - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ - ^yarn-project/Dockerfile$ - ^yarn-project/cli/aztec-cli dependencies: From 5adcb502692af8b97b69815d002b091f13854000 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 15:20:25 +0000 Subject: [PATCH 14/29] fix --- build-system/scripts/ecr_login | 5 ++--- build-system/scripts/remove_old_images | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index b44d06edb9a..8e6c90c175a 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,5 +1,4 @@ #!/usr/bin/env bash set -euo pipefail -REGION=${1:-$ECR_REGION} -aws ecr-public get-login-password --region $REGION \ - | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file +aws ecr-public get-login-password --region us-east-1 \ + | docker login --username AWS --password-stdin public.ecr.aws 2> /dev/null \ No newline at end of file diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images index 7c1c927e6b6..eb8e4c6f046 100755 --- a/build-system/scripts/remove_old_images +++ b/build-system/scripts/remove_old_images @@ -7,7 +7,7 @@ REPOSITORY=$1 shift IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) -for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do +for IMAGE in $(docker images --format "{{.ID}}" aztecprotocol/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do echo "Removing $IMAGE..." docker rmi --force $IMAGE done From cb8a8e6bebb9864ef9a83b4e617922ff8c6c1460 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 15:22:15 +0000 Subject: [PATCH 15/29] [ci rebuild] From 8d66ceaa7b5576a5bc398211f3baf796a3dba996 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 15:36:16 +0000 Subject: [PATCH 16/29] public ecr needs to be us-east-1 --- build-system/scripts/setup_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 56ace5e4121..92583393acd 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -83,7 +83,7 @@ echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV -echo export ECR_REGION=us-east-2 >> $BASH_ENV +echo export ECR_REGION=us-east-1 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV echo export ECR_URL=public.ecr.aws/aztec >> $BASH_ENV echo export ECR_DEPLOY_REGION=eu-west-2 >> $BASH_ENV From 8b5b4592165a9cf6c144e72b0439687d3be3fd76 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 15:36:23 +0000 Subject: [PATCH 17/29] [ci rebuild] From aaf417a0c9d114d79b56cbd4569e1221f5c3e678 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 16:29:17 +0000 Subject: [PATCH 18/29] Use small executor for workflow calcs. Force process pool to 8. --- .circleci/config.yml | 5 +++-- build-system/scripts/generate_circleci_config.py | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 435d9f88a99..f1f4e8313f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: generate-config: docker: - image: aztecprotocol/alpine-build-image - resource_class: xlarge + resource_class: small steps: - *checkout - *setup_env @@ -86,6 +86,7 @@ jobs: build-system/scripts/generate_circleci_config.py > .circleci/generated_config.yml - continuation/continue: configuration_path: .circleci/generated_config.yml + # Noir noir-x86_64: docker: @@ -207,7 +208,7 @@ jobs: name: "Build" command: cond_spot_run_build barretenberg-x86_64-linux-clang-fuzzing 128 aztec_manifest_key: barretenberg-x86_64-linux-clang-fuzzing - + barretenberg-x86_64-linux-clang-assert: docker: - image: aztecprotocol/alpine-build-image diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index f04328646d0..9b0f650f892 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -20,7 +20,7 @@ def get_manifest_job_names(): def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): """ - This function checks if a given CircleCI job is associated with a specific already-built manifest job. + This function checks if a given CircleCI job is associated with a specific already-built manifest job. It does so by checking the job's steps for an 'aztec_manifest_key' that contain references to manifest names. We want to see at least one such key, and for all such keys to be in 'already_built_manifest_jobs'. """ @@ -43,7 +43,7 @@ def is_already_built_circleci_job(circleci_job, already_built_manifest_jobs): return False matching_steps += 1 # All steps have matched - but make sure that's actually more than one step - return matching_steps > 0 + return matching_steps > 0 def get_already_built_circleci_job_names(circleci_jobs): already_built_manifest_jobs = list(get_already_built_manifest_job_names()) @@ -65,7 +65,7 @@ def _get_already_built_manifest_job_names(manifest_name): def get_already_built_manifest_job_names(): manifest_names = get_manifest_job_names() - with ProcessPoolExecutor() as executor: + with ProcessPoolExecutor(max_workers=8) as executor: futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names} for future in as_completed(futures): result = future.result() @@ -75,11 +75,11 @@ def get_already_built_manifest_job_names(): def remove_jobs_from_workflow(jobs, to_remove): """ Removes jobs from a given CircleCI JSON workflow. - + Parameters: jobs (dict): The JSON object representing the CircleCI workflow jobs dependencies portion. to_remove (list): The list of jobs to be removed from the workflow. - + Returns: dict: The new JSON object with specified jobs removed. """ @@ -97,7 +97,7 @@ def remove_jobs_from_workflow(jobs, to_remove): if __name__ == '__main__': # The CircleCI workflow as a JSON string (Replace this with your actual workflow) - + # Convert the JSON string to a Python dictionary workflow_dict = yaml.safe_load(open('.circleci/config.yml')) From 92aa9c764aeaccda9ddf0ef46d24df22ad3a907e Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 16:29:27 +0000 Subject: [PATCH 19/29] ecr-public --- barretenberg/cpp/scripts/bb-tests.sh | 2 +- barretenberg/cpp/scripts/run_tests | 2 +- barretenberg/ts/scripts/run_tests | 2 +- build-system/scripts/clean_image_tags | 2 +- build-system/scripts/ensure_repo | 8 ++++---- build-system/scripts/image_exists | 2 +- build-system/scripts/tag_remote_image | 8 ++++---- build-system/scripts/untag_remote_image | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/barretenberg/cpp/scripts/bb-tests.sh b/barretenberg/cpp/scripts/bb-tests.sh index 0c0d27ed922..56686623580 100755 --- a/barretenberg/cpp/scripts/bb-tests.sh +++ b/barretenberg/cpp/scripts/bb-tests.sh @@ -3,7 +3,7 @@ # Might be better to list exclusions here rather than inclusions as risky to maintain. set -eu -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI diff --git a/barretenberg/cpp/scripts/run_tests b/barretenberg/cpp/scripts/run_tests index e801ad2bde3..b8be58a5af2 100755 --- a/barretenberg/cpp/scripts/run_tests +++ b/barretenberg/cpp/scripts/run_tests @@ -11,7 +11,7 @@ TESTS=$2 shift shift -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/barretenberg/ts/scripts/run_tests b/barretenberg/ts/scripts/run_tests index 0f0d2895cec..ea2b3bf8482 100755 --- a/barretenberg/ts/scripts/run_tests +++ b/barretenberg/ts/scripts/run_tests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -xe -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=bb.js IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) diff --git a/build-system/scripts/clean_image_tags b/build-system/scripts/clean_image_tags index 9c4557e68c6..9da74edda65 100755 --- a/build-system/scripts/clean_image_tags +++ b/build-system/scripts/clean_image_tags @@ -6,7 +6,7 @@ REPOSITORY=$1 EXISTING_TAG=$2 REGION=${3:-$ECR_REGION} -IMAGE_TAGS=$(aws ecr describe-images --repository-name $REPOSITORY --image-ids "imageTag=$EXISTING_TAG" | jq -r '.imageDetails[0].imageTags[]') +IMAGE_TAGS=$(aws ecr-public describe-images --repository-name $REPOSITORY --image-ids "imageTag=$EXISTING_TAG" | jq -r '.imageDetails[0].imageTags[]') TAGS_COUNT=$(echo "$IMAGE_TAGS" | wc -l) echo "Found $TAGS_COUNT for $REPOSITORY:$EXISTING_TAG" diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 260f82fb4c0..11dc55badba 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -29,12 +29,12 @@ REFRESH_LIFECYCLE=${3:-} retry ecr_login $REGION # Create the repository if it doesn't exist. -if ! aws ecr describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then +if ! aws ecr-public describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then echo "Creating repo: $REPOSITORY" - aws ecr create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 - aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 + aws ecr-public create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 + aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 elif [ -n "$REFRESH_LIFECYCLE" ]; then echo "Refreshing lifecycle rules for repo: $REPOSITORY" - aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 + aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 fi echo diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index 0ad9d90ae69..c8816fbc86e 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -eu # Returns true if the given image exists in the current ECR. -aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 +aws ecr-public describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 0bb9e2a390f..3a7c0952035 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -7,7 +7,7 @@ EXISTING_TAG=$2 NEW_TAG=$3 REGION=${4:-$ECR_REGION} -EXISTING_TAG_MANIFEST=$(aws ecr batch-get-image \ +EXISTING_TAG_MANIFEST=$(aws ecr-public batch-get-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-ids imageTag=$EXISTING_TAG \ @@ -19,7 +19,7 @@ if [ -z "$EXISTING_TAG_MANIFEST" ]; then exit 1 fi -NEW_TAG_MANIFEST=$(aws ecr batch-get-image \ +NEW_TAG_MANIFEST=$(aws ecr-public batch-get-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-ids imageTag=$NEW_TAG \ @@ -28,7 +28,7 @@ NEW_TAG_MANIFEST=$(aws ecr batch-get-image \ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then echo "Tagging $1:$EXISTING_TAG as $1:$NEW_TAG..." - TAG_RESULT=$(aws ecr put-image \ + TAG_RESULT=$(aws ecr-public put-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ @@ -40,7 +40,7 @@ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then echo "Failed to tag due to limit exceeded. Starting tag cleanup." clean_image_tags $REPOSITORY $EXISTING_TAG $REGION - aws ecr put-image \ + aws ecr-public put-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ diff --git a/build-system/scripts/untag_remote_image b/build-system/scripts/untag_remote_image index aa913c1200b..51a4c777738 100755 --- a/build-system/scripts/untag_remote_image +++ b/build-system/scripts/untag_remote_image @@ -1,4 +1,4 @@ #!/usr/bin/env bash REPOSITORY=$1 TAG=$2 -aws ecr batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file +aws ecr-public batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file From ad3cdcb862019deebbd552beb719d5d759aca31d Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 16:29:50 +0000 Subject: [PATCH 20/29] [ci rebuild] From af51314af6ae347e99df17701ef683a0ce39072b Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 16:43:52 +0000 Subject: [PATCH 21/29] We can't have lifecycles on public repos. Regretting this. Lets try disabling for now. --- build-system/scripts/ensure_repo | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 11dc55badba..85f4fe6fa15 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -32,9 +32,9 @@ retry ecr_login $REGION if ! aws ecr-public describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then echo "Creating repo: $REPOSITORY" aws ecr-public create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 - aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 -elif [ -n "$REFRESH_LIFECYCLE" ]; then - echo "Refreshing lifecycle rules for repo: $REPOSITORY" - aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 + # aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 +# elif [ -n "$REFRESH_LIFECYCLE" ]; then +# echo "Refreshing lifecycle rules for repo: $REPOSITORY" +# aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 fi echo From 462a4d4090a84901bd74a86db884d8a859fa23b3 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 17:22:39 +0000 Subject: [PATCH 22/29] Update amis to have aws that supports ecr-public --- build-system/scripts/request_spot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 6c4e1d1bb2d..93c5d9a990b 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -32,10 +32,10 @@ fi # Construct the full instance type. We use r6a currently. if [ "$ARCH" == "x86_64" ]; then MACHINE_TYPE="r6a" - AMI="ami-0e5df77ac318c7a18" + AMI="ami-04d8422a9ba4de80f" elif [ "$ARCH" == "arm64" ]; then MACHINE_TYPE="r7g" - AMI="ami-02c5a5064fe72b22e" + AMI="ami-0d8a9b0419ddb331a" else echo "Unknown arch: $ARCH" exit 1 From d82e3233e77bf3a71c082f6c12fef091f6be5d77 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 18:07:46 +0000 Subject: [PATCH 23/29] Giving up on public ecr. Too many compromises. --- barretenberg/cpp/scripts/bb-tests.sh | 2 +- barretenberg/cpp/scripts/run_tests | 2 +- barretenberg/ts/scripts/run_tests | 2 +- build-system/scripts/clean_image_tags | 2 +- build-system/scripts/ecr_login | 5 +++-- build-system/scripts/ensure_repo | 12 ++++++------ build-system/scripts/image_exists | 2 +- build-system/scripts/setup_env | 4 ++-- build-system/scripts/tag_remote_image | 8 ++++---- build-system/scripts/untag_remote_image | 2 +- 10 files changed, 21 insertions(+), 20 deletions(-) diff --git a/barretenberg/cpp/scripts/bb-tests.sh b/barretenberg/cpp/scripts/bb-tests.sh index 56686623580..0c0d27ed922 100755 --- a/barretenberg/cpp/scripts/bb-tests.sh +++ b/barretenberg/cpp/scripts/bb-tests.sh @@ -3,7 +3,7 @@ # Might be better to list exclusions here rather than inclusions as risky to maintain. set -eu -$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI diff --git a/barretenberg/cpp/scripts/run_tests b/barretenberg/cpp/scripts/run_tests index b8be58a5af2..e801ad2bde3 100755 --- a/barretenberg/cpp/scripts/run_tests +++ b/barretenberg/cpp/scripts/run_tests @@ -11,7 +11,7 @@ TESTS=$2 shift shift -$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI REPOSITORY=barretenberg-x86_64-linux-clang-assert diff --git a/barretenberg/ts/scripts/run_tests b/barretenberg/ts/scripts/run_tests index ea2b3bf8482..0f0d2895cec 100755 --- a/barretenberg/ts/scripts/run_tests +++ b/barretenberg/ts/scripts/run_tests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -xe -$(aws ecr-public get-login --region us-east-2 --no-include-email) 2> /dev/null +$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=bb.js IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) diff --git a/build-system/scripts/clean_image_tags b/build-system/scripts/clean_image_tags index 9da74edda65..9c4557e68c6 100755 --- a/build-system/scripts/clean_image_tags +++ b/build-system/scripts/clean_image_tags @@ -6,7 +6,7 @@ REPOSITORY=$1 EXISTING_TAG=$2 REGION=${3:-$ECR_REGION} -IMAGE_TAGS=$(aws ecr-public describe-images --repository-name $REPOSITORY --image-ids "imageTag=$EXISTING_TAG" | jq -r '.imageDetails[0].imageTags[]') +IMAGE_TAGS=$(aws ecr describe-images --repository-name $REPOSITORY --image-ids "imageTag=$EXISTING_TAG" | jq -r '.imageDetails[0].imageTags[]') TAGS_COUNT=$(echo "$IMAGE_TAGS" | wc -l) echo "Found $TAGS_COUNT for $REPOSITORY:$EXISTING_TAG" diff --git a/build-system/scripts/ecr_login b/build-system/scripts/ecr_login index 8e6c90c175a..5d9e6671e59 100755 --- a/build-system/scripts/ecr_login +++ b/build-system/scripts/ecr_login @@ -1,4 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -aws ecr-public get-login-password --region us-east-1 \ - | docker login --username AWS --password-stdin public.ecr.aws 2> /dev/null \ No newline at end of file +REGION=${1:-$ECR_REGION} +aws ecr get-login-password --region $REGION \ + | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$REGION.amazonaws.com 2> /dev/null \ No newline at end of file diff --git a/build-system/scripts/ensure_repo b/build-system/scripts/ensure_repo index 85f4fe6fa15..260f82fb4c0 100755 --- a/build-system/scripts/ensure_repo +++ b/build-system/scripts/ensure_repo @@ -29,12 +29,12 @@ REFRESH_LIFECYCLE=${3:-} retry ecr_login $REGION # Create the repository if it doesn't exist. -if ! aws ecr-public describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then +if ! aws ecr describe-repositories --region $REGION --repository-names $REPOSITORY > /dev/null 2>&1; then echo "Creating repo: $REPOSITORY" - aws ecr-public create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 - # aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 -# elif [ -n "$REFRESH_LIFECYCLE" ]; then -# echo "Refreshing lifecycle rules for repo: $REPOSITORY" -# aws ecr-public put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 + aws ecr create-repository --region $REGION --repository-name $REPOSITORY > /dev/null 2>&1 + aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 +elif [ -n "$REFRESH_LIFECYCLE" ]; then + echo "Refreshing lifecycle rules for repo: $REPOSITORY" + aws ecr put-lifecycle-policy --region $REGION --repository-name $REPOSITORY --lifecycle-policy-text "$LIFECYCLE_POLICY" > /dev/null 2>&1 fi echo diff --git a/build-system/scripts/image_exists b/build-system/scripts/image_exists index c8816fbc86e..0ad9d90ae69 100755 --- a/build-system/scripts/image_exists +++ b/build-system/scripts/image_exists @@ -1,4 +1,4 @@ #!/usr/bin/env bash set -eu # Returns true if the given image exists in the current ECR. -aws ecr-public describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 +aws ecr describe-images --region=$ECR_REGION --repository-name=$1 --image-ids=imageTag=$2 > /dev/null 2>&1 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 92583393acd..11fd7e70bfa 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -83,9 +83,9 @@ echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV -echo export ECR_REGION=us-east-1 >> $BASH_ENV +echo export ECR_REGION=us-east-2 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV -echo export ECR_URL=public.ecr.aws/aztec >> $BASH_ENV +echo export ECR_URL=278380418400.dkr.ecr.us-east-2.amazonaws.com >> $BASH_ENV echo export ECR_DEPLOY_REGION=eu-west-2 >> $BASH_ENV echo export DOCKERHUB_ACCOUNT=aztecprotocol >> $BASH_ENV echo export DOCKERHUB_USERNAME=aztecprotocolci >> $BASH_ENV diff --git a/build-system/scripts/tag_remote_image b/build-system/scripts/tag_remote_image index 3a7c0952035..0bb9e2a390f 100755 --- a/build-system/scripts/tag_remote_image +++ b/build-system/scripts/tag_remote_image @@ -7,7 +7,7 @@ EXISTING_TAG=$2 NEW_TAG=$3 REGION=${4:-$ECR_REGION} -EXISTING_TAG_MANIFEST=$(aws ecr-public batch-get-image \ +EXISTING_TAG_MANIFEST=$(aws ecr batch-get-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-ids imageTag=$EXISTING_TAG \ @@ -19,7 +19,7 @@ if [ -z "$EXISTING_TAG_MANIFEST" ]; then exit 1 fi -NEW_TAG_MANIFEST=$(aws ecr-public batch-get-image \ +NEW_TAG_MANIFEST=$(aws ecr batch-get-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-ids imageTag=$NEW_TAG \ @@ -28,7 +28,7 @@ NEW_TAG_MANIFEST=$(aws ecr-public batch-get-image \ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then echo "Tagging $1:$EXISTING_TAG as $1:$NEW_TAG..." - TAG_RESULT=$(aws ecr-public put-image \ + TAG_RESULT=$(aws ecr put-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ @@ -40,7 +40,7 @@ if [ "$EXISTING_TAG_MANIFEST" != "$NEW_TAG_MANIFEST" ]; then echo "Failed to tag due to limit exceeded. Starting tag cleanup." clean_image_tags $REPOSITORY $EXISTING_TAG $REGION - aws ecr-public put-image \ + aws ecr put-image \ --region $REGION \ --repository-name $REPOSITORY \ --image-tag $NEW_TAG \ diff --git a/build-system/scripts/untag_remote_image b/build-system/scripts/untag_remote_image index 51a4c777738..aa913c1200b 100755 --- a/build-system/scripts/untag_remote_image +++ b/build-system/scripts/untag_remote_image @@ -1,4 +1,4 @@ #!/usr/bin/env bash REPOSITORY=$1 TAG=$2 -aws ecr-public batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file +aws ecr batch-delete-image --region=$ECR_REGION --repository-name $REPOSITORY --image-ids imageTag=$2 > /dev/null \ No newline at end of file From 6fd63b6e91de899471fa1c4471260d2061a5c584 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 26 Feb 2024 18:36:25 +0000 Subject: [PATCH 24/29] fix and debug. --- build-system/scripts/build | 2 +- yarn-project/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index dbff7016daf..5ff4c289e39 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -104,7 +104,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do # Finally attempt to locate x86_64 image tag, as sometimes we build arch specific images from x86_64 images. PARENT_IMAGE_TAG=$(calculate_image_tag $PARENT_REPO x86_64) if ! image_exists $PARENT_REPO $PARENT_IMAGE_TAG; then - echo "Failed to locate multiarch image, arch specific image, or x86_64 image. Aborting." + echo "Failed to locate multiarch image, arch specific image, or x86_64 image for $PARENT_REPO. Aborting." exit 1 fi fi diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 1315b683147..3349532fa20 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -8,7 +8,7 @@ FROM aztecprotocol/l1-contracts as contracts FROM aztecprotocol/noir-projects as noir-projects FROM aztecprotocol/boxes-files as boxes-files -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base +FROM aztecprotocol/yarn-project-base COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects COPY --from=boxes-files /usr/src/boxes /usr/src/boxes From ae51b9563bc2de1308f1b1d68c9a97976305d57d Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 27 Feb 2024 10:36:17 +0000 Subject: [PATCH 25/29] [ci rebuild] From f29ea9c9552a67bd6c7b1e00ea4ac26685fa61d2 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 27 Feb 2024 11:16:22 +0000 Subject: [PATCH 26/29] bootstrap requires explicit command. Warnings around caching. --- avm-transpiler/bootstrap.sh | 2 +- avm-transpiler/bootstrap_cache.sh | 2 - barretenberg/cpp/bootstrap.sh | 8 ++-- barretenberg/cpp/bootstrap_cache.sh | 2 - barretenberg/ts/bootstrap.sh | 2 +- barretenberg/ts/bootstrap_cache.sh | 2 - bootstrap.sh | 61 +++++++++++++++------------ build-system/scripts/can_use_ci_cache | 4 ++ l1-contracts/bootstrap.sh | 2 +- l1-contracts/bootstrap_cache.sh | 2 - noir-projects/bootstrap.sh | 2 +- noir-projects/bootstrap_cache.sh | 2 - noir/bootstrap.sh | 2 +- noir/bootstrap_cache.sh | 2 - 14 files changed, 48 insertions(+), 47 deletions(-) create mode 100755 build-system/scripts/can_use_ci_cache diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 2c726cb4e43..259eab3a37a 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -17,6 +17,6 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh \ No newline at end of file diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index dafe9ef5f0a..080136e2be8 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 32de0856a4d..f7156202726 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -31,8 +31,11 @@ fi # Download ignition transcripts. (cd ./srs_db && ./download_ignition.sh 0) +# Install wasi-sdk. +./scripts/install-wasi-sdk.sh + # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit # Pick native toolchain file. ARCH=$(uname -m) @@ -64,9 +67,6 @@ if [ ! -d ./srs_db/grumpkin ]; then (cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192) fi -# Install wasi-sdk. -./scripts/install-wasi-sdk.sh - # Build WASM. cmake --preset wasm cmake --build --preset wasm diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 13e29707e3e..a739a6cab66 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index f8f4dfd56b8..05f2c14a9eb 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -19,7 +19,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit yarn install --immutable echo "Building with command 'yarn $BUILD_CMD'..." diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index c936d8cc7f5..c5e39052643 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/bootstrap.sh b/bootstrap.sh index adb97e67c76..dc40d2cc757 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -12,39 +12,48 @@ cd "$(dirname "$0")" CMD=${1:-} -if [ -n "$CMD" ]; then - if [ "$CMD" = "clean" ]; then - echo "WARNING: This will erase *all* untracked files, including hooks and submodules." - echo -n "Continue? [y/n] " - read user_input - if [ "$user_input" != "y" ] && [ "$user_input" != "Y" ]; then - exit 1 - fi +YELLOW="\033[93m" +BOLD="\033[1m" +RESET="\033[0m" - # Remove hooks and submodules. - rm -rf .git/hooks/* - rm -rf .git/modules/* - for SUBMODULE in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do - rm -rf $SUBMODULE - done +PATH=$PATH:./build-system/scripts - # Remove all untracked files, directories, nested repos, and .gitignore files. - git clean -ffdx +if [ "$CMD" = "clean" ]; then + echo "WARNING: This will erase *all* untracked files, including hooks and submodules." + echo -n "Continue? [y/n] " + read user_input + if [ "$user_input" != "y" ] && [ "$user_input" != "Y" ]; then + exit 1 + fi + + # Remove hooks and submodules. + rm -rf .git/hooks/* + rm -rf .git/modules/* + for SUBMODULE in $(git config --file .gitmodules --get-regexp path | awk '{print $2}'); do + rm -rf $SUBMODULE + done - exit 0 - elif [ "$CMD" = "full" ]; then - export NO_CACHE=1 - else - echo "Unknown command: $CMD" + # Remove all untracked files, directories, nested repos, and .gitignore files. + git clean -ffdx + + exit 0 +elif [ "$CMD" = "full" ]; then + export USE_CACHE= + if can_use_ci_cache; then + echo -e "${BOLD}${YELLOW}WARNING: Performing a full bootstrap. Consider leveraging './bootstrap.sh fast' to use CI cache.${RESET}" + echo + fi +elif [ "$CMD" = "fast" ]; then + export USE_CACHE=1 + if ! can_use_ci_cache; then + echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials.${RESET}" exit 1 fi +else + echo "usage: $0 " + exit 1 fi -# if [ ! -f ~/.nvm/nvm.sh ]; then -# echo "Nvm not found at ~/.nvm" -# exit 1 -# fi - # Install pre-commit git hooks. HOOKS_DIR=$(git rev-parse --git-path hooks) echo "(cd barretenberg/cpp && ./format.sh staged)" >$HOOKS_DIR/pre-commit diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache new file mode 100755 index 00000000000..27317a87632 --- /dev/null +++ b/build-system/scripts/can_use_ci_cache @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 \ No newline at end of file diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index 489ee2a80eb..ee1c46dba7e 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -19,7 +19,7 @@ fi . ./scripts/install_foundry.sh # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit # Clean rm -rf broadcast cache out serve diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 39ac1ecfc7a..911796ce9b5 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index 2dc019310b0..dd7e0468b6a 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit PROJECTS=( noir-contracts diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 1f0500b21ab..898fbdbea17 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 1f9506904a4..48f85e759fa 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit +[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh \ No newline at end of file diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index d06aa493662..1cec6c81d8e 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eu -[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 - cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null From b90c77072548181715699430fb8a513bb600cb4f Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 27 Feb 2024 11:50:20 +0000 Subject: [PATCH 27/29] Add parallelism to cpp builds native/wasm/wasm-threads. Fix undefined bash cache var. --- avm-transpiler/bootstrap.sh | 2 +- barretenberg/cpp/bootstrap.sh | 40 +++++++++++++++++++++++------------ barretenberg/ts/bootstrap.sh | 2 +- l1-contracts/bootstrap.sh | 2 +- noir-projects/bootstrap.sh | 2 +- noir/bootstrap.sh | 2 +- 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 259eab3a37a..e3446b3e6f1 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -17,6 +17,6 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh \ No newline at end of file diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index f7156202726..80ee4024b63 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -35,7 +35,7 @@ fi ./scripts/install-wasi-sdk.sh # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit # Pick native toolchain file. ARCH=$(uname -m) @@ -57,20 +57,34 @@ echo "# Building with preset: $PRESET" echo "# When running cmake directly, remember to use: --build --preset $PRESET" echo "#################################" -# Build native. -cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert -cmake --build --preset $PRESET --target bb +function build_native { + cmake --preset $PRESET -DCMAKE_BUILD_TYPE=RelWithAssert + cmake --build --preset $PRESET --target bb +} + +function build_wasm { + cmake --preset wasm + cmake --build --preset wasm +} + +function build_wasm_threads { + cmake --preset wasm-threads + cmake --build --preset wasm-threads +} + +g="\033[32m" # Green +b="\033[34m" # Blue +p="\033[35m" # Purple +r="\033[0m" # Reset + +(build_native > >(awk -v g="$g" -v r="$r" '$0=g"native: "r $0')) & +(build_wasm > >(awk -v b="$b" -v r="$r" '$0=b"wasm: "r $0')) & +(build_wasm_threads > >(awk -v p="$p" -v r="$r" '$0=p"wasm_threads: "r $0')) & + +wait if [ ! -d ./srs_db/grumpkin ]; then # The Grumpkin SRS is generated manually at the moment, only up to a large enough size for tests # If tests require more points, the parameter can be increased here. - (cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192) + cd ./build && cmake --build . --parallel --target grumpkin_srs_gen && ./bin/grumpkin_srs_gen 8192 fi - -# Build WASM. -cmake --preset wasm -cmake --build --preset wasm - -# Build WASM with new threading. -cmake --preset wasm-threads -cmake --build --preset wasm-threads diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index 05f2c14a9eb..e503cd4b52b 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -19,7 +19,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit yarn install --immutable echo "Building with command 'yarn $BUILD_CMD'..." diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index ee1c46dba7e..fa8189b567d 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -19,7 +19,7 @@ fi . ./scripts/install_foundry.sh # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit # Clean rm -rf broadcast cache out serve diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index dd7e0468b6a..319d28ae211 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit PROJECTS=( noir-contracts diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 48f85e759fa..b4373eb9a80 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -16,7 +16,7 @@ if [ -n "$CMD" ]; then fi # Attempt to just pull artefacts from CI and exit on success. -[ -n "$USE_CACHE" ] && ./bootstrap_cache.sh && exit +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh \ No newline at end of file From e943d08af94224e2bcd394b6350d362ca9c1d7bd Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 27 Feb 2024 11:53:55 +0000 Subject: [PATCH 28/29] Revert change r.e. setup-workflow. Performance seems bad. --- .circleci/config.yml | 2 +- build-system/scripts/generate_circleci_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f1f4e8313f0..8b5a4e08814 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,7 @@ jobs: generate-config: docker: - image: aztecprotocol/alpine-build-image - resource_class: small + resource_class: xlarge steps: - *checkout - *setup_env diff --git a/build-system/scripts/generate_circleci_config.py b/build-system/scripts/generate_circleci_config.py index 9b0f650f892..51dafde0af1 100755 --- a/build-system/scripts/generate_circleci_config.py +++ b/build-system/scripts/generate_circleci_config.py @@ -65,7 +65,7 @@ def _get_already_built_manifest_job_names(manifest_name): def get_already_built_manifest_job_names(): manifest_names = get_manifest_job_names() - with ProcessPoolExecutor(max_workers=8) as executor: + with ProcessPoolExecutor() as executor: futures = {executor.submit(_get_already_built_manifest_job_names, key): key for key in manifest_names} for future in as_completed(futures): result = future.result() From 07f7e14c73f8b9b547e6043f3bcc33ae103d5035 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Tue, 27 Feb 2024 12:19:18 +0000 Subject: [PATCH 29/29] better msg. --- bootstrap.sh | 5 ++--- build-system/scripts/can_use_ci_cache | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index dc40d2cc757..e6d67f21f65 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -16,7 +16,7 @@ YELLOW="\033[93m" BOLD="\033[1m" RESET="\033[0m" -PATH=$PATH:./build-system/scripts +source ./build-system/scripts/setup_env '' '' '' > /dev/null if [ "$CMD" = "clean" ]; then echo "WARNING: This will erase *all* untracked files, including hooks and submodules." @@ -38,7 +38,6 @@ if [ "$CMD" = "clean" ]; then exit 0 elif [ "$CMD" = "full" ]; then - export USE_CACHE= if can_use_ci_cache; then echo -e "${BOLD}${YELLOW}WARNING: Performing a full bootstrap. Consider leveraging './bootstrap.sh fast' to use CI cache.${RESET}" echo @@ -46,7 +45,7 @@ elif [ "$CMD" = "full" ]; then elif [ "$CMD" = "fast" ]; then export USE_CACHE=1 if ! can_use_ci_cache; then - echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials.${RESET}" + echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}" exit 1 fi else diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache index 27317a87632..2831dabb2bb 100755 --- a/build-system/scripts/can_use_ci_cache +++ b/build-system/scripts/can_use_ci_cache @@ -1,4 +1,3 @@ #!/usr/bin/env bash set -euo pipefail - -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 \ No newline at end of file +type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1 \ No newline at end of file