From 1e3812591c7bbad0a1af7bb7237ddea145a55b83 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 17:22:34 +0000 Subject: [PATCH 01/21] Build nargo against Ubuntu 20 for better compatability --- noir/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noir/Dockerfile b/noir/Dockerfile index 000292e0a47..bd509450287 100644 --- a/noir/Dockerfile +++ b/noir/Dockerfile @@ -1,10 +1,10 @@ -FROM rust:bookworm +FROM rust:bullseye WORKDIR /usr/src/noir COPY . . RUN ./scripts/bootstrap_native.sh # When running the container, mount the users home directory to same location. -FROM ubuntu:lunar +FROM ubuntu:focal # Install Tini as nargo doesn't handle signals properly. # Install git as nargo needs it to clone. RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean From 31cf1a9a2abca62156fac7e9396c17050ad0e558 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 17:58:01 +0000 Subject: [PATCH 02/21] retry a docker create thing. --- build-system/scripts/create_ecr_manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/create_ecr_manifest b/build-system/scripts/create_ecr_manifest index 0495c3c165b..4e258f59df3 100755 --- a/build-system/scripts/create_ecr_manifest +++ b/build-system/scripts/create_ecr_manifest @@ -24,7 +24,7 @@ IFS=',' for A in $ARCH_LIST; do IMAGE_URI=$(calculate_image_uri $REPOSITORY $A) echo "Adding image $IMAGE_URI to manifest list $MULTIARCH_IMAGE_URI..." - docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI + retry docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI done retry docker manifest push --purge $MULTIARCH_IMAGE_URI From 60a38d32a317e99981f49cbaa03f156895bb2980 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 21 Feb 2024 19:10:02 +0000 Subject: [PATCH 03/21] bootstrap scripts try to use cache by default. l1-contracts will attempt to use ci cache. --- avm-transpiler/bootstrap.sh | 3 +++ barretenberg/bootstrap.sh | 3 +++ bootstrap.sh | 18 +++++------------- l1-contracts/Dockerfile | 3 +++ l1-contracts/bootstrap.sh | 3 +++ l1-contracts/bootstrap_cache.sh | 8 ++++++++ noir/bootstrap.sh | 3 +++ noir/bootstrap_cache.sh | 1 - 8 files changed, 28 insertions(+), 14 deletions(-) create mode 100755 l1-contracts/bootstrap_cache.sh diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 916d8107876..2c726cb4e43 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -16,4 +16,7 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + ./scripts/bootstrap_native.sh \ No newline at end of file diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index c16fd294b6c..9d398f7105b 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -eu +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + cd "$(dirname "$0")" (cd cpp && ./bootstrap.sh $@) diff --git a/bootstrap.sh b/bootstrap.sh index 818d5795637..fb1e5c8b2db 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -61,19 +61,11 @@ PROJECTS=( # Build projects locally for P in "${PROJECTS[@]}"; do - if [ -n "${BOOTSTRAP_USE_REMOTE_CACHE:-}" ] && [ -f "$P/bootstrap_cache.sh" ]; then - echo "**************************************" - echo -e "\033[1mBootstrapping $P from remote cache...\033[0m" - echo "**************************************" - echo - $P/bootstrap_cache.sh - else - echo "**************************************" - echo -e "\033[1mBootstrapping $P...\033[0m" - echo "**************************************" - echo - $P/bootstrap.sh - fi + echo "**************************************" + echo -e "\033[1mBootstrapping $P...\033[0m" + echo "**************************************" + echo + $P/bootstrap.sh echo echo done diff --git a/l1-contracts/Dockerfile b/l1-contracts/Dockerfile index 1ced5c8672e..bb0511f6657 100644 --- a/l1-contracts/Dockerfile +++ b/l1-contracts/Dockerfile @@ -13,3 +13,6 @@ RUN forge clean && forge fmt --check && forge build && forge test RUN yarn && yarn lint RUN git add . && yarn slither && yarn slither-has-diff RUN forge build + +FROM scratch +COPY --from=0 /usr/src/l1-contracts/out /usr/src/l1-contracts/out \ No newline at end of file diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index f776b6072dc..c11b7d56383 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -15,6 +15,9 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + # Clean rm -rf broadcast cache out serve diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh new file mode 100755 index 00000000000..5d9d9c7931e --- /dev/null +++ b/l1-contracts/bootstrap_cache.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eu + +cd "$(dirname "$0")" +source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" +extract_repo l1-contracts /usr/src/l1-contracts/out . diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 5ebe7ade090..1f9506904a4 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -15,5 +15,8 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + ./scripts/bootstrap_native.sh ./scripts/bootstrap_packages.sh \ No newline at end of file diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 672702416bd..ac919f3ca65 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -8,4 +8,3 @@ echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" extract_repo noir-packages /usr/src/noir/packages ./ echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" extract_repo noir /usr/src/noir/target/release ./target/ - From 97822b101f5bb7660ed37a2a752b488810ecfdb7 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:04:32 +0000 Subject: [PATCH 04/21] Remove old cache images. --- avm-transpiler/bootstrap_cache.sh | 2 ++ barretenberg/bootstrap.sh | 3 --- barretenberg/cpp/bootstrap.sh | 3 +++ barretenberg/{ => cpp}/bootstrap_cache.sh | 7 +++---- barretenberg/ts/bootstrap.sh | 5 ++++- barretenberg/ts/bootstrap_cache.sh | 12 ++++++++++++ bootstrap.sh | 2 +- build-system/scripts/remove_old_images | 13 +++++++++++++ l1-contracts/bootstrap_cache.sh | 2 ++ l1-contracts/src/core/libraries/ConstantsGen.sol | 12 ++++-------- noir/bootstrap_cache.sh | 3 +++ yarn-project/circuits.js/package.json | 2 +- 12 files changed, 48 insertions(+), 18 deletions(-) rename barretenberg/{ => cpp}/bootstrap_cache.sh (54%) create mode 100755 barretenberg/ts/bootstrap_cache.sh create mode 100755 build-system/scripts/remove_old_images diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index a618d231593..080136e2be8 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -7,3 +7,5 @@ source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" extract_repo avm-transpiler \ /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/ + +remove_old_images avm-transpiler diff --git a/barretenberg/bootstrap.sh b/barretenberg/bootstrap.sh index 9d398f7105b..c16fd294b6c 100755 --- a/barretenberg/bootstrap.sh +++ b/barretenberg/bootstrap.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash set -eu -# Attempt to just pull artefacts from CI and exit on success. -./bootstrap_cache.sh && exit - cd "$(dirname "$0")" (cd cpp && ./bootstrap.sh $@) diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh index 0b9295bafbd..32de0856a4d 100755 --- a/barretenberg/cpp/bootstrap.sh +++ b/barretenberg/cpp/bootstrap.sh @@ -31,6 +31,9 @@ fi # Download ignition transcripts. (cd ./srs_db && ./download_ignition.sh 0) +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + # Pick native toolchain file. ARCH=$(uname -m) if [ "$OS" == "macos" ]; then diff --git a/barretenberg/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh similarity index 54% rename from barretenberg/bootstrap_cache.sh rename to barretenberg/cpp/bootstrap_cache.sh index 71e081f77e8..a739a6cab66 100755 --- a/barretenberg/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -2,12 +2,11 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null +source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m" -extract_repo bb.js \ +extract_repo barretenberg-wasm-linux-clang \ /usr/src/barretenberg/cpp/build-wasm/bin ./cpp/build-wasm \ /usr/src/barretenberg/cpp/build-wasm-threads/bin ./cpp/build-wasm-threads -echo -e "\033[1mBuilding ESM bb.ts...\033[0m" -(cd ts && SKIP_CPP_BUILD=1 ./scripts/build_wasm.sh && ./bootstrap.sh esm) +remove_old_images barretenberg-wasm-linux-clang diff --git a/barretenberg/ts/bootstrap.sh b/barretenberg/ts/bootstrap.sh index 1c0464dc212..f8f4dfd56b8 100755 --- a/barretenberg/ts/bootstrap.sh +++ b/barretenberg/ts/bootstrap.sh @@ -18,10 +18,13 @@ if [ -n "$CMD" ]; then fi fi +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + yarn install --immutable echo "Building with command 'yarn $BUILD_CMD'..." yarn $BUILD_CMD # Make bin globally available. npm link -echo "Barretenberg ts build successful" \ No newline at end of file +echo "Barretenberg ts build successful" diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh new file mode 100755 index 00000000000..c5e39052643 --- /dev/null +++ b/barretenberg/ts/bootstrap_cache.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eu + +cd "$(dirname "$0")" +source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m" +extract_repo bb.js /usr/src/barretenberg/ts/dest . +# Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. +yarn install + +remove_old_images bb.js diff --git a/bootstrap.sh b/bootstrap.sh index fb1e5c8b2db..8cd728f1d02 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -65,7 +65,7 @@ for P in "${PROJECTS[@]}"; do echo -e "\033[1mBootstrapping $P...\033[0m" echo "**************************************" echo - $P/bootstrap.sh + (cd $P && ./bootstrap.sh) echo echo done diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images new file mode 100755 index 00000000000..128fbbdf444 --- /dev/null +++ b/build-system/scripts/remove_old_images @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Removes all cache-* docker images for the given repository that are not the current content hash. +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +shift + +IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) +for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do + echo "Removing $IMAGE..." + docker rmi $IMAGE +done diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 5d9d9c7931e..911796ce9b5 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -6,3 +6,5 @@ source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" extract_repo l1-contracts /usr/src/l1-contracts/out . + +remove_old_images l1-contracts diff --git a/l1-contracts/src/core/libraries/ConstantsGen.sol b/l1-contracts/src/core/libraries/ConstantsGen.sol index f8a7ae2576e..db3f98aacbe 100644 --- a/l1-contracts/src/core/libraries/ConstantsGen.sol +++ b/l1-contracts/src/core/libraries/ConstantsGen.sol @@ -76,14 +76,10 @@ library Constants { uint256 internal constant MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS = 1000; uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_PRIVATE_FUNCTION_IN_FIELDS = 500; uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_UNCONSTRAINED_FUNCTION_IN_FIELDS = 500; - uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = - 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; - uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = - 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; - uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = - 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; - uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = - 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; + uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; + uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; + uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; + uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; uint256 internal constant L1_TO_L2_MESSAGE_ORACLE_CALL_LENGTH = 25; uint256 internal constant MAX_NOTE_FIELDS_LENGTH = 20; uint256 internal constant GET_NOTE_ORACLE_RETURN_LENGTH = 23; diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index ac919f3ca65..1cec6c81d8e 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -8,3 +8,6 @@ echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" extract_repo noir-packages /usr/src/noir/packages ./ echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" extract_repo noir /usr/src/noir/target/release ./target/ + +remove_old_images noir-packages +remove_old_images noir diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index 715e844ad06..a3888d384bb 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -26,7 +26,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts && cd ../../l1-contracts && ./.foundry/bin/forge fmt", + "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "inherits": [ From f4d59db4ddb773144345a4786c151f0e99aa8e5b Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:40:46 +0000 Subject: [PATCH 05/21] Better retry. --- build-system/scripts/retry | 10 ++++++++-- build-system/scripts/setup_env | 7 ++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/build-system/scripts/retry b/build-system/scripts/retry index 0489aa226fd..2172e7dcfd9 100755 --- a/build-system/scripts/retry +++ b/build-system/scripts/retry @@ -1,7 +1,13 @@ +if [ -n "$RETRY_DISABLED" ]; then + "$@" && exit || exit 1 +fi + ATTEMPTS=3 -# Retries up to 3 times with 10 second intervals +# Retries up to 3 times with 5 second intervals for i in $(seq 1 $ATTEMPTS); do - "$@" && exit || sleep 10 + "$@" && exit + [ "$i" != "$ATTEMPTS" ] && sleep 5 done + >&2 echo "$@ failed after $ATTEMPTS attempts" exit 1 diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index dc6ad03d0df..14482b99fb0 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -6,6 +6,10 @@ # The script should be sourced from the root of the repository, e.g: # source ./build-system/scripts/setup_env # This ensures the resultant variables are set in the calling shell. + +# Save current options as we might be sourcing. +CURRENT_SHELL_OPTS=$(set +o) +trap 'eval "$CURRENT_SHELL_OPTS"' EXIT [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu @@ -139,6 +143,3 @@ cat $BASH_ENV # Having written the variables to $BASH_ENV, we now want to set them in this shell context. source $BASH_ENV - -# This script is often "sourced", we don't want calling shell to exit on a subsequent error! -set +e From 6ac200099db87684ec74093699ddd83eb661aca3 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:48:27 +0000 Subject: [PATCH 06/21] Early out if can't use cache. --- avm-transpiler/bootstrap_cache.sh | 2 ++ barretenberg/cpp/bootstrap_cache.sh | 2 ++ barretenberg/ts/bootstrap_cache.sh | 2 ++ l1-contracts/bootstrap_cache.sh | 2 ++ noir/bootstrap_cache.sh | 2 ++ 5 files changed, 10 insertions(+) diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 080136e2be8..96f68ab0e76 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index a739a6cab66..9f420deda2e 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index c5e39052643..ceb9e561db4 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 911796ce9b5..a4fc1b05a3f 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 1cec6c81d8e..7a3523ab93b 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -eu +type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null From bd3a1bdac12c07d4aa1b555f43b074f034d0548c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:49:56 +0000 Subject: [PATCH 07/21] fix unbound --- build-system/scripts/retry | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/retry b/build-system/scripts/retry index 2172e7dcfd9..6cbfdc14649 100755 --- a/build-system/scripts/retry +++ b/build-system/scripts/retry @@ -1,4 +1,4 @@ -if [ -n "$RETRY_DISABLED" ]; then +if [ -n "${RETRY_DISABLED:-}" ]; then "$@" && exit || exit 1 fi From c52fec804e1157e0416935eeabd5da0b2788229c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 09:52:28 +0000 Subject: [PATCH 08/21] Remove codegen sol file. --- .../src/core/libraries/ConstantsGen.sol | 120 ------------------ 1 file changed, 120 deletions(-) delete mode 100644 l1-contracts/src/core/libraries/ConstantsGen.sol diff --git a/l1-contracts/src/core/libraries/ConstantsGen.sol b/l1-contracts/src/core/libraries/ConstantsGen.sol deleted file mode 100644 index db3f98aacbe..00000000000 --- a/l1-contracts/src/core/libraries/ConstantsGen.sol +++ /dev/null @@ -1,120 +0,0 @@ -// GENERATED FILE - DO NOT EDIT, RUN yarn remake-constants in circuits.js -// SPDX-License-Identifier: Apache-2.0 -// Copyright 2023 Aztec Labs. -pragma solidity >=0.8.18; - -/** - * @title Constants Library - * @author Aztec Labs - * @notice Library that contains constants used throughout the Aztec protocol - */ -library Constants { - // Prime field modulus - uint256 internal constant P = - 21888242871839275222246405745257275088548364400416034343698204186575808495617; - uint256 internal constant MAX_FIELD_VALUE = P - 1; - - uint256 internal constant ARGS_LENGTH = 16; - uint256 internal constant RETURN_VALUES_LENGTH = 4; - uint256 internal constant MAX_NEW_COMMITMENTS_PER_CALL = 16; - uint256 internal constant MAX_NEW_NULLIFIERS_PER_CALL = 16; - uint256 internal constant MAX_PRIVATE_CALL_STACK_LENGTH_PER_CALL = 4; - uint256 internal constant MAX_PUBLIC_CALL_STACK_LENGTH_PER_CALL = 4; - uint256 internal constant MAX_NEW_L2_TO_L1_MSGS_PER_CALL = 2; - uint256 internal constant MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_CALL = 16; - uint256 internal constant MAX_PUBLIC_DATA_READS_PER_CALL = 16; - uint256 internal constant MAX_READ_REQUESTS_PER_CALL = 32; - uint256 internal constant MAX_NULLIFIER_KEY_VALIDATION_REQUESTS_PER_CALL = 1; - uint256 internal constant MAX_NEW_COMMITMENTS_PER_TX = 64; - uint256 internal constant MAX_NON_REVERTIBLE_COMMITMENTS_PER_TX = 8; - uint256 internal constant MAX_REVERTIBLE_COMMITMENTS_PER_TX = 56; - uint256 internal constant MAX_NEW_NULLIFIERS_PER_TX = 64; - uint256 internal constant MAX_NON_REVERTIBLE_NULLIFIERS_PER_TX = 8; - uint256 internal constant MAX_REVERTIBLE_NULLIFIERS_PER_TX = 56; - uint256 internal constant MAX_PRIVATE_CALL_STACK_LENGTH_PER_TX = 8; - uint256 internal constant MAX_PUBLIC_CALL_STACK_LENGTH_PER_TX = 8; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX = 3; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_CALL_STACK_LENGTH_PER_TX = 5; - uint256 internal constant MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 32; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 16; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX = 16; - uint256 internal constant MAX_PUBLIC_DATA_READS_PER_TX = 32; - uint256 internal constant MAX_NON_REVERTIBLE_PUBLIC_DATA_READS_PER_TX = 16; - uint256 internal constant MAX_REVERTIBLE_PUBLIC_DATA_READS_PER_TX = 16; - uint256 internal constant MAX_NEW_L2_TO_L1_MSGS_PER_TX = 2; - uint256 internal constant MAX_NEW_CONTRACTS_PER_TX = 1; - uint256 internal constant MAX_READ_REQUESTS_PER_TX = 128; - uint256 internal constant MAX_NULLIFIER_KEY_VALIDATION_REQUESTS_PER_TX = 4; - uint256 internal constant NUM_ENCRYPTED_LOGS_HASHES_PER_TX = 1; - uint256 internal constant NUM_UNENCRYPTED_LOGS_HASHES_PER_TX = 1; - uint256 internal constant NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP = 16; - uint256 internal constant VK_TREE_HEIGHT = 3; - uint256 internal constant FUNCTION_TREE_HEIGHT = 5; - uint256 internal constant CONTRACT_TREE_HEIGHT = 16; - uint256 internal constant NOTE_HASH_TREE_HEIGHT = 32; - uint256 internal constant PUBLIC_DATA_TREE_HEIGHT = 40; - uint256 internal constant NULLIFIER_TREE_HEIGHT = 20; - uint256 internal constant L1_TO_L2_MSG_TREE_HEIGHT = 16; - uint256 internal constant ROLLUP_VK_TREE_HEIGHT = 8; - uint256 internal constant ARTIFACT_FUNCTION_TREE_MAX_HEIGHT = 5; - uint256 internal constant CONTRACT_SUBTREE_HEIGHT = 0; - uint256 internal constant CONTRACT_SUBTREE_SIBLING_PATH_LENGTH = 16; - uint256 internal constant NOTE_HASH_SUBTREE_HEIGHT = 6; - uint256 internal constant NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH = 26; - uint256 internal constant NULLIFIER_SUBTREE_HEIGHT = 6; - uint256 internal constant PUBLIC_DATA_SUBTREE_HEIGHT = 5; - uint256 internal constant ARCHIVE_HEIGHT = 16; - uint256 internal constant NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH = 14; - uint256 internal constant PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH = 35; - uint256 internal constant L1_TO_L2_MSG_SUBTREE_HEIGHT = 4; - uint256 internal constant L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH = 12; - uint256 internal constant FUNCTION_SELECTOR_NUM_BYTES = 4; - uint256 internal constant MAPPING_SLOT_PEDERSEN_SEPARATOR = 4; - uint256 internal constant NUM_FIELDS_PER_SHA256 = 2; - uint256 internal constant ARGS_HASH_CHUNK_LENGTH = 32; - uint256 internal constant ARGS_HASH_CHUNK_COUNT = 32; - uint256 internal constant MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS = 1000; - uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_PRIVATE_FUNCTION_IN_FIELDS = 500; - uint256 internal constant MAX_PACKED_BYTECODE_SIZE_PER_UNCONSTRAINED_FUNCTION_IN_FIELDS = 500; - uint256 internal constant REGISTERER_CONTRACT_CLASS_REGISTERED_MAGIC_VALUE = 0x6999d1e02b08a447a463563453cb36919c9dd7150336fc7c4d2b52f8; - uint256 internal constant REGISTERER_PRIVATE_FUNCTION_BROADCASTED_MAGIC_VALUE = 0x1b70e95fde0b70adc30496b90a327af6a5e383e028e7a43211a07bcd; - uint256 internal constant REGISTERER_UNCONSTRAINED_FUNCTION_BROADCASTED_MAGIC_VALUE = 0xe7af816635466f128568edb04c9fa024f6c87fb9010fdbffa68b3d99; - uint256 internal constant DEPLOYER_CONTRACT_INSTANCE_DEPLOYED_MAGIC_VALUE = 0x85864497636cf755ae7bde03f267ce01a520981c21c3682aaf82a631; - uint256 internal constant L1_TO_L2_MESSAGE_ORACLE_CALL_LENGTH = 25; - uint256 internal constant MAX_NOTE_FIELDS_LENGTH = 20; - uint256 internal constant GET_NOTE_ORACLE_RETURN_LENGTH = 23; - uint256 internal constant MAX_NOTES_PER_PAGE = 10; - uint256 internal constant VIEW_NOTE_ORACLE_RETURN_LENGTH = 212; - uint256 internal constant AZTEC_ADDRESS_LENGTH = 1; - uint256 internal constant CALL_CONTEXT_LENGTH = 8; - uint256 internal constant CONTENT_COMMITMENT_LENGTH = 7; - uint256 internal constant CONTRACT_DEPLOYMENT_DATA_LENGTH = 6; - uint256 internal constant CONTRACT_STORAGE_READ_LENGTH = 2; - uint256 internal constant CONTRACT_STORAGE_UPDATE_REQUEST_LENGTH = 2; - uint256 internal constant ETH_ADDRESS_LENGTH = 1; - uint256 internal constant FUNCTION_DATA_LENGTH = 4; - uint256 internal constant FUNCTION_LEAF_PREIMAGE_LENGTH = 5; - uint256 internal constant GLOBAL_VARIABLES_LENGTH = 6; - uint256 internal constant HEADER_LENGTH = 25; - uint256 internal constant L1_TO_L2_MESSAGE_LENGTH = 8; - uint256 internal constant L2_TO_L1_MESSAGE_LENGTH = 2; - uint256 internal constant NEW_CONTRACT_DATA_LENGTH = 3; - uint256 internal constant NULLIFIER_KEY_VALIDATION_REQUEST_LENGTH = 4; - uint256 internal constant NULLIFIER_KEY_VALIDATION_REQUEST_CONTEXT_LENGTH = 5; - uint256 internal constant PARTIAL_STATE_REFERENCE_LENGTH = 8; - uint256 internal constant PRIVATE_CALL_STACK_ITEM_LENGTH = 219; - uint256 internal constant PRIVATE_CIRCUIT_PUBLIC_INPUTS_LENGTH = 214; - uint256 internal constant PUBLIC_CIRCUIT_PUBLIC_INPUTS_LENGTH = 194; - uint256 internal constant STATE_REFERENCE_LENGTH = 10; - uint256 internal constant TX_CONTEXT_DATA_LENGTH = 11; - uint256 internal constant TX_REQUEST_LENGTH = 17; - uint256 internal constant GET_NOTES_ORACLE_RETURN_LENGTH = 674; - uint256 internal constant COMMITMENTS_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant NULLIFIERS_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant PUBLIC_DATA_WRITES_NUM_BYTES_PER_BASE_ROLLUP = 2048; - uint256 internal constant CONTRACTS_NUM_BYTES_PER_BASE_ROLLUP = 32; - uint256 internal constant CONTRACT_DATA_NUM_BYTES_PER_BASE_ROLLUP = 64; - uint256 internal constant CONTRACT_DATA_NUM_BYTES_PER_BASE_ROLLUP_UNPADDED = 52; - uint256 internal constant L2_TO_L1_MSGS_NUM_BYTES_PER_BASE_ROLLUP = 64; - uint256 internal constant LOGS_HASHES_NUM_BYTES_PER_BASE_ROLLUP = 64; -} From 332e1a39d5b5af3f103b2b65ff644ad059fd884c Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 13:48:42 +0000 Subject: [PATCH 09/21] fix --- build_manifest.yml | 3 ++- l1-contracts/bootstrap.sh | 6 +++--- yarn-project/circuits.js/package.json | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index bac4b7bc86e..9517efe5c8b 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -226,4 +226,5 @@ avm-transpiler: dockerfile: avm-transpiler/Dockerfile rebuildPatterns: - ^avm-transpiler/ - - ^noir/ + dependencies: + - noir diff --git a/l1-contracts/bootstrap.sh b/l1-contracts/bootstrap.sh index c11b7d56383..489ee2a80eb 100755 --- a/l1-contracts/bootstrap.sh +++ b/l1-contracts/bootstrap.sh @@ -15,15 +15,15 @@ if [ -n "$CMD" ]; then fi fi +# Install foundry. +. ./scripts/install_foundry.sh + # Attempt to just pull artefacts from CI and exit on success. ./bootstrap_cache.sh && exit # Clean rm -rf broadcast cache out serve -# Install foundry. -. ./scripts/install_foundry.sh - # Install forge install --no-commit diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index a3888d384bb..715e844ad06 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -26,7 +26,7 @@ "clean": "rm -rf ./dest .tsbuildinfo", "formatting": "run -T prettier --check ./src && run -T eslint ./src", "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", - "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts", + "remake-constants": "node --loader ts-node/esm src/scripts/constants.in.ts && prettier -w src/constants.gen.ts && cd ../../l1-contracts && ./.foundry/bin/forge fmt", "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests" }, "inherits": [ From e40e51d6b9e12aa710afc8648831fe7b9befc2b8 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:08:07 +0000 Subject: [PATCH 10/21] avm-transpiler to target ubuntu 20 --- avm-transpiler/Dockerfile | 6 +++--- build-system/scripts/remove_old_images | 2 +- noir/Dockerfile | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/avm-transpiler/Dockerfile b/avm-transpiler/Dockerfile index 1f55f8ba40d..95a4ead89ee 100644 --- a/avm-transpiler/Dockerfile +++ b/avm-transpiler/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:bookworm +FROM rust:bullseye WORKDIR /usr/src COPY ./avm-transpiler ./avm-transpiler @@ -8,6 +8,6 @@ WORKDIR /usr/src/avm-transpiler RUN apt-get update && apt-get install -y git RUN ./scripts/bootstrap_native.sh -FROM ubuntu:lunar +FROM ubuntu:focal COPY --from=0 /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler -ENTRYPOINT ["sh", "-c"] \ No newline at end of file +ENTRYPOINT ["sh", "-c"] diff --git a/build-system/scripts/remove_old_images b/build-system/scripts/remove_old_images index 128fbbdf444..7c1c927e6b6 100755 --- a/build-system/scripts/remove_old_images +++ b/build-system/scripts/remove_old_images @@ -9,5 +9,5 @@ shift IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY) for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do echo "Removing $IMAGE..." - docker rmi $IMAGE + docker rmi --force $IMAGE done diff --git a/noir/Dockerfile b/noir/Dockerfile index bd509450287..3a478c3f95a 100644 --- a/noir/Dockerfile +++ b/noir/Dockerfile @@ -9,4 +9,4 @@ FROM ubuntu:focal # Install git as nargo needs it to clone. RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean COPY --from=0 /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo -ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"] From 6ece897a4fccf05872d0e64aedd8b0971cb64950 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:14:08 +0000 Subject: [PATCH 11/21] fix --- build_manifest.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build_manifest.yml b/build_manifest.yml index 9517efe5c8b..7a144219eff 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -146,7 +146,7 @@ yarn-project-base: yarn-project: buildDir: yarn-project rebuildPatterns: - - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ - ^yarn-project/Dockerfile$ - ^yarn-project/cli/aztec-cli dependencies: @@ -226,5 +226,4 @@ avm-transpiler: dockerfile: avm-transpiler/Dockerfile rebuildPatterns: - ^avm-transpiler/ - dependencies: - - noir + - ^noir/ From 2465a8d0751be71776e5fc27e9bebd4c7a1e7aab Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 23 Feb 2024 14:20:27 +0000 Subject: [PATCH 12/21] fix --- build_manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_manifest.yml b/build_manifest.yml index 7a144219eff..bac4b7bc86e 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -146,7 +146,7 @@ yarn-project-base: yarn-project: buildDir: yarn-project rebuildPatterns: - - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml)$ + - ^yarn-project/.*\.(ts|tsx|js|cjs|mjs|json|html|md|sh|nr|toml|snap)$ - ^yarn-project/Dockerfile$ - ^yarn-project/cli/aztec-cli dependencies: From 0b0efad65c924b56c79c26884a987bd2d35e5f10 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sat, 24 Feb 2024 19:27:54 +0000 Subject: [PATCH 13/21] remove old cache images when doing bootstrap_docker.sh --- build-system/scripts/build_local | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index d32d70eb349..8f4121dc5e7 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -96,24 +96,27 @@ for E in "${PROJECTS[@]}"; do if (cd $(git rev-parse --show-toplevel) && git diff-index --name-only HEAD; git ls-files --others --exclude-standard) | grep -qE "$GREP_PATTERN"; then echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}" - docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $DEPLOY_IMAGE_URI . + docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI . else if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}" - docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI else if [ -z "$NO_CACHE" ] && [ -f ~/.aws/credentials ] && ecr_login && image_exists $REPO $TAG ; then docker pull $CACHE_IMAGE_URI else docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI . fi - docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI fi fi + # Retag with deployment uri. + docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI + # Retag for aztecprotocol dockerhub. docker tag $DEPLOY_IMAGE_URI aztecprotocol/$REPO:latest + remove_old_images $REPO + echo -e "${BOLD}Tagged${RESET}: aztecprotocol/$REPO:latest" echo -e "${BOLD}SHA256${RESET}: $(docker inspect --format='{{.Id}}' $DEPLOY_IMAGE_URI)" From 5c56c48948b56d820ed9937e8f85c82524014dfb Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 10:41:44 +0000 Subject: [PATCH 14/21] Allow skipping use of cache with NO_CACHE --- avm-transpiler/bootstrap_cache.sh | 2 +- barretenberg/cpp/bootstrap_cache.sh | 2 +- barretenberg/ts/bootstrap_cache.sh | 2 +- l1-contracts/bootstrap_cache.sh | 2 +- noir/bootstrap_cache.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 96f68ab0e76..dafe9ef5f0a 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 9f420deda2e..13e29707e3e 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index ceb9e561db4..c936d8cc7f5 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index a4fc1b05a3f..39ac1ecfc7a 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 7a3523ab93b..d06aa493662 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -eu -type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null From e8b55f4ae5ed58c18e6f001ce8383408603fb946 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 10:43:11 +0000 Subject: [PATCH 15/21] Add ./bootstrap.sh full command. --- bootstrap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bootstrap.sh b/bootstrap.sh index 8cd728f1d02..adb97e67c76 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -32,6 +32,8 @@ if [ -n "$CMD" ]; then git clean -ffdx exit 0 + elif [ "$CMD" = "full" ]; then + export NO_CACHE=1 else echo "Unknown command: $CMD" exit 1 From 4e7f7cb1833bdf12e9a78d79108bd8fdddec802e Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 13:47:35 +0000 Subject: [PATCH 16/21] Make noir-projects cli cache friendly. --- noir-projects/Dockerfile | 11 ++++++----- noir-projects/bootstrap.sh | 15 +++++++++++++++ noir-projects/bootstrap_cache.sh | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100755 noir-projects/bootstrap_cache.sh diff --git a/noir-projects/Dockerfile b/noir-projects/Dockerfile index d4fa7dc97d8..571bc975e83 100644 --- a/noir-projects/Dockerfile +++ b/noir-projects/Dockerfile @@ -1,21 +1,22 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/avm-transpiler as transpiler -FROM ubuntu:lunar +FROM ubuntu:lunar AS builder # Copy in nargo COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo # Copy in transpiler COPY --from=transpiler /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler - -WORKDIR /usr/src/noir-projects # Copy in noir projects +WORKDIR /usr/src/noir-projects COPY . . # Build WORKDIR /usr/src/noir-projects/noir-contracts RUN ./scripts/compile.sh && ./scripts/transpile.sh && ../../noir/target/release/nargo test --silence-warnings - WORKDIR /usr/src/noir-projects/noir-protocol-circuits RUN cd src && ../../../noir/target/release/nargo compile --silence-warnings && ../../../noir/target/release/nargo test --silence-warnings - WORKDIR /usr/src/noir-projects/aztec-nr RUN ../../noir/target/release/nargo compile --silence-warnings && ../../noir/target/release/nargo test --silence-warnings + +FROM scratch +COPY --from=builder /usr/src/noir-projects/noir-contracts/target /usr/src/noir-projects/noir-contracts/target +COPY --from=builder /usr/src/noir-projects/noir-protocol-circuits/src/target /usr/src/noir-projects/noir-protocol-circuits/src/target \ No newline at end of file diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index 20d1da50304..2dc019310b0 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -3,6 +3,21 @@ set -eu cd "$(dirname "$0")" +CMD=${1:-} + +if [ -n "$CMD" ]; then + if [ "$CMD" = "clean" ]; then + git clean -fdx + exit 0 + else + echo "Unknown command: $CMD" + exit 1 + fi +fi + +# Attempt to just pull artefacts from CI and exit on success. +./bootstrap_cache.sh && exit + PROJECTS=( noir-contracts noir-protocol-circuits diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh new file mode 100755 index 00000000000..1f0500b21ab --- /dev/null +++ b/noir-projects/bootstrap_cache.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -eu + +[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1 + +cd "$(dirname "$0")" +source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" +extract_repo noir-projects \ + /usr/src/noir-projects/noir-contracts/target ./noir-contracts \ + /usr/src/noir-projects/noir-protocol-circuits/src/target ./noir-protocol-circuits/src + +remove_old_images noir-projects From 18bfdbb31c5e3bfdc4727229d809682915dac334 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 14:14:56 +0000 Subject: [PATCH 17/21] wip --- .circleci/config.yml | 7 +++---- build_manifest.yml | 8 +++----- yarn-project/Dockerfile | 15 ++++++++++----- yarn-project/yarn-project-base/Dockerfile | 17 ++--------------- 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1cec4bf45e..5a71d16e12a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1282,16 +1282,15 @@ workflows: # Yarn Project - yarn-project-base: requires: - - avm-transpiler - - l1-contracts - bb-js - noir-packages - - noir-projects - - boxes-files <<: *defaults - yarn-project: requires: - yarn-project-base + - l1-contracts + - noir-projects + - boxes-files <<: *defaults - yarn-project-prod: *defaults_yarn_project - yarn-project-formatting: *defaults_yarn_project diff --git a/build_manifest.yml b/build_manifest.yml index bac4b7bc86e..6241bb3cc1b 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -135,13 +135,8 @@ yarn-project-base: - ^yarn-project/yarn.lock - ^yarn-project/.*/package.json$ dependencies: - - l1-contracts - bb.js - - noir - noir-packages - - boxes-files - - avm-transpiler - - noir-projects yarn-project: buildDir: yarn-project @@ -151,6 +146,9 @@ yarn-project: - ^yarn-project/cli/aztec-cli dependencies: - yarn-project-base + - l1-contracts + - boxes-files + - noir-projects yarn-project-prod: buildDir: yarn-project diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 9de7d8776b8..aea67d0c588 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -4,19 +4,24 @@ # - Run the tests. # - Run the formatter checks. # Any subsequent build steps needed to support downstream containers should be done in those containers build files. -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files -# Copy in the entire workspace. +FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base +COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts +COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects +COPY --from=boxes-files /usr/src/boxes /usr/src/boxes COPY . . - +# Generate L1 contract TypeScript artifacts. +RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts # This is actually our code generation tool. Needed to build contract typescript wrappers. RUN yarn workspace @aztec/noir-compiler build -# Builds noir contracts (TODO: move this stage pre yarn-project). Generates typescript wrappers. +# Generates typescript wrappers. RUN yarn workspace @aztec/noir-contracts.js build:contracts # We need to build accounts as it needs to copy in account contracts from noir-contracts. RUN yarn workspace @aztec/accounts build:copy-contracts RUN yarn workspace @aztec/protocol-contracts build:copy-contracts RUN yarn workspace @aztec/noir-protocol-circuits-types build RUN yarn tsc -b - ENTRYPOINT ["yarn"] diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 933ec230be2..3532de89fa6 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -42,25 +42,15 @@ # RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src # - Create final slim image by copying needed dirs into a fresh image. # -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages -FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files FROM node:18.19.0 RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean -# Copy L1 contracts. -COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts -# Copy in bb.js +# Copy in portalled packages. COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts -# Copy in noir packages COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages -# Copy in noir projects -COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects -# Copy in boxes -COPY --from=boxes-files /usr/src/boxes /usr/src/boxes # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution @@ -92,7 +82,4 @@ RUN yarn --immutable && rm -rf /root/.cache/puppeteer && /bin/bash -c '\ RUN echo "enableNetwork: false" >> .yarnrc.yml # Check package.json inheritance and tsconfig project references. -RUN yarn prepare:check - -# Generate L1 contract TypeScript artifacts. -RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts \ No newline at end of file +RUN yarn prepare:check \ No newline at end of file From 40216be425ff32255ee42f0e8ad7c4c64f1d8091 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 17:53:00 +0000 Subject: [PATCH 18/21] fix --- noir-projects/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/noir-projects/Dockerfile b/noir-projects/Dockerfile index 571bc975e83..0d5e3ab2f4e 100644 --- a/noir-projects/Dockerfile +++ b/noir-projects/Dockerfile @@ -18,5 +18,4 @@ WORKDIR /usr/src/noir-projects/aztec-nr RUN ../../noir/target/release/nargo compile --silence-warnings && ../../noir/target/release/nargo test --silence-warnings FROM scratch -COPY --from=builder /usr/src/noir-projects/noir-contracts/target /usr/src/noir-projects/noir-contracts/target -COPY --from=builder /usr/src/noir-projects/noir-protocol-circuits/src/target /usr/src/noir-projects/noir-protocol-circuits/src/target \ No newline at end of file +COPY --from=builder /usr/src/noir-projects/noir-contracts /usr/src/noir-projects/noir-contracts \ No newline at end of file From 7c308e31547076d073ed9e16349412e10e260664 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 17:53:48 +0000 Subject: [PATCH 19/21] fix --- noir-projects/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noir-projects/Dockerfile b/noir-projects/Dockerfile index 0d5e3ab2f4e..2e2f48e4de8 100644 --- a/noir-projects/Dockerfile +++ b/noir-projects/Dockerfile @@ -18,4 +18,4 @@ WORKDIR /usr/src/noir-projects/aztec-nr RUN ../../noir/target/release/nargo compile --silence-warnings && ../../noir/target/release/nargo test --silence-warnings FROM scratch -COPY --from=builder /usr/src/noir-projects/noir-contracts /usr/src/noir-projects/noir-contracts \ No newline at end of file +COPY --from=builder /usr/src/noir-projects /usr/src/noir-projects \ No newline at end of file From b6468a10ea11767785e4d2efe13b1308a35cccb6 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 20:48:57 +0000 Subject: [PATCH 20/21] fix --- boxes/.dockerignore | 3 ++- boxes/Dockerfile | 3 ++- boxes/blank-react/package.json | 2 +- boxes/docker-compose.yml | 6 +++++- yarn-project/Dockerfile | 2 ++ yarn-project/yarn-project-base/Dockerfile | 3 --- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/boxes/.dockerignore b/boxes/.dockerignore index bde85ad2dcd..7eab5f5a5cb 100644 --- a/boxes/.dockerignore +++ b/boxes/.dockerignore @@ -5,4 +5,5 @@ node_modules .tsbuildinfo Dockerfile* .dockerignore -docker-compose.yml \ No newline at end of file +docker-compose.yml +**/artifacts \ No newline at end of file diff --git a/boxes/Dockerfile b/boxes/Dockerfile index 9241170b544..ef381f2fff5 100644 --- a/boxes/Dockerfile +++ b/boxes/Dockerfile @@ -6,6 +6,7 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects # We need yarn. Start fresh container. FROM node:18.19.0 +RUN apt update && apt install netcat-openbsd COPY --from=aztec /usr/src /usr/src COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr @@ -13,4 +14,4 @@ WORKDIR /usr/src/boxes ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest RUN yarn && yarn build -ENTRYPOINT ["yarn", "workspace"] +ENTRYPOINT ["/bin/sh", "-c"] diff --git a/boxes/blank-react/package.json b/boxes/blank-react/package.json index 67a1de85426..e4613de8f60 100644 --- a/boxes/blank-react/package.json +++ b/boxes/blank-react/package.json @@ -14,7 +14,7 @@ "serve": "serve -p 3000 ./dist", "formatting": "prettier --check ./src && eslint ./src", "formatting:fix": "prettier -w ./src", - "test": "yarn prep && NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand" + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand" }, "jest": { "preset": "ts-jest/presets/default-esm", diff --git a/boxes/docker-compose.yml b/boxes/docker-compose.yml index e731d25c01e..333fa2a0b2c 100644 --- a/boxes/docker-compose.yml +++ b/boxes/docker-compose.yml @@ -18,7 +18,11 @@ services: boxes: image: aztecprotocol/boxes - command: "@aztec/$BOX test" + entrypoint: > + sh -c ' + while ! nc -z aztec 8080; do sleep 1; done; + yarn workspace @aztec/$BOX test + ' environment: DEBUG: "aztec:*" DEBUG_COLORS: "true" diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index aea67d0c588..1eccb4284d5 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -13,6 +13,8 @@ COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects COPY --from=boxes-files /usr/src/boxes /usr/src/boxes COPY . . +# If everything's worked properly, we should no longer need access to the network. +RUN echo "enableNetwork: false" >> .yarnrc.yml # Generate L1 contract TypeScript artifacts. RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts # This is actually our code generation tool. Needed to build contract typescript wrappers. diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index 3532de89fa6..b53ed652bc0 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -78,8 +78,5 @@ RUN yarn --immutable && rm -rf /root/.cache/puppeteer && /bin/bash -c '\ [[ $F =~ (.*-) ]] && ln $F /root/.yarn/berry/cache/${BASH_REMATCH[1]}8.zip; \ done' -# If everything's worked properly, we should no longer need access to the network. -RUN echo "enableNetwork: false" >> .yarnrc.yml - # Check package.json inheritance and tsconfig project references. RUN yarn prepare:check \ No newline at end of file From fefccb811db3080a23527350fc3092d6330a65cb Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Sun, 25 Feb 2024 21:11:45 +0000 Subject: [PATCH 21/21] Gah. Cos of arm we actually do need net connection to productionify. Explains why prod step is slow. --- yarn-project/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index 1eccb4284d5..aea67d0c588 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -13,8 +13,6 @@ COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects COPY --from=boxes-files /usr/src/boxes /usr/src/boxes COPY . . -# If everything's worked properly, we should no longer need access to the network. -RUN echo "enableNetwork: false" >> .yarnrc.yml # Generate L1 contract TypeScript artifacts. RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts # This is actually our code generation tool. Needed to build contract typescript wrappers.