fix: Revert "feat: use s3 cache in bootstrap fast"

avm-transpiler/Earthfile

@@ -12,11 +12,7 @@ source:
 build:
     FROM +source
     # build avm transpiler, and make sure the big build and deps folders don't hit cache
-    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
-        --prefix="avm-transpiler" \
-        --rebuild_patterns="../noir/.rebuild_patterns_native .rebuild_patterns" \
-        --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \
-        --build_artifacts="target"
+    RUN ./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
     SAVE ARTIFACT scripts/compile_then_transpile.sh
 

avm-transpiler/bootstrap_cache.sh

@@ -2,7 +2,10 @@
 set -eu
 
 cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m"
-HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" ../build-system/s3-cache-scripts/compute-content-hash.sh)
-../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz
+extract_repo_if_working_copy_clean avm-transpiler \
+  /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/
+
+remove_old_images avm-transpiler

barretenberg/cpp/Earthfile

@@ -131,7 +131,7 @@ preset-sol:
 
 preset-wasm-threads:
   FROM +source
-  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads \
+  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads-v1 \
     --command="cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm"
   SAVE ARTIFACT build/bin
 

barretenberg/cpp/bootstrap_cache.sh

@@ -2,30 +2,16 @@
 set -eu
 
 cd "$(dirname "$0")"
+source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m"
+extract_repo_if_working_copy_clean barretenberg-x86_64-linux-clang \
+  /usr/src/barretenberg/cpp/build/bin ./build \
+  /usr/src/barretenberg/cpp/build-pic/lib ./build-pic
 
-SCRIPTS_PATH=../../build-system/s3-cache-scripts/
-HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns $SCRIPTS_PATH/compute-content-hash.sh)
-TMP=$(mktemp -d)
+echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
+extract_repo_if_working_copy_clean barretenberg-wasm-linux-clang \
+  /usr/src/barretenberg/cpp/build-wasm/bin ./build-wasm \
+  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./build-wasm-threads
 
-function on_exit() {
-  rm -rf "$TMP"
-}
-trap on_exit EXIT
-
-# Parallel download of all the cached builds because they're quite big
-echo "
-barretenberg-preset-wasm
-barretenberg-preset-wasm-threads
-barretenberg-preset-release
-barretenberg-preset-release-world-state
-" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
-
-# # clobber the existing build with the cached build
-cp -r $TMP/barretenberg-preset-wasm/build build-wasm/
-cp -r $TMP/barretenberg-preset-wasm-threads/build build-wasm-threads/
-
-mkdir -p build
-cp -r $TMP/barretenberg-preset-release/build/* build/
-cp -r $TMP/barretenberg-preset-release-world-state/build/* build/
+remove_old_images barretenberg-wasm-linux-clang

barretenberg/ts/Earthfile

@@ -6,11 +6,12 @@ CACHE:
   ARG build_artifacts
   ARG prefix
   # TODO(#8929): reinstate bb.js caching
-  DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
-    --prefix="bb.js-$prefix" \
-    --command="$command" \
-    --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
-    --build_artifacts="$build_artifacts"
+  RUN $command
+  # DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
+  #   --prefix="bb.js-$prefix" \
+  #   --command="$command" \
+  #   --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
+  #   --build_artifacts="$build_artifacts"
 
 deps:
     FROM ../../build-images+from-registry
@@ -33,7 +34,7 @@ deps:
 esm:
     FROM +deps
     DO +CACHE \
-      --prefix="esm" \
+      --prefix="browser" \
       --command="yarn build:esm" \
       --build_artifacts="."
     SAVE ARTIFACT /usr/src/barretenberg/ts build

barretenberg/ts/bootstrap_cache.sh

@@ -4,29 +4,9 @@ set -eu
 cd "$(dirname "$0")"
 source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
-CACHE_SCRIPTS=../../build-system/s3-cache-scripts
-
 echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m"
-TMP=$(mktemp -d)
-
-function on_exit() {
-  rm -rf "$TMP"
-}
-trap on_exit EXIT
-
-HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh)
-
-# Parallel download of all the cached builds because they're quite big
-echo "
-bb.js-esm
-bb.js-cjs
-bb.js-browser
-" | xargs --max-procs 0 -I {} bash -c "$CACHE_SCRIPTS/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
-
-mkdir -p dest
-cp -r $TMP/bb.js-esm/dest/* dest/
-cp -r $TMP/bb.js-cjs/dest/* dest/
-cp -r $TMP/bb.js-browser/dest/* dest/
-
+extract_repo_if_working_copy_clean bb.js /usr/src/barretenberg/ts/dest .
 # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled.
 yarn install
+
+remove_old_images bb.js

bootstrap.sh

@@ -128,7 +128,7 @@ elif [ "$CMD" = "full" ]; then
 elif [ "$CMD" = "fast" ]; then
   export USE_CACHE=1
   if ! can_use_ci_cache; then
-    echo -e "${BOLD}${YELLOW}WARNING: AWS credentials are missing. Note this is for internal aztec devs only.${RESET}"
+    echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}"
     exit 1
   fi
 elif [ "$CMD" = "check" ]; then

build-system/s3-cache-scripts/cache-download.sh

@@ -1,14 +1,13 @@
 #!/bin/bash
 set -eu
 
-if [ "$#" -lt 1 ]; then
+if [ "$#" -ne 1 ]; then
     echo "Usage: $0 <tar.gz_file_to_download_and_extract>"
     exit 1
 fi
 
 # Get the tar.gz file name from the argument
 TAR_FILE="$1"
-OUT_DIR="${2:-.}"
 
 function on_exit() {
   # Cleanup the temporary tar.gz file
@@ -21,7 +20,6 @@ trap on_exit EXIT
 aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet --no-progress
 
 # Extract the cache file
-mkdir -p "$OUT_DIR"
-tar -xzf "$TAR_FILE" -C "$OUT_DIR"
+tar -xzf "$TAR_FILE"
 
-echo "Cache download and extraction of $TAR_FILE complete."
+echo "Cache download and extraction complete."

build-system/s3-cache-scripts/cache-upload.sh

@@ -24,4 +24,4 @@ trap on_exit EXIT
 # Rest of args are our binary paths
 tar -czf "$TAR_FILE" $@
 
-aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"
+aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"

build-system/scripts/can_use_ci_cache

@@ -1,9 +1,3 @@
 #!/usr/bin/env bash
 set -euo pipefail
-if [ ! -z ${AWS_ACCESS_KEY_ID:-} ] && [ ! -z ${AWS_SECRET_ACCESS_KEY:-} ]; then
-  exit 0
-elif [ -f ~/.aws/credentials ]; then
-  exit 0
-else
-  exit 1
-fi
+type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1

l1-contracts/Earthfile

@@ -1,26 +1,15 @@
 VERSION 0.8
 
-src:
+build:
     FROM ../build-images+from-registry
     WORKDIR /usr/src/l1-contracts
     COPY --dir lib src terraform test *.json *.toml *.sh .
-    DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns"
-
-lint:
-    FROM +src
+    #RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff
+    # "slither": "forge clean && forge build --build-info --skip '*/test/**' --force && slither . --checklist --ignore-compile --show-ignored-findings --config-file ./slither.config.json | tee slither_output.md",
+    # "slither-has-diff": "./slither_has_diff.sh"
     RUN solhint --config ./.solhint.json --fix "src/**/*.sol"
     RUN forge clean && forge fmt --check
-
-build:
-    FROM +lint
-    WORKDIR /usr/src/l1-contracts
-
-    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
-        --prefix="l1-contracts" \
-        --rebuild_patterns=".rebuild_patterns" \
-        --command="forge build" \
-        --build_artifacts="out"
-
+    RUN forge build
     SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts
 
 test:

l1-contracts/bootstrap_cache.sh

@@ -2,7 +2,9 @@
 set -eu
 
 cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving contracts from remote cache...\033[0m"
-HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns ../build-system/s3-cache-scripts/compute-content-hash.sh)
-../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz
+extract_repo_if_working_copy_clean l1-contracts /usr/src/l1-contracts/out .
+
+remove_old_images l1-contracts

noir-projects/Earthfile

@@ -1,82 +1,69 @@
 VERSION 0.8
 
-deps:
-  LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
-
+source:
   FROM ../build-images+from-registry
+
+  # Install nargo
   COPY ../noir/+nargo/nargo /usr/bin/nargo
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-  ENV BB_HASH=$bb_source_hash
-  ENV NARGO=nargo
+  # Install world state napi
+  COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node
 
-source:
-  FROM +deps
   WORKDIR /usr/src/noir-projects
 
-  COPY package.json yarn.lock .
-  RUN yarn
-
-  COPY mega_honk_circuits.json .
+  # Copy source.
+  COPY package.json yarn.lock mega_honk_circuits.json .
   COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts .
 
 build-contracts:
   ARG RAYON_NUM_THREADS
-
+  LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+
+  # Install bb
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
 
   # Install transpiler
   COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler
-  ENV TRANSPILER=avm-transpiler
-
-  WORKDIR /usr/src/noir-projects/noir-contracts
-  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
-    --prefix="noir-projects-noir-contracts" \
-    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
-    --command="./bootstrap.sh" \
-    --build_artifacts="target"
 
-  WORKDIR /usr/src/noir-projects
+  RUN yarn
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
+  RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler  ./bootstrap.sh
   SAVE ARTIFACT noir-contracts
 
 build-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  ENV PARALLEL_VK=false
-
-  WORKDIR /usr/src/noir-projects/noir-protocol-circuits
-
-  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
-    --prefix="noir-projects-noir-protocol-circuits" \
-    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
-    --command="./bootstrap.sh" \
-    --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json crates/autogenerated"
+  # Install bb
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  RUN yarn
 
-  WORKDIR /usr/src/noir-projects
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
+  RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
   SAVE ARTIFACT noir-protocol-circuits
 
 build-mock-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  ENV PARALLEL_VK=false
-
-  WORKDIR /usr/src/noir-projects/mock-protocol-circuits
-  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
-    --prefix="noir-projects-mock-protocol-circuits" \
-    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
-    --command="./bootstrap.sh" \
-    --build_artifacts="target"
+  # Install bb
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  RUN yarn
 
-  WORKDIR /usr/src/noir-projects
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
+  RUN echo "building with num threads $RAYON_NUM_THREADS"
+  RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
   SAVE ARTIFACT mock-protocol-circuits
 
 build:
@@ -122,7 +109,7 @@ format:
   FROM +source
 
   WORKDIR /usr/src/noir-projects/noir-protocol-circuits
-  RUN node ./scripts/generate_variants.js
+  RUN yarn && node ./scripts/generate_variants.js
   RUN nargo fmt --check
 
   WORKDIR /usr/src/noir-projects/mock-protocol-circuits

noir-projects/bootstrap_cache.sh

@@ -2,19 +2,16 @@
 set -eu
 
 cd "$(dirname "$0")"
-
-SCRIPTS_PATH=../build-system/s3-cache-scripts/
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m"
+extract_repo_if_working_copy_clean noir-projects \
+  /usr/src/noir-projects/noir-contracts/target ./noir-contracts \
+  /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits
 
-PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
-MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
-CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
-
-echo "
-noir-protocol-circuits $PROTOCOL_CIRCUITS_HASH
-mock-protocol-circuits $MOCK_CIRCUITS_HASH
-noir-contracts $CONTRACTS_HASH
-" | xargs --max-procs 0 --max-args 2 bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-\$0-\$1.tar.gz \$0"
+remove_old_images noir-projects
 
 yarn
+
+./mock-protocol-circuits/bootstrap.sh
+(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)