chore: squash yp ypb + other build improvements.

.circleci/config.yml

@@ -435,77 +435,62 @@ jobs:
           command: build noir-projects
           aztec_manifest_key: noir-projects
 
-  boxes-files:
-    machine:
-      image: default
-    resource_class: medium
+  yarn-project-pre-join:
+    docker:
+      - image: cimg/base:2023.09
+    resource_class: small
     steps:
-      - *checkout
-      - *setup_env
       - run:
-          name: "Build"
-          command: build boxes-files
-          aztec_manifest_key: boxes-files
+          name: "Noop"
+          command: echo Noop
 
-  yarn-project-base:
-    machine:
-      image: default
-    resource_class: large
+  yarn-project-x86_64:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
           name: "Build"
-          command: build yarn-project-base | add_timestamps
-          aztec_manifest_key: yarn-project-base
+          command: cond_spot_run_build yarn-project 64
+          aztec_manifest_key: yarn-project
 
-  yarn-project:
-    machine:
-      image: default
-    resource_class: large
+  yarn-project-arm64:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Build
-          command: build yarn-project | add_timestamps
+          name: "Build"
+          command: cond_spot_run_build yarn-project 64 arm64
           aztec_manifest_key: yarn-project
 
-  yarn-project-prod:
+  yarn-project-ecr-manifest:
     machine:
       image: default
-    resource_class: large
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: Build
-          command: build yarn-project-prod | add_timestamps
-          aztec_manifest_key: yarn-project-prod
-
-  yarn-project-formatting:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
+    resource_class: medium
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Check Formatting
-          command: cond_spot_run_container yarn-project 8 formatting | add_timestamps
+          name: "Create ECR manifest"
+          command: create_ecr_manifest yarn-project x86_64,arm64
           aztec_manifest_key: yarn-project
 
-  yarn-project-tests:
+  yarn-project-test:
     docker:
       - image: aztecprotocol/alpine-build-image
     resource_class: small
     steps:
       - *checkout
       - *setup_env
       - run:
-          name: Test
-          command: cond_spot_run_container yarn-project 64 test | add_timestamps
-          aztec_manifest_key: yarn-project
+          name: "Build and test"
+          command: cond_spot_run_build yarn-project-test 64
+          aztec_manifest_key: yarn-project-test
 
   aztec-package:
     machine:
@@ -711,7 +696,6 @@ jobs:
           command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_authwit.test.ts
           aztec_manifest_key: end-to-end
 
-
   e2e-blacklist-token-contract:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -986,7 +970,7 @@ jobs:
       - *setup_env
       - run:
           name: "Test"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts
+          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_aztec_js_browser.test.ts
           aztec_manifest_key: end-to-end
 
   e2e-card-game:
@@ -1140,9 +1124,6 @@ jobs:
     steps:
       - *checkout
       - *setup_env
-      - run:
-          name: "Copy docs dockerignore"
-          command: cp docs/.dockerignore .
       - run:
           name: "Build docs"
           command: |
@@ -1306,14 +1287,14 @@ defaults: &defaults
         event: fail
         branch_pattern: "master"
 
-defaults_yarn_project: &defaults_yarn_project
+defaults_yarn_project_pre_join: &defaults_yarn_project_pre_join
   requires:
-    - yarn-project
+    - yarn-project-pre-join
   <<: *defaults
 
-defaults_yarn_project_prod: &defaults_yarn_project_prod
+defaults_yarn_project: &defaults_yarn_project
   requires:
-    - yarn-project-prod
+    - yarn-project-ecr-manifest
   <<: *defaults
 
 defaults_deploy: &defaults_deploy
@@ -1407,37 +1388,35 @@ workflows:
 
       - yellow-paper: *defaults
 
-      - boxes-files: *defaults
-
       - noir-projects:
           requires:
             - avm-transpiler
             - noir-ecr-manifest
           <<: *defaults
 
       # Yarn Project
-      - yarn-project-base:
+      - yarn-project-pre-join:
           requires:
             - bb-js
             - noir-packages
-          <<: *defaults
-      - yarn-project:
-          requires:
-            - yarn-project-base
             - l1-contracts
             - noir-projects
-            - boxes-files
           <<: *defaults
-      - yarn-project-prod: *defaults_yarn_project
-      - yarn-project-formatting: *defaults_yarn_project
-      - yarn-project-tests: *defaults_yarn_project
-      - end-to-end: *defaults_yarn_project
-      - build-docs: *defaults_yarn_project
+      - end-to-end: *defaults_yarn_project_pre_join
+      - aztec-faucet: *defaults_yarn_project_pre_join
+      - build-docs: *defaults_yarn_project_pre_join
+      - yarn-project-test: *defaults_yarn_project_pre_join
+      - yarn-project-x86_64: *defaults_yarn_project_pre_join
+      - yarn-project-arm64: *defaults_yarn_project_pre_join
+      - yarn-project-ecr-manifest:
+          requires:
+            - yarn-project-x86_64
+            - yarn-project-arm64
+          <<: *defaults
 
       # Artifacts
-      - aztec-package: *defaults_yarn_project_prod
-      - cli: *defaults_yarn_project_prod
-      - aztec-faucet: *defaults_yarn_project_prod
+      - aztec-package: *defaults_yarn_project
+      - cli: *defaults_yarn_project
 
       # Boxes.
       - boxes:

.gitignore

@@ -15,3 +15,5 @@ cmake-build-debug
 
 .graphite*
 .DS_Store
+
+**/*.dockerignore

barretenberg/sol/Dockerfile

@@ -23,20 +23,14 @@ RUN apk update && apk add git curl build-base openmp-dev bash
 COPY --from=0 /usr/src/barretenberg/cpp/build/bin /usr/src/barretenberg/cpp/build/bin
 COPY --from=0 /usr/src/barretenberg/cpp/srs_db /usr/src/barretenberg/cpp/srs_db
 WORKDIR /usr/src/barretenberg/sol
-RUN git init
 COPY ./sol .
 
 # Copy forge binary directly from foundry
 COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/forge /usr/local/bin/forge
 
-RUN forge install --no-commit \
-  https://github.com/foundry-rs/forge-std \
-  https://github.com/openzeppelin/openzeppelin-contracts \
-  https://github.com/Arachnid/solidity-stringutils
-
 RUN cd ../cpp/srs_db && ./download_ignition.sh 3 && cd ../../sol
 
 RUN ./scripts/init.sh
 
-# TestBase is excluded as it is just boilerplate 
+# TestBase is excluded as it is just boilerplate
 RUN forge test --no-match-contract TestBase

boxes/Dockerfile

@@ -1,4 +1,3 @@
-# Builds the boxes (they were copied into yarn-project-base so the cli can unbox).
 # Produces a container that can be run to test a specific box. See docker-compose.yml.
 FROM aztecprotocol/aztec AS aztec
 FROM aztecprotocol/noir as noir
@@ -10,7 +9,9 @@ RUN apt update && apt install netcat-openbsd
 COPY --from=aztec /usr/src /usr/src
 COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo
 COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr
+COPY --from=noir-projects /usr/src/noir-projects/noir-protocol-circuits/crates/types /usr/src/noir-projects/noir-protocol-circuits/crates/types
 WORKDIR /usr/src/boxes
+COPY . .
 ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
 ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest
 RUN yarn && yarn build

build-system/scripts/build

@@ -31,28 +31,6 @@ echo "Working directory: $PWD"
 echo "Dockerfile: $DOCKERFILE"
 echo "Build directory: $BUILD_DIR"
 
-# Fetch images with retries
-function fetch_image() {
-  echo "Pulling: $1"
-  if ! retry docker pull $1 > /dev/null 2>&1; then
-    echo "Image not found: $1"
-    return 1
-  fi
-  return 0
-}
-
-# Fetch images, but don't assume this will work
-function try_fetch_image() {
-  echo "Pulling: $1"
-  if ! docker pull $1 > /dev/null 2>&1; then
-    echo "Image not found: $1"
-    return 1
-  fi
-  return 0
-}
-
-NOREPO=$(query_manifest noRepo $REPOSITORY)
-
 # Login to ECR and ensure repository exists.
 retry ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle
 # Login to dockerhub.
@@ -91,6 +69,7 @@ if [ -d $ROOT_PATH/$PROJECT_DIR/terraform ]; then
 fi
 
 # For each dependency, substitute references to the dependency in dockerfile, with the relevent built image uri.
+# This is necessary vs pulling and retagging the image, as that doesn't work with buildx.
 # We have to perform a bit of probing to determine which actual image we want to use.
 # When we used buildx to create a multiarch image, there will be no images with "-$ARCH" suffixes (normalise this?).
 # Also we sometimes build an arm image from an x86 parent, so there won't always be an arm parent, and we fallback.
@@ -115,9 +94,12 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do
   # Substitute references to parent repo, with the relevent built image uri.
   DEPLOY_URI=aztecprotocol/$PARENT_REPO
   PARENT_IMAGE_URI=$ECR_URL/$PARENT_REPO:$PARENT_IMAGE_TAG
-  awk '{if ($1 == "FROM" && $2 == "'$DEPLOY_URI'") $2 = "'$PARENT_IMAGE_URI'"; print $0}' $DOCKERFILE > _temp && mv _temp $DOCKERFILE
+  sed -i "s#^FROM \\(.*\\)${DEPLOY_URI}\\( \|$\\)#FROM \\1${PARENT_IMAGE_URI}\\2#" $DOCKERFILE
 done
 
+# Build a dockerignore file that only permits git files and untracked files.
+create_dockerignore $REPOSITORY
+
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false)
 echo "Commit tag version: $COMMIT_TAG_VERSION"