chore: decouple ypb (#4749)

This:
* Decouples (partially) yarn-project-base build from rest of pipeline.
Actually it's not quite what I hoped as I realised that at present to
build ypb we need bb.js and noir packages to have been built first
(which in turn require bb and noir). This maybe able to be worked
around. My hope was that I fully decouple it so that we would only
perform the expensive yarn install, if the package.json files had
changed. Practically this partial decoupling probably won't improve CI
times that much, but it still makes sense to do what's done here.
* Fixes bug in boxes whereby it didn't wait for aztec to fully start
before running test.
* Remove `prep` call which compiles from `test`. Think this was added by
@signorecello but running the container tests should not be doing any
builds.

.circleci/config.yml

@@ -1389,16 +1389,15 @@ workflows:
       # Yarn Project
       - yarn-project-base:
           requires:
-            - avm-transpiler
-            - l1-contracts
             - bb-js
             - noir-packages
-            - noir-projects
-            - boxes-files
           <<: *defaults
       - yarn-project:
           requires:
             - yarn-project-base
+            - l1-contracts
+            - noir-projects
+            - boxes-files
           <<: *defaults
       - yarn-project-prod: *defaults_yarn_project
       - yarn-project-formatting: *defaults_yarn_project

boxes/.dockerignore

@@ -5,4 +5,5 @@ node_modules
 .tsbuildinfo
 Dockerfile*
 .dockerignore
-docker-compose.yml
+docker-compose.yml
+**/artifacts

boxes/Dockerfile

@@ -6,11 +6,12 @@ FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects
 
 # We need yarn. Start fresh container.
 FROM node:18.19.0
+RUN apt update && apt install netcat-openbsd
 COPY --from=aztec /usr/src /usr/src
 COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
 COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr
 WORKDIR /usr/src/boxes
 ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo
 ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest
 RUN yarn && yarn build
-ENTRYPOINT ["yarn", "workspace"]
+ENTRYPOINT ["/bin/sh", "-c"]

boxes/docker-compose.yml

@@ -18,7 +18,11 @@ services:
 
   boxes:
     image: aztecprotocol/boxes
-    command: "@aztec/$BOX test"
+    entrypoint: >
+      sh -c '
+        while ! nc -z aztec 8080; do sleep 1; done;
+        yarn workspace @aztec/$BOX test
+      '
     environment:
       DEBUG: "aztec:*"
       DEBUG_COLORS: "true"

boxes/react/package.json

@@ -14,7 +14,7 @@
     "serve": "serve -p 3000 ./dist",
     "formatting": "prettier --check ./src && eslint ./src",
     "formatting:fix": "prettier -w ./src",
-    "test": "yarn prep && NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand"
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --runInBand"
   },
   "jest": {
     "preset": "ts-jest/presets/default-esm",

build_manifest.yml

@@ -135,13 +135,8 @@ yarn-project-base:
     - ^yarn-project/yarn.lock
     - ^yarn-project/.*/package.json$
   dependencies:
-    - l1-contracts
     - bb.js
-    - noir
     - noir-packages
-    - boxes-files
-    - avm-transpiler
-    - noir-projects
 
 yarn-project:
   buildDir: yarn-project
@@ -151,6 +146,9 @@ yarn-project:
     - ^yarn-project/cli/aztec-cli
   dependencies:
     - yarn-project-base
+    - l1-contracts
+    - boxes-files
+    - noir-projects
 
 yarn-project-prod:
   buildDir: yarn-project

yarn-project/Dockerfile

@@ -4,19 +4,24 @@
 #   - Run the tests.
 #   - Run the formatter checks.
 # Any subsequent build steps needed to support downstream containers should be done in those containers build files.
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base as builder
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files
 
-# Copy in the entire workspace.
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base
+COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts
+COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects
+COPY --from=boxes-files /usr/src/boxes /usr/src/boxes
 COPY . .
-
+# Generate L1 contract TypeScript artifacts.
+RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts
 # This is actually our code generation tool. Needed to build contract typescript wrappers.
 RUN yarn workspace @aztec/noir-compiler build
-# Builds noir contracts (TODO: move this stage pre yarn-project). Generates typescript wrappers.
+# Generates typescript wrappers.
 RUN yarn workspace @aztec/noir-contracts.js build:contracts
 # We need to build accounts as it needs to copy in account contracts from noir-contracts.
 RUN yarn workspace @aztec/accounts build:copy-contracts
 RUN yarn workspace @aztec/protocol-contracts build:copy-contracts
 RUN yarn workspace @aztec/noir-protocol-circuits-types build
 RUN yarn tsc -b
-
 ENTRYPOINT ["yarn"]

yarn-project/yarn-project-base/Dockerfile

@@ -42,25 +42,15 @@
 #      RUN yarn workspaces focus --production && yarn cache clean && rm -rf ../**/src
 #  - Create final slim image by copying needed dirs into a fresh image.
 #
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/l1-contracts as contracts
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js as bb.js
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-projects as noir-projects
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-packages as noir-packages
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/boxes-files as boxes-files
 
 FROM node:18.19.0
 RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean
 
-# Copy L1 contracts.
-COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts
-# Copy in bb.js
+# Copy in portalled packages.
 COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts
-# Copy in noir packages
 COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages
-# Copy in noir projects
-COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects
-# Copy in boxes
-COPY --from=boxes-files /usr/src/boxes /usr/src/boxes
 
 # We install a symlink to yarn-project's node_modules at a location that all portalled packages can find as they
 # walk up the tree as part of module resolution. The supposedly idiomatic way of supporting module resolution
@@ -88,11 +78,5 @@ RUN yarn --immutable && rm -rf /root/.cache/puppeteer && /bin/bash -c '\
   [[ $F =~ (.*-) ]] && ln $F /root/.yarn/berry/cache/${BASH_REMATCH[1]}8.zip; \
   done'
 
-# If everything's worked properly, we should no longer need access to the network.
-RUN echo "enableNetwork: false" >> .yarnrc.yml
-
 # Check package.json inheritance and tsconfig project references.
-RUN yarn prepare:check
-
-# Generate L1 contract TypeScript artifacts.
-RUN cd l1-artifacts && ./scripts/generate-artifacts.sh && rm -rf /usr/src/l1-contracts
+RUN yarn prepare:check