chore: Build the acir test vectors as part of CI.

.circleci/config.yml

@@ -114,6 +114,17 @@ jobs:
           name: "Build"
           command: cond_spot_run_build noir-packages 32
 
+  noir-acir-tests:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build"
+          command: cond_spot_run_build noir-acir-tests 32
+
   # Barretenberg
   barretenberg-wasm-linux-clang:
     docker:
@@ -992,6 +1003,10 @@ workflows:
             - noir-arm64
           <<: *defaults
       - noir-packages: *defaults
+      - noir-acir-tests:
+          requires:
+            - noir-ecr-manifest
+          <<: *defaults
 
       # Barretenberg
       - barretenberg-x86_64-linux-gcc: *defaults
@@ -1010,6 +1025,7 @@ workflows:
       - barretenberg-acir-tests-bb:
           requires:
             - barretenberg-x86_64-linux-clang-assert
+            - noir-acir-tests
           <<: *defaults
       - bb-js:
           requires:
@@ -1022,6 +1038,7 @@ workflows:
       - bb-js-acir-tests:
           requires:
             - bb-js
+            - noir-acir-tests
           <<: *defaults
 
       - l1-contracts: *defaults

barretenberg/acir_tests/Dockerfile.bb

@@ -1,8 +1,10 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests
 
 FROM node:18-alpine
 RUN apk update && apk add git bash curl jq coreutils
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
+COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .
 # Run every acir test through native bb build prove_then_verify flow.

barretenberg/acir_tests/Dockerfile.bb.js

@@ -1,7 +1,9 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests
 
 FROM node:18
 COPY --from=0 /usr/src/barretenberg/ts /usr/src/barretenberg/ts
+COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
 RUN apt update && apt install -y lsof jq
 WORKDIR /usr/src/barretenberg/acir_tests
 # Build/install ts apps.

barretenberg/acir_tests/Dockerfile.bb.sol

@@ -1,13 +1,15 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-sol
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir-acir-tests as noir-acir-tests
 
 FROM node:18-alpine
 RUN apk update && apk add git bash curl jq
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
 COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol
+COPY --from=noir-acir-tests /usr/src/noir/tooling/nargo_cli/tests /usr/src/noir/tooling/nargo_cli/tests
 COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .
-# Run every acir test through a solidity verifier".
+# Run every acir test through a solidity verifier.
 RUN (cd sol-test && yarn)
 RUN PARALLEL=1 FLOW=sol ./run_acir_tests.sh

barretenberg/acir_tests/Dockerfile.noir_acir_tests

@@ -0,0 +1,10 @@
+# This is a slightly odd location for this Dockerfile, as ostensibly it should belong next to the tests themselves.
+# However, the noir subrepo has no concept of the aztec build pipeline, so the Dockerfile wouldn't make sense there.
+# So, it lives here.
+# This chains off the nargo build, and creates a container with a compiled set of acir tests.
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir
+RUN apk add bash jq
+ENV PATH="/usr/src/noir/target/release:${PATH}"
+WORKDIR /usr/src/noir/tooling/nargo_cli/tests
+COPY . .
+RUN ./rebuild.sh

barretenberg/acir_tests/clone_test_vectors.sh

@@ -1,19 +1,8 @@
 #!/bin/bash
 set -eu
 
-# Pull down the test vectors from the noir repo, if we don't have the folder already.
+TEST_SRC=${TEST_SRC:-../../noir/tooling/nargo_cli/tests/acir_artifacts}
+
 if [ ! -d acir_tests ]; then
-  if [ -n "${TEST_SRC:-}" ]; then
-    cp -R $TEST_SRC acir_tests
-  else
-    rm -rf noir
-    git clone -b $BRANCH --filter=blob:none --no-checkout https://github.com/noir-lang/noir.git
-    cd noir
-    git sparse-checkout init --cone
-    git sparse-checkout set test_programs/acir_artifacts
-    git checkout
-    cd ..
-    mv noir/test_programs/acir_artifacts acir_tests
-    rm -rf noir
-  fi
-fi
+  cp -R $TEST_SRC acir_tests
+fi

build_manifest.yml

@@ -1,3 +1,21 @@
+noir:
+  # Add less aggressive rebuildPatterns.
+  buildDir: noir
+
+noir-packages:
+  buildDir: noir
+  dockerfile: Dockerfile.packages
+
+noir-acir-tests:
+  buildDir: noir/tooling/nargo_cli/tests
+  # Awkward. Maybe change dockerfile paths to be absolute.
+  dockerfile: ../../../../barretenberg/acir_tests/Dockerfile.noir_acir_tests
+  rebuildPatterns:
+    - ^noir/tooling/nargo_cli/tests
+    - ^barretenberg/acir_tests/Dockerfile.noir_acir_tests
+  dependencies:
+    - noir
+
 barretenberg-x86_64-linux-clang:
   buildDir: barretenberg/cpp
   dockerfile: dockerfiles/Dockerfile.x86_64-linux-clang
@@ -38,19 +56,22 @@ barretenberg-acir-tests-bb:
   dockerfile: Dockerfile.bb
   dependencies:
     - barretenberg-x86_64-linux-clang-assert
+    - noir-acir-tests
 
 barretenberg-acir-tests-bb-sol:
   buildDir: barretenberg/acir_tests
   dockerfile: Dockerfile.bb.sol
   dependencies:
     - barretenberg-x86_64-linux-clang-assert
     - barretenberg-x86_64-linux-clang-sol
+    - noir-acir-tests
 
 barretenberg-acir-tests-bb.js:
   buildDir: barretenberg/acir_tests
   dockerfile: Dockerfile.bb.js
   dependencies:
     - bb.js
+    - noir-acir-tests
 
 l1-contracts:
   buildDir: l1-contracts
@@ -149,10 +170,3 @@ docs:
     - ^.*.nr$
   dependencies:
     - yarn-project
-
-noir:
-  buildDir: noir
-
-noir-packages:
-  buildDir: noir
-  dockerfile: Dockerfile.packages

noir/tooling/nargo_cli/tests/rebuild.sh

@@ -1,56 +1,72 @@
 #!/bin/bash
 set -e
 
-excluded_dirs=("workspace" "workspace_default_member")
+process_dir() {
+    local dir=$1
+    local current_dir=$2
+    local dir_name=$(basename "$dir")
+
+    if [[ ! -d "$current_dir/acir_artifacts/$dir_name" ]]; then
+      mkdir -p $current_dir/acir_artifacts/$dir_name
+    fi
+
+    cd $dir
+    if [ -d ./target/ ]; then
+      rm -r ./target/
+    fi
+    nargo compile && nargo execute witness
+
+    if [ -f ./target/witness.tr ]; then
+      mv ./target/witness.tr ./target/witness.gz
+    fi
+
+    if [ -f ./target/${dir_name}.json ]; then
+        jq -r '.bytecode' ./target/${dir_name}.json | base64 -d > ./target/acir.gz
+    fi
+
+    rm ./target/${dir_name}.json
+
+    if [ -d "$current_dir/acir_artifacts/$dir_name/target" ]; then
+      rm -r "$current_dir/acir_artifacts/$dir_name/target"
+    fi
+    mkdir $current_dir/acir_artifacts/$dir_name/target
+
+    mv ./target/*.gz $current_dir/acir_artifacts/$dir_name/target/
 
+    cd $current_dir
+}
+
+export -f process_dir
+
+excluded_dirs=("workspace" "workspace_default_member")
 current_dir=$(pwd)
 base_path="$current_dir/execution_success"
 
-# Clear the acir_artifacts directory of any existing artifacts
 rm -rf $current_dir/acir_artifacts
 mkdir -p $current_dir/acir_artifacts
 
-# Loop over every directory
+# Gather directories to process.
+dirs_to_process=()
 for dir in $base_path/*; do
-  if [[ ! -d $dir ]]; then
-    continue
-  fi
-
-  dir_name=$(basename "$dir")
-
-  if [[ ! " ${excluded_dirs[@]} " =~ " ${dir_name} " ]]; then
-      if [[ ! -d "$current_dir/acir_artifacts/$dir_name" ]]; then
-        mkdir -p $current_dir/acir_artifacts/$dir_name
-      fi
-
-      cd $dir
-      if [ -d ./target/ ]; then
-        rm -r ./target/
-      fi
-      nargo compile && nargo execute witness
-
-      # Rename witness.tr to witness.gz
-      if [ -f ./target/witness.tr ]; then
-        mv ./target/witness.tr ./target/witness.gz
-      fi
-
-      # Extract bytecode field from JSON, base64 decode it, and save it to the target directory
-      if [ -f ./target/${dir_name}.json ]; then
-          jq -r '.bytecode' ./target/${dir_name}.json | base64 -d > ./target/acir.gz
-      fi
-
-      # Delete the JSON file after extracting bytecode field
-      rm ./target/${dir_name}.json
-
-      # Clear the target directory in acir_artifacts
-      if [ -d "$current_dir/acir_artifacts/$dir_name/target" ]; then
-        rm -r "$current_dir/acir_artifacts/$dir_name/target"
-      fi
-      mkdir $current_dir/acir_artifacts/$dir_name/target
-
-      # Move the artifacts from the target directory to the corresponding directory in acir_artifacts
-      mv ./target/*.gz $current_dir/acir_artifacts/$dir_name/target/
-
-      cd $base_path
-  fi
+    if [[ ! -d $dir ]] || [[ " ${excluded_dirs[@]} " =~ " $(basename "$dir") " ]]; then
+        continue
+    fi
+    dirs_to_process+=("$dir")
+done
+
+# Process each directory in parallel
+pids=()
+for dir in "${dirs_to_process[@]}"; do
+    process_dir "$dir" "$current_dir" &
+    pids+=($!)
+done
+
+# Check the exit status of each background job.
+for pid in "${pids[@]}"; do
+    wait $pid || exit_status=$?
 done
+
+# Exit with a failure status if any job failed.
+if [ ! -z "$exit_status" ]; then
+    exit $exit_status
+fi