feat: run noir-packages-test in Earthly (#6174)

.circleci/config.yml

@@ -254,18 +254,6 @@ jobs:
           command: cond_spot_run_build noir-packages 32
           aztec_manifest_key: noir-packages
 
-  noir-packages-tests:
-    docker:
-      - image: aztecprotocol/alpine-build-image
-    resource_class: small
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Build"
-          command: cond_spot_run_build noir-packages-tests 32
-          aztec_manifest_key: noir-packages-tests
-
   avm-transpiler:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -521,12 +509,6 @@ workflows:
           requires:
             - bb-js
           <<: *defaults
-      - noir-packages-tests:
-          requires:
-            - bb-js
-            - noir-ecr-manifest
-            - noir-packages
-          <<: *defaults
 
       # Transpiler
       - avm-transpiler: *defaults
@@ -588,7 +570,6 @@ workflows:
             - barretenberg-x86_64-linux-clang-fuzzing
             - barretenberg-wasm-linux-clang
             - barretenberg-docs
-            - noir-packages-tests
             - e2e-join
             - aztec-builder
           <<: *defaults

.github/workflows/ci.yml

@@ -145,6 +145,21 @@ jobs:
         timeout-minutes: 25
         run: earthly-ci --no-output ./+test
 
+  noir-packages-test:
+    needs: setup
+    runs-on: ${{ inputs.username || github.actor }}-x86
+    steps:
+      - {
+          uses: actions/checkout@v4,
+          with: { ref: "${{ github.event.pull_request.head.sha }}" },
+        }
+      - uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+          concurrency_key: noir-${{ inputs.username || github.actor }}-x86
+      - name: "Test Noir JS packages"
+        run: earthly-ci --no-output ./noir+packages-test
+
   noir-projects:
     needs: setup
     runs-on: ${{ inputs.username || github.actor }}-x86
@@ -348,7 +363,7 @@ jobs:
         working-directory: ./barretenberg/cpp/
         timeout-minutes: 15
         run: earthly-ci --no-output +bench-ultra-honk --bench_mode=cache
-  
+
   protocol-circuits-gates-report:
     needs: setup
     runs-on: ${{ inputs.username || github.actor }}-x86
@@ -393,6 +408,7 @@ jobs:
       - yarn-project-formatting
       - yarn-project-test
       - prover-client-test
+      - noir-packages-test
     if: always()
     steps:
       - run: |

barretenberg/ts/Earthfile

@@ -16,7 +16,7 @@ COPY --dir src *.json *.js *.cjs .
 COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg-threads.wasm
 COPY ../cpp/+preset-wasm/bin/barretenberg.wasm src/barretenberg_wasm/barretenberg.wasm
 COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node/barretenberg_wasm/barretenberg-threads.wasm
-COPY ../cpp/+preset-wasm/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm
+COPY ../cpp/+preset-wasm-threads/bin/barretenberg.wasm dest/node-cjs/barretenberg_wasm/barretenberg-threads.wasm
 
 esm:
     RUN yarn build:esm

noir/.earthlyignore

@@ -0,0 +1,87 @@
+# Must include the .gitignore for all child projects as this is used by Earthly
+# Note due to how we use Eartlhy each .gitignore MUST accompany any earthfile that might actually copy these artifacts
+**/Earthfile
+**/Readme.md
+**/Dockerfile*
+**/docker-compose*.yml
+
+# root .gitignore contents
+dest
+node_modules
+.cache
+scripts/.earthly
+.pnp.cjs
+.pnp.loader.mjs
+build/
+.idea
+cmake-build-debug
+.terraform*
+.bootstrapped
+.tsbuildinfo
+
+# Local Netlify folder
+.netlify
+
+.graphite*
+.DS_Store
+
+**/*.dockerignore
+
+# Earthly
+.arg
+.secret
+
+# ./ .gitignore contents:
+**/package.tgz
+packages
+
+# ./noir-repo/ .gitignore contents:
+/target
+.DS_Store
+examples/**/target/
+examples/9
+node_modules
+pkg/
+.idea
+
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+# Noir.js
+tooling/noir_js/lib
+
+# Nargo output
+*.proof
+*.acir
+*.acir.sha256
+*.tr
+*.pk
+*.vk
+**/Verifier.toml
+**/contract
+**/target
+!test_programs/acir_artifacts/*/target
+!test_programs/acir_artifacts/*/target/witness.gz
+!compiler/wasm/noir-script/target
+
+gates_report.json
+
+# Github Actions scratch space
+# This gives a location to download artifacts into the repository in CI without making git dirty.
+libbarretenberg-wasm32
+
+# Wasm build atifacts
+compiler/wasm/nodejs
+compiler/wasm/web
+tooling/noirc_abi_wasm/nodejs
+tooling/noirc_abi_wasm/web
+tooling/noir_js/lib
+
+# docs autogen build
+/docs/docs/noir_js/reference/

noir/Earthfile

@@ -1,6 +1,7 @@
 VERSION 0.8
 
-nargo:
+nargo-src:
+
     FROM ../build-images/+build
     WORKDIR /usr/src
     # Relevant source (TODO finer-grained 'tooling')
@@ -14,26 +15,58 @@ nargo:
       noir-repo/utils \
       noir-repo/Cargo.lock \
       noir-repo/Cargo.toml \
+      noir-repo/.github \
       noir-repo
 
-    # TODO(AD) is this OK as a content hash?
+    # NOTE: we use a fake commit hash here
+    # we don't want Noir to rebuild everytime the parent repo changes
+    # just only when it changes
+    # the commit hash gets injected into version strings
     ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}')
+    RUN echo $COMMIT_HASH > .content-hash
+
+    # # borrow Nix's approach to build everything in 1970
+    ENV SOURCE_TIMESTAMP=1
+    ENV SOURCE_DATE_EPOCH=1
+
+nargo:
+    FROM +nargo-src
+    RUN ./noir-repo/.github/scripts/wasm-bindgen-install.sh
     COPY ./scripts/bootstrap_native.sh ./scripts/bootstrap_native.sh
     RUN ./scripts/bootstrap_native.sh
-    RUN echo CONTENT HASH $COMMIT_HASH | tee .content-hash
     SAVE ARTIFACT /usr/src/noir-repo/target/release/nargo nargo
     SAVE ARTIFACT /usr/src/noir-repo/target/release/acvm acvm
     SAVE IMAGE aztecprotocol/nargo
 
-packages:
+packages-deps:
     BUILD ../barretenberg/ts/+build # prefetch
-    FROM ../build-images/+build
 
-    # `noir-repo` is nested inside of `noir` so we copy `bb.js` as such to account
-    # for the extra nested folder specified in portalled package paths
-    COPY ../barretenberg/ts/+build/build /usr/src/../barretenberg/ts
+    LOCALLY
+    LET packages = $(git ls-files "**/package*.json" package*.json)
+    LET tsconfigs = $(git ls-files "**/tsconfig*.json" tsconfig*.json)
 
-    WORKDIR /usr/src
+    FROM +nargo-src
+
+    COPY ../barretenberg/ts/+build/build /usr/src/barretenberg/ts
+
+    WORKDIR /usr/src/noir
+
+    COPY --dir \
+      ./noir-repo/package.json \
+      ./noir-repo/yarn.lock \
+      ./noir-repo/.yarnrc.yml \
+      ./noir-repo/.yarn \
+      ./noir-repo
+
+    FOR file IN $packages
+        COPY $file $file
+    END
+
+    RUN cd noir-repo && yarn install --immutable && cd ../
+
+    FOR file IN $tsconfigs
+      COPY $file $file
+    END
 
     # Relevant source (TODO finer-grained)
     COPY --dir \
@@ -45,7 +78,7 @@ packages:
       noir-repo/scripts \
       noir-repo/test_programs \
       noir-repo/tooling \
-      noir-repo/utils \      
+      noir-repo/utils \
       noir-repo/Cargo.lock \
       noir-repo/.yarnrc.yml \
       noir-repo/.yarn \
@@ -58,16 +91,74 @@ packages:
       noir-repo/.github \
       noir-repo
 
-    COPY noir-repo/.github/scripts noir-repo/.github/scripts
-    COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh
+packages:
+    FROM +packages-deps
 
-    # TODO(AD) is this OK as a content hash?
-    ENV COMMIT_HASH=$(find . -type f -exec sha256sum {} ';' | sort | sha256sum | awk '{print $1}')
-    RUN echo CONTENT HASH $COMMIT_HASH | tee .content-hash
+    COPY ./scripts/bootstrap_packages.sh ./scripts/bootstrap_packages.sh
     RUN ./scripts/bootstrap_packages.sh
     SAVE ARTIFACT packages
     SAVE IMAGE --cache-hint
 
+packages-test-build:
+  FROM +packages-deps
+
+  COPY +nargo/nargo /usr/src/noir/noir-repo/target/release/nargo
+  COPY +nargo/acvm /usr/src/noir/noir-repo/target/release/acvm
+
+  ENV NARGO_BACKEND_PATH=/usr/src/barretenberg/ts/dest/node/main.js
+  ENV PATH=$PATH:/usr/src/noir/noir-repo/target/release
+
+  WORKDIR /usr/src/barretenberg/ts
+  RUN yarn --immutable
+
+  WORKDIR /usr/src/noir/noir-repo
+  COPY --dir noir-repo/.github/scripts/wasm-bindgen-install.sh ./.github/scripts/wasm-bindgen-install.sh
+  RUN ./.github/scripts/wasm-bindgen-install.sh
+
+  ENV SOURCE_DATE_EPOCH=$(date +%s)
+  ENV GIT_DIRTY=false
+  ENV GIT_COMMIT=$COMMIT_HASH
+  RUN yarn build
+  # this builds text fixtures to be used in tests
+  RUN yarn workspace @noir-lang/noir_wasm run test:build_fixtures
+
+  SAVE ARTIFACT /usr/src /usr/src
+
+packages-test-node:
+  FROM +packages-test-build
+  ENV NODE_OPTIONS=--max_old_space_size=8192
+  WORKDIR /usr/src/noir/noir-repo
+  RUN yarn workspaces foreach \
+    --parallel \
+    --verbose \
+    --exclude @noir-lang/root \ # foreach includes the root workspace, ignore it
+    --exclude @noir-lang/noir_js \ # noir_js OOMs
+    --exclude integration-tests \ # separate node and browser tests
+    --exclude @noir-lang/noir_wasm \
+    run test
+  RUN yarn workspaces foreach \
+    --parallel \
+    --verbose \
+    --include integration-tests \
+    --include @noir-lang/noir_wasm \
+    run test:node
+
+packages-test-browser:
+  FROM node:18
+  COPY --dir +packages-test-build/usr/src /usr
+  WORKDIR /usr/src/noir/noir-repo
+  RUN ./.github/scripts/playwright-install.sh
+  RUN yarn workspaces foreach \
+    --parallel \
+    --verbose \
+    --include integration-tests \
+    --include @noir-lang/noir_wasm \
+    run test:browser
+
+packages-test:
+  BUILD +packages-test-node
+  BUILD +packages-test-browser
+
 run:
     # When running the container, mount the users home directory to same location.
     FROM ubuntu:noble