Merge branch 'master' into sync-noir

* master: (207 commits)
  refactor: disallow prune in devnet + add onlyOwners (#8134)
  refactor: add partial note support for value note (#8141)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  git subrepo push --branch=master barretenberg
  Update external-ci-approved.yml
  fix: Fixes for the nightly test run against Sepolia (#8229)
  fix: Docker containers healthchecks (#8228)
  chore: Create external-ci-approved.yml (#8235)
  refactor: rework balances map (#8127)
  feat: Add CLI command for gathering proving metrics (#8221)
  chore: Less noisy AVM failures in proving (#8227)
  chore: open an issue if publishing bb fails (#8223)
  feat: spartan token transfer (#8163)
  fix: Attempt to fix nightly test (#8222)
  feat: initial validator set (#8133)
  chore: Configure interval mining for anvil (#8211)
  fix: remove devnet ARM builds for now (#8202)
  fix: Export brillig names in contract functions (#8212)
  ...

.github/RELEASE_FAILED.md

@@ -0,0 +1,8 @@
+---
+title: "Nightly release failed"
+assignees: fcarreiro, ludamad
+---
+
+The most recent nightly release failed. This means that no `bb` artifacts were published (which are required by Noir).
+
+Check the [{{env.WORKFLOW_NAME}}]({{env.WORKFLOW_URL}}) workflow for details.

.github/workflows/ci.yml

@@ -78,6 +78,7 @@ jobs:
               - barretenberg/cpp/pil/**
               - barretenberg/cpp/src/barretenberg/vm/**
               - barretenberg/cpp/src/barretenberg/**/generated/*
+              - barretenberg/cpp/src/barretenberg/client_ivc.{hpp,cpp}
             non-docs:
               - '!(docs/**)'
             non-misc-ci:
@@ -125,7 +126,7 @@ jobs:
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of non-bench end-to-end jobs
         id: e2e_list
-        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' | grep -v 'devnet' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
+        run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | grep -v '+bench' | grep -v "+network" | grep -v 'devnet' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
       - name: Create list of bench end-to-end jobs
         id: bench_list
         run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep '+bench' |  sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT
@@ -160,6 +161,35 @@ jobs:
             export FORCE_COLOR=1
             ../../scripts/earthly-ci -P --no-output +${{ matrix.test }}
 
+  # all the network end-to-end tests for aztec (not required to merge)
+  network-e2e:
+    needs: [build, changes]
+    if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }}
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 45
+        with:
+          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 40
+          run: |
+            set -eux
+            cd ./yarn-project/end-to-end/
+            ./scripts/setup_local_k8s.sh
+            export FORCE_COLOR=1
+            export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
+            ../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer
+
   # all the benchmarking end-to-end integration tests for aztec (not required to merge)
   bench-e2e:
     needs: [build, changes]
@@ -276,7 +306,7 @@ jobs:
         # limit our parallelism to half our cores
         run: earthly-ci --no-output +preset-gcc
 
-  # barretenberg (prover) native and AVM (public VM) tests
+  # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
     needs: [build-images, changes]
@@ -577,6 +607,7 @@ jobs:
       # https://aztecprotocol.github.io/aztec-packages/dev/bench/ with new benchmark data.
       # This also creates an alert if benchmarks exceed the threshold specified below.
       - name: Store benchmark result
+        if: ${{ github.ref == 'refs/heads/master' }}
         uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
         with:
           name: C++ Benchmark
@@ -591,6 +622,7 @@ jobs:
           comment-on-alert: true
           fail-on-alert: false
           alert-comment-cc-users: "@ludamad @codygunton"
+          max-items-in-chart: 50
 
   boxes:
     needs: build

.github/workflows/external-ci-approved.yml

@@ -0,0 +1,45 @@
+# Mirror PRs labeled external-ci-approved. It will sync on every-reapply.
+name: Mirror PRs for External CI
+
+on:
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  mirror-pr:
+    runs-on: ubuntu-latest
+
+    if: contains(github.event.pull_request.labels.*.name, 'external-ci-approved')
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        # TODO remove guesswork
+        ref: ${{ github.event.pull_request.head.sha || github.event.pull_request_target.head.sha }}
+        token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+
+    - name: Set up Git
+      run: |
+        git config user.name "${{ github.actor }}"
+        git config user.email "${{ github.actor }}@users.noreply.github.com"
+
+    - name: Create or update the mirrored branch
+      run: |
+        HASH=$(git rev-parse HEAD)
+        git checkout -b mirrored/${{ github.head_ref }} || git checkout mirrored/${{ github.head_ref }}
+        git reset --hard $HASH
+        git push origin mirrored/${{ github.head_ref }} --force
+
+    - name: Create or update the mirrored pull request
+      uses: peter-evans/create-pull-request@v5
+      with:
+        token:  ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+        base: master
+        head: ${{ github.head_ref }}
+        title: "Mirror PR: ${{ github.event.pull_request.title }}"
+        body: "This PR has been created because #${{ github.event.pull_request.number }} has been labeled external-ci-approved. It will sync on every-reapply."
+        draft: false
+        branch: mirrored/${{ github.head_ref }}
+        update_existing_pr: true

.github/workflows/publish-bb.yml

@@ -93,7 +93,7 @@ jobs:
           sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/* /usr/local/lib/
           sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/share/* /usr/local/share/
 
-      - name: Install yarn # Needed to call 'yarn build' on barretenberg/ts 
+      - name: Install yarn # Needed to call 'yarn build' on barretenberg/ts
         run: |
           curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
           echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
@@ -152,7 +152,7 @@ jobs:
           export PATH="/usr/local/opt/llvm@16/bin:$PATH"
           export LDFLAGS="-L/usr/local/opt/llvm@16/lib"
           export CPPFLAGS="-I/usr/local/opt/llvm@16/include"
-          cmake -DCMAKE_BUILD_TYPE=RelWithAssert --preset default 
+          cmake -DCMAKE_BUILD_TYPE=RelWithAssert --preset default
           cmake --build --preset default --target bb
 
       - name: Package barretenberg artifact
@@ -170,7 +170,7 @@ jobs:
           retention-days: 3
 
   build-mac-m1:
-    name: Build on Mac aarch64-apple-darwin 
+    name: Build on Mac aarch64-apple-darwin
     runs-on: macos-14
     steps:
       - name: Checkout
@@ -201,9 +201,44 @@ jobs:
           path: ./barretenberg/cpp/build/bin/barretenberg-aarch64-apple-darwin.tar.gz
           retention-days: 3
 
+  build-check:
+    name: Check builds are successful
+    needs: [build-x86_64-linux-gnu, build-mac-intel, build-mac-m1,build-wasm-ts]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report overall success
+        env:
+          FAIL: ${{ contains(needs.*.result, 'failure') }}
+        run: |
+          if [[ $FAIL == true ]]; then
+              echo "At least one job failed, release is unsuccessful."
+              exit 1
+          else
+              echo "All jobs succeeded, uploading artifacts to release."
+              exit 0
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        if: ${{ failure() }}
+        with:
+          ref: ${{ inputs.tag || env.GITHUB_REF }}
+
+      # Raise an issue if the release failed
+      - name: Alert on dead links
+        uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
+        if: ${{ failure() }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_NAME: ${{ github.workflow }}
+          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        with:
+          update_existing: true
+          filename: .github/RELEASE_FAILED.md
+
   release:
     name: Publish
-    needs: [build-x86_64-linux-gnu, build-mac-intel, build-mac-m1,build-wasm-ts]
+    needs: [build-check]
     runs-on: ubuntu-latest
     steps:
       - name: Download files from Linux Runner

.github/workflows/pull-noir.yml

@@ -119,7 +119,11 @@ jobs:
             # Fix PR branch
             git fetch # see recent change
             git checkout $BRANCH || git checkout -b $BRANCH
-            git merge -Xtheirs origin/$STAGING_BRANCH -m "$COMMIT_MESSAGE"
+            if ! git merge -Xtheirs origin/$STAGING_BRANCH -m "$COMMIT_MESSAGE" ; then
+              # resolve modify/delete conflicts
+              git diff --name-only --diff-filter=U | xargs git rm
+              git commit -m "$COMMIT_MESSAGE"
+            fi
             git push origin $BRANCH
           }
           force_sync_staging

.github/workflows/sepolia-test.yml

@@ -0,0 +1,63 @@
+name: Run public testnet test
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 08 * * 1-5"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  GIT_COMMIT: ${{ github.sha }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+  # Address 0x652575Ff941e7c2850fB89f2B207efF6B06BC7B4
+  SEQ_PUBLISHER_PRIVATE_KEY: ${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }}
+
+  # Address 0xE3b8F9F23b8D4BD7d437218Bff3bcED1ce5E70B3
+  PROVER_PUBLISHER_PRIVATE_KEY: ${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }}
+
+  ETHEREUM_HOST: "https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}"
+  L1_CHAIN_ID: "11155111"
+
+jobs:
+  setup:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      username: ${{ github.event.pull_request.user.login || github.actor }}
+      runner_type: builder-x86
+    secrets: inherit
+
+  build-and-test:
+    needs: setup
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-test-artifacts-${{ github.actor }}
+
+      - name: "Build E2E Image"
+        timeout-minutes: 40
+        run: |
+          earthly-ci ./yarn-project+export-e2e-test-images
+
+      - name: Run Testnet Test
+        timeout-minutes: 40
+        run: |
+          set -eux
+          cd ./yarn-project/end-to-end/
+          export FORCE_COLOR=1
+          ../../scripts/earthly-ci -P --no-output +e2e-public-testnet \
+          --SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_SEQ_PRIVATE_KEY }} \
+          --PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PRIVATE_KEY }} \
+          --ETHEREUM_HOST="https://sepolia.infura.io/v3/${{ secrets.SEPOLIA_API_KEY }}" \
+          --L1_CHAIN_ID="11155111"
+
+

.noir-sync-commit

@@ -1 +1 @@
-82eb1581251faa9716d762a673fa1b871b3e7be2
+86c151aa43ba990a0e900995064a88fc2ae6637d

.release-please-manifest.json

@@ -1,7 +1,7 @@
 {
-  ".": "0.48.0",
+  ".": "0.51.0",
   "yarn-project/cli": "0.35.1",
-  "yarn-project/aztec": "0.48.0",
-  "barretenberg": "0.48.0",
-  "barretenberg/ts": "0.48.0"
+  "yarn-project/aztec": "0.51.0",
+  "barretenberg": "0.51.0",
+  "barretenberg/ts": "0.51.0"
 }

.vscode/settings.json

@@ -169,6 +169,7 @@
     "**/dest/**": true,
     "**/noir/noir-repo/docs/versioned_docs/**": true
   },
+  "files.trimTrailingWhitespace": true,
   "cmake.sourceDirectory": "${workspaceFolder}/barretenberg/cpp",
   "typescript.tsserver.maxTsServerMemory": 4096,
 }