Merge branch 'master' into gd/noir_issue_4330

* master: (176 commits)
  refactor: counters (#7342)
  chore: notify Noir team on changes to the stdlib (#7387)
  chore(bb): do not instantiate Relation (#7389)
  chore(avm): avoid including flavor where possible (#7361)
  chore(avm): better log_derivative_inverse_round (#7360)
  chore(avm): smaller prover (#7359)
  chore(avm): smaller transcript (#7357)
  feat: Sync from noir (#7352)
  feat: VK tree (#6914)
  chore(docs): Update token tutorial (#7241)
  chore: sync external contributions (#6984)
  fix: privacy leak in private refund (#7358)
  fix: included argshash computation in public call_interfaces and cleanup (#7354)
  docs: making private refunds smooth-brain friendly (#7343)
  refactor: optimize public call stack item hashing (#7330)
  fix: Allow importing notes from other contracts and inject them in the macros (#7349)
  refactor: Optimize private call stack item hash for gate count (#7285)
  git subrepo push --branch=master noir-projects/aztec-nr
  git_subrepo.sh: Fix parent in .gitrepo file. [skip ci]
  chore: replace relative paths to noir-protocol-circuits
  ...

.circleci/config.yml

@@ -440,16 +440,6 @@ jobs:
           command: |
             should_release || exit 0
             yarn-project/deploy_npm.sh latest
-      - run:
-          name: "Release canary to NPM: l1-contracts"
-          command: |
-            should_release || exit 0
-            deploy_npm l1-contracts canary
-      - run:
-          name: "Release latest to NPM: l1-contracts"
-          command: |
-            should_release || exit 0
-            deploy_npm l1-contracts latest
       - run:
           name: "Update aztec-up"
           command: |

.devcontainer/scripts/onCreateCommand.sh

@@ -11,11 +11,11 @@ if ! grep -q "PXE_URL" ~/.bashrc; then
 fi
 
 if ! grep -q "alias sandbox" ~/.bashrc; then
-    echo "alias sandbox=\"npx create-aztec-app sandbox\"" >> ~/.bashrc
+    echo "alias sandbox=\"npx aztec-app sandbox\"" >> ~/.bashrc
 fi
 
 source ~/.bashrc
-yes | npx create-aztec-app -t $TYPE -n $NAME -s
+yes | npx aztec-app -t $TYPE -n $NAME -s
 mv $NAME/* $NAME/.* .
 rm -rf $NAME
 

.devcontainer/scripts/postAttachCommand.sh

@@ -5,7 +5,7 @@ NAME=$2
 apt install gh
 gh codespace ports visibility 8080:public -c $CODESPACE_NAME
 
-npx create-aztec-app sandbox start
+npx aztec-app sandbox start
 
 r=$(tput sgr0)       # Reset color
 bold=$(tput bold)    # Bold text

.github/scripts/extract_l1_addresses.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+FILE_PATH=$1
+
+# Read the file line by line
+while IFS= read -r line; do
+  # Extract the hexadecimal address using awk
+  address=$(echo "$line" | awk '{print $NF}')
+
+  # Assign the address to the respective variable based on the line content
+  if [[ $line == *"Rollup Address"* ]]; then
+    export TF_VAR_ROLLUP_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_ROLLUP_CONTRACT_ADDRESS=$TF_VAR_ROLLUP_CONTRACT_ADDRESS"
+  elif [[ $line == *"Registry Address"* ]]; then
+    export TF_VAR_REGISTRY_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_REGISTRY_CONTRACT_ADDRESS=$TF_VAR_REGISTRY_CONTRACT_ADDRESS"
+  elif [[ $line == *"Inbox Address"* ]]; then
+    export TF_VAR_INBOX_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_INBOX_CONTRACT_ADDRESS=$TF_VAR_INBOX_CONTRACT_ADDRESS"
+  elif [[ $line == *"Outbox Address"* ]]; then
+    export TF_VAR_OUTBOX_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_OUTBOX_CONTRACT_ADDRESS=$TF_VAR_OUTBOX_CONTRACT_ADDRESS"
+  elif [[ $line == *"Oracle Address"* ]]; then
+    export TF_VAR_AVAILABILITY_ORACLE_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_AVAILABILITY_ORACLE_CONTRACT_ADDRESS=$TF_VAR_AVAILABILITY_ORACLE_CONTRACT_ADDRESS"
+  elif [[ $line == *"Gas Token Address"* ]]; then
+    export TF_VAR_GAS_TOKEN_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_GAS_TOKEN_CONTRACT_ADDRESS=$TF_VAR_GAS_TOKEN_CONTRACT_ADDRESS"
+  elif [[ $line == *"Gas Portal Address"* ]]; then
+    export TF_VAR_GAS_PORTAL_CONTRACT_ADDRESS=$address
+    echo "TF_VAR_GAS_PORTAL_CONTRACT_ADDRESS=$TF_VAR_GAS_PORTAL_CONTRACT_ADDRESS"
+  else
+    echo "Unknown contract address: $line"
+  fi
+done <"$FILE_PATH"
+
+# echo all addresses into github env
+echo "TF_VAR_ROLLUP_CONTRACT_ADDRESS=$TF_VAR_ROLLUP_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_REGISTRY_CONTRACT_ADDRESS=$TF_VAR_REGISTRY_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_INBOX_CONTRACT_ADDRESS=$TF_VAR_INBOX_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_OUTBOX_CONTRACT_ADDRESS=$TF_VAR_OUTBOX_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_AVAILABILITY_ORACLE_CONTRACT_ADDRESS=$TF_VAR_AVAILABILITY_ORACLE_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_GAS_TOKEN_CONTRACT_ADDRESS=$TF_VAR_GAS_TOKEN_CONTRACT_ADDRESS" >>$GITHUB_ENV
+echo "TF_VAR_GAS_PORTAL_CONTRACT_ADDRESS=$TF_VAR_GAS_PORTAL_CONTRACT_ADDRESS" >>$GITHUB_ENV
+
+# Set global variable for redeployment of contracts
+echo "CONTRACTS_DEPLOYED=1" >>$GITHUB_ENV

.github/workflows/ci.yml

@@ -2,7 +2,8 @@ name: CI
 on:
   push:
     branches: [master]
-  pull_request: {}
+  pull_request:
+    branches-ignore: [devnet]
   workflow_dispatch:
     inputs: {}
 
@@ -97,7 +98,11 @@ jobs:
       # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        run: earthly-ci ./yarn-project+export-e2e-test-images
+        run: |
+          earthly-ci \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          ./yarn-project+export-e2e-test-images
       # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end
       # (Note ARM uses just 2 tests as a smoketest)
       - name: Create list of non-bench end-to-end jobs
@@ -362,7 +367,7 @@ jobs:
   noir-format:
     needs: [setup, changes]
     runs-on: ${{ github.event.pull_request.user.login || github.actor }}-x86
-    if: ${{ needs.changes.outputs.noir == 'true' }}
+    if: ${{ needs.changes.outputs.noir == 'true' || needs.changes.outputs.noir-projects == 'true' }}
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -376,7 +381,11 @@ jobs:
       - name: "Format noir-projects"
         working-directory: ./noir-projects/
         timeout-minutes: 40
-        run: earthly-ci --no-output ./+format
+        run: | 
+          earthly-ci --no-output \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          ./+format
 
   noir-test:
     needs: [setup, changes]
@@ -429,7 +438,11 @@ jobs:
           concurrency_key: noir-projects-x86
       - name: "Noir Projects"
         timeout-minutes: 40
-        run: earthly-ci --no-output ./noir-projects/+test
+        run: |
+          earthly-ci --no-output \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          ./noir-projects/+test
 
   avm-format:
     needs: [setup, changes]
@@ -514,7 +527,14 @@ jobs:
       - name: "Docs Preview"
         if: github.event.number
         timeout-minutes: 40
-        run: earthly-ci --no-output ./docs/+deploy-preview --ENV=staging --PR=${{ github.event.number }} --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }}
+        run: | 
+          earthly-ci --no-output \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          ./docs/+deploy-preview --ENV=staging --PR=${{ github.event.number }} \
+          --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} \
+          --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} \
+          --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }}
 
   bb-bench:
     runs-on: ubuntu-20.04
@@ -616,7 +636,10 @@ jobs:
         working-directory: ./noir-projects/
         timeout-minutes: 40
         run: |
-          earthly-ci --artifact +gates-report/gates_report.json
+          earthly-ci \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          --artifact +gates-report/gates_report.json 
           mv gates_report.json ../protocol_circuits_report.json
 
       - name: Compare gates reports

.github/workflows/codeql.yml

@@ -0,0 +1,87 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on: workflow_dispatch
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: javascript-typescript
+          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/devnet-deploys.yml

@@ -3,9 +3,15 @@ on:
   push:
     branches: [devnet]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
   GIT_COMMIT: ${{ github.sha }}
+  DEPLOY_TAG: devnet
+  FILE_PATH: ./l1-contracts/addresses.txt
   # TF Vars
   TF_VAR_DOCKERHUB_ACCOUNT: aztecprotocol
   TF_VAR_CHAIN_ID: 31337
@@ -25,26 +31,49 @@ jobs:
     secrets: inherit
 
   build:
+    needs: setup
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GIT_COMMIT }}" }
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+          fetch-depth: 0
       - uses: ./.github/ci-setup-action
         with:
-          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
           concurrency_key: build-release-artifacts-${{ github.actor }}
-      - name: "Build & Push images"
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+      - name: "Build & Push aztec images"
+        timeout-minutes: 40
+        # Run the build steps for each image with version and arch, push to dockerhub
+        run: |
+          earthly-ci --no-output --push ./yarn-project+export-aztec-arch --DIST_TAG=${{ env.DEPLOY_TAG }}
+
+      - name: Check if L1 contracts need deployment
+        id: check_changes_build
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { execSync } = require('child_process');
+            const changedFiles = execSync('git diff --name-only ${{ github.event.before }} ${{ github.sha }}').toString().split('\n');
+            const fileChanged = changedFiles.includes('l1-contracts/REDEPLOY');
+            return fileChanged
+
+      - name: "Build & Push cli image"
+        if: steps.check_changes_build.outputs.result == 'true'
         timeout-minutes: 40
         # Run the build steps for each image with version and arch, push to dockerhub
         run: |
-          earthly-ci --no-output --push ./yarn-project+export-aztec-arch --DIST_TAG=devnet
+          earthly-ci --no-output --push ./yarn-project+export-cli --DIST_TAG=${{ env.DEPLOY_TAG }}
 
   terraform_deploy:
     runs-on: ubuntu-latest
     needs: build
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GIT_COMMIT }}" }
+        with:
+          ref: "${{ env.GIT_COMMIT }}"
+          fetch-depth: 0
+      - uses: ./.github/ci-setup-action
       - uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.7.5
@@ -56,14 +85,56 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
 
-      - name: Deploy Bootstrap Nodes
-        working-directory: ./yarn-project/aztec/terraform/node
+      - name: Check if L1 contracts need deployment
+        id: check_changes_release
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const { execSync } = require('child_process');
+            const changedFiles = execSync('git diff --name-only ${{ github.event.before }} ${{ github.sha }}').toString().split('\n');
+            const fileChanged = changedFiles.includes('l1-contracts/REDEPLOY');
+            return fileChanged
+      - name: Deploy L1 Contracts
+        if: steps.check_changes_release.outputs.result == 'true'
         run: |
-          terraform init -input=false -backend-config="key=devnet/aztec-node"
+          docker pull aztecprotocol/cli:${{ env.DEPLOY_TAG }}
+          docker run aztecprotocol/cli:${{ env.DEPLOY_TAG }} \
+            deploy-l1-contracts -p ${{ secrets.SEQ_1_PUBLISHER_PRIVATE_KEY }} \
+            -u https://${{ env.DEPLOY_TAG }}-mainnet-fork.aztec.network:8545/${{ secrets.FORK_API_KEY }} \
+            | tee ${{ env.FILE_PATH }}
+          ./.github/scripts/extract_l1_addresses.sh ${{ env.FILE_PATH }}
+
+      - name: Apply l1-contracts Terraform
+        if: steps.check_changes_release.outputs.result == 'true'
+        working-directory: ./l1-contracts/terraform
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/l1-contracts"
           terraform apply -input=false -auto-approve
 
+      - name: Deploy P2P Bootstrap Nodes
+        working-directory: ./yarn-project/p2p-bootstrap/terraform
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/p2p-bootstrap"
+          terraform apply -input=false -auto-approve
+
+      - name: Init Aztec Node Terraform
+        working-directory: ./yarn-project/aztec/terraform/node
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node"
+
+      - name: Taint node filesystem if L1 contracts are redeployed
+        if: steps.check_changes_release.outputs.result == 'true'
+        working-directory: ./yarn-project/aztec/terraform/node
+        run: |
+          terraform state list | grep 'aws_efs_file_system.node_data_store' | xargs -n1 terraform taint
+
       - name: Deploy Aztec Nodes
         working-directory: ./yarn-project/aztec/terraform/node
         run: |
-          terraform init -input=false -backend-config="key=devnet/aztec-node"
+          terraform apply -input=false -auto-approve
+
+      - name: Deploy Provers
+        working-directory: ./yarn-project/aztec/terraform/prover
+        run: |
+          terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/prover"
           terraform apply -input=false -auto-approve