Merge branch 'master' into tf/evaluator-changes

* master: (67 commits) chore: Fix bad merge on AztecLMDBStore initializer feat: add persisted database of proving jobs (#9942) chore: Clean up data configuration (#9973) chore: remove public kernels (#10027) chore: misc cleanup, docs and renaming (#9968) feat: IPA Accumulator in Builder (#9846) chore(docs): Updates to token contract (#9954) test(avm): minor benchmarking (#9869) chore(ci): run `l1-contracts` CI in parallel with `build` step (#10024) chore: build acir test programs in parallel to e2e build step (#9988) chore: pull out `array_set` pass changes (#9993) feat(avm): ephemeral avm tree (#9798) fix: don't take down runners with faulty runner check (#10019) feat(docs): add transaction profiler docs (#9932) chore: hotfix runner wait (#10018) refactor: remove EnqueuedCallSimulator (#10015) refactor: stop calling public kernels (#9971) git subrepo push --branch=master noir-projects/aztec-nr git_subrepo.sh: Fix parent in .gitrepo file. [skip ci] chore: replace relative paths to noir-protocol-circuits ...
AztecProtocol · Nov 19, 2024 · afda45d · afda45d
2 parents 7a8df32 + c8de6cf
commit afda45d
Show file tree

Hide file tree

Showing 787 changed files with 62,642 additions and 37,814 deletions.
diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js
diff --git a/.github/spot-runner-action/src/main.ts b/.github/spot-runner-action/src/main.ts
@@ -23,10 +23,11 @@ async function pollSpotStatus(
     }
     try {
       core.info("Found ec2 instance, looking for runners.");
-      if (process.env.WAIT_FOR_RUNNERS === "false" || await ghClient.hasRunner([config.githubJobId])) {
+      // TODO find out whatever happened here but we seem to not be able to wait for runners
+      //if (process.env.WAIT_FOR_RUNNERS === "false" || await ghClient.hasRunner([config.githubJobId])) {
         // we have runners
         return instances[0].InstanceId!;
-      }
+      //}
     } catch (err) {}
     // wait 10 seconds
     await new Promise((r) => setTimeout(r, 10000));
@@ -174,11 +175,11 @@ async function startWithGithubRunners(config: ActionConfig) {
       return false;
     }
     await setupGithubRunners(ip, config);
-    if (instanceId) await ghClient.pollForRunnerCreation([config.githubJobId]);
-    else {
-      core.error("Instance failed to register with Github Actions");
-      throw Error("Instance failed to register with Github Actions");
-    }
+    // if (instanceId) await ghClient.pollForRunnerCreation([config.githubJobId]);
+    // else {
+    //  core.error("Instance failed to register with Github Actions");
+    //  throw Error("Instance failed to register with Github Actions");
+    // }
     core.info("Done setting up runner.")
   }
   // Export to github environment

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -385,7 +385,7 @@ jobs:
         run: earthly-ci --no-output ./+test
 
   noir-build-acir-tests:
-    needs: [build, configure]
+    needs: [build-images, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -399,7 +399,7 @@ jobs:
         run: earthly-ci --no-output ./noir/+build-acir-tests
 
   bb-acir-tests-bb:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -414,7 +414,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb
 
   bb-acir-tests-bb-ultra-plonk:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -429,7 +429,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-ultra-plonk
 
   bb-acir-tests-bb-ultra-honk:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -444,7 +444,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-ultra-honk
 
   bb-acir-tests-bb-mega-honk:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -459,7 +459,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb-mega-honk
 
   bb-acir-tests-sol:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -474,7 +474,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-sol
 
   bb-acir-tests-sol-honk:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -489,7 +489,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-sol-honk
 
   bb-acir-tests-bb-js:
-    needs: [noir-build-acir-tests, configure]
+    needs: [noir-build-acir-tests, build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
     steps:
@@ -504,7 +504,7 @@ jobs:
         run: earthly-ci --no-output ./+barretenberg-acir-tests-bb.js
 
   noir-format:
-    needs: [build, configure]
+    needs: [build-images, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.noir == 'true' || needs.configure.outputs.noir-projects == 'true'
     steps:
@@ -524,7 +524,7 @@ jobs:
           earthly-ci --no-output ./+format
 
   noir-test:
-    needs: [build, configure]
+    needs: [build-images, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.noir == 'true'
     steps:
@@ -537,6 +537,7 @@ jobs:
         run: earthly-ci --no-output ./noir+test
 
   noir-examples:
+    # We delay this job until after the `build` job has completed as it depends on bb being built.
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.noir == 'true'
@@ -550,6 +551,7 @@ jobs:
         run: earthly-ci --no-output ./noir+examples
 
   noir-packages-test:
+    # We delay this job until after the `build` job has completed as it depends on bb.js being built.
     needs: [build, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.barretenberg == 'true' || needs.configure.outputs.noir == 'true'
@@ -666,7 +668,7 @@ jobs:
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
-          concurrency_key: kind-network-test-${{ matrix.test }}
+          concurrency_key: kind-smoke-test
       - name: Setup and KIND Network Test
         timeout-minutes: 60
         uses: ./.github/ensure-tester-with-images
@@ -700,9 +702,11 @@ jobs:
             values: 16-validators
             runner_type: 16core-tester-x86
             timeout: 60
-          - test: transfer.test.ts
-            values: 48-validators
-            runner_type: 32core-tester-x86
+          # TODO: Enable this once stable
+          # - test: transfer.test.ts
+          #   values: 48-validators
+          #   runner_type: 32core-tester-x86
+          #   timeout: 90
           - test: reorg.test.ts
             values: 16-validators
             runner_type: 16core-tester-x86-high-memory
@@ -748,7 +752,7 @@ jobs:
           path: network-test.log
 
   l1-contracts-test:
-    needs: [build, configure]
+    needs: [build-images, configure]
     runs-on: ${{ needs.configure.outputs.username }}-x86
     if: needs.configure.outputs.l1-contracts == 'true'
     steps:

diff --git a/.github/workflows/nightly-kind-test.yml b/.github/workflows/nightly-kind-test.yml
@@ -53,7 +53,9 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        test: ["transfer", "reorg"]
+        test:
+          - transfer
+          - reorg
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -62,7 +64,7 @@ jobs:
         uses: ./.github/ensure-tester-with-images
         timeout-minutes: 90
         with:
-          runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
+          runner_type: 16core-tester-x86
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
@@ -73,13 +75,42 @@ jobs:
           run: |
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
+            cd yarn-project/end-to-end
             export FORCE_COLOR=1
-            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
+            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true CLEANUP_CLUSTER=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
+
+  proving-test:
+    needs: build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+      - name: Setup and Test
+        uses: ./.github/ensure-tester-with-images
+        timeout-minutes: 90
+        with:
+          runner_type: 128core-tester-x86
+          builder_type: builder-x86
+          # these are copied to the tester and expected by the earthly command below
+          # if they fail to copy, it will try to build them on the tester and fail
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
+          # command to produce the images in case they don't exist
+          builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
+          tester_ttl: 90
+          run: |
+            set -eux
+            ./spartan/scripts/setup_local_k8s.sh
+            cd yarn-project/end-to-end
+            export FORCE_COLOR=1
+            NAMESPACE=proving FRESH_INSTALL=true CLEANUP_CLUSTER=true INSTALL_TIMEOUT=45m VALUES_FILE="1-validator-with-proving.yaml" ./scripts/network_test.sh ./src/spartan/proving.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04
     needs:
       - test
+      - proving-test
     if: always()
     steps:
       - name: Report overall success

diff --git a/.github/workflows/publish-aztec-packages.yml b/.github/workflows/publish-aztec-packages.yml
@@ -322,7 +322,11 @@ jobs:
         working-directory: ./aztec-up/terraform
         run: |
           terraform init
-          TAG=${{ env.DEPLOY_TAG }}
+          if [ "${{ github.ref_name }}" == "master" ]; then
+            TAG=master
+          else
+            TAG=${{ env.DEPLOY_TAG }}
+          fi
           export TF_VAR_VERSION=${TAG#aztec-packages-v}
           terraform apply -auto-approve
 

diff --git a/avm-transpiler/Cargo.lock b/avm-transpiler/Cargo.lock
diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo
@@ -6,7 +6,7 @@
 [subrepo]
 remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 350fdbe3462299fa87b2e40113448bcbb9754e24
-	parent = 9325f6ff987022da1a4dabb771781cdc999af18e
+	commit = 2147bc98c5a3cafbd8c06867dbd010c2dfeb5373
+	parent = beed29383fb8e2fb7e2ecd1c394908e098a494be
 	method = merge
 	cmdver = 0.4.6
diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt
@@ -31,7 +31,7 @@ option(DISABLE_AZTEC_VM "Don't build Aztec VM (acceptable if iterating on core p
 option(MULTITHREADING "Enable multi-threading" ON)
 option(OMP_MULTITHREADING "Enable OMP multi-threading" OFF)
 option(FUZZING "Build ONLY fuzzing harnesses" OFF)
-option(ENABLE_PAR_ALGOS "Enable parallel algorithms" ON)
+option(ENABLE_PAR_ALGOS "Enable parallel algorithms" OFF)
 option(COVERAGE "Enable collecting coverage from tests" OFF)
 option(ENABLE_ASAN "Address sanitizer for debugging tricky memory corruption" OFF)
 option(ENABLE_HEAVY_TESTS "Enable heavy tests when collecting coverage" OFF)

diff --git a/barretenberg/cpp/pil/avm/alu.pil b/barretenberg/cpp/pil/avm/alu.pil
@@ -147,6 +147,7 @@ namespace alu(256);
 
     // This holds the product over the integers
     // (u1 multiplication only cares about a_lo and b_lo)
+    // TODO(9937): The following is not well constrained as this expression overflows the field.
     pol PRODUCT = a_lo * b_lo + (1 - u1_tag) * (LIMB_BITS_POW * partial_prod_lo + MAX_BITS_POW * (partial_prod_hi + a_hi * b_hi));
 
     // =============== ADDITION/SUBTRACTION Operation Constraints ================================================= 

diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt
@@ -73,7 +73,7 @@ add_subdirectory(barretenberg/dsl)
 add_subdirectory(barretenberg/ecc)
 add_subdirectory(barretenberg/eccvm)
 add_subdirectory(barretenberg/env)
-add_subdirectory(barretenberg/execution_trace)
+add_subdirectory(barretenberg/trace_to_polynomials)
 add_subdirectory(barretenberg/examples)
 add_subdirectory(barretenberg/flavor)
 add_subdirectory(barretenberg/goblin)
@@ -128,7 +128,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:dsl_objects>
     $<TARGET_OBJECTS:ecc_objects>
     $<TARGET_OBJECTS:eccvm_objects>
-    $<TARGET_OBJECTS:execution_trace_objects>
+    $<TARGET_OBJECTS:trace_to_polynomials_objects>
     $<TARGET_OBJECTS:simple_example_objects>
     $<TARGET_OBJECTS:flavor_objects>
     $<TARGET_OBJECTS:goblin_objects>

diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp
@@ -357,9 +357,7 @@ void client_ivc_prove_output_all_msgpack(const std::string& bytecodePath,
     }
     // TODO(#7371) dedupe this with the rest of the similar code
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode
-    ClientIVC ivc;
-    ivc.auto_verify_mode = true;
-    ivc.trace_settings.structure = TraceStructure::E2E_FULL_TEST;
+    ClientIVC ivc{ { E2E_FULL_TEST_STRUCTURE }, /*auto_verify_mode=*/true };
 
     // Accumulate the entire program stack into the IVC
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1116): remove manual setting of is_kernel once databus
@@ -446,9 +444,7 @@ bool foldAndVerifyProgram(const std::string& bytecodePath, const std::string& wi
     init_bn254_crs(1 << 22);
     init_grumpkin_crs(1 << 16);
 
-    ClientIVC ivc;
-    ivc.auto_verify_mode = true;
-    ivc.trace_settings.structure = TraceStructure::SMALL_TEST;
+    ClientIVC ivc{ { SMALL_TEST_STRUCTURE }, /*auto_verify_mode=*/true };
 
     auto program_stack = acir_format::get_acir_program_stack(
         bytecodePath, witnessPath, false); // TODO(https://github.com/AztecProtocol/barretenberg/issues/1013): this
@@ -499,9 +495,7 @@ void client_ivc_prove_output_all(const std::string& bytecodePath,
     init_grumpkin_crs(1 << 16);
 
     // TODO(https://github.com/AztecProtocol/barretenberg/issues/1101): remove use of auto_verify_mode
-    ClientIVC ivc;
-    ivc.auto_verify_mode = true;
-    ivc.trace_settings.structure = TraceStructure::E2E_FULL_TEST;
+    ClientIVC ivc{ { E2E_FULL_TEST_STRUCTURE }, /*auto_verify_mode=*/true };
 
     auto program_stack = acir_format::get_acir_program_stack(
         bytecodePath, witnessPath, false); // TODO(https://github.com/AztecProtocol/barretenberg/issues/1013): this
@@ -596,7 +590,7 @@ void prove_tube(const std::string& output_path)
     }
     ClientIVC verifier{ builder, input };
 
-    verifier.verify(proof);
+    ClientIVC::Output client_ivc_rec_verifier_output = verifier.verify(proof);
 
     PairingPointAccumulatorIndices current_aggregation_object =
         stdlib::recursion::init_default_agg_obj_indices<Builder>(*builder);
@@ -605,6 +599,10 @@ void prove_tube(const std::string& output_path)
     // This is currently just setting the aggregation object to the default one.
     builder->add_pairing_point_accumulator(current_aggregation_object);
 
+    // The tube only calls an IPA recursive verifier once, so we can just add this IPA claim and proof
+    builder->add_ipa_claim(client_ivc_rec_verifier_output.opening_claim.get_witness_indices());
+    builder->ipa_proof = convert_stdlib_proof_to_native(client_ivc_rec_verifier_output.ipa_transcript->proof_data);
+
     using Prover = UltraProver_<UltraFlavor>;
     using Verifier = UltraVerifier_<UltraFlavor>;
     Prover tube_prover{ *builder };
@@ -628,8 +626,9 @@ void prove_tube(const std::string& output_path)
     write_file(tubeAsFieldsVkPath, { data.begin(), data.end() });
 
     info("Native verification of the tube_proof");
-    Verifier tube_verifier(tube_verification_key);
-    bool verified = tube_verifier.verify_proof(tube_proof);
+    auto ipa_verification_key = std::make_shared<VerifierCommitmentKey<curve::Grumpkin>>(1 << CONST_ECCVM_LOG_N);
+    Verifier tube_verifier(tube_verification_key, ipa_verification_key);
+    bool verified = tube_verifier.verify_proof(tube_proof, builder->ipa_proof);
     info("Tube proof verification: ", verified);
 }
 
@@ -1138,14 +1137,22 @@ template <IsUltraFlavor Flavor> bool verify_honk(const std::string& proof_path,
 {
     using VerificationKey = Flavor::VerificationKey;
     using Verifier = UltraVerifier_<Flavor>;
-    using VerifierCommitmentKey = bb::VerifierCommitmentKey<curve::BN254>;
 
     auto g2_data = get_bn254_g2_data(CRS_PATH);
     srs::init_crs_factory({}, g2_data);
     auto proof = from_buffer<std::vector<bb::fr>>(read_file(proof_path));
     auto vk = std::make_shared<VerificationKey>(from_buffer<VerificationKey>(read_file(vk_path)));
-    vk->pcs_verification_key = std::make_shared<VerifierCommitmentKey>();
-    Verifier verifier{ vk };
+    vk->pcs_verification_key = std::make_shared<VerifierCommitmentKey<curve::BN254>>();
+
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1154): Remove this and pass in the IPA proof to the
+    // verifier.
+    std::shared_ptr<VerifierCommitmentKey<curve::Grumpkin>> ipa_verification_key = nullptr;
+    if constexpr (HasIPAAccumulatorFlavor<Flavor>) {
+        init_grumpkin_crs(1 << 16);
+        vk->contains_ipa_claim = false;
+        ipa_verification_key = std::make_shared<VerifierCommitmentKey<curve::Grumpkin>>(1 << CONST_ECCVM_LOG_N);
+    }
+    Verifier verifier{ vk, ipa_verification_key };
 
     bool verified = verifier.verify_proof(proof);
 

diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt
@@ -17,3 +17,4 @@ add_subdirectory(append_only_tree_bench)
 add_subdirectory(ultra_bench)
 add_subdirectory(stdlib_hash)
 add_subdirectory(circuit_construction_bench)
+add_subdirectory(mega_memory_bench)