feat: swap polys to facilitate dynamic trace overflow

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -209,10 +209,10 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<Verific
 
         initialized = true;
     } else { // Otherwise, fold the new key into the accumulator
+        vinfo("in folding prover");
         FoldingProver folding_prover({ fold_output.accumulator, proving_key }, trace_usage_tracker);
-        vinfo("constructed folding prover");
         fold_output = folding_prover.prove();
-        vinfo("constructed folding proof");
+        info("constructed folding proof");
 
         // Add fold proof and corresponding verification key to the verification queue
         verification_queue.push_back(bb::ClientIVC::VerifierInputs{ fold_output.proof, honk_vk, QUEUE_TYPE::PG });
@@ -279,6 +279,8 @@ HonkProof ClientIVC::construct_and_prove_hiding_circuit()
     merge_verification_queue.emplace_back(merge_proof);
 
     auto decider_pk = std::make_shared<DeciderProvingKey>(builder);
+    // WORKTODO: This fails in the dynamic accum expansion case
+    ASSERT(CircuitChecker::check(builder));
     honk_vk = std::make_shared<VerificationKey>(decider_pk->proving_key);
     MegaProver prover(decider_pk);
 

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.test.cpp

@@ -31,6 +31,45 @@ class ClientIVCTests : public ::testing::Test {
     using DeciderVerificationKeys = DeciderVerificationKeys_<Flavor>;
     using FoldingVerifier = ProtogalaxyVerifier_<DeciderVerificationKeys>;
 
+    // WORKTODO: duplicate of similar logic in pg test suite; put in some more central location for use by all modules
+    static void check_accumulator_target_sum_consistency(const std::shared_ptr<DeciderProvingKey>& accumulator)
+
+    {
+        info("in check accumulator target sum consistency, size: ", accumulator->proving_key.circuit_size);
+        ProtogalaxyProverInternal<DeciderProvingKeys> pg_internal;
+        auto expected_honk_evals = pg_internal.compute_row_evaluations(
+            accumulator->proving_key.polynomials, accumulator->alphas, accumulator->relation_parameters);
+        // Construct pow(\vec{betas*}) as in the paper
+        GateSeparatorPolynomial expected_gate_separators(accumulator->gate_challenges,
+                                                         accumulator->gate_challenges.size());
+
+        // Compute the corresponding target sum and create a dummy accumulator
+        FF expected_target_sum{ 0 };
+        for (size_t idx = 0; idx < accumulator->proving_key.circuit_size; idx++) {
+            expected_target_sum += expected_honk_evals[idx] * expected_gate_separators[idx];
+        }
+        info("expected target sum: ", expected_target_sum);
+        info("acc target sum: ", accumulator->target_sum);
+        EXPECT_TRUE(accumulator->target_sum == expected_target_sum);
+    }
+
+    // WORKTODO: similar to above, this doesnt really belong here
+    static void fold_verify_then_decider_prove_and_verify(const ClientIVC& ivc)
+    {
+        const auto& queue_entry = ivc.verification_queue.back();
+        const auto& fold_proof = queue_entry.proof;
+        auto key_to_fold = std::make_shared<ClientIVC::DeciderVerificationKey>(queue_entry.honk_verification_key);
+        const auto& prover_accumulator = ivc.fold_output.accumulator;
+
+        ClientIVC::FoldingVerifier folding_verifier({ ivc.verifier_accumulator, key_to_fold });
+        auto verifier_accumulator = folding_verifier.verify_folding_proof(fold_proof);
+
+        DeciderProver decider_prover(prover_accumulator);
+        DeciderVerifier decider_verifier(verifier_accumulator);
+        HonkProof decider_proof = decider_prover.construct_proof();
+        EXPECT_TRUE(decider_verifier.verify_proof(decider_proof));
+    }
+
     /**
      * @brief Construct mock circuit with arithmetic gates and goblin ops
      * @details Defaulted to add 2^16 gates (which will bump to next power of two with the addition of dummy gates).
@@ -403,5 +442,72 @@ TEST_F(ClientIVCTests, StructuredTraceOverflow)
         log2_num_gates += 1;
     }
 
+    EXPECT_TRUE(ivc.prove_and_verify());
+};
+
+/**
+ * @brief Test dynamic structured trace overflow block mechanism
+ * @details Tests the case where the required overflow capacity is not known until runtime. Accumulates two circuits,
+ * the second of which overflows the trace but not enough to change the dyadic circuit size and thus there is no need
+ * for a virtual size increase of the first key.
+ *
+ */
+TEST_F(ClientIVCTests, DynamicOverflow)
+{
+    // Define trace settings with zero overflow capacity
+    ClientIVC ivc{ { SMALL_TEST_STRUCTURE, /*overflow_capacity=*/0 } };
+
+    MockCircuitProducer circuit_producer;
+
+    size_t NUM_CIRCUITS = 2;
+
+    // define parameters for two circuits; the first fits within the structured trace, the second overflows
+    std::vector<size_t> log2_num_arith_gates = { 14, 16 };
+    // Accumulate
+    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+        auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_arith_gates[idx]);
+        ivc.accumulate(circuit);
+    }
+
+    // DEBUG: check consistency of the target sum computed internal to ivc
+    check_accumulator_target_sum_consistency(ivc.fold_output.accumulator);
+
+    // DEBUG: run native pg verifier then native decider prover/verifier
+    fold_verify_then_decider_prove_and_verify(ivc);
+
+    EXPECT_TRUE(ivc.prove_and_verify());
+};
+
+/**
+ * @brief Test dynamic trace overflow where the dyadic circuit size also increases
+ * @details Accumulates two circuits, the second of which overflows the trace structure and leads to an increased dyadic
+ * circuit size. This requires the virtual size of the polynomials in the first key to be increased accordingly which
+ * should be handled automatically in PG/ClientIvc.
+ *
+ */
+TEST_F(ClientIVCTests, DynamicOverflowCircuitSizeChange)
+{
+    uint32_t overflow_capacity = 0;
+    // uint32_t overflow_capacity = 1 << 1;
+    ClientIVC ivc{ { SMALL_TEST_STRUCTURE, overflow_capacity } };
+
+    MockCircuitProducer circuit_producer;
+
+    size_t NUM_CIRCUITS = 2;
+
+    // define parameters for two circuits; the first fits within the structured trace, the second overflows
+    std::vector<size_t> log2_num_arith_gates = { 14, 18, 16 };
+    // Accumulate
+    for (size_t idx = 0; idx < NUM_CIRCUITS; ++idx) {
+        auto circuit = circuit_producer.create_next_circuit(ivc, log2_num_arith_gates[idx]);
+        ivc.accumulate(circuit);
+    }
+
+    check_accumulator_target_sum_consistency(ivc.fold_output.accumulator);
+    // DEBUG: check consistency of the target sum computed internal to ivc
+
+    // DEBUG: run native pg verifier then native decider prover/verifier
+    fold_verify_then_decider_prove_and_verify(ivc);
+
     EXPECT_TRUE(ivc.prove_and_verify());
 };

barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.test.cpp

@@ -213,7 +213,7 @@ TEST_F(IvcRecursionConstraintTest, GenerateVK)
         // Construct kernel consisting only of the kernel completion logic
         AcirProgram program = construct_mock_kernel_program(ivc.verification_queue, { num_app_public_inputs });
         Builder kernel = acir_format::create_kernel_circuit(program.constraints, ivc);
-        // WORKTODO: this would normally happen in accumulate()
+        // Mimic what would normally happen in accumulate()
         kernel.add_pairing_point_accumulator(stdlib::recursion::init_default_agg_obj_indices<Builder>(kernel));
 
         auto proving_key = std::make_shared<DeciderProvingKey_<MegaFlavor>>(kernel, trace_settings);

barretenberg/cpp/src/barretenberg/honk/proof_system/logderivative_library.hpp

@@ -29,7 +29,7 @@ void compute_logderivative_inverse(Polynomials& polynomials, auto& relation_para
     using Accumulator = typename Relation::ValueAccumulator0;
     constexpr size_t READ_TERMS = Relation::READ_TERMS;
     constexpr size_t WRITE_TERMS = Relation::WRITE_TERMS;
-
+    static_cast<void>(circuit_size);
     auto& inverse_polynomial = Relation::template get_inverse_polynomial(polynomials);
     for (size_t i = 0; i < circuit_size; ++i) {
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/940): avoid get_row if possible.

barretenberg/cpp/src/barretenberg/plonk_honk_shared/execution_trace/execution_trace_usage_tracker.hpp

@@ -18,8 +18,9 @@ struct ExecutionTraceUsageTracker {
     using MegaTraceActiveRanges = MegaTraceBlockData<Range>;
     using MegaTraceFixedBlockSizes = MegaExecutionTraceBlocks;
 
-    TraceStructure max_sizes;             // max utilization of each block
-    MegaTraceFixedBlockSizes fixed_sizes; // fixed size of each block prescribed by structuring
+    TraceStructure max_sizes; // max utilization of each block
+    // Fixed size of each block prescribed by strucuring and the highest size of an overflow block encountered
+    MegaTraceFixedBlockSizes fixed_sizes;
     // Store active ranges based on the most current accumulator and those based on all but the most recently
     // accumulated circuit. The former is needed for the combiner calculation and the latter for the perturbator.
     std::vector<Range> active_ranges;
@@ -48,7 +49,7 @@ struct ExecutionTraceUsageTracker {
     }
 
     // Update the max block utilization and active trace ranges based on the data from a provided circuit
-    void update(const Builder& circuit)
+    void update(Builder& circuit)
     {
         // Update the max utilization of each gate block
         for (auto [block, max_size] : zip_view(circuit.blocks.get(), max_sizes.get())) {
@@ -72,8 +73,11 @@ struct ExecutionTraceUsageTracker {
         }
 
         // The active ranges must also include the rows where the actual databus and lookup table data are stored.
-        // (Note: lookup tables are constructed at the end of the trace; databus data is constructed at the start).
-        size_t dyadic_circuit_size = fixed_sizes.get_structured_dyadic_size();
+        // (Note: lookup tables are constructed at the end of the trace; databus data is constructed at the start) so we
+        // need to determine the dyadic size for this. We call the size function on the current circuit which will have
+        // the same fixed block sizes but might also have an overflow block potentially influencing the dyadic circuit
+        // size.
+        size_t dyadic_circuit_size = circuit.blocks.get_structured_dyadic_size();
 
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1152): should be able to use simply Range{ 0,
         // max_databus_size } but this breaks for certain choices of num_threads.
@@ -111,6 +115,13 @@ struct ExecutionTraceUsageTracker {
                                            "lookup",
                                            "overflow" };
 
+    std::vector<std::string> active_ranges_labels = [this] {
+        std::vector<std::string> result = block_labels;
+        result.push_back("databus table data");
+        result.push_back("lookup table data");
+        return result;
+    }();
+
     void print()
     {
         info("Minimum required block sizes for structured trace: ");
@@ -123,7 +134,17 @@ struct ExecutionTraceUsageTracker {
     void print_active_ranges()
     {
         info("Active regions of accumulator: ");
-        for (auto [label, range] : zip_view(block_labels, active_ranges)) {
+        for (auto [label, range] : zip_view(active_ranges_labels, active_ranges)) {
+            std::cout << std::left << std::setw(20) << (label + ":") << "(" << range.first << ", " << range.second
+                      << ")" << std::endl;
+        }
+        info("");
+    }
+
+    void print_previous_active_ranges()
+    {
+        info("Active regions of previous accumulator: ");
+        for (auto [label, range] : zip_view(active_ranges_labels, previous_active_ranges)) {
             std::cout << std::left << std::setw(20) << (label + ":") << "(" << range.first << ", " << range.second
                       << ")" << std::endl;
         }

barretenberg/cpp/src/barretenberg/plonk_honk_shared/execution_trace/mega_execution_trace.hpp

@@ -30,10 +30,17 @@ template <typename T> struct MegaTraceBlockData {
 
     std::vector<std::string_view> get_labels() const
     {
-        return { "ecc_op",     "pub_inputs",         "busread",
-                 "arithmetic", "delta_range",        "elliptic",
-                 "aux",        "poseidon2_external", "poseidon2_internal",
-                 "lookup" };
+        return { "ecc_op",
+                 "pub_inputs",
+                 "busread",
+                 "arithmetic",
+                 "delta_range",
+                 "elliptic",
+                 "aux",
+                 "poseidon2_external",
+                 "poseidon2_internal",
+                 "lookup",
+                 "overflow" };
     }
 
     auto get()
@@ -174,7 +181,6 @@ class MegaExecutionTraceBlocks : public MegaTraceBlockData<MegaTraceBlock> {
         this->overflow.fixed_size = settings.overflow_capacity;
     }
 
-    // WORKTODO: use or remove
     MegaExecutionTraceBlocks(const TraceSettings& settings)
         : MegaExecutionTraceBlocks()
     {
@@ -217,6 +223,7 @@ class MegaExecutionTraceBlocks : public MegaTraceBlockData<MegaTraceBlock> {
         for (auto block : this->get()) {
             total_size += block.get_fixed_size();
         }
+        info("total size: ");
 
         auto log2_n = static_cast<size_t>(numeric::get_msb(total_size));
         if ((1UL << log2_n) != (total_size)) {

barretenberg/cpp/src/barretenberg/plonk_honk_shared/library/grand_product_delta.hpp

@@ -48,6 +48,9 @@ typename Flavor::FF compute_public_input_delta(std::span<const typename Flavor::
     // Note: The public inputs may be offset from the 0th index of the wires, for example due to the inclusion of an
     // initial zero row or Goblin-stlye ECC op gates. Accordingly, the indices i in the above formulas are given by i =
     // [0, m-1] + offset, i.e. i = offset, 1 + offset, …, m - 1 + offset.
+
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1158): Ensure correct construction of public input
+    // delta in the face of increases to virtual size caused by execution trace overflow
     Field numerator_acc = gamma + (beta * Field(domain_size + offset));
     Field denominator_acc = gamma - beta * Field(1 + offset);
 

barretenberg/cpp/src/barretenberg/plonk_honk_shared/proving_key_inspector.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "barretenberg/common/assert.hpp"
 #include "barretenberg/common/log.hpp"
 
 namespace bb::proving_key_inspector {

barretenberg/cpp/src/barretenberg/plonk_honk_shared/relation_checker.cpp

@@ -0,0 +1,3 @@
+#include "relation_checker.hpp"
+
+// Hack to make the module compile.