lookup into dynamic tables

barretenberg/cpp/pil/vm2/execution.pil

@@ -46,6 +46,8 @@ last * sel' = 0;
 
 // These are needed to have a non-empty set of columns for each type.
 pol public input;
-#[LOOKUP_DUMMY]
+#[LOOKUP_DUMMY_PRECOMPUTED]
 sel {/*will be 1=OR*/ sel, clk, clk, clk} in
-precomputed.sel_bitwise {precomputed.bitwise_op_id, precomputed.bitwise_input_a, precomputed.bitwise_input_b, precomputed.bitwise_output};
+precomputed.sel_bitwise {precomputed.bitwise_op_id, precomputed.bitwise_input_a, precomputed.bitwise_input_b, precomputed.bitwise_output};
+#[LOOKUP_DUMMY_DYNAMIC]  // Just a self-lookup for now, for testing.
+sel {op1, op2, op3, op4} in sel {op1, op2, op3, op4};

barretenberg/cpp/src/barretenberg/vm2/generated/columns.hpp

@@ -8,8 +8,8 @@ namespace bb::avm2 {
 // The entities that will be used in the flavor.
 // clang-format off
 #define AVM2_PRECOMPUTED_ENTITIES precomputed_bitwise_input_a, precomputed_bitwise_input_b, precomputed_bitwise_op_id, precomputed_bitwise_output, precomputed_clk, precomputed_first_row, precomputed_sel_bitwise
-#define AVM2_WIRE_ENTITIES execution_input, alu_dst_addr, alu_ia, alu_ia_addr, alu_ib, alu_ib_addr, alu_ic, alu_op, alu_sel_op_add, execution_addressing_error_idx, execution_addressing_error_kind, execution_clk, execution_ex_opcode, execution_indirect, execution_last, execution_op1, execution_op1_after_relative, execution_op2, execution_op2_after_relative, execution_op3, execution_op3_after_relative, execution_op4, execution_op4_after_relative, execution_pc, execution_rop1, execution_rop2, execution_rop3, execution_rop4, execution_sel, execution_sel_addressing_error, execution_sel_op1_is_address, execution_sel_op2_is_address, execution_sel_op3_is_address, execution_sel_op4_is_address, execution_stack_pointer_tag, execution_stack_pointer_val, lookup_dummy_counts
-#define AVM2_DERIVED_WITNESS_ENTITIES lookup_dummy_inv
+#define AVM2_WIRE_ENTITIES execution_input, alu_dst_addr, alu_ia, alu_ia_addr, alu_ib, alu_ib_addr, alu_ic, alu_op, alu_sel_op_add, execution_addressing_error_idx, execution_addressing_error_kind, execution_clk, execution_ex_opcode, execution_indirect, execution_last, execution_op1, execution_op1_after_relative, execution_op2, execution_op2_after_relative, execution_op3, execution_op3_after_relative, execution_op4, execution_op4_after_relative, execution_pc, execution_rop1, execution_rop2, execution_rop3, execution_rop4, execution_sel, execution_sel_addressing_error, execution_sel_op1_is_address, execution_sel_op2_is_address, execution_sel_op3_is_address, execution_sel_op4_is_address, execution_stack_pointer_tag, execution_stack_pointer_val, lookup_dummy_precomputed_counts, lookup_dummy_dynamic_counts
+#define AVM2_DERIVED_WITNESS_ENTITIES lookup_dummy_precomputed_inv, lookup_dummy_dynamic_inv
 #define AVM2_SHIFTED_ENTITIES execution_sel_shift
 #define AVM2_TO_BE_SHIFTED(e) e.execution_sel
 #define AVM2_ALL_ENTITIES AVM2_PRECOMPUTED_ENTITIES, AVM2_WIRE_ENTITIES, AVM2_DERIVED_WITNESS_ENTITIES, AVM2_SHIFTED_ENTITIES

barretenberg/cpp/src/barretenberg/vm2/generated/flavor.cpp

@@ -48,9 +48,11 @@ AvmFlavor::AllConstRefValues::AllConstRefValues(
     , execution_sel_op4_is_address(il[40])
     , execution_stack_pointer_tag(il[41])
     , execution_stack_pointer_val(il[42])
-    , lookup_dummy_counts(il[43])
-    , lookup_dummy_inv(il[44])
-    , execution_sel_shift(il[45])
+    , lookup_dummy_precomputed_counts(il[43])
+    , lookup_dummy_dynamic_counts(il[44])
+    , lookup_dummy_precomputed_inv(il[45])
+    , lookup_dummy_dynamic_inv(il[46])
+    , execution_sel_shift(il[47])
 {}
 
 AvmFlavor::ProverPolynomials::ProverPolynomials(ProvingKey& proving_key)
@@ -110,8 +112,10 @@ AvmFlavor::AllConstRefValues AvmFlavor::ProverPolynomials::get_row(size_t row_id
                      execution_sel_op4_is_address[row_idx],
                      execution_stack_pointer_tag[row_idx],
                      execution_stack_pointer_val[row_idx],
-                     lookup_dummy_counts[row_idx],
-                     lookup_dummy_inv[row_idx],
+                     lookup_dummy_precomputed_counts[row_idx],
+                     lookup_dummy_dynamic_counts[row_idx],
+                     lookup_dummy_precomputed_inv[row_idx],
+                     lookup_dummy_dynamic_inv[row_idx],
                      execution_sel_shift[row_idx] };
 }
 
@@ -160,8 +164,10 @@ AvmFlavor::CommitmentLabels::CommitmentLabels()
     Base::execution_sel_op4_is_address = "EXECUTION_SEL_OP4_IS_ADDRESS";
     Base::execution_stack_pointer_tag = "EXECUTION_STACK_POINTER_TAG";
     Base::execution_stack_pointer_val = "EXECUTION_STACK_POINTER_VAL";
-    Base::lookup_dummy_inv = "LOOKUP_DUMMY_INV";
-    Base::lookup_dummy_counts = "LOOKUP_DUMMY_COUNTS";
+    Base::lookup_dummy_precomputed_inv = "LOOKUP_DUMMY_PRECOMPUTED_INV";
+    Base::lookup_dummy_dynamic_inv = "LOOKUP_DUMMY_DYNAMIC_INV";
+    Base::lookup_dummy_precomputed_counts = "LOOKUP_DUMMY_PRECOMPUTED_COUNTS";
+    Base::lookup_dummy_dynamic_counts = "LOOKUP_DUMMY_DYNAMIC_COUNTS";
 };
 
 void AvmFlavor::Transcript::deserialize_full_transcript()

barretenberg/cpp/src/barretenberg/vm2/generated/flavor.hpp

@@ -21,7 +21,8 @@
 #include "relations/execution.hpp"
 
 // Lookup and permutation relations
-#include "relations/lookup_dummy.hpp"
+#include "relations/lookup_dummy_dynamic.hpp"
+#include "relations/lookup_dummy_precomputed.hpp"
 
 // Metaprogramming to concatenate tuple types.
 template <typename... input_t> using tuple_cat_t = decltype(std::tuple_cat(std::declval<input_t>()...));
@@ -51,12 +52,12 @@ class AvmFlavor {
     static constexpr bool HasZK = false;
 
     static constexpr size_t NUM_PRECOMPUTED_ENTITIES = 7;
-    static constexpr size_t NUM_WITNESS_ENTITIES = 38;
+    static constexpr size_t NUM_WITNESS_ENTITIES = 40;
     static constexpr size_t NUM_SHIFTED_ENTITIES = 1;
     static constexpr size_t NUM_WIRES = NUM_WITNESS_ENTITIES + NUM_PRECOMPUTED_ENTITIES;
     // We have two copies of the witness entities, so we subtract the number of fixed ones (they have no shift), one for
     // the unshifted and one for the shifted
-    static constexpr size_t NUM_ALL_ENTITIES = 46;
+    static constexpr size_t NUM_ALL_ENTITIES = 48;
     // The total number of witnesses including shifts and derived entities.
     static constexpr size_t NUM_ALL_WITNESS_ENTITIES = NUM_WITNESS_ENTITIES + NUM_SHIFTED_ENTITIES;
 
@@ -73,7 +74,8 @@ class AvmFlavor {
     template <typename FF_>
     using LookupRelations_ = std::tuple<
         // Lookups
-        lookup_dummy_relation<FF_>>;
+        lookup_dummy_dynamic_relation<FF_>,
+        lookup_dummy_precomputed_relation<FF_>>;
 
     using LookupRelations = LookupRelations_<FF>;
 

barretenberg/cpp/src/barretenberg/vm2/generated/full_row.cpp

@@ -62,8 +62,10 @@ template <typename FF> std::vector<std::string> AvmFullRow<FF>::names()
              "execution_sel_op4_is_address",
              "execution_stack_pointer_tag",
              "execution_stack_pointer_val",
-             "lookup_dummy_inv",
-             "lookup_dummy_counts" };
+             "lookup_dummy_precomputed_inv",
+             "lookup_dummy_dynamic_inv",
+             "lookup_dummy_precomputed_counts",
+             "lookup_dummy_dynamic_counts" };
 }
 
 template <typename FF> RefVector<const FF> AvmFullRow<FF>::as_vector() const
@@ -112,8 +114,10 @@ template <typename FF> RefVector<const FF> AvmFullRow<FF>::as_vector() const
         execution_sel_op4_is_address,
         execution_stack_pointer_tag,
         execution_stack_pointer_val,
-        lookup_dummy_inv,
-        lookup_dummy_counts,
+        lookup_dummy_precomputed_inv,
+        lookup_dummy_dynamic_inv,
+        lookup_dummy_precomputed_counts,
+        lookup_dummy_dynamic_counts,
     };
 }
 

barretenberg/cpp/src/barretenberg/vm2/generated/full_row.hpp

@@ -17,7 +17,7 @@ template <typename FF_> struct AvmFullRow {
 
     RefVector<const FF> as_vector() const;
     static std::vector<std::string> names();
-    static constexpr size_t SIZE = 45;
+    static constexpr size_t SIZE = 47;
 
     // Risky but oh so efficient.
     FF& get_column(ColumnAndShifts col)

barretenberg/cpp/src/barretenberg/vm2/generated/relations/lookup_dummy_dynamic.hpp

@@ -0,0 +1,83 @@
+// AUTOGENERATED FILE
+#pragma once
+
+#include "../columns.hpp"
+#include "barretenberg/relations/generic_lookup/generic_lookup_relation.hpp"
+
+#include <cstddef>
+#include <tuple>
+
+namespace bb::avm2 {
+
+class lookup_dummy_dynamic_lookup_settings {
+  public:
+    static constexpr size_t READ_TERMS = 1;
+    static constexpr size_t WRITE_TERMS = 1;
+    static constexpr size_t READ_TERM_TYPES[READ_TERMS] = { 0 };
+    static constexpr size_t WRITE_TERM_TYPES[WRITE_TERMS] = { 0 };
+    static constexpr size_t LOOKUP_TUPLE_SIZE = 4;
+    static constexpr size_t INVERSE_EXISTS_POLYNOMIAL_DEGREE = 4;
+    static constexpr size_t READ_TERM_DEGREE = 0;
+    static constexpr size_t WRITE_TERM_DEGREE = 0;
+
+    // Columns using the Column enum.
+    static constexpr Column SRC_SELECTOR = Column::execution_sel;
+    static constexpr Column DST_SELECTOR = Column::execution_sel;
+    static constexpr Column COUNTS = Column::lookup_dummy_dynamic_counts;
+    static constexpr Column INVERSES = Column::lookup_dummy_dynamic_inv;
+    static constexpr std::array<Column, LOOKUP_TUPLE_SIZE> SRC_COLUMNS = {
+        Column::execution_op1, Column::execution_op2, Column::execution_op3, Column::execution_op4
+    };
+    static constexpr std::array<Column, LOOKUP_TUPLE_SIZE> DST_COLUMNS = {
+        Column::execution_op1, Column::execution_op2, Column::execution_op3, Column::execution_op4
+    };
+
+    template <typename AllEntities> static inline auto inverse_polynomial_is_computed_at_row(const AllEntities& in)
+    {
+        return (in.execution_sel == 1 || in.execution_sel == 1);
+    }
+
+    template <typename Accumulator, typename AllEntities>
+    static inline auto compute_inverse_exists(const AllEntities& in)
+    {
+        using View = typename Accumulator::View;
+        const auto is_operation = View(in.execution_sel);
+        const auto is_table_entry = View(in.execution_sel);
+        return (is_operation + is_table_entry - is_operation * is_table_entry);
+    }
+
+    template <typename AllEntities> static inline auto get_const_entities(const AllEntities& in)
+    {
+        return get_entities(in);
+    }
+
+    template <typename AllEntities> static inline auto get_nonconst_entities(AllEntities& in)
+    {
+        return get_entities(in);
+    }
+
+    template <typename AllEntities> static inline auto get_entities(AllEntities&& in)
+    {
+        return std::forward_as_tuple(in.lookup_dummy_dynamic_inv,
+                                     in.lookup_dummy_dynamic_counts,
+                                     in.execution_sel,
+                                     in.execution_sel,
+                                     in.execution_op1,
+                                     in.execution_op2,
+                                     in.execution_op3,
+                                     in.execution_op4,
+                                     in.execution_op1,
+                                     in.execution_op2,
+                                     in.execution_op3,
+                                     in.execution_op4);
+    }
+};
+
+template <typename FF_>
+class lookup_dummy_dynamic_relation : public GenericLookupRelation<lookup_dummy_dynamic_lookup_settings, FF_> {
+  public:
+    static constexpr const char* NAME = "LOOKUP_DUMMY_DYNAMIC";
+};
+template <typename FF_> using lookup_dummy_dynamic = GenericLookup<lookup_dummy_dynamic_lookup_settings, FF_>;
+
+} // namespace bb::avm2

barretenberg/cpp/src/barretenberg/vm2/generated/relations/lookup_dummy.hpp → barretenberg/cpp/src/barretenberg/vm2/generated/relations/lookup_dummy_precomputed.hpp

@@ -9,7 +9,7 @@
 
 namespace bb::avm2 {
 
-class lookup_dummy_lookup_settings {
+class lookup_dummy_precomputed_lookup_settings {
   public:
     static constexpr size_t READ_TERMS = 1;
     static constexpr size_t WRITE_TERMS = 1;
@@ -23,8 +23,8 @@ class lookup_dummy_lookup_settings {
     // Columns using the Column enum.
     static constexpr Column SRC_SELECTOR = Column::execution_sel;
     static constexpr Column DST_SELECTOR = Column::precomputed_sel_bitwise;
-    static constexpr Column COUNTS = Column::lookup_dummy_counts;
-    static constexpr Column INVERSES = Column::lookup_dummy_inv;
+    static constexpr Column COUNTS = Column::lookup_dummy_precomputed_counts;
+    static constexpr Column INVERSES = Column::lookup_dummy_precomputed_inv;
     static constexpr std::array<Column, LOOKUP_TUPLE_SIZE> SRC_COLUMNS = {
         Column::execution_sel, Column::execution_clk, Column::execution_clk, Column::execution_clk
     };
@@ -59,8 +59,8 @@ class lookup_dummy_lookup_settings {
 
     template <typename AllEntities> static inline auto get_entities(AllEntities&& in)
     {
-        return std::forward_as_tuple(in.lookup_dummy_inv,
-                                     in.lookup_dummy_counts,
+        return std::forward_as_tuple(in.lookup_dummy_precomputed_inv,
+                                     in.lookup_dummy_precomputed_counts,
                                      in.execution_sel,
                                      in.precomputed_sel_bitwise,
                                      in.execution_sel,
@@ -74,10 +74,11 @@ class lookup_dummy_lookup_settings {
     }
 };
 
-template <typename FF_> class lookup_dummy_relation : public GenericLookupRelation<lookup_dummy_lookup_settings, FF_> {
+template <typename FF_>
+class lookup_dummy_precomputed_relation : public GenericLookupRelation<lookup_dummy_precomputed_lookup_settings, FF_> {
   public:
-    static constexpr const char* NAME = "LOOKUP_DUMMY";
+    static constexpr const char* NAME = "LOOKUP_DUMMY_PRECOMPUTED";
 };
-template <typename FF_> using lookup_dummy = GenericLookup<lookup_dummy_lookup_settings, FF_>;
+template <typename FF_> using lookup_dummy_precomputed = GenericLookup<lookup_dummy_precomputed_lookup_settings, FF_>;
 
 } // namespace bb::avm2

barretenberg/cpp/src/barretenberg/vm2/proving_helper.cpp

@@ -65,7 +65,6 @@ AvmProver::ProverPolynomials compute_polynomials(tracegen::TraceContainer& trace
     AVM_TRACK_TIME("proving/init_polys_unshifted", ({
                        auto unshifted = polys.get_unshifted();
 
-                       // FIXME: We don't support handling of derived polynomials.
                        // Derived polynomials will be empty.
                        bb::parallel_for(unshifted.size(), [&](size_t i) {
                            auto& poly = unshifted[i];

barretenberg/cpp/src/barretenberg/vm2/tracegen/lib/lookup_builder.hpp

@@ -2,9 +2,10 @@
 
 #include <array>
 #include <cstddef>
-#include <span>
+#include <stdexcept>
 
 #include "barretenberg/vm2/common/field.hpp"
+#include "barretenberg/vm2/common/map.hpp"
 #include "barretenberg/vm2/generated/columns.hpp"
 #include "barretenberg/vm2/tracegen/trace_container.hpp"
 
@@ -16,6 +17,8 @@ template <typename LookupSettings_> class BaseLookupTraceBuilder {
 
     void process(TraceContainer& trace)
     {
+        init(trace);
+
         // Let "src_sel {c1, c2, ...} in dst_sel {d1, d2, ...}" be a lookup,
         // For each row that has a 1 in the src_sel, we take the values of {c1, c2, ...},
         // find a row dst_row in the target columns {d1, d2, ...} where the values match.
@@ -42,6 +45,55 @@ template <typename LookupSettings_> class BaseLookupTraceBuilder {
   protected:
     using LookupSettings = LookupSettings_;
     virtual uint32_t find_in_dst(const std::array<FF, LookupSettings::LOOKUP_TUPLE_SIZE>& tup) const = 0;
+    virtual void init(TraceContainer&){}; // Optional initialization step.
+};
+
+// This class is used when the lookup is into a non-precomputed table.
+// It calculates the counts by trying to find the tuple in the destination columns.
+// It creates an index of the destination columns on init, and uses it to find the tuple efficiently.
+template <typename LookupSettings_> class LookupIntoDynamicTable : public BaseLookupTraceBuilder<LookupSettings_> {
+  public:
+    virtual ~LookupIntoDynamicTable() = default;
+
+  protected:
+    using LookupSettings = LookupSettings_;
+
+    void init(TraceContainer& trace) override
+    {
+        row_idx.reserve(trace.get_column_rows(LookupSettings::DST_SELECTOR));
+        trace.visit_column(LookupSettings::DST_SELECTOR, [&](uint32_t row, const FF& dst_sel_value) {
+            assert(dst_sel_value == 1);
+            auto dst_values = trace.get_multiple(LookupSettings::DST_COLUMNS, row);
+            row_idx.insert({ get_key(dst_values), row });
+        });
+    }
+
+    uint32_t find_in_dst(const std::array<FF, LookupSettings::LOOKUP_TUPLE_SIZE>& tup) const override
+    {
+        auto it = row_idx.find(get_key(tup));
+        if (it != row_idx.end()) {
+            return it->second;
+        }
+        // throw std::runtime_error("Failed computing counts for " + std::string(LookupSettings::NAME) +
+        //                          ". Could not find tuple in destination.");
+        throw std::runtime_error("Failed computing counts. Could not find tuple in destination.");
+    }
+
+  private:
+    FF get_key(const std::array<FF, LookupSettings::LOOKUP_TUPLE_SIZE>& tup) const
+    {
+        FF acc = 0;
+        for (const auto& el : tup) {
+            acc = acc * beta + el;
+        }
+        return acc + gamma;
+    }
+
+    // We use an RLC for the key instead of the tuple, to save memory.
+    // FIXME: reconsider, what if beta is 0.
+    unordered_flat_map<FF, uint32_t> row_idx;
+    FF beta = FF::random_element();
+    FF gamma = FF::random_element();
 };
 
 } // namespace bb::avm2::tracegen

barretenberg/cpp/src/barretenberg/vm2/tracegen_helper.cpp

@@ -9,7 +9,8 @@
 #include "barretenberg/common/thread.hpp"
 #include "barretenberg/numeric/bitop/get_msb.hpp"
 #include "barretenberg/vm/stats.hpp"
-#include "barretenberg/vm2/generated/relations/lookup_dummy.hpp"
+#include "barretenberg/vm2/generated/relations/lookup_dummy_dynamic.hpp"
+#include "barretenberg/vm2/generated/relations/lookup_dummy_precomputed.hpp"
 #include "barretenberg/vm2/tracegen/alu_trace.hpp"
 #include "barretenberg/vm2/tracegen/execution_trace.hpp"
 #include "barretenberg/vm2/tracegen/lib/lookup_into_bitwise.hpp"
@@ -73,11 +74,15 @@ TraceContainer AvmTraceGenHelper::generate_trace(EventsContainer&& events)
 
     // Now we can compute lookups.
     {
-        auto jobs_lookups = std::array<std::function<void()>, 1>{
+        auto jobs_lookups = std::array<std::function<void()>, 2>{
             [&]() {
-                LookupIntoBitwise<lookup_dummy_lookup_settings> lookup_execution_bitwise;
+                LookupIntoBitwise<lookup_dummy_precomputed_lookup_settings> lookup_execution_bitwise;
                 lookup_execution_bitwise.process(trace);
             },
+            [&]() {
+                LookupIntoDynamicTable<lookup_dummy_dynamic_lookup_settings> lookup_execution_execution;
+                lookup_execution_execution.process(trace);
+            },
         };
         AVM_TRACK_TIME("tracegen/lookups", execute_jobs(jobs_lookups));
     }