diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo index 91492d1465b..4f192f1c1a9 100644 --- a/barretenberg/.gitrepo +++ b/barretenberg/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/barretenberg branch = master - commit = 5e751b81cd5e5f34d9286d5a19ea6d4853566480 - parent = 887c01103255ea4cbbb6cb33c8771d47123b3bff + commit = 0fe0a5d6dff43547aaec45256440982184e93bb0 + parent = da1470d074f4884e61b51e450a661432c6f0a10f method = merge cmdver = 0.4.6 diff --git a/barretenberg/cpp/scripts/merkle_tree_tests.sh b/barretenberg/cpp/scripts/merkle_tree_tests.sh index f53e2750d5a..9e5e0f0b3c9 100755 --- a/barretenberg/cpp/scripts/merkle_tree_tests.sh +++ b/barretenberg/cpp/scripts/merkle_tree_tests.sh @@ -5,7 +5,7 @@ set -e # run commands relative to parent directory cd $(dirname $0)/.. -DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* +DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* TEST=${1:-$DEFAULT_TESTS} PRESET=${PRESET:-clang16} diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index 72c6c44fff0..a994b0b5929 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -1465,6 +1465,12 @@ int main(int argc, char* argv[]) } else if (command == "prove_ultra_honk_output_all") { std::string output_path = get_option(args, "-o", "./proofs"); prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + } else if (command == "prove_ultra_rollup_honk_output_all") { + std::string output_path = get_option(args, "-o", "./proofs/proof"); + prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + } else if (command == "prove_ultra_keccak_honk_output_all") { + std::string output_path = get_option(args, "-o", "./proofs/proof"); + prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); } else if (command == "prove_mega_honk_output_all") { std::string output_path = get_option(args, "-o", "./proofs"); prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); @@ -1526,9 +1532,9 @@ int main(int argc, char* argv[]) } else if (command == "prove_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); prove_honk(bytecode_path, witness_path, output_path, recursive); - } else if (command == "prove_ultra_keccak_honk_output_all") { + } else if (command == "prove_ultra_rollup_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); - prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + prove_honk(bytecode_path, witness_path, output_path, recursive); } else if (command == "verify_ultra_honk") { return verify_honk(proof_path, vk_path) ? 0 : 1; } else if (command == "verify_ultra_keccak_honk") { @@ -1539,6 +1545,9 @@ int main(int argc, char* argv[]) } else if (command == "write_vk_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", "./target/vk"); write_vk_honk(bytecode_path, output_path, recursive); + } else if (command == "write_vk_ultra_rollup_honk") { + std::string output_path = get_option(args, "-o", "./target/vk"); + write_vk_honk(bytecode_path, output_path, recursive); } else if (command == "prove_mega_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); prove_honk(bytecode_path, witness_path, output_path, recursive); @@ -1553,12 +1562,15 @@ int main(int argc, char* argv[]) } else if (command == "vk_as_fields_ultra_honk") { std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); vk_as_fields_honk(vk_path, output_path); - } else if (command == "vk_as_fields_mega_honk") { - std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); - vk_as_fields_honk(vk_path, output_path); } else if (command == "vk_as_fields_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); vk_as_fields_honk(vk_path, output_path); + } else if (command == "vk_as_fields_ultra_rollup_honk") { + std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); + vk_as_fields_honk(vk_path, output_path); + } else if (command == "vk_as_fields_mega_honk") { + std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); + vk_as_fields_honk(vk_path, output_path); } else { std::cerr << "Unknown command: " << command << "\n"; return 1; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp index 27917a072fa..1afb02a3c66 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp @@ -60,12 +60,10 @@ template class OpeningClaim { opening_pair.challenge.binary_basis_limbs[1].element.normalize().witness_index, opening_pair.challenge.binary_basis_limbs[2].element.normalize().witness_index, opening_pair.challenge.binary_basis_limbs[3].element.normalize().witness_index, - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1153): Uncomment this when we turn the - // eval into witnesses. - // opening_pair.evaluation.binary_basis_limbs[0].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[1].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[2].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[3].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[0].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[1].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[2].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[3].element.normalize().witness_index, commitment.x.normalize().witness_index, // no idea if we need these normalize() calls... commitment.y.normalize().witness_index }; } diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp index 0cb225dd7cf..a66387474d1 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp @@ -113,6 +113,7 @@ std::vector::Claim> GeminiProver_::prove( std::move(batched_to_be_shifted), std::move(batched_concatenated)); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t l = 0; l < CONST_PROOF_SIZE_LOG_N - 1; l++) { if (l < log_n - 1) { transcript->send_to_verifier("Gemini:FOLD_" + std::to_string(l + 1), diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp index c104f84a7d7..6c9aea63316 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp @@ -200,6 +200,7 @@ template class IPA { // Iterate for log(poly_degree) rounds to compute the round commitments. auto log_poly_length = static_cast(numeric::get_msb(poly_length)); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constant from IPA. if (log_poly_length > CONST_ECCVM_LOG_N) { throw_or_abort("IPA log_poly_length is too large"); } diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp index 3e40157661a..0abcb594b50 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp @@ -159,6 +159,7 @@ TEST_F(IPATest, AIsZeroAfterOneRound) // initialize an empty mock transcript auto transcript = std::make_shared(); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constant from IPA. const size_t num_challenges = CONST_ECCVM_LOG_N + 1; std::vector random_vector(num_challenges); diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp index 83ced38c1f4..1f594bbd3eb 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp @@ -469,6 +469,7 @@ template class ShpleminiVerifier_ { // Initialize batching challenge as ν² Fr current_batching_challenge = shplonk_batching_challenge.sqr(); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t j = 0; j < CONST_PROOF_SIZE_LOG_N - 1; ++j) { // Compute the scaling factor (ν²⁺ⁱ) / (z + r²⁽ⁱ⁺²⁾) for i = 0, … , d-2 Fr scaling_factor = current_batching_challenge * inverse_vanishing_evals[j + 2]; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp index 527e1d0b73e..5e90d4a00bb 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp @@ -2,6 +2,7 @@ #include "barretenberg/commitment_schemes/claim.hpp" #include "barretenberg/commitment_schemes/commitment_key.hpp" #include "barretenberg/commitment_schemes/verification_key.hpp" +#include "barretenberg/stdlib/primitives/curves/bn254.hpp" #include "barretenberg/transcript/transcript.hpp" /** @@ -224,7 +225,9 @@ template class ShplonkVerifier_ { // [G] = [Q] - ∑ⱼ (1/zⱼ(r))[Bⱼ] + ( ∑ⱼ (1/zⱼ(r)) Tⱼ(r) )[1] // = [Q] - ∑ⱼ (1/zⱼ(r))[Bⱼ] + G₀ [1] // G₀ = ∑ⱼ ρʲ ⋅ vⱼ / (z − xⱼ ) - auto G_commitment_constant = Fr(0); + Fr G_commitment_constant(0); + + Fr evaluation(0); // TODO(#673): The recursive and non-recursive (native) logic is completely separated via the following // conditional. Much of the logic could be shared, but I've chosen to do it this way since soon the "else" @@ -274,6 +277,8 @@ template class ShplonkVerifier_ { // [G] += G₀⋅[1] = [G] + (∑ⱼ νʲ ⋅ vⱼ / (z − xⱼ ))⋅[1] G_commitment = GroupElement::batch_mul(commitments, scalars); + // Set evaluation to constant witness + evaluation.convert_constant_to_fixed_witness(z_challenge.get_context()); } else { // [G] = [Q] - ∑ⱼ νʲ / (z − xⱼ )⋅[fⱼ] + G₀⋅[1] // = [Q] - [∑ⱼ νʲ ⋅ ( fⱼ(X) − vⱼ) / (z − xⱼ )] @@ -309,7 +314,7 @@ template class ShplonkVerifier_ { } // Return opening pair (z, 0) and commitment [G] - return { { z_challenge, Fr(0) }, G_commitment }; + return { { z_challenge, evaluation }, G_commitment }; }; /** * @brief Computes \f$ \frac{1}{z - r}, \frac{1}{z+r}, \ldots, \frac{1}{z+r^{2^{d-1}}} \f$. diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp index ff178e0267a..53f8a39c993 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp @@ -421,6 +421,7 @@ template class ZeroMorphProver_ { transcript->send_to_verifier(label, q_k_commitment); } // Add buffer elements to remove log_N dependence in proof + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t idx = log_N; idx < CONST_PROOF_SIZE_LOG_N; ++idx) { auto buffer_element = Commitment::one(); std::string label = "ZM:C_q_" + std::to_string(idx); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 7dc41b0abc1..d645b460708 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -39,16 +39,17 @@ template class ContentAddressedAppendOn using StoreType = Store; // Asynchronous methods accept these callback function types as arguments - using AppendCompletionCallback = std::function&)>; - using MetaDataCallback = std::function&)>; - using HashPathCallback = std::function&)>; - using FindLeafCallback = std::function&)>; - using GetLeafCallback = std::function&)>; + using AppendCompletionCallback = std::function&)>; + using MetaDataCallback = std::function&)>; + using HashPathCallback = std::function&)>; + using FindLeafCallback = std::function&)>; + using GetLeafCallback = std::function&)>; using CommitCallback = std::function&)>; - using RollbackCallback = std::function; + using RollbackCallback = std::function; using RemoveHistoricBlockCallback = std::function&)>; using UnwindBlockCallback = std::function&)>; - using FinaliseBlockCallback = std::function; + using FinaliseBlockCallback = std::function; + using GetBlockForIndexCallback = std::function&)>; // Only construct from provided store and thread pool, no copies or moves ContentAddressedAppendOnlyTree(std::unique_ptr store, @@ -90,7 +91,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes */ void get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const; @@ -131,7 +132,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes * @param on_completion Callback to be called on completion */ - void get_meta_data(const index_t& blockNumber, + void get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const; @@ -151,7 +152,7 @@ template class ContentAddressedAppendOn * @param on_completion Callback to be called on completion */ void get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& completion) const; @@ -164,7 +165,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree */ void find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -181,10 +182,23 @@ template class ContentAddressedAppendOn */ void find_leaf_index_from(const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Returns the block numbers that correspond to the given indices values + */ + void find_block_numbers(const std::vector& indices, const GetBlockForIndexCallback& on_completion) const; + + /** + * @brief Returns the block numbers that correspond to the given indices values, from the perspective of a + * historical block number + */ + void find_block_numbers(const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const; + /** * @brief Commit the tree to the backing store */ @@ -200,11 +214,11 @@ template class ContentAddressedAppendOn */ uint32_t depth() const { return depth_; } - void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); + void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); - void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion); + void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion); - void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion); + void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion); protected: using ReadTransaction = typename Store::ReadTransaction; @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree::get_meta_data(bool in } template -void ContentAddressedAppendOnlyTree::get_meta_data(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const { @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons template void ContentAddressedAppendOnlyTree::get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const { @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons workers_->enqueue(job); } +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + index_t maxIndex = meta.committedSize; + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + BlockPayload blockPayload; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + if (!store_->get_block_data(blockNumber, blockPayload, *tx)) { + throw std::runtime_error(format("Unable to find block numbers for indices for block ", + blockNumber, + ", failed to get block data.")); + } + index_t maxIndex = std::min(meta.committedSize, blockPayload.size); + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + template void ContentAddressedAppendOnlyTree::get_subtree_sibling_path( uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const @@ -473,7 +543,7 @@ std::optional ContentAddressedAppendOnlyTree::find_lea NodePayload nodePayload; bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted); if (!success) { - // std::cout << "No root" << std::endl; + // std::cout << "No root " << hash << std::endl; return std::nullopt; } // std::cout << "Found root at depth " << i << " : " << hash << std::endl; @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ template void ContentAddressedAppendOnlyTree::get_leaf(const index_t& leaf_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& on_completion) const { @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree::find_leaf_index(const template void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -696,7 +766,7 @@ template void ContentAddressedAppendOnlyTree::find_leaf_index_from( const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree::rollback(const Rollba template void ContentAddressedAppendOnlyTree::remove_historic_block( - const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) + const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) { auto job = [=, this]() { execute_and_report( @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree::remove_historic_block } template -void ContentAddressedAppendOnlyTree::unwind_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion) { auto job = [=, this]() { @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree::unwind_block(const in } template -void ContentAddressedAppendOnlyTree::finalise_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion) { auto job = [=, this]() { @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree::add_batch_internal( new_root = new_hash; meta.root = new_hash; meta.size = new_size; - // std::cout << "New size: " << meta.size << std::endl; + // std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl; store_->put_meta(meta); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 5ac60d919b7..83f72c9ca1f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -123,7 +123,7 @@ void check_sibling_path(TreeType& tree, void check_historic_sibling_path(TreeType& tree, index_t index, fr_sibling_path expected_sibling_path, - index_t blockNumber, + block_number_t blockNumber, bool expected_success = true) { Signal signal; @@ -160,7 +160,7 @@ void rollback_tree(TreeType& tree) signal.wait_for_level(); } -void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -171,7 +171,7 @@ void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expe signal.wait_for_level(); } -void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -206,7 +206,7 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } -void finalise_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -312,7 +312,7 @@ void check_leaf( } void check_historic_leaf(TreeType& tree, - const index_t& blockNumber, + const block_number_t& blockNumber, const fr& leaf, index_t leaf_index, bool expected_success, @@ -350,6 +350,31 @@ void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_pat EXPECT_EQ(hash, expected_root); } +void get_blocks_for_indices(TreeType& tree, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + +void get_blocks_for_indices(TreeType& tree, + const block_number_t& blockNumber, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, blockNumber, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -1284,7 +1309,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_da for (uint32_t i = 0; i < historicPathsZeroIndex.size(); i++) { // retrieving historic data should fail if the block is outside of the window - const index_t blockNumber = i + 1; + const block_number_t blockNumber = i + 1; const bool expectedSuccess = expectedBlockHeight <= windowSize || blockNumber > (expectedBlockHeight - windowSize); check_historic_sibling_path(tree, 0, historicPathsZeroIndex[i], blockNumber, expectedSuccess); @@ -1399,7 +1424,7 @@ void test_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); // attempting to unwind a block that is not the tip should fail @@ -1513,6 +1538,101 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_bl } } +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_retrieve_block_numbers_by_index) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + const size_t block_size = 32; + + for (size_t i = 0; i < 5; i++) { + std::vector values = create_values(block_size); + add_values(tree, values); + commit_tree(tree); + } + std::vector indices{ 12, 33, 63, 64, 65, 80, 96, 159, 160 }; + std::vector> blockNumbers; + + // All but the last block number should be valid when looking at latest + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + index_t maxIndex = 5 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now get blocks for indices from the perspective of block 2 + get_blocks_for_indices(tree, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = 2 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + unwind_block(tree, 5); + unwind_block(tree, 4); + + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + maxIndex = 3 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // fork from block 1 + std::unique_ptr forkStore = std::make_unique(name, depth, 1, db); + TreeType treeFork(std::move(forkStore), pool); + + // Now, using the fork, get block indices but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now, using the fork, get block indics from the perspective of block 2, but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks) { std::string name = random_string(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 197f784d59f..45cc232e8e1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -47,14 +47,13 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>&)>; + using AddCompletionCallbackWithWitness = std::function>&)>; using AddSequentiallyCompletionCallbackWithWitness = - std::function>&)>; - using AddCompletionCallback = std::function&)>; + std::function>&)>; + using AddCompletionCallback = std::function&)>; - using LeafCallback = std::function>&)>; - using FindLowLeafCallback = std::function&)>; + using LeafCallback = std::function>&)>; + using FindLowLeafCallback = std::function&)>; ContentAddressedIndexedTree(std::unique_ptr store, std::shared_ptr workers, @@ -151,7 +150,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -169,7 +168,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -178,7 +177,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::get_leaf(const index_t& template void ContentAddressedIndexedTree::get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const LeafCallback& completion) const { @@ -458,7 +457,7 @@ void ContentAddressedIndexedTree::find_leaf_index( template void ContentAddressedIndexedTree::find_leaf_index( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const { @@ -496,7 +495,7 @@ void ContentAddressedIndexedTree::find_leaf_index_from( template void ContentAddressedIndexedTree::find_leaf_index_from( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, const index_t& start_index, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const @@ -559,7 +558,7 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& template void ContentAddressedIndexedTree::find_low_leaf(const fr& leaf_key, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLowLeafCallback& on_completion) const { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 5177b370833..8a7443ef05c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -104,8 +104,11 @@ template void check_root(TypeOfTree& tree, fr expected_roo } template -fr_sibling_path get_historic_sibling_path( - TypeOfTree& tree, index_t blockNumber, index_t index, bool includeUncommitted = true, bool expected_success = true) +fr_sibling_path get_historic_sibling_path(TypeOfTree& tree, + block_number_t blockNumber, + index_t index, + bool includeUncommitted = true, + bool expected_success = true) { fr_sibling_path h; Signal signal; @@ -177,7 +180,7 @@ GetLowIndexedLeafResponse get_low_leaf(TypeOfTree& tree, const LeafValueType& le template GetLowIndexedLeafResponse get_historic_low_leaf(TypeOfTree& tree, - index_t blockNumber, + block_number_t blockNumber, const LeafValueType& leaf, bool includeUncommitted = true) { @@ -236,7 +239,7 @@ void check_find_leaf_index_from(TypeOfTree& tree, template void check_historic_find_leaf_index(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t expected_index, bool expected_success, bool includeUncommitted = true) @@ -257,7 +260,7 @@ void check_historic_find_leaf_index(TypeOfTree& tree, template void check_historic_find_leaf_index_from(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t start_index, index_t expected_index, bool expected_success, @@ -280,7 +283,7 @@ template void check_historic_leaf(TypeOfTree& tree, const LeafValueType& leaf, index_t expected_index, - index_t blockNumber, + block_number_t blockNumber, bool expected_success, bool includeUncommitted = true) { @@ -300,7 +303,7 @@ void check_historic_leaf(TypeOfTree& tree, template void check_historic_sibling_path(TypeOfTree& tree, index_t index, - index_t blockNumber, + block_number_t blockNumber, const fr_sibling_path& expected_sibling_path, bool includeUncommitted = true, bool expected_success = true) @@ -413,7 +416,7 @@ void block_sync_values_sequential(TypeOfTree& tree, } template -void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -425,7 +428,7 @@ void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool ex } template -void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -437,7 +440,7 @@ void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_ } template -void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -2596,7 +2599,7 @@ void test_nullifier_tree_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); unwind_block(tree, blockNumber); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index cf2a55c1285..9bbea8ea42e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -46,6 +46,8 @@ class LMDBTransaction { template bool get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; template bool get_value(T& key, index_t& data, const LMDBDatabase& db) const; @@ -88,6 +90,12 @@ bool LMDBTransaction::get_value_or_previous(T& key, K& data, const LMDBDatabase& return lmdb_queries::get_value_or_previous(key, data, db, *this); } +template +bool LMDBTransaction::get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const +{ + return lmdb_queries::get_value_or_greater(key, data, db, *this); +} + template bool LMDBTransaction::get_value_or_previous(T& key, K& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 3f4f07aa829..09f87cae606 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -55,29 +55,35 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m { LMDBDatabaseCreationTransaction tx(_environment); - _blockDatabase = std::make_unique( - _environment, tx, _name + std::string("blocks"), false, false, block_key_cmp); + _blockDatabase = + std::make_unique(_environment, tx, _name + BLOCKS_DB, false, false, block_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _nodeDatabase = - std::make_unique(_environment, tx, _name + std::string("nodes"), false, false, fr_key_cmp); + _nodeDatabase = std::make_unique(_environment, tx, _name + NODES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafKeyToIndexDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf indices"), false, false, fr_key_cmp); + _leafKeyToIndexDatabase = + std::make_unique(_environment, tx, _name + LEAF_INDICES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafHashToPreImageDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); + _leafHashToPreImageDatabase = + std::make_unique(_environment, tx, _name + LEAF_PREIMAGES_DB, false, false, fr_key_cmp); + tx.commit(); + } + + { + LMDBDatabaseCreationTransaction tx(_environment); + _indexToBlockDatabase = + std::make_unique(_environment, tx, _name + BLOCK_INDICES_DB, false, false, index_key_cmp); tx.commit(); } } @@ -107,9 +113,11 @@ void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) stats.leafIndicesDBStats = DBStats(LEAF_INDICES_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); stats.nodesDBStats = DBStats(NODES_DB, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _indexToBlockDatabase->underlying(), &stat); + stats.blockIndicesDBStats = DBStats(BLOCK_INDICES_DB, stat); } -void LMDBTreeStore::write_block_data(uint64_t blockNumber, +void LMDBTreeStore::write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, LMDBTreeStore::WriteTransaction& tx) { @@ -120,13 +128,15 @@ void LMDBTreeStore::write_block_data(uint64_t blockNumber, tx.put_value(key, encoded, *_blockDatabase); } -void LMDBTreeStore::delete_block_data(uint64_t blockNumber, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::delete_block_data(const block_number_t& blockNumber, LMDBTreeStore::WriteTransaction& tx) { BlockMetaKeyType key(blockNumber); tx.delete_value(key, *_blockDatabase); } -bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockData, LMDBTreeStore::ReadTransaction& tx) +bool LMDBTreeStore::read_block_data(const block_number_t& blockNumber, + BlockPayload& blockData, + LMDBTreeStore::ReadTransaction& tx) { BlockMetaKeyType key(blockNumber); std::vector data; @@ -137,6 +147,87 @@ bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockDat return success; } +void LMDBTreeStore::write_block_index_data(const block_number_t& blockNumber, + const index_t& sizeAtBlock, + WriteTransaction& tx) +{ + // There can be multiple block numbers aganst the same index (zero size blocks) + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Read the block index payload + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + BlockIndexPayload payload; + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + } + + // Double check it's not already present (it shouldn't be) + // We then add the block number and sort + // Sorting shouldn't be necessary as we add blocks in ascending order, but we will make sure + // Sorting here and when we unwind blocks means that looking up the block number for an index becomes O(1) + // These lookups are much more frequent than adds or deletes so we take the hit here + if (!payload.contains(blockNumber)) { + payload.blockNumbers.emplace_back(blockNumber); + payload.sort(); + } + + // Write the new payload back down + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + +bool LMDBTreeStore::find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx) +{ + LeafIndexKeyType key(index + 1); + std::vector data; + // Retrieve the payload + bool success = tx.get_value_or_greater(key, data, *_indexToBlockDatabase); + if (!success) { + return false; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + if (payload.blockNumbers.empty()) { + return false; + } + // The block numbers are sorted so we simply return the lowest + blockNumber = payload.blockNumbers[0]; + return true; +} + +void LMDBTreeStore::delete_block_index(const index_t& sizeAtBlock, + const block_number_t& blockNumber, + WriteTransaction& tx) +{ + // To delete a block number form an index we retieve all the block numbers from that index + // Then we find and remove the block number in question + // Then we write back down + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Retrieve the data + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + if (!success) { + return; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + + payload.delete_block(blockNumber); + + // if it's now empty, delete it + if (payload.blockNumbers.empty()) { + tx.delete_value(key, *_indexToBlockDatabase); + return; + } + // not empty write it back + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + void LMDBTreeStore::write_meta_data(const TreeMeta& metaData, LMDBTreeStore::WriteTransaction& tx) { msgpack::sbuffer buffer; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index ab4dfb7316c..a28ce245967 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -25,7 +25,7 @@ namespace bb::crypto::merkle_tree { struct BlockPayload { index_t size; - index_t blockNumber; + block_number_t blockNumber; fr root; MSGPACK_FIELDS(size, blockNumber, root) @@ -55,6 +55,49 @@ struct NodePayload { return left == other.left && right == other.right && ref == other.ref; } }; + +struct BlockIndexPayload { + std::vector blockNumbers; + + MSGPACK_FIELDS(blockNumbers) + + bool operator==(const BlockIndexPayload& other) const { return blockNumbers == other.blockNumbers; } + + void sort() { std::sort(blockNumbers.begin(), blockNumbers.end()); } + + bool contains(const block_number_t& blockNumber) + { + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return false; + } + return *it == blockNumber; + } + + void delete_block(const block_number_t& blockNumber) + { + if (blockNumbers.empty()) { + return; + } + // shuffle the block number down, removing the one we want to remove and then pop the end item + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return; + } + // It could be a block higher than the one we are looking for + if (*it != blockNumber) { + return; + } + // we have found our block, shuffle blocks after this one down + auto readIt = it + 1; + while (readIt != blockNumbers.end()) { + *it++ = *readIt++; + } + blockNumbers.pop_back(); + } +}; /** * Creates an abstraction against a collection of LMDB databases within a single environment used to store merkle tree * data @@ -78,11 +121,18 @@ class LMDBTreeStore { void get_stats(TreeDBStats& stats, ReadTransaction& tx); - void write_block_data(uint64_t blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + void write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + + bool read_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx); + + void delete_block_data(const block_number_t& blockNumber, WriteTransaction& tx); + + void write_block_index_data(const block_number_t& blockNumber, const index_t& sizeAtBlock, WriteTransaction& tx); - bool read_block_data(uint64_t blockNumber, BlockPayload& blockData, ReadTransaction& tx); + // index here is 0 based + bool find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx); - void delete_block_data(uint64_t blockNumber, WriteTransaction& tx); + void delete_block_index(const index_t& sizeAtBlock, const block_number_t& blockNumber, WriteTransaction& tx); void write_meta_data(const TreeMeta& metaData, WriteTransaction& tx); @@ -136,6 +186,7 @@ class LMDBTreeStore { LMDBDatabase::Ptr _nodeDatabase; LMDBDatabase::Ptr _leafKeyToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; + LMDBDatabase::Ptr _indexToBlockDatabase; template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cb4d342b09a..e9b85aa5bbc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" @@ -255,3 +257,254 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) EXPECT_FALSE(success); } } + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_by_index) +{ + struct BlockAndIndex { + block_number_t blockNumber; + // this block contains leaves up to index (0 based) + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 82 }, + BlockAndIndex{ .blockNumber = 4, .index = 114 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_TRUE(store.find_block_for_index(130, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete the last block + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[4].index + 1, blocks[4].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete non-exisatent indices to check it does nothing + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + store.delete_block_index(21, 1, *transaction); + store.delete_block_index(150, 6, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } +} + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_with_duplicate_indices) +{ + struct BlockAndIndex { + block_number_t blockNumber; + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 60 }, + BlockAndIndex{ .blockNumber = 4, .index = 60 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete block 2 at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, blocks[1].blockNumber, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // try and delete blocks that don't exist at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, 2, *transaction); + store.delete_block_index(blocks[1].index + 1, 5, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp index aa97a2d2518..c26768fa8ec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -177,6 +177,45 @@ bool get_value_or_previous(TKey& key, return success; } +template +bool get_value_or_greater(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +{ + bool success = false; + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize == dbKey.mv_size) { + // key is the same size, so this contains the data we are looking for + copy_to_vector(dbVal, data); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // no key greater than or equal, nothing to extract + } else { + throw_error("get_value_or_greater::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + template void get_all_values_greater_or_equal_key(const TKey& key, std::vector>& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 013f1cc8478..abaec64a3c7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -147,7 +147,7 @@ template class ContentAddressedCachedTreeStore { /** * @brief Reads the tree meta data, including uncommitted data if requested */ - bool get_block_data(const index_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; + bool get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; /** * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. @@ -198,13 +198,15 @@ template class ContentAddressedCachedTreeStore { fr get_current_root(ReadTransaction& tx, bool includeUncommitted) const; - void remove_historical_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); - void unwind_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); std::optional get_fork_block() const; - void advance_finalised_block(const index_t& blockNumber); + void advance_finalised_block(const block_number_t& blockNumber); + + std::optional find_block_for_index(const index_t& index, ReadTransaction& tx) const; private: std::string name_; @@ -233,7 +235,7 @@ template class ContentAddressedCachedTreeStore { void initialise(); - void initialise_from_block(const index_t& blockNumber); + void initialise_from_block(const block_number_t& blockNumber); bool read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const; @@ -258,6 +260,10 @@ template class ContentAddressedCachedTreeStore { void extract_db_stats(TreeDBStats& stats); + void persist_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + + void delete_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + index_t constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const; WriteTransactionPtr create_write_transaction() const { return dataStore_->create_write_transaction(); } @@ -304,6 +310,31 @@ index_t ContentAddressedCachedTreeStore::constrain_tree_size(cons return sizeLimit; } +template +std::optional ContentAddressedCachedTreeStore::find_block_for_index( + const index_t& index, ReadTransaction& tx) const +{ + block_number_t blockNumber = 0; + bool success = dataStore_->find_block_for_index(index, blockNumber, tx); + return success ? std::make_optional(blockNumber) : std::nullopt; +} + +template +void ContentAddressedCachedTreeStore::persist_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->write_block_index_data(blockNumber, index, tx); +} + +template +void ContentAddressedCachedTreeStore::delete_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->delete_block_index(index, blockNumber, tx); +} + template std::pair ContentAddressedCachedTreeStore::find_low_value( const fr& new_leaf_key, const RequestContext& requestContext, ReadTransaction& tx) const @@ -553,7 +584,7 @@ void ContentAddressedCachedTreeStore::get_meta(TreeMeta& m, } template -bool ContentAddressedCachedTreeStore::get_block_data(const index_t& blockNumber, +bool ContentAddressedCachedTreeStore::get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const { @@ -649,6 +680,7 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, .blockNumber = uncommittedMeta.unfinalisedBlockHeight, .root = uncommittedMeta.root }; dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx); + dataStore_->write_block_index_data(block.blockNumber, block.size, *tx); } uncommittedMeta.committedSize = uncommittedMeta.size; @@ -767,7 +799,7 @@ void ContentAddressedCachedTreeStore::persist_meta(TreeMeta& m, W } template -void ContentAddressedCachedTreeStore::advance_finalised_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::advance_finalised_block(const block_number_t& blockNumber) { TreeMeta committedMeta; TreeMeta uncommittedMeta; @@ -827,7 +859,7 @@ void ContentAddressedCachedTreeStore::advance_finalised_block(con } template -void ContentAddressedCachedTreeStore::unwind_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -894,6 +926,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); // remove the block from the block data table dataStore_->delete_block_data(blockNumber, *writeTx); + dataStore_->delete_block_index(blockData.size, blockData.blockNumber, *writeTx); uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; uncommittedMeta.size = previousBlockData.size; uncommittedMeta.committedSize = previousBlockData.size; @@ -916,7 +949,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& } template -void ContentAddressedCachedTreeStore::remove_historical_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -1107,7 +1140,7 @@ template void ContentAddressedCachedTreeStore -void ContentAddressedCachedTreeStore::initialise_from_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::initialise_from_block(const block_number_t& blockNumber) { // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during // construction then we throw diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index 164a6b254cf..6b77a6a5ebd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { @@ -16,9 +17,9 @@ struct TreeMeta { bb::fr root; index_t initialSize; bb::fr initialRoot; - uint64_t oldestHistoricBlock; - uint64_t unfinalisedBlockHeight; - uint64_t finalisedBlockHeight; + block_number_t oldestHistoricBlock; + block_number_t unfinalisedBlockHeight; + block_number_t finalisedBlockHeight; MSGPACK_FIELDS(name, depth, @@ -31,6 +32,34 @@ struct TreeMeta { unfinalisedBlockHeight, finalisedBlockHeight) + TreeMeta(std::string n, + uint32_t d, + const index_t& s, + const index_t& c, + const bb::fr& r, + const index_t& is, + const bb::fr& ir, + const block_number_t& o, + const block_number_t& u, + const block_number_t& f) + : name(std::move(n)) + , depth(d) + , size(s) + , committedSize(c) + , root(r) + , initialSize(is) + , initialRoot(ir) + , oldestHistoricBlock(o) + , unfinalisedBlockHeight(u) + , finalisedBlockHeight(f) + {} + TreeMeta() = default; + ~TreeMeta() = default; + TreeMeta(const TreeMeta& other) = default; + TreeMeta(TreeMeta&& other) noexcept { *this = std::move(other); } + TreeMeta& operator=(const TreeMeta& other) = default; + TreeMeta& operator=(TreeMeta&& other) noexcept = default; + bool operator==(const TreeMeta& other) const { return name == other.name && depth == other.depth && size == other.size && @@ -50,12 +79,4 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) return os; } -struct LeavesMeta { - index_t size; - - MSGPACK_FIELDS(size) - - bool operator==(const LeavesMeta& other) const { return size == other.size; } -}; - } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 5fdfd19b85b..d525acb8672 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -12,19 +12,41 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { struct TreeMetaResponse { TreeMeta meta; + + TreeMetaResponse() = default; + ~TreeMetaResponse() = default; + TreeMetaResponse(const TreeMetaResponse& other) = default; + TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; + TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; + TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; }; struct AddDataResponse { index_t size; fr root; + + AddDataResponse() = default; + ~AddDataResponse() = default; + AddDataResponse(const AddDataResponse& other) = default; + AddDataResponse(AddDataResponse&& other) noexcept = default; + AddDataResponse& operator=(const AddDataResponse& other) = default; + AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; }; struct GetSiblingPathResponse { fr_sibling_path path; + + GetSiblingPathResponse() = default; + ~GetSiblingPathResponse() = default; + GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; + GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; }; template struct LeafUpdateWitnessData { @@ -32,6 +54,18 @@ template struct LeafUpdateWitnessData { index_t index; fr_sibling_path path; + LeafUpdateWitnessData(const IndexedLeaf& l, const index_t& i, fr_sibling_path p) + : leaf(l) + , index(i) + , path(std::move(p)) + {} + LeafUpdateWitnessData() = default; + ~LeafUpdateWitnessData() = default; + LeafUpdateWitnessData(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData(LeafUpdateWitnessData&& other) noexcept = default; + LeafUpdateWitnessData& operator=(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; + MSGPACK_FIELDS(leaf, index, path); }; @@ -40,20 +74,59 @@ template struct AddIndexedDataResponse { fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; std::shared_ptr>> low_leaf_witness_data; + + AddIndexedDataResponse() = default; + ~AddIndexedDataResponse() = default; + AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; + AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; }; template struct AddIndexedDataSequentiallyResponse { AddDataResponse add_data_result; std::shared_ptr>> low_leaf_witness_data; std::shared_ptr>> insertion_witness_data; + + AddIndexedDataSequentiallyResponse() = default; + ~AddIndexedDataSequentiallyResponse() = default; + AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; + AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; +}; + +struct BlockForIndexResponse { + std::vector> blockNumbers; + + BlockForIndexResponse() = default; + ~BlockForIndexResponse() = default; + BlockForIndexResponse(const BlockForIndexResponse& other) = default; + BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; + BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; + BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; }; struct FindLeafIndexResponse { index_t leaf_index; + + FindLeafIndexResponse() = default; + ~FindLeafIndexResponse() = default; + FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; + FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; }; struct GetLeafResponse { std::optional leaf; + + GetLeafResponse() = default; + ~GetLeafResponse() = default; + GetLeafResponse(const GetLeafResponse& other) = default; + GetLeafResponse(GetLeafResponse&& other) noexcept = default; + GetLeafResponse& operator=(const GetLeafResponse& other) = default; + GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; }; template struct GetIndexedLeafResponse { @@ -66,6 +139,17 @@ struct GetLowIndexedLeafResponse { MSGPACK_FIELDS(is_already_present, index); + GetLowIndexedLeafResponse(bool p, const index_t& i) + : is_already_present(p) + , index(i) + {} + GetLowIndexedLeafResponse() = default; + ~GetLowIndexedLeafResponse() = default; + GetLowIndexedLeafResponse(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse(GetLowIndexedLeafResponse&& other) noexcept = default; + GetLowIndexedLeafResponse& operator=(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse& operator=(GetLowIndexedLeafResponse&& other) noexcept = default; + bool operator==(const GetLowIndexedLeafResponse& other) const { return is_already_present == other.is_already_present && index == other.index; @@ -75,27 +159,66 @@ struct GetLowIndexedLeafResponse { struct CommitResponse { TreeMeta meta; TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse& other) = default; + CommitResponse(CommitResponse&& other) noexcept = default; + CommitResponse& operator=(const CommitResponse& other) = default; + CommitResponse& operator=(CommitResponse&& other) noexcept = default; }; struct UnwindResponse { TreeMeta meta; TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse& other) = default; + UnwindResponse(UnwindResponse&& other) noexcept = default; + UnwindResponse& operator=(const UnwindResponse& other) = default; + UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; }; struct RemoveHistoricResponse { TreeMeta meta; TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; + RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; }; template struct TypedResponse { ResponseType inner; bool success{ true }; std::string message; + + TypedResponse() = default; + ~TypedResponse() = default; + TypedResponse(const TypedResponse& other) = default; + TypedResponse(TypedResponse&& other) noexcept = default; + TypedResponse& operator=(const TypedResponse& other) = default; + TypedResponse& operator=(TypedResponse&& other) noexcept = default; }; struct Response { bool success; std::string message; + + Response(bool s, std::string m) + : success(s) + , message(std::move(m)) + {} + Response() = default; + ~Response() = default; + Response(const Response& other) = default; + Response(Response&& other) noexcept = default; + Response& operator=(const Response& other) = default; + Response& operator=(Response&& other) noexcept = default; }; template @@ -116,8 +239,7 @@ void execute_and_report(const std::function&)>& } } -inline void execute_and_report(const std::function& f, - const std::function& on_completion) +inline void execute_and_report(const std::function& f, const std::function& on_completion) { Response response{ true, "" }; try { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index 1aa333d061b..354ba949c20 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -10,7 +10,10 @@ namespace bb::crypto::merkle_tree { -void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess) +void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, + block_number_t blockNumber, + fr root, + bool expectedSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -25,7 +28,7 @@ void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t block } void inline check_block_and_root_data( - LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) + LMDBTreeStore::SharedPtr db, block_number_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -40,7 +43,7 @@ void inline check_block_and_root_data( } void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, - index_t blockNumber, + block_number_t blockNumber, index_t expectedSize, bool expectedSuccess) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 8d9c5de4933..c8ce520fb9c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -6,10 +6,11 @@ #include namespace bb::crypto::merkle_tree { using index_t = uint64_t; +using block_number_t = uint64_t; struct RequestContext { bool includeUncommitted; - std::optional blockNumber; + std::optional blockNumber; bb::fr root; }; @@ -17,6 +18,7 @@ const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; +const std::string BLOCK_INDICES_DB = "block indices"; struct DBStats { std::string name; @@ -71,6 +73,7 @@ struct TreeDBStats { DBStats nodesDBStats; DBStats leafPreimagesDBStats; DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; TreeDBStats() = default; TreeDBStats(uint64_t mapSize) @@ -80,24 +83,27 @@ struct TreeDBStats { const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats) + const DBStats& leafIndicesStats, + const DBStats& blockIndicesStats) : mapSize(mapSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) , leafIndicesDBStats(leafIndicesStats) + , blockIndicesDBStats(blockIndicesStats) {} TreeDBStats(const TreeDBStats& other) = default; TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats) + MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats, blockIndicesDBStats) bool operator==(const TreeDBStats& other) const { return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats; + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; } TreeDBStats& operator=(TreeDBStats&& other) noexcept @@ -108,6 +114,7 @@ struct TreeDBStats { nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); } return *this; } @@ -118,7 +125,7 @@ struct TreeDBStats { { os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " - << stats.leafIndicesDBStats; + << stats.leafIndicesDBStats << ", Block Indices DB " << stats.blockIndicesDBStats; return os; } }; diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp index 99608cc626f..3c44307c8e9 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp @@ -14,7 +14,7 @@ using PairingPointAccumulatorIndices = std::array; -static constexpr uint32_t IPA_CLAIM_SIZE = 6; +static constexpr uint32_t IPA_CLAIM_SIZE = 10; using IPAClaimIndices = std::array; using IPAClaimPubInputIndices = std::array; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp b/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp index 6d79834aa95..a812ec7811e 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp @@ -53,16 +53,16 @@ template (builder, 0); + state[i] = witness_t::create_constant_witness(builder, 0); } - state[rate] = witness_t(builder, domain_iv.get_value()); + state[rate] = witness_t::create_constant_witness(builder, domain_iv.get_value()); } std::array perform_duplex() { // zero-pad the cache for (size_t i = cache_size; i < rate; ++i) { - cache[i] = witness_t(builder, 0); + cache[i] = witness_t::create_constant_witness(builder, 0); } // add the cache into sponge state for (size_t i = 0; i < rate; ++i) { @@ -122,7 +122,7 @@ template (builder, 0); + cache[cache_size] = witness_t::create_constant_witness(builder, 0); return result; } diff --git a/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp b/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp index 2cd6e8050d6..c8170f4373c 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp @@ -133,7 +133,7 @@ UltraRecursiveVerifier_::Output UltraRecursiveVerifier_::verify_ // Extract the IPA claim from the public inputs // Parse out the nested IPA claim using key->ipa_claim_public_input_indices and runs the native IPA verifier. if constexpr (HasIPAAccumulator) { - const auto recover_fq_from_public_inputs = [](std::array& limbs) { + const auto recover_fq_from_public_inputs = [](std::array& limbs) { for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { limbs[k].create_range_constraint(Curve::BaseField::NUM_LIMB_BITS, "limb_" + std::to_string(k)); } @@ -142,18 +142,25 @@ UltraRecursiveVerifier_::Output UltraRecursiveVerifier_::verify_ if (verification_key->verification_key->contains_ipa_claim) { OpeningClaim> ipa_claim; - std::array bigfield_limbs; - for (size_t k = 0; k < 4; k++) { - bigfield_limbs[k] = + std::array challenge_bigfield_limbs; + for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { + challenge_bigfield_limbs[k] = verification_key ->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[k]]; } - ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(bigfield_limbs); - ipa_claim.opening_pair.evaluation = 0; + std::array evaluation_bigfield_limbs; + for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { + evaluation_bigfield_limbs[k] = + verification_key + ->public_inputs[verification_key->verification_key + ->ipa_claim_public_input_indices[Curve::BaseField::NUM_LIMBS + k]]; + } + ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(challenge_bigfield_limbs); + ipa_claim.opening_pair.evaluation = recover_fq_from_public_inputs(evaluation_bigfield_limbs); ipa_claim.commitment = { - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[4]], - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[5]], - false // WORKTODO: make this a witness? + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[8]], + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[9]], + false }; output.ipa_opening_claim = std::move(ipa_claim); } diff --git a/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp b/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp index dd3011149d8..06db1ba8d94 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp @@ -450,7 +450,7 @@ template class bigfield { void set_origin_tag(const bb::OriginTag& tag) const { - for (size_t i = 0; i < 4; i++) { + for (size_t i = 0; i < NUM_LIMBS; i++) { binary_basis_limbs[i].element.set_origin_tag(tag); } prime_basis_limb.set_origin_tag(tag); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp index b14c4a6774d..3c4278c7439 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp @@ -34,18 +34,27 @@ template bool UltraVerifier_::verify_proof(const HonkP // Parse out the nested IPA claim using key->ipa_claim_public_input_indices and runs the native IPA verifier. if constexpr (HasIPAAccumulator) { if (verification_key->verification_key->contains_ipa_claim) { + + constexpr size_t NUM_LIMBS = 4; OpeningClaim ipa_claim; - std::array bigfield_limbs; - for (size_t k = 0; k < 4; k++) { - bigfield_limbs[k] = + + std::array challenge_bigfield_limbs; + std::array evaluation_bigfield_limbs; + for (size_t k = 0; k < NUM_LIMBS; k++) { + challenge_bigfield_limbs[k] = verification_key ->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[k]]; } - ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(bigfield_limbs); - ipa_claim.opening_pair.evaluation = 0; + for (size_t k = 0; k < NUM_LIMBS; k++) { + evaluation_bigfield_limbs[k] = + verification_key->public_inputs[verification_key->verification_key + ->ipa_claim_public_input_indices[NUM_LIMBS + k]]; + } + ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(challenge_bigfield_limbs); + ipa_claim.opening_pair.evaluation = recover_fq_from_public_inputs(evaluation_bigfield_limbs); ipa_claim.commitment = { - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[4]], - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[5]] + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[8]], + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[9]] }; // verify the ipa_proof with this claim diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp index 34e74a0631c..f5fbbcbf257 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -32,7 +32,7 @@ using StateReference = std::unordered_map; struct WorldStateRevision { index_t forkId{ 0 }; - index_t blockNumber{ 0 }; + block_number_t blockNumber{ 0 }; bool includeUncommitted{ false }; MSGPACK_FIELDS(forkId, blockNumber, includeUncommitted) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 9cb3b36bf6c..7e74fe44896 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -142,7 +142,7 @@ Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const } uint64_t WorldState::create_fork(const std::optional& blockNumber) { - index_t blockNumberForFork = 0; + block_number_t blockNumberForFork = 0; if (!blockNumber.has_value()) { // we are forking at latest WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; @@ -159,7 +159,7 @@ uint64_t WorldState::create_fork(const std::optional& blockNumber) return forkId; } -void WorldState::remove_forks_for_block(const index_t& blockNumber) +void WorldState::remove_forks_for_block(const block_number_t& blockNumber) { // capture the shared pointers outside of the lock scope so we are not under the lock when the objects are destroyed std::vector forks; @@ -191,7 +191,7 @@ void WorldState::delete_fork(const uint64_t& forkId) } } -Fork::SharedPtr WorldState::create_new_fork(const index_t& blockNumber) +Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); fork->_blockNumber = blockNumber; @@ -241,10 +241,10 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M return std::visit( [=](auto&& wrapper) { Signal signal(1); - TreeMetaResponse response; + TypedResponse local; - auto callback = [&](const TypedResponse& meta) { - response = meta.inner; + auto callback = [&](TypedResponse& meta) { + local = std::move(meta); signal.signal_level(0); }; @@ -255,12 +255,15 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M } signal.wait_for_level(0); - return response; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner; }, fork->_trees.at(tree_id)); } -void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const +void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const { Fork::SharedPtr fork = retrieve_fork(revision.forkId); @@ -271,13 +274,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra Signal signal(static_cast(tree_ids.size())); std::mutex mutex; + std::unordered_map> local; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &responses, &mutex, id](const TypedResponse& meta) { + auto callback = [&signal, &local, &mutex, id](TypedResponse& meta) { { std::lock_guard lock(mutex); - responses[id] = meta.inner.meta; + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -293,6 +297,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + responses[tree_id] = std::move(m.inner.meta); + } } StateReference WorldState::get_state_reference(const WorldStateRevision& revision) const @@ -324,19 +336,15 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio Signal signal(static_cast(tree_ids.size())); StateReference state_reference; + std::unordered_map> local; std::mutex state_ref_mutex; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &state_reference, &state_ref_mutex, initial_state, id]( - const TypedResponse& meta) { + auto callback = [&signal, &local, &state_ref_mutex, id](TypedResponse& meta) { { std::lock_guard lock(state_ref_mutex); - if (initial_state) { - state_reference.insert({ id, { meta.inner.meta.initialRoot, meta.inner.meta.initialSize } }); - } else { - state_reference.insert({ id, { meta.inner.meta.root, meta.inner.meta.size } }); - } + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -352,6 +360,19 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + if (initial_state) { + state_reference[tree_id] = std::make_pair(m.inner.meta.initialRoot, m.inner.meta.initialSize); + continue; + } + state_reference[tree_id] = std::make_pair(m.inner.meta.root, m.inner.meta.size); + } + return state_reference; } @@ -364,10 +385,10 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, return std::visit( [leaf_index, revision](auto&& wrapper) { Signal signal(1); - fr_sibling_path path; + TypedResponse local; - auto callback = [&signal, &path](const TypedResponse& response) { - path = response.inner.path; + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; @@ -378,7 +399,42 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, } signal.wait_for_level(0); - return path; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner.path; + }, + fork->_trees.at(tree_id)); +} + +void WorldState::get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const +{ + Fork::SharedPtr fork = retrieve_fork(revision.forkId); + + std::visit( + [&leafIndices, revision, &blockNumbers](auto&& wrapper) { + Signal signal(1); + TypedResponse local; + + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); + signal.signal_level(); + }; + + if (revision.blockNumber) { + wrapper.tree->find_block_numbers(leafIndices, revision.blockNumber, callback); + } else { + wrapper.tree->find_block_numbers(leafIndices, callback); + } + signal.wait_for_level(0); + + if (!local.success) { + throw std::runtime_error(local.message); + } + blockNumbers = std::move(local.inner.blockNumbers); }, fork->_trees.at(tree_id)); } @@ -534,7 +590,15 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re { auto& wrapper = std::get>(fork->_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); - PublicDataTree::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_decrement(); }; + PublicDataTree::AddCompletionCallback completion = [&](const auto& resp) -> void { + // take the first error + bool expected = true; + if (!resp.success && success.compare_exchange_strong(expected, false)) { + err_message = resp.message; + } + + signal.signal_decrement(); + }; wrapper.tree->add_or_update_values_sequentially(public_writes, completion); } @@ -566,9 +630,9 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi { Fork::SharedPtr fork = retrieve_fork(revision.forkId); Signal signal; - GetLowIndexedLeafResponse low_leaf_info; - auto callback = [&signal, &low_leaf_info](const TypedResponse& response) { - low_leaf_info = response.inner; + TypedResponse low_leaf_info; + auto callback = [&signal, &low_leaf_info](TypedResponse& response) { + low_leaf_info = std::move(response); signal.signal_level(); }; @@ -591,7 +655,11 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi } signal.wait_for_level(); - return low_leaf_info; + + if (!low_leaf_info.success) { + throw std::runtime_error(low_leaf_info.message); + } + return low_leaf_info.inner; } WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockNumber) @@ -599,11 +667,13 @@ WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockN WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); if (toBlockNumber <= archive_state.meta.finalisedBlockHeight) { - throw std::runtime_error("Unable to finalise block, already finalised"); - } - if (!set_finalised_block(toBlockNumber)) { - throw std::runtime_error("Failed to set finalised block"); + throw std::runtime_error(format("Unable to finalise blocks to block number ", + toBlockNumber, + ", current finalised block: ", + archive_state.meta.finalisedBlockHeight)); } + // This will throw if it fails + set_finalised_block(toBlockNumber); WorldStateStatusSummary status; get_status_summary(status); return status; @@ -616,10 +686,10 @@ WorldStateStatusFull WorldState::unwind_blocks(const index_t& toBlockNumber) throw std::runtime_error("Unable to unwind block, block not found"); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; blockNumber--) { - if (!unwind_block(blockNumber, status)) { - throw std::runtime_error("Failed to unwind block"); - } + for (block_number_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; + blockNumber--) { + // This will throw if it fails + unwind_block(blockNumber, status); } populate_status_summary(status); return status; @@ -635,35 +705,43 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock archive_state.meta.oldestHistoricBlock)); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { - if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error(format( - "Failed to remove historical block ", blockNumber, " when removing blocks up to ", toBlockNumber)); - } + for (block_number_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; + blockNumber++) { + // This will throw if it fails + remove_historical_block(blockNumber, status); } populate_status_summary(status); return status; } -bool WorldState::set_finalised_block(const index_t& blockNumber) +bool WorldState::set_finalised_block(const block_number_t& blockNumber) { - std::atomic_bool success = true; Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); Signal signal(static_cast(fork->_trees.size())); + std::array local; + std::mutex mtx; for (auto& [id, tree] : fork->_trees) { std::visit( - [&signal, &success, blockNumber](auto&& wrapper) { - wrapper.tree->finalise_block(blockNumber, [&signal, &success](const Response& resp) { - success = success && resp.success; + [&signal, &local, blockNumber, id, &mtx](auto&& wrapper) { + wrapper.tree->finalise_block(blockNumber, [&signal, &local, &mtx, id](Response& resp) { + { + std::lock_guard lock(mtx); + local[id] = std::move(resp); + } signal.signal_decrement(); }); }, tree); } signal.wait_for_level(); - return success; + for (auto& m : local) { + if (!m.success) { + throw std::runtime_error(m.message); + } + } + return true; } -bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -723,10 +801,13 @@ bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } -bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -786,8 +867,11 @@ bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateS blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) @@ -829,7 +913,12 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { - std::optional leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + std::optional leaf_index = std::nullopt; + + try { + leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + } catch (std::runtime_error&) { + } if (!leaf_index.has_value()) { return false; @@ -887,7 +976,7 @@ void WorldState::validate_trees_are_equally_synched() bool WorldState::determine_if_synched(std::array& metaResponses) { - index_t blockNumber = metaResponses[0].unfinalisedBlockHeight; + block_number_t blockNumber = metaResponses[0].unfinalisedBlockHeight; for (size_t i = 1; i < metaResponses.size(); i++) { if (blockNumber != metaResponses[i].unfinalisedBlockHeight) { return false; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 5afa4a010ef..c66412aae77 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -107,6 +107,11 @@ class WorldState { MerkleTreeId tree_id, index_t leaf_index) const; + void get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const; + /** * @brief Get the leaf preimage object * @@ -259,12 +264,12 @@ class WorldState { uint64_t maxReaders); Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; - Fork::SharedPtr create_new_fork(const index_t& blockNumber); - void remove_forks_for_block(const index_t& blockNumber); + Fork::SharedPtr create_new_fork(const block_number_t& blockNumber); + void remove_forks_for_block(const block_number_t& blockNumber); - bool unwind_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool set_finalised_block(const index_t& blockNumber); + bool unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool set_finalised_block(const block_number_t& blockNumber); void get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const; @@ -304,7 +309,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); template void remove_historic_block_for_tree(TreeDBStats& dbStats, @@ -313,7 +318,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); }; template @@ -342,7 +347,7 @@ void WorldState::unwind_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.unwind_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -362,7 +367,7 @@ void WorldState::remove_historic_block_for_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.remove_historic_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -384,15 +389,13 @@ std::optional> WorldState::get_indexed_leaf( using Tree = ContentAddressedIndexedTree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse> local; if (auto* const wrapper = std::get_if>(&fork->_trees.at(id))) { - std::optional> value; - Signal signal; - auto callback = [&](const TypedResponse>& response) { - if (response.inner.indexed_leaf.has_value()) { - value = response.inner.indexed_leaf; - } + Signal signal; + auto callback = [&](TypedResponse>& response) { + local = std::move(response); signal.signal_level(0); }; @@ -403,7 +406,11 @@ std::optional> WorldState::get_indexed_leaf( } signal.wait_for_level(); - return value; + if (!local.success) { + throw std::runtime_error("Failed to find indexed leaf: " + local.message); + } + + return local.inner.indexed_leaf; } throw std::runtime_error("Invalid tree type"); @@ -419,12 +426,17 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, Fork::SharedPtr fork = retrieve_fork(revision.forkId); std::optional leaf; + bool success = true; + std::string error_msg; Signal signal; if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse& resp) { - if (resp.inner.leaf.has_value()) { - leaf = resp.inner.leaf.value(); + auto callback = [&signal, &leaf, &success, &error_msg](const TypedResponse& response) { + if (!response.success || !response.inner.leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.leaf; } signal.signal_level(); }; @@ -439,12 +451,16 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, using Tree = ContentAddressedIndexedTree; auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse>& resp) { - if (resp.inner.indexed_leaf.has_value()) { - leaf = resp.inner.indexed_leaf.value().value; - } - signal.signal_level(); - }; + auto callback = + [&signal, &leaf, &success, &error_msg](const TypedResponse>& response) { + if (!response.success || !response.inner.indexed_leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.indexed_leaf.value().value; + } + signal.signal_level(); + }; if (revision.blockNumber) { wrapper.tree->get_leaf(leaf_index, revision.blockNumber, revision.includeUncommitted, callback); @@ -454,6 +470,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, } signal.wait_for_level(); + return leaf; } @@ -464,15 +481,13 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev index_t start_index) const { using namespace crypto::merkle_tree; - std::optional index; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse local; Signal signal; - auto callback = [&](const TypedResponse& response) { - if (response.success) { - index = response.inner.leaf_index; - } + auto callback = [&](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; if constexpr (std::is_same_v) { @@ -496,7 +511,12 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev } signal.wait_for_level(0); - return index; + + if (!local.success) { + return std::nullopt; + } + + return local.inner.leaf_index; } template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) @@ -537,7 +557,7 @@ template void WorldState::append_leaves(MerkleTreeId id, const std: signal.wait_for_level(0); if (!success) { - throw std::runtime_error("Failed to append leaves: " + error_msg); + throw std::runtime_error(error_msg); } } @@ -576,7 +596,7 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to batch insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; @@ -615,7 +635,7 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to sequentially insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 024ffbf4ac0..a5ced2921ad 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -1,6 +1,7 @@ #include "barretenberg/world_state/world_state.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" @@ -11,6 +12,7 @@ #include #include #include +#include #include #include @@ -79,6 +81,10 @@ void assert_leaf_index( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) { std::optional index = ws.find_leaf_index(revision, tree_id, value); + EXPECT_TRUE(index.has_value()); + if (!index.has_value()) { + return; + } EXPECT_EQ(index.value(), expected_index); } @@ -363,6 +369,7 @@ TEST_F(WorldStateTest, NullifierTree) auto test_leaf = ws.get_indexed_leaf(WorldStateRevision::committed(), tree_id, 128); // at this point 142 should be the biggest leaf so it wraps back to 0 + EXPECT_TRUE(test_leaf.has_value()); EXPECT_EQ(test_leaf.value(), IndexedLeaf(test_nullifier, 0, 0)); auto predecessor_of_142_again = @@ -524,6 +531,19 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) for (const auto& [tree_id, snapshot] : block_state_ref) { EXPECT_EQ(state_ref.at(tree_id), snapshot); } + + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, { 0 }, blockNumbers); + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_EQ(blockNumbers[0], 1); + + EXPECT_THROW(ws.get_block_numbers_for_leaf_indices( + WorldStateRevision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 2, .includeUncommitted = false }, + MerkleTreeId::NOTE_HASH_TREE, + { 0 }, + blockNumbers), + std::runtime_error); } TEST_F(WorldStateTest, SyncBlockFromDirtyState) @@ -789,3 +809,43 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); } + +TEST_F(WorldStateTest, GetBlockForIndex) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + // bb::fr block_hash(1); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x187a19972150cd1e76d8201d720da7682fcf4d93ec6a3c7b0d84bbefde5bd927"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x2467e5f90736b4ea977e7d21cfb3714181e16b7d6cd867768b59e2ea90fa3eaf"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x24ffd0fab86555ab2e86cffc706d4cfb4b8c405c3966af805de954504ffc27ac"), 1 } }, + }; + + WorldStateStatusFull status = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + WorldStateStatusSummary expected{ 1, 0, 1, true }; + EXPECT_EQ(status.summary, expected); + + StateReference state_ref = ws.get_state_reference(WorldStateRevision::committed()); + + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, + MerkleTreeId::NOTE_HASH_TREE, + MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + }; + + for (const auto& id : tree_ids) { + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), id, { state_ref[id].second - 1 }, blockNumbers); + + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_TRUE(blockNumbers[0].has_value()); + EXPECT_EQ(blockNumbers[0].value(), 1); + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index d8ac68d0db1..f3290da5e9e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -150,6 +150,11 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::GET_SIBLING_PATH, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); + _dispatcher.registerTarget(WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { + return get_block_numbers_for_leaf_indices(obj, buffer); + }); + _dispatcher.registerTarget( WorldStateMessageType::FIND_LEAF_INDEX, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); @@ -387,6 +392,24 @@ bool WorldStateAddon::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& b return true; } +bool WorldStateAddon::get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + GetBlockNumbersForLeafIndicesResponse response; + _ws->get_block_numbers_for_leaf_indices( + request.value.revision, request.value.treeId, request.value.leafIndices, response.blockNumbers); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, header, response); + + msgpack::pack(buffer, resp_msg); + + return true; +} + bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const { TypedMessage request; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 1a077a946e9..d0b33be2532 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -38,6 +38,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index c876f5993bc..b98a8c6a69d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -1,5 +1,6 @@ #pragma once #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" @@ -20,6 +21,7 @@ enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -54,7 +56,7 @@ struct TreeIdOnlyRequest { struct CreateForkRequest { bool latest; - index_t blockNumber; + block_number_t blockNumber; MSGPACK_FIELDS(latest, blockNumber); }; @@ -129,6 +131,18 @@ struct GetSiblingPathRequest { MSGPACK_FIELDS(treeId, revision, leafIndex); }; +struct GetBlockNumbersForLeafIndicesRequest { + MerkleTreeId treeId; + WorldStateRevision revision; + std::vector leafIndices; + MSGPACK_FIELDS(treeId, revision, leafIndices); +}; + +struct GetBlockNumbersForLeafIndicesResponse { + std::vector> blockNumbers; + MSGPACK_FIELDS(blockNumbers); +}; + template struct FindLeafIndexRequest { MerkleTreeId treeId; WorldStateRevision revision; diff --git a/noir-projects/aztec-nr/.gitrepo b/noir-projects/aztec-nr/.gitrepo index 4696364255e..e3be0061d71 100644 --- a/noir-projects/aztec-nr/.gitrepo +++ b/noir-projects/aztec-nr/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/aztec-nr branch = master - commit = 6344f8f2ef40886a0047d49c5685adec9ff1911f + commit = 6e54884e4b0ea5204966b98c490f6e93722b7ad9 method = merge cmdver = 0.4.6 - parent = b535302b6f9b3ff8d27c1c104060ab7b19af0fb7 + parent = 9d0a890cedaa29019e6312fbb24164b8adbd8bc4 diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index f0ee82aabe0..00e585513a6 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -168,6 +168,8 @@ spec: value: "{{ .Values.aztec.epochDuration }}" - name: AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS value: "{{ .Values.aztec.epochProofClaimWindow }}" + - name: PEER_ID_PRIVATE_KEY + value: "{{ .Values.bootNode.peerIdPrivateKey }}" ports: - containerPort: {{ .Values.bootNode.service.nodePort }} - containerPort: {{ .Values.bootNode.service.p2pTcpPort }} diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index 245b51f9435..005d3061137 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -35,6 +35,7 @@ aztec: epochProofClaimWindow: 13 # in L2 slots bootNode: + peerIdPrivateKey: "" externalHost: "" replicas: 1 service: diff --git a/spartan/aztec-network/values/release.yaml b/spartan/aztec-network/values/release.yaml index e3e34dac3b8..b48f9cf2640 100644 --- a/spartan/aztec-network/values/release.yaml +++ b/spartan/aztec-network/values/release.yaml @@ -119,6 +119,7 @@ validator: bootNode: realProofs: true + peerIdPrivateKey: 080212200ba8451c6d62b03c4441f0a466c0bce7a3a595f2cf50a055ded3305c77aa3af0 validator: disabled: true diff --git a/spartan/releases/rough-rhino/full-node.sh b/spartan/releases/rough-rhino/full-node.sh index cf13a1d3a45..75a0d33bca1 100755 --- a/spartan/releases/rough-rhino/full-node.sh +++ b/spartan/releases/rough-rhino/full-node.sh @@ -22,8 +22,8 @@ docker run --rm --network=host \ -e AZTEC_SLOT_DURATION=36 \ -e AZTEC_EPOCH_DURATION=32 \ -e AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS=13 \ - -e ETHEREUM_HOST=http://35.221.3.35:8545 \ - -e BOOTSTRAP_NODES=enr:-Jq4QKIJisajcICBVMoMwFtbmPgmHt3KoonypbBIQCAMNjhMc6DKW0J4vJzDpGPFUX7T2fzyyjezHgKKzeZY_DbRz_kGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCPdAyOJc2VjcDI1NmsxoQK92C7GObzDvCt9uwzW0lhKJKGCvOWkmAZjd2E2w-svuoN0Y3CCndCDdWRwgp3Q \ + -e ETHEREUM_HOST=http://34.48.76.131:8545 \ + -e BOOTSTRAP_NODES=enr:-Jq4QO_3szmgtG2cbEdnFDIhpGAQkc1HwfNy4-M6sG9QmQbPTmp9PMOHR3xslfR23hORiU-GpA7uM9uXw49lFcnuuvYGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCIwTIOJc2VjcDI1NmsxoQKQTN17XKCwjYSSwmTc-6YzCMhd3v6Ofl8TS-WunX6LCoN0Y3CCndCDdWRwgp3Q \ -e REGISTRY_CONTRACT_ADDRESS=0x5fbdb2315678afecb367f032d93f642f64180aa3 \ -e GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ -e FEE_JUICE_CONTRACT_ADDRESS=0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 \ diff --git a/spartan/releases/rough-rhino/validator.sh b/spartan/releases/rough-rhino/validator.sh index 98e81124227..246e59527bb 100755 --- a/spartan/releases/rough-rhino/validator.sh +++ b/spartan/releases/rough-rhino/validator.sh @@ -25,8 +25,8 @@ docker run --rm --network=host \ -e AZTEC_SLOT_DURATION=36 \ -e AZTEC_EPOCH_DURATION=32 \ -e AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS=13 \ - -e ETHEREUM_HOST=http://35.221.3.35:8545 \ - -e BOOTSTRAP_NODES=enr:-Jq4QKIJisajcICBVMoMwFtbmPgmHt3KoonypbBIQCAMNjhMc6DKW0J4vJzDpGPFUX7T2fzyyjezHgKKzeZY_DbRz_kGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCPdAyOJc2VjcDI1NmsxoQK92C7GObzDvCt9uwzW0lhKJKGCvOWkmAZjd2E2w-svuoN0Y3CCndCDdWRwgp3Q \ + -e ETHEREUM_HOST=http://34.48.76.131:8545 \ + -e BOOTSTRAP_NODES=enr:-Jq4QO_3szmgtG2cbEdnFDIhpGAQkc1HwfNy4-M6sG9QmQbPTmp9PMOHR3xslfR23hORiU-GpA7uM9uXw49lFcnuuvYGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCIwTIOJc2VjcDI1NmsxoQKQTN17XKCwjYSSwmTc-6YzCMhd3v6Ofl8TS-WunX6LCoN0Y3CCndCDdWRwgp3Q \ -e REGISTRY_CONTRACT_ADDRESS=0x5fbdb2315678afecb367f032d93f642f64180aa3 \ -e GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ -e FEE_JUICE_CONTRACT_ADDRESS=0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 \ diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 57f576c55d3..587d97371a4 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -435,6 +435,22 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + /** + * Find the block numbers of the given leaf indices in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + public async findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const committedDb = await this.#getWorldState(blockNumber); + return await committedDb.getBlockNumbersForLeafIndices(treeId, leafIndices); + } + public async findNullifiersIndexesWithBlock( blockNumber: L2BlockNumber, nullifiers: Fr[], @@ -836,10 +852,12 @@ export class AztecNodeService implements AztecNode { // TODO(#10007): Remove this method public addContractClass(contractClass: ContractClassPublic): Promise { + this.log.info(`Adding contract class via API ${contractClass.id}`); return this.contractDataSource.addContractClass(contractClass); } public addContractArtifact(address: AztecAddress, artifact: ContractArtifact): Promise { + this.log.info(`Adding contract artifact ${artifact.name} for ${address.toString()} via API`); // TODO: Node should validate the artifact before accepting it return this.contractDataSource.addContractArtifact(address, artifact); } diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 4e3f9bd4a8d..c2db85e5250 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -91,6 +91,11 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); + it('findBlockNumbersForIndexes', async () => { + const response = await context.client.findBlockNumbersForIndexes(1, MerkleTreeId.ARCHIVE, [5n, 58n]); + expect(response).toEqual([3n, 9n]); + }); + it('findNullifiersIndexesWithBlock', async () => { const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); expect(response).toEqual([ @@ -349,6 +354,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + + findBlockNumbersForIndexes( + _blockNumber: number | 'latest', + _treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + expect(leafIndices).toEqual([5n, 58n]); + return Promise.resolve([3n, 9n]); + } findNullifiersIndexesWithBlock( blockNumber: number | 'latest', nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 8e5b2bda55f..457d188acf0 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -61,7 +61,7 @@ export interface AztecNode * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data * @param treeId - The tree to search in. - * @param leafValue - The values to search for + * @param leafValues - The values to search for * @returns The indexes of the given leaves in the given tree or undefined if not found. */ findLeavesIndexes( @@ -70,6 +70,19 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Find the indexes of the given leaves in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; + /** * Returns the indexes of the given nullifiers in the nullifier tree, * scoped to the block they were included in. @@ -430,6 +443,11 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findBlockNumbersForIndexes: z + .function() + .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.BigInt)) + .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z .function() .args(L2BlockNumberSchema, z.array(schemas.Fr)) diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 8520da7ca55..9017c1a6a84 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -199,6 +199,16 @@ export interface MerkleTreeReadOperations { treeId: ID, index: bigint, ): Promise | undefined>; + + /** + * Get the block numbers for a set of leaf indices + * @param treeId - The tree for which the block numbers should be returned. + * @param leafIndices - The indices to be queried. + */ + getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; } export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations { diff --git a/yarn-project/circuit-types/src/tx_execution_request.ts b/yarn-project/circuit-types/src/tx_execution_request.ts index 2a99636da2e..bdd90e28550 100644 --- a/yarn-project/circuit-types/src/tx_execution_request.ts +++ b/yarn-project/circuit-types/src/tx_execution_request.ts @@ -4,6 +4,7 @@ import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import { type FieldsOf } from '@aztec/foundation/types'; +import { inspect } from 'util'; import { z } from 'zod'; import { AuthWitness } from './auth_witness.js'; @@ -141,4 +142,8 @@ export class TxExecutionRequest { [AuthWitness.random()], ); } + + [inspect.custom]() { + return `TxExecutionRequest(${this.origin} called ${this.functionSelector})`; + } } diff --git a/yarn-project/end-to-end/src/spartan/utils.ts b/yarn-project/end-to-end/src/spartan/utils.ts index fe34c31534a..3152de9c53d 100644 --- a/yarn-project/end-to-end/src/spartan/utils.ts +++ b/yarn-project/end-to-end/src/spartan/utils.ts @@ -355,7 +355,11 @@ export async function awaitL2BlockNumber( await sleep(1000); tips = await rollupCheatCodes.getTips(); } - logger.info(`Reached L2 Block ${tips.pending}`); + if (tips.pending < blockNumber) { + throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`); + } else { + logger.info(`Reached L2 Block ${tips.pending}`); + } } export async function restartBot(namespace: string, logger: Logger) { diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 4a5ab7a7576..201d69ba2b0 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -67,6 +67,8 @@ import { } from '@aztec/protocol-contracts'; import { type AcirSimulator } from '@aztec/simulator'; +import { inspect } from 'util'; + import { type PXEServiceConfig, getPackageInfo } from '../config/index.js'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; import { IncomingNoteDao } from '../database/incoming_note_dao.js'; @@ -519,8 +521,7 @@ export class PXEService implements PXE { return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!); }) .catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError(err, inspect(txRequest), inspect(privateExecutionResult)); }); } @@ -576,8 +577,15 @@ export class PXEService implements PXE { ); }) .catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError( + err, + inspect(txRequest), + `simulatePublic=${simulatePublic}`, + `msgSender=${msgSender?.toString() ?? 'undefined'}`, + `skipTxValidation=${skipTxValidation}`, + `profile=${profile}`, + `scopes=${scopes?.map(s => s.toString()).join(', ') ?? 'undefined'}`, + ); }); } @@ -588,8 +596,7 @@ export class PXEService implements PXE { } this.log.info(`Sending transaction ${txHash}`); await this.node.sendTx(tx).catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError(err, inspect(tx)); }); this.log.info(`Sent transaction ${txHash}`); return txHash; @@ -613,8 +620,12 @@ export class PXEService implements PXE { return executionResult; }) .catch(err => { - this.log.error(err); - throw err; + const stringifiedArgs = args.map(arg => arg.toString()).join(', '); + throw this.contextualizeError( + err, + `simulateUnconstrained ${to}:${functionName}(${stringifiedArgs})`, + `scopes=${scopes?.map(s => s.toString()).join(', ') ?? 'undefined'}`, + ); }); } @@ -986,4 +997,13 @@ export class PXEService implements PXE { async resetNoteSyncData() { return await this.db.resetNoteSyncData(); } + + private contextualizeError(err: Error, ...context: string[]): Error { + this.log.error(err.name, err); + this.log.debug('Context:'); + for (const c of context) { + this.log.debug(c); + } + return err; + } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index e5a55462b97..5acbbd261f6 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -843,7 +843,7 @@ class TestSubject extends Sequencer { } public override doRealWork() { - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); return super.doRealWork(); } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 8c9750ff043..325a2dd2d44 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -189,7 +189,7 @@ export class Sequencer { public start() { this.runningPromise = new RunningPromise(this.work.bind(this), this.pollingIntervalMs); this.runningPromise.start(); - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); this.log.info('Sequencer started'); return Promise.resolve(); } @@ -201,7 +201,7 @@ export class Sequencer { this.log.debug(`Stopping sequencer`); await this.runningPromise?.stop(); this.publisher.interrupt(); - this.setState(SequencerState.STOPPED, 0, true /** force */); + this.setState(SequencerState.STOPPED, 0n, true /** force */); this.log.info('Stopped sequencer'); } @@ -212,7 +212,7 @@ export class Sequencer { this.log.info('Restarting sequencer'); this.publisher.restart(); this.runningPromise!.start(); - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); } /** @@ -232,7 +232,7 @@ export class Sequencer { * - If our block for some reason is not included, revert the state */ protected async doRealWork() { - this.setState(SequencerState.SYNCHRONIZING, 0); + this.setState(SequencerState.SYNCHRONIZING, 0n); // Update state when the previous block has been synced const prevBlockSynced = await this.isBlockSynced(); // Do not go forward with new block if the previous one has not been mined and processed @@ -243,7 +243,7 @@ export class Sequencer { this.log.debug('Previous block has been mined and processed'); - this.setState(SequencerState.PROPOSER_CHECK, 0); + this.setState(SequencerState.PROPOSER_CHECK, 0n); const chainTip = await this.l2BlockSource.getBlock(-1); const historicalHeader = chainTip?.header; @@ -277,9 +277,8 @@ export class Sequencer { if (!this.shouldProposeBlock(historicalHeader, {})) { return; } - const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration, Number(slot)); - this.setState(SequencerState.WAITING_FOR_TXS, secondsIntoSlot); + this.setState(SequencerState.WAITING_FOR_TXS, slot); // Get txs to build the new block. const pendingTxs = this.p2pClient.getTxs('pending'); @@ -325,7 +324,7 @@ export class Sequencer { } catch (err) { this.log.error(`Error assembling block`, (err as any).stack); } - this.setState(SequencerState.IDLE, 0); + this.setState(SequencerState.IDLE, 0n); } protected async work() { @@ -339,7 +338,7 @@ export class Sequencer { throw err; } } finally { - this.setState(SequencerState.IDLE, 0); + this.setState(SequencerState.IDLE, 0n); } } @@ -398,13 +397,23 @@ export class Sequencer { return true; } - setState(proposedState: SequencerState, secondsIntoSlot: number, force: boolean = false) { + /** + * Sets the sequencer state and checks if we have enough time left in the slot to transition to the new state. + * @param proposedState - The new state to transition to. + * @param currentSlotNumber - The current slot number. + * @param force - Whether to force the transition even if the sequencer is stopped. + * + * @dev If the `currentSlotNumber` doesn't matter (e.g. transitioning to IDLE), pass in `0n`; + * it is only used to check if we have enough time left in the slot to transition to the new state. + */ + setState(proposedState: SequencerState, currentSlotNumber: bigint, force: boolean = false) { if (this.state === SequencerState.STOPPED && force !== true) { this.log.warn( `Cannot set sequencer from ${this.state} to ${proposedState} as it is stopped. Set force=true to override.`, ); return; } + const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration, Number(currentSlotNumber)); if (!this.doIHaveEnoughTimeLeft(proposedState, secondsIntoSlot)) { throw new SequencerTooSlowError(this.state, proposedState, this.timeTable[proposedState], secondsIntoSlot); } @@ -567,12 +576,7 @@ export class Sequencer { this.metrics.recordNewBlock(newGlobalVariables.blockNumber.toNumber(), validTxs.length); const workTimer = new Timer(); - const secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - newGlobalVariables.slotNumber.toNumber(), - ); - this.setState(SequencerState.CREATING_BLOCK, secondsIntoSlot); + this.setState(SequencerState.CREATING_BLOCK, newGlobalVariables.slotNumber.toBigInt()); this.log.info( `Building blockNumber=${newGlobalVariables.blockNumber.toNumber()} txCount=${ validTxs.length @@ -688,23 +692,13 @@ export class Sequencer { this.log.info('Creating block proposal'); const proposal = await this.validatorClient.createBlockProposal(block.header, block.archive.root, txHashes); - let secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); + const slotNumber = block.header.globalVariables.slotNumber.toBigInt(); - this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS, secondsIntoSlot); + this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS, slotNumber); this.log.info('Broadcasting block proposal to validators'); this.validatorClient.broadcastBlockProposal(proposal); - secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); - - this.setState(SequencerState.WAITING_FOR_ATTESTATIONS, secondsIntoSlot); + this.setState(SequencerState.WAITING_FOR_ATTESTATIONS, slotNumber); const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations); this.log.info(`Collected attestations from validators, number of attestations: ${attestations.length}`); @@ -761,13 +755,8 @@ export class Sequencer { txHashes?: TxHash[], proofQuote?: EpochProofQuote, ) { - const secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); // Publishes new block to the network and awaits the tx to be mined - this.setState(SequencerState.PUBLISHING_BLOCK, secondsIntoSlot); + this.setState(SequencerState.PUBLISHING_BLOCK, block.header.globalVariables.slotNumber.toBigInt()); const publishedL2Block = await this.publisher.proposeL2Block(block, attestations, txHashes, proofQuote); if (!publishedL2Block) { diff --git a/yarn-project/sequencer-client/src/sequencer/utils.ts b/yarn-project/sequencer-client/src/sequencer/utils.ts index 4c16e8c8a9b..8bb4b440dc2 100644 --- a/yarn-project/sequencer-client/src/sequencer/utils.ts +++ b/yarn-project/sequencer-client/src/sequencer/utils.ts @@ -75,5 +75,5 @@ export function orderAttestations(attestations: BlockAttestation[], orderAddress export function getSecondsIntoSlot(l1GenesisTime: number, aztecSlotDuration: number, slotNumber: number): number { const slotStartTimestamp = l1GenesisTime + slotNumber * aztecSlotDuration; - return Date.now() / 1000 - slotStartTimestamp; + return Number((Date.now() / 1000 - slotStartTimestamp).toFixed(3)); } diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 01f3e483497..853ce0bb58f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -173,6 +173,24 @@ export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state. export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message'; export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER = + 'aztec.world_state.db_used_size.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA = + 'aztec.world_state.db_used_size.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH = + 'aztec.world_state.db_used_size.block_indices.note_hash'; + +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER = + 'aztec.world_state.db_num_items.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = + 'aztec.world_state.db_num_items.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH = + 'aztec.world_state.db_num_items.block_indices.note_hash'; + export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 49d53e8f3a5..9ce07806d0e 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -166,6 +166,19 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { treeId, }; } + + async getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const response = await this.instance.call(WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, { + treeId, + revision: this.revision, + leafIndices, + }); + + return response.blockNumbers.map(x => (x === undefined || x === null ? undefined : BigInt(x))); + } } export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 12c4c410edc..6f95755ce7c 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -54,6 +54,7 @@ export enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -139,6 +140,8 @@ export interface TreeDBStats { leafPreimagesDBStats: DBStats; /** Stats for the 'leaf indices' DB */ leafIndicesDBStats: DBStats; + /** Stats for the 'block indices' DB */ + blockIndicesDBStats: DBStats; } export interface WorldStateMeta { @@ -189,6 +192,7 @@ export function buildEmptyTreeDBStats() { leafIndicesDBStats: buildEmptyDBStats(), leafKeysDBStats: buildEmptyDBStats(), leafPreimagesDBStats: buildEmptyDBStats(), + blockIndicesDBStats: buildEmptyDBStats(), } as TreeDBStats; } @@ -271,6 +275,7 @@ export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blocksDBStats = sanitiseDBStats(stats.blocksDBStats); stats.leafIndicesDBStats = sanitiseDBStats(stats.leafIndicesDBStats); stats.leafPreimagesDBStats = sanitiseDBStats(stats.leafPreimagesDBStats); + stats.blockIndicesDBStats = sanitiseDBStats(stats.blockIndicesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); return stats; @@ -344,6 +349,14 @@ interface GetTreeInfoResponse { root: Buffer; } +interface GetBlockNumbersForLeafIndicesRequest extends WithTreeId, WithWorldStateRevision { + leafIndices: bigint[]; +} + +interface GetBlockNumbersForLeafIndicesResponse { + blockNumbers: bigint[]; +} + interface GetSiblingPathRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} type GetSiblingPathResponse = Buffer[]; @@ -446,6 +459,7 @@ export type WorldStateRequest = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafRequest; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageRequest; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesRequest; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; @@ -481,6 +495,7 @@ export type WorldStateResponse = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafResponse; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageResponse; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesResponse; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index d18593fcbf0..91044fdef56 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -490,10 +490,75 @@ describe('NativeWorldState', () => { }); }); + describe('block numbers for indices', () => { + let block: L2Block; + let messages: Fr[]; + let noteHashes: number; + let nullifiers: number; + let publicTree: number; + + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + + it('correctly reports block numbers', async () => { + const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + const statuses = []; + const numBlocks = 2; + const txsPerBlock = 2; + for (let i = 0; i < numBlocks; i++) { + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, txsPerBlock, fork)); + noteHashes = block.body.txEffects[0].noteHashes.length; + nullifiers = block.body.txEffects[0].nullifiers.length; + publicTree = block.body.txEffects[0].publicDataWrites.length; + await fork.close(); + const status = await ws.handleL2BlockAndMessages(block, messages); + statuses.push(status); + } + + const checkTree = async ( + treeId: MerkleTreeId, + itemsLength: number, + blockNumber: number, + initialSize: number, + numPerBlock: number, + ) => { + const before = initialSize + itemsLength * blockNumber * numPerBlock - 2; + const on = before + 1; + const after = on + 1; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + treeId, + [before, on, after].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([blockNumber, blockNumber, blockNumber + 1].map(x => BigInt(x))); + }; + + for (let i = 0; i < numBlocks - 1; i++) { + await checkTree(MerkleTreeId.NOTE_HASH_TREE, noteHashes, i + 1, 0, 2); + await checkTree(MerkleTreeId.NULLIFIER_TREE, nullifiers, i + 1, 128, 2); + await checkTree(MerkleTreeId.PUBLIC_DATA_TREE, publicTree, i + 1, 128, 2); + await checkTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messages.length, i + 1, 0, 1); + } + + const lastStatus = statuses[statuses.length - 1]; + const before = Number(lastStatus.meta.noteHashTreeMeta.committedSize) - 2; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + MerkleTreeId.NOTE_HASH_TREE, + [before, before + 1, before + 2].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([2, 2, undefined].map(x => (x == undefined ? x : BigInt(x)))); + }); + }); + describe('status reporting', () => { let block: L2Block; let messages: Fr[]; + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + it('correctly reports status', async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); const statuses = []; diff --git a/yarn-project/world-state/src/synchronizer/instrumentation.ts b/yarn-project/world-state/src/synchronizer/instrumentation.ts index d52dca0aef4..9b4fb6a3480 100644 --- a/yarn-project/world-state/src/synchronizer/instrumentation.ts +++ b/yarn-project/world-state/src/synchronizer/instrumentation.ts @@ -5,7 +5,7 @@ import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/ import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js'; type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data'; -type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks'; +type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices'; class TreeDBInstrumentation { private dbNumItems: Gauge; @@ -70,6 +70,7 @@ class TreeInstrumentation { this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes')); this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage')); this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices')); + this.treeDbInstrumentation.set('block_indices', new TreeDBInstrumentation(meter, treeName, 'block_indices')); } private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) { @@ -92,6 +93,7 @@ class TreeInstrumentation { this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats); this.updateDBMetrics('blocks', treeDbStats.blocksDBStats); this.updateDBMetrics('nodes', treeDbStats.nodesDBStats); + this.updateDBMetrics('block_indices', treeDbStats.blockIndicesDBStats); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 82f5934fb18..a1b93999290 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -181,6 +181,13 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations throw new Error('Method not implemented in legacy merkle tree'); } + getBlockNumbersForLeafIndices( + _treeId: ID, + _leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + throw new Error('Method not implemented in legacy merkle tree'); + } + close(): Promise { return Promise.resolve(); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 90560ef0960..7f4e4bc9d6a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -101,6 +101,10 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati }; } + getBlockNumbersForLeafIndices(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> { + throw new Error('Not implemented'); + } + async getStateReference(): Promise { const snapshots = await Promise.all([ this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE),