From cf05a7a346ea11853e940d5e9ac105ef0d629d35 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:27:20 +0000 Subject: [PATCH] feat: Allow querying block number for tree indices (#10332) This PR make the following changes. 1. Captures and propagates more of the errors generated in the merkle trees out to the TS interface. 2. Introduces the `block_number_t` typedef within the native world state. 3. Introduces a new DB in the native world state. This DB maps block numbers to the size of the tree at that block. It then uses this DB to fulfill queries looking to identify which block given notes were included within. --- barretenberg/cpp/scripts/merkle_tree_tests.sh | 2 +- .../content_addressed_append_only_tree.hpp | 120 +++++++-- ...ontent_addressed_append_only_tree.test.cpp | 134 +++++++++- .../content_addressed_indexed_tree.hpp | 27 +- .../content_addressed_indexed_tree.test.cpp | 25 +- .../lmdb_store/lmdb_transaction.hpp | 8 + .../lmdb_store/lmdb_tree_store.cpp | 113 +++++++- .../lmdb_store/lmdb_tree_store.hpp | 59 +++- .../lmdb_store/lmdb_tree_store.test.cpp | 253 ++++++++++++++++++ .../crypto/merkle_tree/lmdb_store/queries.hpp | 39 +++ .../cached_content_addressed_tree_store.hpp | 53 +++- .../merkle_tree/node_store/tree_meta.hpp | 43 ++- .../crypto/merkle_tree/response.hpp | 126 ++++++++- .../crypto/merkle_tree/test_fixtures.hpp | 9 +- .../barretenberg/crypto/merkle_tree/types.hpp | 17 +- .../src/barretenberg/world_state/types.hpp | 2 +- .../barretenberg/world_state/world_state.cpp | 191 +++++++++---- .../barretenberg/world_state/world_state.hpp | 88 +++--- .../world_state/world_state.test.cpp | 60 +++++ .../barretenberg/world_state_napi/addon.cpp | 23 ++ .../barretenberg/world_state_napi/addon.hpp | 1 + .../barretenberg/world_state_napi/message.hpp | 16 +- .../aztec-node/src/aztec-node/server.ts | 16 ++ .../src/interfaces/aztec-node.test.ts | 14 + .../src/interfaces/aztec-node.ts | 20 +- .../src/interfaces/merkle_tree_operations.ts | 10 + yarn-project/telemetry-client/src/metrics.ts | 18 ++ .../src/native/merkle_trees_facade.ts | 13 + .../world-state/src/native/message.ts | 15 ++ .../src/native/native_world_state.test.ts | 65 +++++ .../src/synchronizer/instrumentation.ts | 4 +- .../merkle_tree_operations_facade.ts | 7 + .../merkle_tree_snapshot_operations_facade.ts | 4 + 33 files changed, 1402 insertions(+), 193 deletions(-) diff --git a/barretenberg/cpp/scripts/merkle_tree_tests.sh b/barretenberg/cpp/scripts/merkle_tree_tests.sh index f53e2750d5a..9e5e0f0b3c9 100755 --- a/barretenberg/cpp/scripts/merkle_tree_tests.sh +++ b/barretenberg/cpp/scripts/merkle_tree_tests.sh @@ -5,7 +5,7 @@ set -e # run commands relative to parent directory cd $(dirname $0)/.. -DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* +DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* TEST=${1:-$DEFAULT_TESTS} PRESET=${PRESET:-clang16} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 7dc41b0abc1..d645b460708 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -39,16 +39,17 @@ template class ContentAddressedAppendOn using StoreType = Store; // Asynchronous methods accept these callback function types as arguments - using AppendCompletionCallback = std::function&)>; - using MetaDataCallback = std::function&)>; - using HashPathCallback = std::function&)>; - using FindLeafCallback = std::function&)>; - using GetLeafCallback = std::function&)>; + using AppendCompletionCallback = std::function&)>; + using MetaDataCallback = std::function&)>; + using HashPathCallback = std::function&)>; + using FindLeafCallback = std::function&)>; + using GetLeafCallback = std::function&)>; using CommitCallback = std::function&)>; - using RollbackCallback = std::function; + using RollbackCallback = std::function; using RemoveHistoricBlockCallback = std::function&)>; using UnwindBlockCallback = std::function&)>; - using FinaliseBlockCallback = std::function; + using FinaliseBlockCallback = std::function; + using GetBlockForIndexCallback = std::function&)>; // Only construct from provided store and thread pool, no copies or moves ContentAddressedAppendOnlyTree(std::unique_ptr store, @@ -90,7 +91,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes */ void get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const; @@ -131,7 +132,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes * @param on_completion Callback to be called on completion */ - void get_meta_data(const index_t& blockNumber, + void get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const; @@ -151,7 +152,7 @@ template class ContentAddressedAppendOn * @param on_completion Callback to be called on completion */ void get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& completion) const; @@ -164,7 +165,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree */ void find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -181,10 +182,23 @@ template class ContentAddressedAppendOn */ void find_leaf_index_from(const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Returns the block numbers that correspond to the given indices values + */ + void find_block_numbers(const std::vector& indices, const GetBlockForIndexCallback& on_completion) const; + + /** + * @brief Returns the block numbers that correspond to the given indices values, from the perspective of a + * historical block number + */ + void find_block_numbers(const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const; + /** * @brief Commit the tree to the backing store */ @@ -200,11 +214,11 @@ template class ContentAddressedAppendOn */ uint32_t depth() const { return depth_; } - void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); + void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); - void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion); + void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion); - void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion); + void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion); protected: using ReadTransaction = typename Store::ReadTransaction; @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree::get_meta_data(bool in } template -void ContentAddressedAppendOnlyTree::get_meta_data(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const { @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons template void ContentAddressedAppendOnlyTree::get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const { @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons workers_->enqueue(job); } +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + index_t maxIndex = meta.committedSize; + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + BlockPayload blockPayload; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + if (!store_->get_block_data(blockNumber, blockPayload, *tx)) { + throw std::runtime_error(format("Unable to find block numbers for indices for block ", + blockNumber, + ", failed to get block data.")); + } + index_t maxIndex = std::min(meta.committedSize, blockPayload.size); + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + template void ContentAddressedAppendOnlyTree::get_subtree_sibling_path( uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const @@ -473,7 +543,7 @@ std::optional ContentAddressedAppendOnlyTree::find_lea NodePayload nodePayload; bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted); if (!success) { - // std::cout << "No root" << std::endl; + // std::cout << "No root " << hash << std::endl; return std::nullopt; } // std::cout << "Found root at depth " << i << " : " << hash << std::endl; @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ template void ContentAddressedAppendOnlyTree::get_leaf(const index_t& leaf_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& on_completion) const { @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree::find_leaf_index(const template void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -696,7 +766,7 @@ template void ContentAddressedAppendOnlyTree::find_leaf_index_from( const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree::rollback(const Rollba template void ContentAddressedAppendOnlyTree::remove_historic_block( - const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) + const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) { auto job = [=, this]() { execute_and_report( @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree::remove_historic_block } template -void ContentAddressedAppendOnlyTree::unwind_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion) { auto job = [=, this]() { @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree::unwind_block(const in } template -void ContentAddressedAppendOnlyTree::finalise_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion) { auto job = [=, this]() { @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree::add_batch_internal( new_root = new_hash; meta.root = new_hash; meta.size = new_size; - // std::cout << "New size: " << meta.size << std::endl; + // std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl; store_->put_meta(meta); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 5ac60d919b7..83f72c9ca1f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -123,7 +123,7 @@ void check_sibling_path(TreeType& tree, void check_historic_sibling_path(TreeType& tree, index_t index, fr_sibling_path expected_sibling_path, - index_t blockNumber, + block_number_t blockNumber, bool expected_success = true) { Signal signal; @@ -160,7 +160,7 @@ void rollback_tree(TreeType& tree) signal.wait_for_level(); } -void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -171,7 +171,7 @@ void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expe signal.wait_for_level(); } -void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -206,7 +206,7 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } -void finalise_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -312,7 +312,7 @@ void check_leaf( } void check_historic_leaf(TreeType& tree, - const index_t& blockNumber, + const block_number_t& blockNumber, const fr& leaf, index_t leaf_index, bool expected_success, @@ -350,6 +350,31 @@ void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_pat EXPECT_EQ(hash, expected_root); } +void get_blocks_for_indices(TreeType& tree, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + +void get_blocks_for_indices(TreeType& tree, + const block_number_t& blockNumber, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, blockNumber, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -1284,7 +1309,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_da for (uint32_t i = 0; i < historicPathsZeroIndex.size(); i++) { // retrieving historic data should fail if the block is outside of the window - const index_t blockNumber = i + 1; + const block_number_t blockNumber = i + 1; const bool expectedSuccess = expectedBlockHeight <= windowSize || blockNumber > (expectedBlockHeight - windowSize); check_historic_sibling_path(tree, 0, historicPathsZeroIndex[i], blockNumber, expectedSuccess); @@ -1399,7 +1424,7 @@ void test_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); // attempting to unwind a block that is not the tip should fail @@ -1513,6 +1538,101 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_bl } } +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_retrieve_block_numbers_by_index) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + const size_t block_size = 32; + + for (size_t i = 0; i < 5; i++) { + std::vector values = create_values(block_size); + add_values(tree, values); + commit_tree(tree); + } + std::vector indices{ 12, 33, 63, 64, 65, 80, 96, 159, 160 }; + std::vector> blockNumbers; + + // All but the last block number should be valid when looking at latest + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + index_t maxIndex = 5 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now get blocks for indices from the perspective of block 2 + get_blocks_for_indices(tree, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = 2 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + unwind_block(tree, 5); + unwind_block(tree, 4); + + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + maxIndex = 3 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // fork from block 1 + std::unique_ptr forkStore = std::make_unique(name, depth, 1, db); + TreeType treeFork(std::move(forkStore), pool); + + // Now, using the fork, get block indices but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now, using the fork, get block indics from the perspective of block 2, but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks) { std::string name = random_string(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 197f784d59f..45cc232e8e1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -47,14 +47,13 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>&)>; + using AddCompletionCallbackWithWitness = std::function>&)>; using AddSequentiallyCompletionCallbackWithWitness = - std::function>&)>; - using AddCompletionCallback = std::function&)>; + std::function>&)>; + using AddCompletionCallback = std::function&)>; - using LeafCallback = std::function>&)>; - using FindLowLeafCallback = std::function&)>; + using LeafCallback = std::function>&)>; + using FindLowLeafCallback = std::function&)>; ContentAddressedIndexedTree(std::unique_ptr store, std::shared_ptr workers, @@ -151,7 +150,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -169,7 +168,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -178,7 +177,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::get_leaf(const index_t& template void ContentAddressedIndexedTree::get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const LeafCallback& completion) const { @@ -458,7 +457,7 @@ void ContentAddressedIndexedTree::find_leaf_index( template void ContentAddressedIndexedTree::find_leaf_index( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const { @@ -496,7 +495,7 @@ void ContentAddressedIndexedTree::find_leaf_index_from( template void ContentAddressedIndexedTree::find_leaf_index_from( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, const index_t& start_index, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const @@ -559,7 +558,7 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& template void ContentAddressedIndexedTree::find_low_leaf(const fr& leaf_key, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLowLeafCallback& on_completion) const { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 5177b370833..8a7443ef05c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -104,8 +104,11 @@ template void check_root(TypeOfTree& tree, fr expected_roo } template -fr_sibling_path get_historic_sibling_path( - TypeOfTree& tree, index_t blockNumber, index_t index, bool includeUncommitted = true, bool expected_success = true) +fr_sibling_path get_historic_sibling_path(TypeOfTree& tree, + block_number_t blockNumber, + index_t index, + bool includeUncommitted = true, + bool expected_success = true) { fr_sibling_path h; Signal signal; @@ -177,7 +180,7 @@ GetLowIndexedLeafResponse get_low_leaf(TypeOfTree& tree, const LeafValueType& le template GetLowIndexedLeafResponse get_historic_low_leaf(TypeOfTree& tree, - index_t blockNumber, + block_number_t blockNumber, const LeafValueType& leaf, bool includeUncommitted = true) { @@ -236,7 +239,7 @@ void check_find_leaf_index_from(TypeOfTree& tree, template void check_historic_find_leaf_index(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t expected_index, bool expected_success, bool includeUncommitted = true) @@ -257,7 +260,7 @@ void check_historic_find_leaf_index(TypeOfTree& tree, template void check_historic_find_leaf_index_from(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t start_index, index_t expected_index, bool expected_success, @@ -280,7 +283,7 @@ template void check_historic_leaf(TypeOfTree& tree, const LeafValueType& leaf, index_t expected_index, - index_t blockNumber, + block_number_t blockNumber, bool expected_success, bool includeUncommitted = true) { @@ -300,7 +303,7 @@ void check_historic_leaf(TypeOfTree& tree, template void check_historic_sibling_path(TypeOfTree& tree, index_t index, - index_t blockNumber, + block_number_t blockNumber, const fr_sibling_path& expected_sibling_path, bool includeUncommitted = true, bool expected_success = true) @@ -413,7 +416,7 @@ void block_sync_values_sequential(TypeOfTree& tree, } template -void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -425,7 +428,7 @@ void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool ex } template -void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -437,7 +440,7 @@ void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_ } template -void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -2596,7 +2599,7 @@ void test_nullifier_tree_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); unwind_block(tree, blockNumber); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index cf2a55c1285..9bbea8ea42e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -46,6 +46,8 @@ class LMDBTransaction { template bool get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; template bool get_value(T& key, index_t& data, const LMDBDatabase& db) const; @@ -88,6 +90,12 @@ bool LMDBTransaction::get_value_or_previous(T& key, K& data, const LMDBDatabase& return lmdb_queries::get_value_or_previous(key, data, db, *this); } +template +bool LMDBTransaction::get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const +{ + return lmdb_queries::get_value_or_greater(key, data, db, *this); +} + template bool LMDBTransaction::get_value_or_previous(T& key, K& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 3f4f07aa829..09f87cae606 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -55,29 +55,35 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m { LMDBDatabaseCreationTransaction tx(_environment); - _blockDatabase = std::make_unique( - _environment, tx, _name + std::string("blocks"), false, false, block_key_cmp); + _blockDatabase = + std::make_unique(_environment, tx, _name + BLOCKS_DB, false, false, block_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _nodeDatabase = - std::make_unique(_environment, tx, _name + std::string("nodes"), false, false, fr_key_cmp); + _nodeDatabase = std::make_unique(_environment, tx, _name + NODES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafKeyToIndexDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf indices"), false, false, fr_key_cmp); + _leafKeyToIndexDatabase = + std::make_unique(_environment, tx, _name + LEAF_INDICES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafHashToPreImageDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); + _leafHashToPreImageDatabase = + std::make_unique(_environment, tx, _name + LEAF_PREIMAGES_DB, false, false, fr_key_cmp); + tx.commit(); + } + + { + LMDBDatabaseCreationTransaction tx(_environment); + _indexToBlockDatabase = + std::make_unique(_environment, tx, _name + BLOCK_INDICES_DB, false, false, index_key_cmp); tx.commit(); } } @@ -107,9 +113,11 @@ void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) stats.leafIndicesDBStats = DBStats(LEAF_INDICES_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); stats.nodesDBStats = DBStats(NODES_DB, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _indexToBlockDatabase->underlying(), &stat); + stats.blockIndicesDBStats = DBStats(BLOCK_INDICES_DB, stat); } -void LMDBTreeStore::write_block_data(uint64_t blockNumber, +void LMDBTreeStore::write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, LMDBTreeStore::WriteTransaction& tx) { @@ -120,13 +128,15 @@ void LMDBTreeStore::write_block_data(uint64_t blockNumber, tx.put_value(key, encoded, *_blockDatabase); } -void LMDBTreeStore::delete_block_data(uint64_t blockNumber, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::delete_block_data(const block_number_t& blockNumber, LMDBTreeStore::WriteTransaction& tx) { BlockMetaKeyType key(blockNumber); tx.delete_value(key, *_blockDatabase); } -bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockData, LMDBTreeStore::ReadTransaction& tx) +bool LMDBTreeStore::read_block_data(const block_number_t& blockNumber, + BlockPayload& blockData, + LMDBTreeStore::ReadTransaction& tx) { BlockMetaKeyType key(blockNumber); std::vector data; @@ -137,6 +147,87 @@ bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockDat return success; } +void LMDBTreeStore::write_block_index_data(const block_number_t& blockNumber, + const index_t& sizeAtBlock, + WriteTransaction& tx) +{ + // There can be multiple block numbers aganst the same index (zero size blocks) + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Read the block index payload + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + BlockIndexPayload payload; + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + } + + // Double check it's not already present (it shouldn't be) + // We then add the block number and sort + // Sorting shouldn't be necessary as we add blocks in ascending order, but we will make sure + // Sorting here and when we unwind blocks means that looking up the block number for an index becomes O(1) + // These lookups are much more frequent than adds or deletes so we take the hit here + if (!payload.contains(blockNumber)) { + payload.blockNumbers.emplace_back(blockNumber); + payload.sort(); + } + + // Write the new payload back down + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + +bool LMDBTreeStore::find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx) +{ + LeafIndexKeyType key(index + 1); + std::vector data; + // Retrieve the payload + bool success = tx.get_value_or_greater(key, data, *_indexToBlockDatabase); + if (!success) { + return false; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + if (payload.blockNumbers.empty()) { + return false; + } + // The block numbers are sorted so we simply return the lowest + blockNumber = payload.blockNumbers[0]; + return true; +} + +void LMDBTreeStore::delete_block_index(const index_t& sizeAtBlock, + const block_number_t& blockNumber, + WriteTransaction& tx) +{ + // To delete a block number form an index we retieve all the block numbers from that index + // Then we find and remove the block number in question + // Then we write back down + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Retrieve the data + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + if (!success) { + return; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + + payload.delete_block(blockNumber); + + // if it's now empty, delete it + if (payload.blockNumbers.empty()) { + tx.delete_value(key, *_indexToBlockDatabase); + return; + } + // not empty write it back + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + void LMDBTreeStore::write_meta_data(const TreeMeta& metaData, LMDBTreeStore::WriteTransaction& tx) { msgpack::sbuffer buffer; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index ab4dfb7316c..a28ce245967 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -25,7 +25,7 @@ namespace bb::crypto::merkle_tree { struct BlockPayload { index_t size; - index_t blockNumber; + block_number_t blockNumber; fr root; MSGPACK_FIELDS(size, blockNumber, root) @@ -55,6 +55,49 @@ struct NodePayload { return left == other.left && right == other.right && ref == other.ref; } }; + +struct BlockIndexPayload { + std::vector blockNumbers; + + MSGPACK_FIELDS(blockNumbers) + + bool operator==(const BlockIndexPayload& other) const { return blockNumbers == other.blockNumbers; } + + void sort() { std::sort(blockNumbers.begin(), blockNumbers.end()); } + + bool contains(const block_number_t& blockNumber) + { + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return false; + } + return *it == blockNumber; + } + + void delete_block(const block_number_t& blockNumber) + { + if (blockNumbers.empty()) { + return; + } + // shuffle the block number down, removing the one we want to remove and then pop the end item + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return; + } + // It could be a block higher than the one we are looking for + if (*it != blockNumber) { + return; + } + // we have found our block, shuffle blocks after this one down + auto readIt = it + 1; + while (readIt != blockNumbers.end()) { + *it++ = *readIt++; + } + blockNumbers.pop_back(); + } +}; /** * Creates an abstraction against a collection of LMDB databases within a single environment used to store merkle tree * data @@ -78,11 +121,18 @@ class LMDBTreeStore { void get_stats(TreeDBStats& stats, ReadTransaction& tx); - void write_block_data(uint64_t blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + void write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + + bool read_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx); + + void delete_block_data(const block_number_t& blockNumber, WriteTransaction& tx); + + void write_block_index_data(const block_number_t& blockNumber, const index_t& sizeAtBlock, WriteTransaction& tx); - bool read_block_data(uint64_t blockNumber, BlockPayload& blockData, ReadTransaction& tx); + // index here is 0 based + bool find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx); - void delete_block_data(uint64_t blockNumber, WriteTransaction& tx); + void delete_block_index(const index_t& sizeAtBlock, const block_number_t& blockNumber, WriteTransaction& tx); void write_meta_data(const TreeMeta& metaData, WriteTransaction& tx); @@ -136,6 +186,7 @@ class LMDBTreeStore { LMDBDatabase::Ptr _nodeDatabase; LMDBDatabase::Ptr _leafKeyToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; + LMDBDatabase::Ptr _indexToBlockDatabase; template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cb4d342b09a..e9b85aa5bbc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" @@ -255,3 +257,254 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) EXPECT_FALSE(success); } } + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_by_index) +{ + struct BlockAndIndex { + block_number_t blockNumber; + // this block contains leaves up to index (0 based) + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 82 }, + BlockAndIndex{ .blockNumber = 4, .index = 114 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_TRUE(store.find_block_for_index(130, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete the last block + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[4].index + 1, blocks[4].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete non-exisatent indices to check it does nothing + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + store.delete_block_index(21, 1, *transaction); + store.delete_block_index(150, 6, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } +} + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_with_duplicate_indices) +{ + struct BlockAndIndex { + block_number_t blockNumber; + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 60 }, + BlockAndIndex{ .blockNumber = 4, .index = 60 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete block 2 at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, blocks[1].blockNumber, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // try and delete blocks that don't exist at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, 2, *transaction); + store.delete_block_index(blocks[1].index + 1, 5, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp index aa97a2d2518..c26768fa8ec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -177,6 +177,45 @@ bool get_value_or_previous(TKey& key, return success; } +template +bool get_value_or_greater(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +{ + bool success = false; + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize == dbKey.mv_size) { + // key is the same size, so this contains the data we are looking for + copy_to_vector(dbVal, data); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // no key greater than or equal, nothing to extract + } else { + throw_error("get_value_or_greater::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + template void get_all_values_greater_or_equal_key(const TKey& key, std::vector>& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 013f1cc8478..abaec64a3c7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -147,7 +147,7 @@ template class ContentAddressedCachedTreeStore { /** * @brief Reads the tree meta data, including uncommitted data if requested */ - bool get_block_data(const index_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; + bool get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; /** * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. @@ -198,13 +198,15 @@ template class ContentAddressedCachedTreeStore { fr get_current_root(ReadTransaction& tx, bool includeUncommitted) const; - void remove_historical_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); - void unwind_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); std::optional get_fork_block() const; - void advance_finalised_block(const index_t& blockNumber); + void advance_finalised_block(const block_number_t& blockNumber); + + std::optional find_block_for_index(const index_t& index, ReadTransaction& tx) const; private: std::string name_; @@ -233,7 +235,7 @@ template class ContentAddressedCachedTreeStore { void initialise(); - void initialise_from_block(const index_t& blockNumber); + void initialise_from_block(const block_number_t& blockNumber); bool read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const; @@ -258,6 +260,10 @@ template class ContentAddressedCachedTreeStore { void extract_db_stats(TreeDBStats& stats); + void persist_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + + void delete_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + index_t constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const; WriteTransactionPtr create_write_transaction() const { return dataStore_->create_write_transaction(); } @@ -304,6 +310,31 @@ index_t ContentAddressedCachedTreeStore::constrain_tree_size(cons return sizeLimit; } +template +std::optional ContentAddressedCachedTreeStore::find_block_for_index( + const index_t& index, ReadTransaction& tx) const +{ + block_number_t blockNumber = 0; + bool success = dataStore_->find_block_for_index(index, blockNumber, tx); + return success ? std::make_optional(blockNumber) : std::nullopt; +} + +template +void ContentAddressedCachedTreeStore::persist_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->write_block_index_data(blockNumber, index, tx); +} + +template +void ContentAddressedCachedTreeStore::delete_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->delete_block_index(index, blockNumber, tx); +} + template std::pair ContentAddressedCachedTreeStore::find_low_value( const fr& new_leaf_key, const RequestContext& requestContext, ReadTransaction& tx) const @@ -553,7 +584,7 @@ void ContentAddressedCachedTreeStore::get_meta(TreeMeta& m, } template -bool ContentAddressedCachedTreeStore::get_block_data(const index_t& blockNumber, +bool ContentAddressedCachedTreeStore::get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const { @@ -649,6 +680,7 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, .blockNumber = uncommittedMeta.unfinalisedBlockHeight, .root = uncommittedMeta.root }; dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx); + dataStore_->write_block_index_data(block.blockNumber, block.size, *tx); } uncommittedMeta.committedSize = uncommittedMeta.size; @@ -767,7 +799,7 @@ void ContentAddressedCachedTreeStore::persist_meta(TreeMeta& m, W } template -void ContentAddressedCachedTreeStore::advance_finalised_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::advance_finalised_block(const block_number_t& blockNumber) { TreeMeta committedMeta; TreeMeta uncommittedMeta; @@ -827,7 +859,7 @@ void ContentAddressedCachedTreeStore::advance_finalised_block(con } template -void ContentAddressedCachedTreeStore::unwind_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -894,6 +926,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); // remove the block from the block data table dataStore_->delete_block_data(blockNumber, *writeTx); + dataStore_->delete_block_index(blockData.size, blockData.blockNumber, *writeTx); uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; uncommittedMeta.size = previousBlockData.size; uncommittedMeta.committedSize = previousBlockData.size; @@ -916,7 +949,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& } template -void ContentAddressedCachedTreeStore::remove_historical_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -1107,7 +1140,7 @@ template void ContentAddressedCachedTreeStore -void ContentAddressedCachedTreeStore::initialise_from_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::initialise_from_block(const block_number_t& blockNumber) { // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during // construction then we throw diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index 164a6b254cf..6b77a6a5ebd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { @@ -16,9 +17,9 @@ struct TreeMeta { bb::fr root; index_t initialSize; bb::fr initialRoot; - uint64_t oldestHistoricBlock; - uint64_t unfinalisedBlockHeight; - uint64_t finalisedBlockHeight; + block_number_t oldestHistoricBlock; + block_number_t unfinalisedBlockHeight; + block_number_t finalisedBlockHeight; MSGPACK_FIELDS(name, depth, @@ -31,6 +32,34 @@ struct TreeMeta { unfinalisedBlockHeight, finalisedBlockHeight) + TreeMeta(std::string n, + uint32_t d, + const index_t& s, + const index_t& c, + const bb::fr& r, + const index_t& is, + const bb::fr& ir, + const block_number_t& o, + const block_number_t& u, + const block_number_t& f) + : name(std::move(n)) + , depth(d) + , size(s) + , committedSize(c) + , root(r) + , initialSize(is) + , initialRoot(ir) + , oldestHistoricBlock(o) + , unfinalisedBlockHeight(u) + , finalisedBlockHeight(f) + {} + TreeMeta() = default; + ~TreeMeta() = default; + TreeMeta(const TreeMeta& other) = default; + TreeMeta(TreeMeta&& other) noexcept { *this = std::move(other); } + TreeMeta& operator=(const TreeMeta& other) = default; + TreeMeta& operator=(TreeMeta&& other) noexcept = default; + bool operator==(const TreeMeta& other) const { return name == other.name && depth == other.depth && size == other.size && @@ -50,12 +79,4 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) return os; } -struct LeavesMeta { - index_t size; - - MSGPACK_FIELDS(size) - - bool operator==(const LeavesMeta& other) const { return size == other.size; } -}; - } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 5fdfd19b85b..d525acb8672 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -12,19 +12,41 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { struct TreeMetaResponse { TreeMeta meta; + + TreeMetaResponse() = default; + ~TreeMetaResponse() = default; + TreeMetaResponse(const TreeMetaResponse& other) = default; + TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; + TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; + TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; }; struct AddDataResponse { index_t size; fr root; + + AddDataResponse() = default; + ~AddDataResponse() = default; + AddDataResponse(const AddDataResponse& other) = default; + AddDataResponse(AddDataResponse&& other) noexcept = default; + AddDataResponse& operator=(const AddDataResponse& other) = default; + AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; }; struct GetSiblingPathResponse { fr_sibling_path path; + + GetSiblingPathResponse() = default; + ~GetSiblingPathResponse() = default; + GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; + GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; }; template struct LeafUpdateWitnessData { @@ -32,6 +54,18 @@ template struct LeafUpdateWitnessData { index_t index; fr_sibling_path path; + LeafUpdateWitnessData(const IndexedLeaf& l, const index_t& i, fr_sibling_path p) + : leaf(l) + , index(i) + , path(std::move(p)) + {} + LeafUpdateWitnessData() = default; + ~LeafUpdateWitnessData() = default; + LeafUpdateWitnessData(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData(LeafUpdateWitnessData&& other) noexcept = default; + LeafUpdateWitnessData& operator=(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; + MSGPACK_FIELDS(leaf, index, path); }; @@ -40,20 +74,59 @@ template struct AddIndexedDataResponse { fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; std::shared_ptr>> low_leaf_witness_data; + + AddIndexedDataResponse() = default; + ~AddIndexedDataResponse() = default; + AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; + AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; }; template struct AddIndexedDataSequentiallyResponse { AddDataResponse add_data_result; std::shared_ptr>> low_leaf_witness_data; std::shared_ptr>> insertion_witness_data; + + AddIndexedDataSequentiallyResponse() = default; + ~AddIndexedDataSequentiallyResponse() = default; + AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; + AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; +}; + +struct BlockForIndexResponse { + std::vector> blockNumbers; + + BlockForIndexResponse() = default; + ~BlockForIndexResponse() = default; + BlockForIndexResponse(const BlockForIndexResponse& other) = default; + BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; + BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; + BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; }; struct FindLeafIndexResponse { index_t leaf_index; + + FindLeafIndexResponse() = default; + ~FindLeafIndexResponse() = default; + FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; + FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; }; struct GetLeafResponse { std::optional leaf; + + GetLeafResponse() = default; + ~GetLeafResponse() = default; + GetLeafResponse(const GetLeafResponse& other) = default; + GetLeafResponse(GetLeafResponse&& other) noexcept = default; + GetLeafResponse& operator=(const GetLeafResponse& other) = default; + GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; }; template struct GetIndexedLeafResponse { @@ -66,6 +139,17 @@ struct GetLowIndexedLeafResponse { MSGPACK_FIELDS(is_already_present, index); + GetLowIndexedLeafResponse(bool p, const index_t& i) + : is_already_present(p) + , index(i) + {} + GetLowIndexedLeafResponse() = default; + ~GetLowIndexedLeafResponse() = default; + GetLowIndexedLeafResponse(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse(GetLowIndexedLeafResponse&& other) noexcept = default; + GetLowIndexedLeafResponse& operator=(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse& operator=(GetLowIndexedLeafResponse&& other) noexcept = default; + bool operator==(const GetLowIndexedLeafResponse& other) const { return is_already_present == other.is_already_present && index == other.index; @@ -75,27 +159,66 @@ struct GetLowIndexedLeafResponse { struct CommitResponse { TreeMeta meta; TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse& other) = default; + CommitResponse(CommitResponse&& other) noexcept = default; + CommitResponse& operator=(const CommitResponse& other) = default; + CommitResponse& operator=(CommitResponse&& other) noexcept = default; }; struct UnwindResponse { TreeMeta meta; TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse& other) = default; + UnwindResponse(UnwindResponse&& other) noexcept = default; + UnwindResponse& operator=(const UnwindResponse& other) = default; + UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; }; struct RemoveHistoricResponse { TreeMeta meta; TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; + RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; }; template struct TypedResponse { ResponseType inner; bool success{ true }; std::string message; + + TypedResponse() = default; + ~TypedResponse() = default; + TypedResponse(const TypedResponse& other) = default; + TypedResponse(TypedResponse&& other) noexcept = default; + TypedResponse& operator=(const TypedResponse& other) = default; + TypedResponse& operator=(TypedResponse&& other) noexcept = default; }; struct Response { bool success; std::string message; + + Response(bool s, std::string m) + : success(s) + , message(std::move(m)) + {} + Response() = default; + ~Response() = default; + Response(const Response& other) = default; + Response(Response&& other) noexcept = default; + Response& operator=(const Response& other) = default; + Response& operator=(Response&& other) noexcept = default; }; template @@ -116,8 +239,7 @@ void execute_and_report(const std::function&)>& } } -inline void execute_and_report(const std::function& f, - const std::function& on_completion) +inline void execute_and_report(const std::function& f, const std::function& on_completion) { Response response{ true, "" }; try { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index 1aa333d061b..354ba949c20 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -10,7 +10,10 @@ namespace bb::crypto::merkle_tree { -void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess) +void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, + block_number_t blockNumber, + fr root, + bool expectedSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -25,7 +28,7 @@ void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t block } void inline check_block_and_root_data( - LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) + LMDBTreeStore::SharedPtr db, block_number_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -40,7 +43,7 @@ void inline check_block_and_root_data( } void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, - index_t blockNumber, + block_number_t blockNumber, index_t expectedSize, bool expectedSuccess) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 8d9c5de4933..c8ce520fb9c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -6,10 +6,11 @@ #include namespace bb::crypto::merkle_tree { using index_t = uint64_t; +using block_number_t = uint64_t; struct RequestContext { bool includeUncommitted; - std::optional blockNumber; + std::optional blockNumber; bb::fr root; }; @@ -17,6 +18,7 @@ const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; +const std::string BLOCK_INDICES_DB = "block indices"; struct DBStats { std::string name; @@ -71,6 +73,7 @@ struct TreeDBStats { DBStats nodesDBStats; DBStats leafPreimagesDBStats; DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; TreeDBStats() = default; TreeDBStats(uint64_t mapSize) @@ -80,24 +83,27 @@ struct TreeDBStats { const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats) + const DBStats& leafIndicesStats, + const DBStats& blockIndicesStats) : mapSize(mapSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) , leafIndicesDBStats(leafIndicesStats) + , blockIndicesDBStats(blockIndicesStats) {} TreeDBStats(const TreeDBStats& other) = default; TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats) + MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats, blockIndicesDBStats) bool operator==(const TreeDBStats& other) const { return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats; + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; } TreeDBStats& operator=(TreeDBStats&& other) noexcept @@ -108,6 +114,7 @@ struct TreeDBStats { nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); } return *this; } @@ -118,7 +125,7 @@ struct TreeDBStats { { os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " - << stats.leafIndicesDBStats; + << stats.leafIndicesDBStats << ", Block Indices DB " << stats.blockIndicesDBStats; return os; } }; diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp index 34e74a0631c..f5fbbcbf257 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -32,7 +32,7 @@ using StateReference = std::unordered_map; struct WorldStateRevision { index_t forkId{ 0 }; - index_t blockNumber{ 0 }; + block_number_t blockNumber{ 0 }; bool includeUncommitted{ false }; MSGPACK_FIELDS(forkId, blockNumber, includeUncommitted) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 9cb3b36bf6c..7e74fe44896 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -142,7 +142,7 @@ Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const } uint64_t WorldState::create_fork(const std::optional& blockNumber) { - index_t blockNumberForFork = 0; + block_number_t blockNumberForFork = 0; if (!blockNumber.has_value()) { // we are forking at latest WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; @@ -159,7 +159,7 @@ uint64_t WorldState::create_fork(const std::optional& blockNumber) return forkId; } -void WorldState::remove_forks_for_block(const index_t& blockNumber) +void WorldState::remove_forks_for_block(const block_number_t& blockNumber) { // capture the shared pointers outside of the lock scope so we are not under the lock when the objects are destroyed std::vector forks; @@ -191,7 +191,7 @@ void WorldState::delete_fork(const uint64_t& forkId) } } -Fork::SharedPtr WorldState::create_new_fork(const index_t& blockNumber) +Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); fork->_blockNumber = blockNumber; @@ -241,10 +241,10 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M return std::visit( [=](auto&& wrapper) { Signal signal(1); - TreeMetaResponse response; + TypedResponse local; - auto callback = [&](const TypedResponse& meta) { - response = meta.inner; + auto callback = [&](TypedResponse& meta) { + local = std::move(meta); signal.signal_level(0); }; @@ -255,12 +255,15 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M } signal.wait_for_level(0); - return response; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner; }, fork->_trees.at(tree_id)); } -void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const +void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const { Fork::SharedPtr fork = retrieve_fork(revision.forkId); @@ -271,13 +274,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra Signal signal(static_cast(tree_ids.size())); std::mutex mutex; + std::unordered_map> local; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &responses, &mutex, id](const TypedResponse& meta) { + auto callback = [&signal, &local, &mutex, id](TypedResponse& meta) { { std::lock_guard lock(mutex); - responses[id] = meta.inner.meta; + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -293,6 +297,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + responses[tree_id] = std::move(m.inner.meta); + } } StateReference WorldState::get_state_reference(const WorldStateRevision& revision) const @@ -324,19 +336,15 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio Signal signal(static_cast(tree_ids.size())); StateReference state_reference; + std::unordered_map> local; std::mutex state_ref_mutex; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &state_reference, &state_ref_mutex, initial_state, id]( - const TypedResponse& meta) { + auto callback = [&signal, &local, &state_ref_mutex, id](TypedResponse& meta) { { std::lock_guard lock(state_ref_mutex); - if (initial_state) { - state_reference.insert({ id, { meta.inner.meta.initialRoot, meta.inner.meta.initialSize } }); - } else { - state_reference.insert({ id, { meta.inner.meta.root, meta.inner.meta.size } }); - } + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -352,6 +360,19 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + if (initial_state) { + state_reference[tree_id] = std::make_pair(m.inner.meta.initialRoot, m.inner.meta.initialSize); + continue; + } + state_reference[tree_id] = std::make_pair(m.inner.meta.root, m.inner.meta.size); + } + return state_reference; } @@ -364,10 +385,10 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, return std::visit( [leaf_index, revision](auto&& wrapper) { Signal signal(1); - fr_sibling_path path; + TypedResponse local; - auto callback = [&signal, &path](const TypedResponse& response) { - path = response.inner.path; + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; @@ -378,7 +399,42 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, } signal.wait_for_level(0); - return path; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner.path; + }, + fork->_trees.at(tree_id)); +} + +void WorldState::get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const +{ + Fork::SharedPtr fork = retrieve_fork(revision.forkId); + + std::visit( + [&leafIndices, revision, &blockNumbers](auto&& wrapper) { + Signal signal(1); + TypedResponse local; + + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); + signal.signal_level(); + }; + + if (revision.blockNumber) { + wrapper.tree->find_block_numbers(leafIndices, revision.blockNumber, callback); + } else { + wrapper.tree->find_block_numbers(leafIndices, callback); + } + signal.wait_for_level(0); + + if (!local.success) { + throw std::runtime_error(local.message); + } + blockNumbers = std::move(local.inner.blockNumbers); }, fork->_trees.at(tree_id)); } @@ -534,7 +590,15 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re { auto& wrapper = std::get>(fork->_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); - PublicDataTree::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_decrement(); }; + PublicDataTree::AddCompletionCallback completion = [&](const auto& resp) -> void { + // take the first error + bool expected = true; + if (!resp.success && success.compare_exchange_strong(expected, false)) { + err_message = resp.message; + } + + signal.signal_decrement(); + }; wrapper.tree->add_or_update_values_sequentially(public_writes, completion); } @@ -566,9 +630,9 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi { Fork::SharedPtr fork = retrieve_fork(revision.forkId); Signal signal; - GetLowIndexedLeafResponse low_leaf_info; - auto callback = [&signal, &low_leaf_info](const TypedResponse& response) { - low_leaf_info = response.inner; + TypedResponse low_leaf_info; + auto callback = [&signal, &low_leaf_info](TypedResponse& response) { + low_leaf_info = std::move(response); signal.signal_level(); }; @@ -591,7 +655,11 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi } signal.wait_for_level(); - return low_leaf_info; + + if (!low_leaf_info.success) { + throw std::runtime_error(low_leaf_info.message); + } + return low_leaf_info.inner; } WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockNumber) @@ -599,11 +667,13 @@ WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockN WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); if (toBlockNumber <= archive_state.meta.finalisedBlockHeight) { - throw std::runtime_error("Unable to finalise block, already finalised"); - } - if (!set_finalised_block(toBlockNumber)) { - throw std::runtime_error("Failed to set finalised block"); + throw std::runtime_error(format("Unable to finalise blocks to block number ", + toBlockNumber, + ", current finalised block: ", + archive_state.meta.finalisedBlockHeight)); } + // This will throw if it fails + set_finalised_block(toBlockNumber); WorldStateStatusSummary status; get_status_summary(status); return status; @@ -616,10 +686,10 @@ WorldStateStatusFull WorldState::unwind_blocks(const index_t& toBlockNumber) throw std::runtime_error("Unable to unwind block, block not found"); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; blockNumber--) { - if (!unwind_block(blockNumber, status)) { - throw std::runtime_error("Failed to unwind block"); - } + for (block_number_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; + blockNumber--) { + // This will throw if it fails + unwind_block(blockNumber, status); } populate_status_summary(status); return status; @@ -635,35 +705,43 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock archive_state.meta.oldestHistoricBlock)); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { - if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error(format( - "Failed to remove historical block ", blockNumber, " when removing blocks up to ", toBlockNumber)); - } + for (block_number_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; + blockNumber++) { + // This will throw if it fails + remove_historical_block(blockNumber, status); } populate_status_summary(status); return status; } -bool WorldState::set_finalised_block(const index_t& blockNumber) +bool WorldState::set_finalised_block(const block_number_t& blockNumber) { - std::atomic_bool success = true; Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); Signal signal(static_cast(fork->_trees.size())); + std::array local; + std::mutex mtx; for (auto& [id, tree] : fork->_trees) { std::visit( - [&signal, &success, blockNumber](auto&& wrapper) { - wrapper.tree->finalise_block(blockNumber, [&signal, &success](const Response& resp) { - success = success && resp.success; + [&signal, &local, blockNumber, id, &mtx](auto&& wrapper) { + wrapper.tree->finalise_block(blockNumber, [&signal, &local, &mtx, id](Response& resp) { + { + std::lock_guard lock(mtx); + local[id] = std::move(resp); + } signal.signal_decrement(); }); }, tree); } signal.wait_for_level(); - return success; + for (auto& m : local) { + if (!m.success) { + throw std::runtime_error(m.message); + } + } + return true; } -bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -723,10 +801,13 @@ bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } -bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -786,8 +867,11 @@ bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateS blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) @@ -829,7 +913,12 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { - std::optional leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + std::optional leaf_index = std::nullopt; + + try { + leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + } catch (std::runtime_error&) { + } if (!leaf_index.has_value()) { return false; @@ -887,7 +976,7 @@ void WorldState::validate_trees_are_equally_synched() bool WorldState::determine_if_synched(std::array& metaResponses) { - index_t blockNumber = metaResponses[0].unfinalisedBlockHeight; + block_number_t blockNumber = metaResponses[0].unfinalisedBlockHeight; for (size_t i = 1; i < metaResponses.size(); i++) { if (blockNumber != metaResponses[i].unfinalisedBlockHeight) { return false; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 5afa4a010ef..c66412aae77 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -107,6 +107,11 @@ class WorldState { MerkleTreeId tree_id, index_t leaf_index) const; + void get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const; + /** * @brief Get the leaf preimage object * @@ -259,12 +264,12 @@ class WorldState { uint64_t maxReaders); Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; - Fork::SharedPtr create_new_fork(const index_t& blockNumber); - void remove_forks_for_block(const index_t& blockNumber); + Fork::SharedPtr create_new_fork(const block_number_t& blockNumber); + void remove_forks_for_block(const block_number_t& blockNumber); - bool unwind_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool set_finalised_block(const index_t& blockNumber); + bool unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool set_finalised_block(const block_number_t& blockNumber); void get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const; @@ -304,7 +309,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); template void remove_historic_block_for_tree(TreeDBStats& dbStats, @@ -313,7 +318,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); }; template @@ -342,7 +347,7 @@ void WorldState::unwind_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.unwind_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -362,7 +367,7 @@ void WorldState::remove_historic_block_for_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.remove_historic_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -384,15 +389,13 @@ std::optional> WorldState::get_indexed_leaf( using Tree = ContentAddressedIndexedTree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse> local; if (auto* const wrapper = std::get_if>(&fork->_trees.at(id))) { - std::optional> value; - Signal signal; - auto callback = [&](const TypedResponse>& response) { - if (response.inner.indexed_leaf.has_value()) { - value = response.inner.indexed_leaf; - } + Signal signal; + auto callback = [&](TypedResponse>& response) { + local = std::move(response); signal.signal_level(0); }; @@ -403,7 +406,11 @@ std::optional> WorldState::get_indexed_leaf( } signal.wait_for_level(); - return value; + if (!local.success) { + throw std::runtime_error("Failed to find indexed leaf: " + local.message); + } + + return local.inner.indexed_leaf; } throw std::runtime_error("Invalid tree type"); @@ -419,12 +426,17 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, Fork::SharedPtr fork = retrieve_fork(revision.forkId); std::optional leaf; + bool success = true; + std::string error_msg; Signal signal; if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse& resp) { - if (resp.inner.leaf.has_value()) { - leaf = resp.inner.leaf.value(); + auto callback = [&signal, &leaf, &success, &error_msg](const TypedResponse& response) { + if (!response.success || !response.inner.leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.leaf; } signal.signal_level(); }; @@ -439,12 +451,16 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, using Tree = ContentAddressedIndexedTree; auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse>& resp) { - if (resp.inner.indexed_leaf.has_value()) { - leaf = resp.inner.indexed_leaf.value().value; - } - signal.signal_level(); - }; + auto callback = + [&signal, &leaf, &success, &error_msg](const TypedResponse>& response) { + if (!response.success || !response.inner.indexed_leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.indexed_leaf.value().value; + } + signal.signal_level(); + }; if (revision.blockNumber) { wrapper.tree->get_leaf(leaf_index, revision.blockNumber, revision.includeUncommitted, callback); @@ -454,6 +470,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, } signal.wait_for_level(); + return leaf; } @@ -464,15 +481,13 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev index_t start_index) const { using namespace crypto::merkle_tree; - std::optional index; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse local; Signal signal; - auto callback = [&](const TypedResponse& response) { - if (response.success) { - index = response.inner.leaf_index; - } + auto callback = [&](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; if constexpr (std::is_same_v) { @@ -496,7 +511,12 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev } signal.wait_for_level(0); - return index; + + if (!local.success) { + return std::nullopt; + } + + return local.inner.leaf_index; } template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) @@ -537,7 +557,7 @@ template void WorldState::append_leaves(MerkleTreeId id, const std: signal.wait_for_level(0); if (!success) { - throw std::runtime_error("Failed to append leaves: " + error_msg); + throw std::runtime_error(error_msg); } } @@ -576,7 +596,7 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to batch insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; @@ -615,7 +635,7 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to sequentially insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 024ffbf4ac0..a5ced2921ad 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -1,6 +1,7 @@ #include "barretenberg/world_state/world_state.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" @@ -11,6 +12,7 @@ #include #include #include +#include #include #include @@ -79,6 +81,10 @@ void assert_leaf_index( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) { std::optional index = ws.find_leaf_index(revision, tree_id, value); + EXPECT_TRUE(index.has_value()); + if (!index.has_value()) { + return; + } EXPECT_EQ(index.value(), expected_index); } @@ -363,6 +369,7 @@ TEST_F(WorldStateTest, NullifierTree) auto test_leaf = ws.get_indexed_leaf(WorldStateRevision::committed(), tree_id, 128); // at this point 142 should be the biggest leaf so it wraps back to 0 + EXPECT_TRUE(test_leaf.has_value()); EXPECT_EQ(test_leaf.value(), IndexedLeaf(test_nullifier, 0, 0)); auto predecessor_of_142_again = @@ -524,6 +531,19 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) for (const auto& [tree_id, snapshot] : block_state_ref) { EXPECT_EQ(state_ref.at(tree_id), snapshot); } + + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, { 0 }, blockNumbers); + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_EQ(blockNumbers[0], 1); + + EXPECT_THROW(ws.get_block_numbers_for_leaf_indices( + WorldStateRevision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 2, .includeUncommitted = false }, + MerkleTreeId::NOTE_HASH_TREE, + { 0 }, + blockNumbers), + std::runtime_error); } TEST_F(WorldStateTest, SyncBlockFromDirtyState) @@ -789,3 +809,43 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); } + +TEST_F(WorldStateTest, GetBlockForIndex) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + // bb::fr block_hash(1); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x187a19972150cd1e76d8201d720da7682fcf4d93ec6a3c7b0d84bbefde5bd927"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x2467e5f90736b4ea977e7d21cfb3714181e16b7d6cd867768b59e2ea90fa3eaf"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x24ffd0fab86555ab2e86cffc706d4cfb4b8c405c3966af805de954504ffc27ac"), 1 } }, + }; + + WorldStateStatusFull status = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + WorldStateStatusSummary expected{ 1, 0, 1, true }; + EXPECT_EQ(status.summary, expected); + + StateReference state_ref = ws.get_state_reference(WorldStateRevision::committed()); + + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, + MerkleTreeId::NOTE_HASH_TREE, + MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + }; + + for (const auto& id : tree_ids) { + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), id, { state_ref[id].second - 1 }, blockNumbers); + + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_TRUE(blockNumbers[0].has_value()); + EXPECT_EQ(blockNumbers[0].value(), 1); + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index d8ac68d0db1..f3290da5e9e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -150,6 +150,11 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::GET_SIBLING_PATH, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); + _dispatcher.registerTarget(WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { + return get_block_numbers_for_leaf_indices(obj, buffer); + }); + _dispatcher.registerTarget( WorldStateMessageType::FIND_LEAF_INDEX, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); @@ -387,6 +392,24 @@ bool WorldStateAddon::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& b return true; } +bool WorldStateAddon::get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + GetBlockNumbersForLeafIndicesResponse response; + _ws->get_block_numbers_for_leaf_indices( + request.value.revision, request.value.treeId, request.value.leafIndices, response.blockNumbers); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, header, response); + + msgpack::pack(buffer, resp_msg); + + return true; +} + bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const { TypedMessage request; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 1a077a946e9..d0b33be2532 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -38,6 +38,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index c876f5993bc..b98a8c6a69d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -1,5 +1,6 @@ #pragma once #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" @@ -20,6 +21,7 @@ enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -54,7 +56,7 @@ struct TreeIdOnlyRequest { struct CreateForkRequest { bool latest; - index_t blockNumber; + block_number_t blockNumber; MSGPACK_FIELDS(latest, blockNumber); }; @@ -129,6 +131,18 @@ struct GetSiblingPathRequest { MSGPACK_FIELDS(treeId, revision, leafIndex); }; +struct GetBlockNumbersForLeafIndicesRequest { + MerkleTreeId treeId; + WorldStateRevision revision; + std::vector leafIndices; + MSGPACK_FIELDS(treeId, revision, leafIndices); +}; + +struct GetBlockNumbersForLeafIndicesResponse { + std::vector> blockNumbers; + MSGPACK_FIELDS(blockNumbers); +}; + template struct FindLeafIndexRequest { MerkleTreeId treeId; WorldStateRevision revision; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 57f576c55d3..4434785d41c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -435,6 +435,22 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + /** + * Find the block numbers of the given leaf indices in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + public async findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const committedDb = await this.#getWorldState(blockNumber); + return await committedDb.getBlockNumbersForLeafIndices(treeId, leafIndices); + } + public async findNullifiersIndexesWithBlock( blockNumber: L2BlockNumber, nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 4e3f9bd4a8d..c2db85e5250 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -91,6 +91,11 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); + it('findBlockNumbersForIndexes', async () => { + const response = await context.client.findBlockNumbersForIndexes(1, MerkleTreeId.ARCHIVE, [5n, 58n]); + expect(response).toEqual([3n, 9n]); + }); + it('findNullifiersIndexesWithBlock', async () => { const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); expect(response).toEqual([ @@ -349,6 +354,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + + findBlockNumbersForIndexes( + _blockNumber: number | 'latest', + _treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + expect(leafIndices).toEqual([5n, 58n]); + return Promise.resolve([3n, 9n]); + } findNullifiersIndexesWithBlock( blockNumber: number | 'latest', nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 8e5b2bda55f..457d188acf0 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -61,7 +61,7 @@ export interface AztecNode * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data * @param treeId - The tree to search in. - * @param leafValue - The values to search for + * @param leafValues - The values to search for * @returns The indexes of the given leaves in the given tree or undefined if not found. */ findLeavesIndexes( @@ -70,6 +70,19 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Find the indexes of the given leaves in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; + /** * Returns the indexes of the given nullifiers in the nullifier tree, * scoped to the block they were included in. @@ -430,6 +443,11 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findBlockNumbersForIndexes: z + .function() + .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.BigInt)) + .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z .function() .args(L2BlockNumberSchema, z.array(schemas.Fr)) diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 8520da7ca55..9017c1a6a84 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -199,6 +199,16 @@ export interface MerkleTreeReadOperations { treeId: ID, index: bigint, ): Promise | undefined>; + + /** + * Get the block numbers for a set of leaf indices + * @param treeId - The tree for which the block numbers should be returned. + * @param leafIndices - The indices to be queried. + */ + getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; } export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 01f3e483497..853ce0bb58f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -173,6 +173,24 @@ export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state. export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message'; export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER = + 'aztec.world_state.db_used_size.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA = + 'aztec.world_state.db_used_size.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH = + 'aztec.world_state.db_used_size.block_indices.note_hash'; + +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER = + 'aztec.world_state.db_num_items.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = + 'aztec.world_state.db_num_items.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH = + 'aztec.world_state.db_num_items.block_indices.note_hash'; + export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 49d53e8f3a5..9ce07806d0e 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -166,6 +166,19 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { treeId, }; } + + async getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const response = await this.instance.call(WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, { + treeId, + revision: this.revision, + leafIndices, + }); + + return response.blockNumbers.map(x => (x === undefined || x === null ? undefined : BigInt(x))); + } } export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 12c4c410edc..6f95755ce7c 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -54,6 +54,7 @@ export enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -139,6 +140,8 @@ export interface TreeDBStats { leafPreimagesDBStats: DBStats; /** Stats for the 'leaf indices' DB */ leafIndicesDBStats: DBStats; + /** Stats for the 'block indices' DB */ + blockIndicesDBStats: DBStats; } export interface WorldStateMeta { @@ -189,6 +192,7 @@ export function buildEmptyTreeDBStats() { leafIndicesDBStats: buildEmptyDBStats(), leafKeysDBStats: buildEmptyDBStats(), leafPreimagesDBStats: buildEmptyDBStats(), + blockIndicesDBStats: buildEmptyDBStats(), } as TreeDBStats; } @@ -271,6 +275,7 @@ export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blocksDBStats = sanitiseDBStats(stats.blocksDBStats); stats.leafIndicesDBStats = sanitiseDBStats(stats.leafIndicesDBStats); stats.leafPreimagesDBStats = sanitiseDBStats(stats.leafPreimagesDBStats); + stats.blockIndicesDBStats = sanitiseDBStats(stats.blockIndicesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); return stats; @@ -344,6 +349,14 @@ interface GetTreeInfoResponse { root: Buffer; } +interface GetBlockNumbersForLeafIndicesRequest extends WithTreeId, WithWorldStateRevision { + leafIndices: bigint[]; +} + +interface GetBlockNumbersForLeafIndicesResponse { + blockNumbers: bigint[]; +} + interface GetSiblingPathRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} type GetSiblingPathResponse = Buffer[]; @@ -446,6 +459,7 @@ export type WorldStateRequest = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafRequest; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageRequest; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesRequest; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; @@ -481,6 +495,7 @@ export type WorldStateResponse = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafResponse; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageResponse; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesResponse; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index d18593fcbf0..91044fdef56 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -490,10 +490,75 @@ describe('NativeWorldState', () => { }); }); + describe('block numbers for indices', () => { + let block: L2Block; + let messages: Fr[]; + let noteHashes: number; + let nullifiers: number; + let publicTree: number; + + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + + it('correctly reports block numbers', async () => { + const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + const statuses = []; + const numBlocks = 2; + const txsPerBlock = 2; + for (let i = 0; i < numBlocks; i++) { + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, txsPerBlock, fork)); + noteHashes = block.body.txEffects[0].noteHashes.length; + nullifiers = block.body.txEffects[0].nullifiers.length; + publicTree = block.body.txEffects[0].publicDataWrites.length; + await fork.close(); + const status = await ws.handleL2BlockAndMessages(block, messages); + statuses.push(status); + } + + const checkTree = async ( + treeId: MerkleTreeId, + itemsLength: number, + blockNumber: number, + initialSize: number, + numPerBlock: number, + ) => { + const before = initialSize + itemsLength * blockNumber * numPerBlock - 2; + const on = before + 1; + const after = on + 1; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + treeId, + [before, on, after].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([blockNumber, blockNumber, blockNumber + 1].map(x => BigInt(x))); + }; + + for (let i = 0; i < numBlocks - 1; i++) { + await checkTree(MerkleTreeId.NOTE_HASH_TREE, noteHashes, i + 1, 0, 2); + await checkTree(MerkleTreeId.NULLIFIER_TREE, nullifiers, i + 1, 128, 2); + await checkTree(MerkleTreeId.PUBLIC_DATA_TREE, publicTree, i + 1, 128, 2); + await checkTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messages.length, i + 1, 0, 1); + } + + const lastStatus = statuses[statuses.length - 1]; + const before = Number(lastStatus.meta.noteHashTreeMeta.committedSize) - 2; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + MerkleTreeId.NOTE_HASH_TREE, + [before, before + 1, before + 2].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([2, 2, undefined].map(x => (x == undefined ? x : BigInt(x)))); + }); + }); + describe('status reporting', () => { let block: L2Block; let messages: Fr[]; + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + it('correctly reports status', async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); const statuses = []; diff --git a/yarn-project/world-state/src/synchronizer/instrumentation.ts b/yarn-project/world-state/src/synchronizer/instrumentation.ts index d52dca0aef4..9b4fb6a3480 100644 --- a/yarn-project/world-state/src/synchronizer/instrumentation.ts +++ b/yarn-project/world-state/src/synchronizer/instrumentation.ts @@ -5,7 +5,7 @@ import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/ import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js'; type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data'; -type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks'; +type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices'; class TreeDBInstrumentation { private dbNumItems: Gauge; @@ -70,6 +70,7 @@ class TreeInstrumentation { this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes')); this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage')); this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices')); + this.treeDbInstrumentation.set('block_indices', new TreeDBInstrumentation(meter, treeName, 'block_indices')); } private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) { @@ -92,6 +93,7 @@ class TreeInstrumentation { this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats); this.updateDBMetrics('blocks', treeDbStats.blocksDBStats); this.updateDBMetrics('nodes', treeDbStats.nodesDBStats); + this.updateDBMetrics('block_indices', treeDbStats.blockIndicesDBStats); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 82f5934fb18..a1b93999290 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -181,6 +181,13 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations throw new Error('Method not implemented in legacy merkle tree'); } + getBlockNumbersForLeafIndices( + _treeId: ID, + _leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + throw new Error('Method not implemented in legacy merkle tree'); + } + close(): Promise { return Promise.resolve(); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 90560ef0960..7f4e4bc9d6a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -101,6 +101,10 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati }; } + getBlockNumbersForLeafIndices(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> { + throw new Error('Not implemented'); + } + async getStateReference(): Promise { const snapshots = await Promise.all([ this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE),