From cf05a7a346ea11853e940d5e9ac105ef0d629d35 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:27:20 +0000 Subject: [PATCH 01/10] feat: Allow querying block number for tree indices (#10332) This PR make the following changes. 1. Captures and propagates more of the errors generated in the merkle trees out to the TS interface. 2. Introduces the `block_number_t` typedef within the native world state. 3. Introduces a new DB in the native world state. This DB maps block numbers to the size of the tree at that block. It then uses this DB to fulfill queries looking to identify which block given notes were included within. --- barretenberg/cpp/scripts/merkle_tree_tests.sh | 2 +- .../content_addressed_append_only_tree.hpp | 120 +++++++-- ...ontent_addressed_append_only_tree.test.cpp | 134 +++++++++- .../content_addressed_indexed_tree.hpp | 27 +- .../content_addressed_indexed_tree.test.cpp | 25 +- .../lmdb_store/lmdb_transaction.hpp | 8 + .../lmdb_store/lmdb_tree_store.cpp | 113 +++++++- .../lmdb_store/lmdb_tree_store.hpp | 59 +++- .../lmdb_store/lmdb_tree_store.test.cpp | 253 ++++++++++++++++++ .../crypto/merkle_tree/lmdb_store/queries.hpp | 39 +++ .../cached_content_addressed_tree_store.hpp | 53 +++- .../merkle_tree/node_store/tree_meta.hpp | 43 ++- .../crypto/merkle_tree/response.hpp | 126 ++++++++- .../crypto/merkle_tree/test_fixtures.hpp | 9 +- .../barretenberg/crypto/merkle_tree/types.hpp | 17 +- .../src/barretenberg/world_state/types.hpp | 2 +- .../barretenberg/world_state/world_state.cpp | 191 +++++++++---- .../barretenberg/world_state/world_state.hpp | 88 +++--- .../world_state/world_state.test.cpp | 60 +++++ .../barretenberg/world_state_napi/addon.cpp | 23 ++ .../barretenberg/world_state_napi/addon.hpp | 1 + .../barretenberg/world_state_napi/message.hpp | 16 +- .../aztec-node/src/aztec-node/server.ts | 16 ++ .../src/interfaces/aztec-node.test.ts | 14 + .../src/interfaces/aztec-node.ts | 20 +- .../src/interfaces/merkle_tree_operations.ts | 10 + yarn-project/telemetry-client/src/metrics.ts | 18 ++ .../src/native/merkle_trees_facade.ts | 13 + .../world-state/src/native/message.ts | 15 ++ .../src/native/native_world_state.test.ts | 65 +++++ .../src/synchronizer/instrumentation.ts | 4 +- .../merkle_tree_operations_facade.ts | 7 + .../merkle_tree_snapshot_operations_facade.ts | 4 + 33 files changed, 1402 insertions(+), 193 deletions(-) diff --git a/barretenberg/cpp/scripts/merkle_tree_tests.sh b/barretenberg/cpp/scripts/merkle_tree_tests.sh index f53e2750d5a..9e5e0f0b3c9 100755 --- a/barretenberg/cpp/scripts/merkle_tree_tests.sh +++ b/barretenberg/cpp/scripts/merkle_tree_tests.sh @@ -5,7 +5,7 @@ set -e # run commands relative to parent directory cd $(dirname $0)/.. -DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* +DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.* TEST=${1:-$DEFAULT_TESTS} PRESET=${PRESET:-clang16} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index 7dc41b0abc1..d645b460708 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -39,16 +39,17 @@ template class ContentAddressedAppendOn using StoreType = Store; // Asynchronous methods accept these callback function types as arguments - using AppendCompletionCallback = std::function&)>; - using MetaDataCallback = std::function&)>; - using HashPathCallback = std::function&)>; - using FindLeafCallback = std::function&)>; - using GetLeafCallback = std::function&)>; + using AppendCompletionCallback = std::function&)>; + using MetaDataCallback = std::function&)>; + using HashPathCallback = std::function&)>; + using FindLeafCallback = std::function&)>; + using GetLeafCallback = std::function&)>; using CommitCallback = std::function&)>; - using RollbackCallback = std::function; + using RollbackCallback = std::function; using RemoveHistoricBlockCallback = std::function&)>; using UnwindBlockCallback = std::function&)>; - using FinaliseBlockCallback = std::function; + using FinaliseBlockCallback = std::function; + using GetBlockForIndexCallback = std::function&)>; // Only construct from provided store and thread pool, no copies or moves ContentAddressedAppendOnlyTree(std::unique_ptr store, @@ -90,7 +91,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes */ void get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const; @@ -131,7 +132,7 @@ template class ContentAddressedAppendOn * @param includeUncommitted Whether to include uncommitted changes * @param on_completion Callback to be called on completion */ - void get_meta_data(const index_t& blockNumber, + void get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const; @@ -151,7 +152,7 @@ template class ContentAddressedAppendOn * @param on_completion Callback to be called on completion */ void get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& completion) const; @@ -164,7 +165,7 @@ template class ContentAddressedAppendOn * @brief Returns the index of the provided leaf in the tree */ void find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; @@ -181,10 +182,23 @@ template class ContentAddressedAppendOn */ void find_leaf_index_from(const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const; + /** + * @brief Returns the block numbers that correspond to the given indices values + */ + void find_block_numbers(const std::vector& indices, const GetBlockForIndexCallback& on_completion) const; + + /** + * @brief Returns the block numbers that correspond to the given indices values, from the perspective of a + * historical block number + */ + void find_block_numbers(const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const; + /** * @brief Commit the tree to the backing store */ @@ -200,11 +214,11 @@ template class ContentAddressedAppendOn */ uint32_t depth() const { return depth_; } - void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); + void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion); - void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion); + void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion); - void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion); + void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion); protected: using ReadTransaction = typename Store::ReadTransaction; @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree::get_meta_data(bool in } template -void ContentAddressedAppendOnlyTree::get_meta_data(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::get_meta_data(const block_number_t& blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const { @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons template void ContentAddressedAppendOnlyTree::get_sibling_path(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, const HashPathCallback& on_completion, bool includeUncommitted) const { @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree::get_sibling_path(cons workers_->enqueue(job); } +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + index_t maxIndex = meta.committedSize; + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + +template +void ContentAddressedAppendOnlyTree::find_block_numbers( + const std::vector& indices, + const block_number_t& blockNumber, + const GetBlockForIndexCallback& on_completion) const +{ + auto job = [=, this]() { + execute_and_report( + [=, this](TypedResponse& response) { + response.inner.blockNumbers.reserve(indices.size()); + TreeMeta meta; + BlockPayload blockPayload; + ReadTransactionPtr tx = store_->create_read_transaction(); + store_->get_meta(meta, *tx, true); + if (!store_->get_block_data(blockNumber, blockPayload, *tx)) { + throw std::runtime_error(format("Unable to find block numbers for indices for block ", + blockNumber, + ", failed to get block data.")); + } + index_t maxIndex = std::min(meta.committedSize, blockPayload.size); + for (index_t index : indices) { + bool outOfRange = index >= maxIndex; + std::optional block = + outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx); + response.inner.blockNumbers.emplace_back(block); + } + }, + on_completion); + }; + workers_->enqueue(job); +} + template void ContentAddressedAppendOnlyTree::get_subtree_sibling_path( uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const @@ -473,7 +543,7 @@ std::optional ContentAddressedAppendOnlyTree::find_lea NodePayload nodePayload; bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted); if (!success) { - // std::cout << "No root" << std::endl; + // std::cout << "No root " << hash << std::endl; return std::nullopt; } // std::cout << "Found root at depth " << i << " : " << hash << std::endl; @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree::get_leaf(const index_ template void ContentAddressedAppendOnlyTree::get_leaf(const index_t& leaf_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const GetLeafCallback& on_completion) const { @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree::find_leaf_index(const template void ContentAddressedAppendOnlyTree::find_leaf_index(const fr& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -696,7 +766,7 @@ template void ContentAddressedAppendOnlyTree::find_leaf_index_from( const fr& leaf, const index_t& start_index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLeafCallback& on_completion) const { @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree::rollback(const Rollba template void ContentAddressedAppendOnlyTree::remove_historic_block( - const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) + const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion) { auto job = [=, this]() { execute_and_report( @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree::remove_historic_block } template -void ContentAddressedAppendOnlyTree::unwind_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion) { auto job = [=, this]() { @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree::unwind_block(const in } template -void ContentAddressedAppendOnlyTree::finalise_block(const index_t& blockNumber, +void ContentAddressedAppendOnlyTree::finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion) { auto job = [=, this]() { @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree::add_batch_internal( new_root = new_hash; meta.root = new_hash; meta.size = new_size; - // std::cout << "New size: " << meta.size << std::endl; + // std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl; store_->put_meta(meta); } diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index 5ac60d919b7..83f72c9ca1f 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -123,7 +123,7 @@ void check_sibling_path(TreeType& tree, void check_historic_sibling_path(TreeType& tree, index_t index, fr_sibling_path expected_sibling_path, - index_t blockNumber, + block_number_t blockNumber, bool expected_success = true) { Signal signal; @@ -160,7 +160,7 @@ void rollback_tree(TreeType& tree) signal.wait_for_level(); } -void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -171,7 +171,7 @@ void remove_historic_block(TreeType& tree, const index_t& blockNumber, bool expe signal.wait_for_level(); } -void unwind_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -206,7 +206,7 @@ void add_values(TreeType& tree, const std::vector& values) signal.wait_for_level(); } -void finalise_block(TreeType& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TreeType& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -312,7 +312,7 @@ void check_leaf( } void check_historic_leaf(TreeType& tree, - const index_t& blockNumber, + const block_number_t& blockNumber, const fr& leaf, index_t leaf_index, bool expected_success, @@ -350,6 +350,31 @@ void check_sibling_path(fr expected_root, fr node, index_t index, fr_sibling_pat EXPECT_EQ(hash, expected_root); } +void get_blocks_for_indices(TreeType& tree, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + +void get_blocks_for_indices(TreeType& tree, + const block_number_t& blockNumber, + const std::vector& indices, + std::vector>& blockNumbers) +{ + Signal signal; + tree.find_block_numbers(indices, blockNumber, [&](const TypedResponse& response) { + blockNumbers = response.inner.blockNumbers; + signal.signal_level(); + }); + signal.wait_for_level(); +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_create) { constexpr size_t depth = 10; @@ -1284,7 +1309,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_remove_historic_block_da for (uint32_t i = 0; i < historicPathsZeroIndex.size(); i++) { // retrieving historic data should fail if the block is outside of the window - const index_t blockNumber = i + 1; + const block_number_t blockNumber = i + 1; const bool expectedSuccess = expectedBlockHeight <= windowSize || blockNumber > (expectedBlockHeight - windowSize); check_historic_sibling_path(tree, 0, historicPathsZeroIndex[i], blockNumber, expectedSuccess); @@ -1399,7 +1424,7 @@ void test_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); // attempting to unwind a block that is not the tip should fail @@ -1513,6 +1538,101 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_sync_and_unwind_large_bl } } +TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_retrieve_block_numbers_by_index) +{ + std::string name = random_string(); + constexpr uint32_t depth = 10; + LMDBTreeStore::SharedPtr db = std::make_shared(_directory, name, _mapSize, _maxReaders); + std::unique_ptr store = std::make_unique(name, depth, db); + ThreadPoolPtr pool = make_thread_pool(1); + TreeType tree(std::move(store), pool); + + const size_t block_size = 32; + + for (size_t i = 0; i < 5; i++) { + std::vector values = create_values(block_size); + add_values(tree, values); + commit_tree(tree); + } + std::vector indices{ 12, 33, 63, 64, 65, 80, 96, 159, 160 }; + std::vector> blockNumbers; + + // All but the last block number should be valid when looking at latest + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + index_t maxIndex = 5 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now get blocks for indices from the perspective of block 2 + get_blocks_for_indices(tree, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = 2 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + unwind_block(tree, 5); + unwind_block(tree, 4); + + get_blocks_for_indices(tree, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + maxIndex = 3 * block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // fork from block 1 + std::unique_ptr forkStore = std::make_unique(name, depth, 1, db); + TreeType treeFork(std::move(forkStore), pool); + + // Now, using the fork, get block indices but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } + + // Now, using the fork, get block indics from the perspective of block 2, but find it's limited to those of block 1 + get_blocks_for_indices(treeFork, 2, indices, blockNumbers); + EXPECT_EQ(blockNumbers.size(), indices.size()); + + maxIndex = block_size - 1; + for (size_t i = 0; i < blockNumbers.size(); i++) { + bool present = indices[i] <= maxIndex; + if (present) { + block_number_t expected = 1 + indices[i] / block_size; + EXPECT_EQ(blockNumbers[i].value(), expected); + } + EXPECT_EQ(blockNumbers[i].has_value(), present); + } +} + TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_advance_finalised_blocks) { std::string name = random_string(); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 197f784d59f..45cc232e8e1 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -47,14 +47,13 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree>&)>; + using AddCompletionCallbackWithWitness = std::function>&)>; using AddSequentiallyCompletionCallbackWithWitness = - std::function>&)>; - using AddCompletionCallback = std::function&)>; + std::function>&)>; + using AddCompletionCallback = std::function&)>; - using LeafCallback = std::function>&)>; - using FindLowLeafCallback = std::function&)>; + using LeafCallback = std::function>&)>; + using FindLowLeafCallback = std::function&)>; ContentAddressedIndexedTree(std::unique_ptr store, std::shared_ptr workers, @@ -151,7 +150,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -169,7 +168,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const; @@ -178,7 +177,7 @@ class ContentAddressedIndexedTree : public ContentAddressedAppendOnlyTree::get_leaf(const index_t& template void ContentAddressedIndexedTree::get_leaf(const index_t& index, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const LeafCallback& completion) const { @@ -458,7 +457,7 @@ void ContentAddressedIndexedTree::find_leaf_index( template void ContentAddressedIndexedTree::find_leaf_index( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const { @@ -496,7 +495,7 @@ void ContentAddressedIndexedTree::find_leaf_index_from( template void ContentAddressedIndexedTree::find_leaf_index_from( const LeafValueType& leaf, - const index_t& blockNumber, + const block_number_t& blockNumber, const index_t& start_index, bool includeUncommitted, const ContentAddressedAppendOnlyTree::FindLeafCallback& on_completion) const @@ -559,7 +558,7 @@ void ContentAddressedIndexedTree::find_low_leaf(const fr& template void ContentAddressedIndexedTree::find_low_leaf(const fr& leaf_key, - const index_t& blockNumber, + const block_number_t& blockNumber, bool includeUncommitted, const FindLowLeafCallback& on_completion) const { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 5177b370833..8a7443ef05c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -104,8 +104,11 @@ template void check_root(TypeOfTree& tree, fr expected_roo } template -fr_sibling_path get_historic_sibling_path( - TypeOfTree& tree, index_t blockNumber, index_t index, bool includeUncommitted = true, bool expected_success = true) +fr_sibling_path get_historic_sibling_path(TypeOfTree& tree, + block_number_t blockNumber, + index_t index, + bool includeUncommitted = true, + bool expected_success = true) { fr_sibling_path h; Signal signal; @@ -177,7 +180,7 @@ GetLowIndexedLeafResponse get_low_leaf(TypeOfTree& tree, const LeafValueType& le template GetLowIndexedLeafResponse get_historic_low_leaf(TypeOfTree& tree, - index_t blockNumber, + block_number_t blockNumber, const LeafValueType& leaf, bool includeUncommitted = true) { @@ -236,7 +239,7 @@ void check_find_leaf_index_from(TypeOfTree& tree, template void check_historic_find_leaf_index(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t expected_index, bool expected_success, bool includeUncommitted = true) @@ -257,7 +260,7 @@ void check_historic_find_leaf_index(TypeOfTree& tree, template void check_historic_find_leaf_index_from(TypeOfTree& tree, const LeafValueType& leaf, - index_t blockNumber, + block_number_t blockNumber, index_t start_index, index_t expected_index, bool expected_success, @@ -280,7 +283,7 @@ template void check_historic_leaf(TypeOfTree& tree, const LeafValueType& leaf, index_t expected_index, - index_t blockNumber, + block_number_t blockNumber, bool expected_success, bool includeUncommitted = true) { @@ -300,7 +303,7 @@ void check_historic_leaf(TypeOfTree& tree, template void check_historic_sibling_path(TypeOfTree& tree, index_t index, - index_t blockNumber, + block_number_t blockNumber, const fr_sibling_path& expected_sibling_path, bool includeUncommitted = true, bool expected_success = true) @@ -413,7 +416,7 @@ void block_sync_values_sequential(TypeOfTree& tree, } template -void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void remove_historic_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -425,7 +428,7 @@ void remove_historic_block(TypeOfTree& tree, const index_t& blockNumber, bool ex } template -void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void finalise_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const Response& response) -> void { @@ -437,7 +440,7 @@ void finalise_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_ } template -void unwind_block(TypeOfTree& tree, const index_t& blockNumber, bool expected_success = true) +void unwind_block(TypeOfTree& tree, const block_number_t& blockNumber, bool expected_success = true) { Signal signal; auto completion = [&](const TypedResponse& response) -> void { @@ -2596,7 +2599,7 @@ void test_nullifier_tree_unwind(std::string directory, const uint32_t blocksToRemove = numBlocksToUnwind; for (uint32_t i = 0; i < blocksToRemove; i++) { - const index_t blockNumber = numBlocks - i; + const block_number_t blockNumber = numBlocks - i; check_block_and_root_data(db, blockNumber, roots[blockNumber - 1], true); unwind_block(tree, blockNumber); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp index cf2a55c1285..9bbea8ea42e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_transaction.hpp @@ -46,6 +46,8 @@ class LMDBTransaction { template bool get_value_or_previous(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const; + template bool get_value(T& key, std::vector& data, const LMDBDatabase& db) const; template bool get_value(T& key, index_t& data, const LMDBDatabase& db) const; @@ -88,6 +90,12 @@ bool LMDBTransaction::get_value_or_previous(T& key, K& data, const LMDBDatabase& return lmdb_queries::get_value_or_previous(key, data, db, *this); } +template +bool LMDBTransaction::get_value_or_greater(T& key, K& data, const LMDBDatabase& db) const +{ + return lmdb_queries::get_value_or_greater(key, data, db, *this); +} + template bool LMDBTransaction::get_value_or_previous(T& key, K& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 3f4f07aa829..09f87cae606 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -55,29 +55,35 @@ LMDBTreeStore::LMDBTreeStore(std::string directory, std::string name, uint64_t m { LMDBDatabaseCreationTransaction tx(_environment); - _blockDatabase = std::make_unique( - _environment, tx, _name + std::string("blocks"), false, false, block_key_cmp); + _blockDatabase = + std::make_unique(_environment, tx, _name + BLOCKS_DB, false, false, block_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _nodeDatabase = - std::make_unique(_environment, tx, _name + std::string("nodes"), false, false, fr_key_cmp); + _nodeDatabase = std::make_unique(_environment, tx, _name + NODES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafKeyToIndexDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf indices"), false, false, fr_key_cmp); + _leafKeyToIndexDatabase = + std::make_unique(_environment, tx, _name + LEAF_INDICES_DB, false, false, fr_key_cmp); tx.commit(); } { LMDBDatabaseCreationTransaction tx(_environment); - _leafHashToPreImageDatabase = std::make_unique( - _environment, tx, _name + std::string("leaf pre-images"), false, false, fr_key_cmp); + _leafHashToPreImageDatabase = + std::make_unique(_environment, tx, _name + LEAF_PREIMAGES_DB, false, false, fr_key_cmp); + tx.commit(); + } + + { + LMDBDatabaseCreationTransaction tx(_environment); + _indexToBlockDatabase = + std::make_unique(_environment, tx, _name + BLOCK_INDICES_DB, false, false, index_key_cmp); tx.commit(); } } @@ -107,9 +113,11 @@ void LMDBTreeStore::get_stats(TreeDBStats& stats, ReadTransaction& tx) stats.leafIndicesDBStats = DBStats(LEAF_INDICES_DB, stat); call_lmdb_func(mdb_stat, tx.underlying(), _nodeDatabase->underlying(), &stat); stats.nodesDBStats = DBStats(NODES_DB, stat); + call_lmdb_func(mdb_stat, tx.underlying(), _indexToBlockDatabase->underlying(), &stat); + stats.blockIndicesDBStats = DBStats(BLOCK_INDICES_DB, stat); } -void LMDBTreeStore::write_block_data(uint64_t blockNumber, +void LMDBTreeStore::write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, LMDBTreeStore::WriteTransaction& tx) { @@ -120,13 +128,15 @@ void LMDBTreeStore::write_block_data(uint64_t blockNumber, tx.put_value(key, encoded, *_blockDatabase); } -void LMDBTreeStore::delete_block_data(uint64_t blockNumber, LMDBTreeStore::WriteTransaction& tx) +void LMDBTreeStore::delete_block_data(const block_number_t& blockNumber, LMDBTreeStore::WriteTransaction& tx) { BlockMetaKeyType key(blockNumber); tx.delete_value(key, *_blockDatabase); } -bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockData, LMDBTreeStore::ReadTransaction& tx) +bool LMDBTreeStore::read_block_data(const block_number_t& blockNumber, + BlockPayload& blockData, + LMDBTreeStore::ReadTransaction& tx) { BlockMetaKeyType key(blockNumber); std::vector data; @@ -137,6 +147,87 @@ bool LMDBTreeStore::read_block_data(uint64_t blockNumber, BlockPayload& blockDat return success; } +void LMDBTreeStore::write_block_index_data(const block_number_t& blockNumber, + const index_t& sizeAtBlock, + WriteTransaction& tx) +{ + // There can be multiple block numbers aganst the same index (zero size blocks) + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Read the block index payload + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + BlockIndexPayload payload; + if (success) { + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + } + + // Double check it's not already present (it shouldn't be) + // We then add the block number and sort + // Sorting shouldn't be necessary as we add blocks in ascending order, but we will make sure + // Sorting here and when we unwind blocks means that looking up the block number for an index becomes O(1) + // These lookups are much more frequent than adds or deletes so we take the hit here + if (!payload.contains(blockNumber)) { + payload.blockNumbers.emplace_back(blockNumber); + payload.sort(); + } + + // Write the new payload back down + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + +bool LMDBTreeStore::find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx) +{ + LeafIndexKeyType key(index + 1); + std::vector data; + // Retrieve the payload + bool success = tx.get_value_or_greater(key, data, *_indexToBlockDatabase); + if (!success) { + return false; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + if (payload.blockNumbers.empty()) { + return false; + } + // The block numbers are sorted so we simply return the lowest + blockNumber = payload.blockNumbers[0]; + return true; +} + +void LMDBTreeStore::delete_block_index(const index_t& sizeAtBlock, + const block_number_t& blockNumber, + WriteTransaction& tx) +{ + // To delete a block number form an index we retieve all the block numbers from that index + // Then we find and remove the block number in question + // Then we write back down + LeafIndexKeyType key(sizeAtBlock); + std::vector data; + // Retrieve the data + bool success = tx.get_value(key, data, *_indexToBlockDatabase); + if (!success) { + return; + } + BlockIndexPayload payload; + msgpack::unpack((const char*)data.data(), data.size()).get().convert(payload); + + payload.delete_block(blockNumber); + + // if it's now empty, delete it + if (payload.blockNumbers.empty()) { + tx.delete_value(key, *_indexToBlockDatabase); + return; + } + // not empty write it back + msgpack::sbuffer buffer; + msgpack::pack(buffer, payload); + std::vector encoded(buffer.data(), buffer.data() + buffer.size()); + tx.put_value(key, encoded, *_indexToBlockDatabase); +} + void LMDBTreeStore::write_meta_data(const TreeMeta& metaData, LMDBTreeStore::WriteTransaction& tx) { msgpack::sbuffer buffer; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp index ab4dfb7316c..a28ce245967 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -25,7 +25,7 @@ namespace bb::crypto::merkle_tree { struct BlockPayload { index_t size; - index_t blockNumber; + block_number_t blockNumber; fr root; MSGPACK_FIELDS(size, blockNumber, root) @@ -55,6 +55,49 @@ struct NodePayload { return left == other.left && right == other.right && ref == other.ref; } }; + +struct BlockIndexPayload { + std::vector blockNumbers; + + MSGPACK_FIELDS(blockNumbers) + + bool operator==(const BlockIndexPayload& other) const { return blockNumbers == other.blockNumbers; } + + void sort() { std::sort(blockNumbers.begin(), blockNumbers.end()); } + + bool contains(const block_number_t& blockNumber) + { + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return false; + } + return *it == blockNumber; + } + + void delete_block(const block_number_t& blockNumber) + { + if (blockNumbers.empty()) { + return; + } + // shuffle the block number down, removing the one we want to remove and then pop the end item + auto it = std::lower_bound(blockNumbers.begin(), blockNumbers.end(), blockNumber); + if (it == blockNumbers.end()) { + // The block was not found, we can return + return; + } + // It could be a block higher than the one we are looking for + if (*it != blockNumber) { + return; + } + // we have found our block, shuffle blocks after this one down + auto readIt = it + 1; + while (readIt != blockNumbers.end()) { + *it++ = *readIt++; + } + blockNumbers.pop_back(); + } +}; /** * Creates an abstraction against a collection of LMDB databases within a single environment used to store merkle tree * data @@ -78,11 +121,18 @@ class LMDBTreeStore { void get_stats(TreeDBStats& stats, ReadTransaction& tx); - void write_block_data(uint64_t blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + void write_block_data(const block_number_t& blockNumber, const BlockPayload& blockData, WriteTransaction& tx); + + bool read_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx); + + void delete_block_data(const block_number_t& blockNumber, WriteTransaction& tx); + + void write_block_index_data(const block_number_t& blockNumber, const index_t& sizeAtBlock, WriteTransaction& tx); - bool read_block_data(uint64_t blockNumber, BlockPayload& blockData, ReadTransaction& tx); + // index here is 0 based + bool find_block_for_index(const index_t& index, block_number_t& blockNumber, ReadTransaction& tx); - void delete_block_data(uint64_t blockNumber, WriteTransaction& tx); + void delete_block_index(const index_t& sizeAtBlock, const block_number_t& blockNumber, WriteTransaction& tx); void write_meta_data(const TreeMeta& metaData, WriteTransaction& tx); @@ -136,6 +186,7 @@ class LMDBTreeStore { LMDBDatabase::Ptr _nodeDatabase; LMDBDatabase::Ptr _leafKeyToIndexDatabase; LMDBDatabase::Ptr _leafHashToPreImageDatabase; + LMDBDatabase::Ptr _indexToBlockDatabase; template bool get_node_data(const fr& nodeHash, NodePayload& nodeData, TxType& tx); }; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cb4d342b09a..e9b85aa5bbc 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -14,6 +15,7 @@ #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" @@ -255,3 +257,254 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) EXPECT_FALSE(success); } } + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_by_index) +{ + struct BlockAndIndex { + block_number_t blockNumber; + // this block contains leaves up to index (0 based) + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 82 }, + BlockAndIndex{ .blockNumber = 4, .index = 114 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_TRUE(store.find_block_for_index(130, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete the last block + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[4].index + 1, blocks[4].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(83, readBack, *transaction)); + EXPECT_EQ(readBack, 4); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete non-exisatent indices to check it does nothing + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + store.delete_block_index(21, 1, *transaction); + store.delete_block_index(150, 6, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_FALSE(store.find_block_for_index(82, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(83, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(130, readBack, *transaction)); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } +} + +TEST_F(LMDBTreeStoreTest, can_write_and_retrieve_block_numbers_with_duplicate_indices) +{ + struct BlockAndIndex { + block_number_t blockNumber; + index_t index; + }; + + std::vector blocks{ BlockAndIndex{ .blockNumber = 1, .index = 25 }, + BlockAndIndex{ .blockNumber = 2, .index = 60 }, + BlockAndIndex{ .blockNumber = 3, .index = 60 }, + BlockAndIndex{ .blockNumber = 4, .index = 60 }, + BlockAndIndex{ .blockNumber = 5, .index = 130 } }; + LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); + { + // write all of the blocks. we will write them in reverse order + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + for (int i = int(blocks.size()) - 1; i >= 0; i--) { + // the arg is block size so add 1 + const BlockAndIndex& block = blocks[size_t(i)]; + store.write_block_index_data(block.blockNumber, block.index + 1, *transaction); + } + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 2); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete block 2 at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, blocks[1].blockNumber, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // try and delete blocks that don't exist at index 60 + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[1].index + 1, 2, *transaction); + store.delete_block_index(blocks[1].index + 1, 5, *transaction); + transaction->commit(); + } + + { + // read back some blocks and check them + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + // should be the new lowest block at this index + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 3); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_FALSE(store.find_block_for_index(131, readBack, *transaction)); + } + + { + // delete 2 more blocks + LMDBTreeWriteTransaction::Ptr transaction = store.create_write_transaction(); + // the arg is block size so add 1 + store.delete_block_index(blocks[3].index + 1, blocks[3].blockNumber, *transaction); + store.delete_block_index(blocks[2].index + 1, blocks[2].blockNumber, *transaction); + transaction->commit(); + } + + { + // check the blocks again + LMDBTreeReadTransaction::Ptr transaction = store.create_read_transaction(); + block_number_t readBack = 0; + EXPECT_TRUE(store.find_block_for_index(5, readBack, *transaction)); + EXPECT_EQ(readBack, 1); + + EXPECT_TRUE(store.find_block_for_index(30, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + + EXPECT_TRUE(store.find_block_for_index(82, readBack, *transaction)); + EXPECT_EQ(readBack, 5); + } +} diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp index aa97a2d2518..c26768fa8ec 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/queries.hpp @@ -177,6 +177,45 @@ bool get_value_or_previous(TKey& key, return success; } +template +bool get_value_or_greater(TKey& key, std::vector& data, const LMDBDatabase& db, const TxType& tx) +{ + bool success = false; + std::vector keyBuffer = serialise_key(key); + uint32_t keySize = static_cast(keyBuffer.size()); + MDB_cursor* cursor = nullptr; + call_lmdb_func("mdb_cursor_open", mdb_cursor_open, tx.underlying(), db.underlying(), &cursor); + + try { + MDB_val dbKey; + dbKey.mv_size = keySize; + dbKey.mv_data = (void*)keyBuffer.data(); + + MDB_val dbVal; + // Look for the key >= to that provided + int code = mdb_cursor_get(cursor, &dbKey, &dbVal, MDB_SET_RANGE); + + if (code == 0) { + // found a key >= our key. if it is not the same size, it must be out of range for what we are looking + // for, this means no more data available + if (keySize == dbKey.mv_size) { + // key is the same size, so this contains the data we are looking for + copy_to_vector(dbVal, data); + success = true; + } + } else if (code == MDB_NOTFOUND) { + // no key greater than or equal, nothing to extract + } else { + throw_error("get_value_or_greater::mdb_cursor_get", code); + } + } catch (std::exception& e) { + call_lmdb_func(mdb_cursor_close, cursor); + throw; + } + call_lmdb_func(mdb_cursor_close, cursor); + return success; +} + template void get_all_values_greater_or_equal_key(const TKey& key, std::vector>& data, diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 013f1cc8478..abaec64a3c7 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -147,7 +147,7 @@ template class ContentAddressedCachedTreeStore { /** * @brief Reads the tree meta data, including uncommitted data if requested */ - bool get_block_data(const index_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; + bool get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const; /** * @brief Finds the index of the given leaf value in the tree if available. Includes uncommitted data if requested. @@ -198,13 +198,15 @@ template class ContentAddressedCachedTreeStore { fr get_current_root(ReadTransaction& tx, bool includeUncommitted) const; - void remove_historical_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); - void unwind_block(const index_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); + void unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats); std::optional get_fork_block() const; - void advance_finalised_block(const index_t& blockNumber); + void advance_finalised_block(const block_number_t& blockNumber); + + std::optional find_block_for_index(const index_t& index, ReadTransaction& tx) const; private: std::string name_; @@ -233,7 +235,7 @@ template class ContentAddressedCachedTreeStore { void initialise(); - void initialise_from_block(const index_t& blockNumber); + void initialise_from_block(const block_number_t& blockNumber); bool read_persisted_meta(TreeMeta& m, ReadTransaction& tx) const; @@ -258,6 +260,10 @@ template class ContentAddressedCachedTreeStore { void extract_db_stats(TreeDBStats& stats); + void persist_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + + void delete_block_for_index(const block_number_t& blockNumber, const index_t& index, WriteTransaction& tx); + index_t constrain_tree_size(const RequestContext& requestContext, ReadTransaction& tx) const; WriteTransactionPtr create_write_transaction() const { return dataStore_->create_write_transaction(); } @@ -304,6 +310,31 @@ index_t ContentAddressedCachedTreeStore::constrain_tree_size(cons return sizeLimit; } +template +std::optional ContentAddressedCachedTreeStore::find_block_for_index( + const index_t& index, ReadTransaction& tx) const +{ + block_number_t blockNumber = 0; + bool success = dataStore_->find_block_for_index(index, blockNumber, tx); + return success ? std::make_optional(blockNumber) : std::nullopt; +} + +template +void ContentAddressedCachedTreeStore::persist_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->write_block_index_data(blockNumber, index, tx); +} + +template +void ContentAddressedCachedTreeStore::delete_block_for_index(const block_number_t& blockNumber, + const index_t& index, + WriteTransaction& tx) +{ + dataStore_->delete_block_index(index, blockNumber, tx); +} + template std::pair ContentAddressedCachedTreeStore::find_low_value( const fr& new_leaf_key, const RequestContext& requestContext, ReadTransaction& tx) const @@ -553,7 +584,7 @@ void ContentAddressedCachedTreeStore::get_meta(TreeMeta& m, } template -bool ContentAddressedCachedTreeStore::get_block_data(const index_t& blockNumber, +bool ContentAddressedCachedTreeStore::get_block_data(const block_number_t& blockNumber, BlockPayload& blockData, ReadTransaction& tx) const { @@ -649,6 +680,7 @@ void ContentAddressedCachedTreeStore::commit(TreeMeta& finalMeta, .blockNumber = uncommittedMeta.unfinalisedBlockHeight, .root = uncommittedMeta.root }; dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx); + dataStore_->write_block_index_data(block.blockNumber, block.size, *tx); } uncommittedMeta.committedSize = uncommittedMeta.size; @@ -767,7 +799,7 @@ void ContentAddressedCachedTreeStore::persist_meta(TreeMeta& m, W } template -void ContentAddressedCachedTreeStore::advance_finalised_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::advance_finalised_block(const block_number_t& blockNumber) { TreeMeta committedMeta; TreeMeta uncommittedMeta; @@ -827,7 +859,7 @@ void ContentAddressedCachedTreeStore::advance_finalised_block(con } template -void ContentAddressedCachedTreeStore::unwind_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::unwind_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -894,6 +926,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& remove_node(std::optional(blockData.root), 0, maxIndex, *writeTx); // remove the block from the block data table dataStore_->delete_block_data(blockNumber, *writeTx); + dataStore_->delete_block_index(blockData.size, blockData.blockNumber, *writeTx); uncommittedMeta.unfinalisedBlockHeight = previousBlockData.blockNumber; uncommittedMeta.size = previousBlockData.size; uncommittedMeta.committedSize = previousBlockData.size; @@ -916,7 +949,7 @@ void ContentAddressedCachedTreeStore::unwind_block(const index_t& } template -void ContentAddressedCachedTreeStore::remove_historical_block(const index_t& blockNumber, +void ContentAddressedCachedTreeStore::remove_historical_block(const block_number_t& blockNumber, TreeMeta& finalMeta, TreeDBStats& dbStats) { @@ -1107,7 +1140,7 @@ template void ContentAddressedCachedTreeStore -void ContentAddressedCachedTreeStore::initialise_from_block(const index_t& blockNumber) +void ContentAddressedCachedTreeStore::initialise_from_block(const block_number_t& blockNumber) { // Read the persisted meta data, if the name or depth of the tree is not consistent with what was provided during // construction then we throw diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp index 164a6b254cf..6b77a6a5ebd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { @@ -16,9 +17,9 @@ struct TreeMeta { bb::fr root; index_t initialSize; bb::fr initialRoot; - uint64_t oldestHistoricBlock; - uint64_t unfinalisedBlockHeight; - uint64_t finalisedBlockHeight; + block_number_t oldestHistoricBlock; + block_number_t unfinalisedBlockHeight; + block_number_t finalisedBlockHeight; MSGPACK_FIELDS(name, depth, @@ -31,6 +32,34 @@ struct TreeMeta { unfinalisedBlockHeight, finalisedBlockHeight) + TreeMeta(std::string n, + uint32_t d, + const index_t& s, + const index_t& c, + const bb::fr& r, + const index_t& is, + const bb::fr& ir, + const block_number_t& o, + const block_number_t& u, + const block_number_t& f) + : name(std::move(n)) + , depth(d) + , size(s) + , committedSize(c) + , root(r) + , initialSize(is) + , initialRoot(ir) + , oldestHistoricBlock(o) + , unfinalisedBlockHeight(u) + , finalisedBlockHeight(f) + {} + TreeMeta() = default; + ~TreeMeta() = default; + TreeMeta(const TreeMeta& other) = default; + TreeMeta(TreeMeta&& other) noexcept { *this = std::move(other); } + TreeMeta& operator=(const TreeMeta& other) = default; + TreeMeta& operator=(TreeMeta&& other) noexcept = default; + bool operator==(const TreeMeta& other) const { return name == other.name && depth == other.depth && size == other.size && @@ -50,12 +79,4 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) return os; } -struct LeavesMeta { - index_t size; - - MSGPACK_FIELDS(size) - - bool operator==(const LeavesMeta& other) const { return size == other.size; } -}; - } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 5fdfd19b85b..d525acb8672 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -12,19 +12,41 @@ #include #include #include +#include namespace bb::crypto::merkle_tree { struct TreeMetaResponse { TreeMeta meta; + + TreeMetaResponse() = default; + ~TreeMetaResponse() = default; + TreeMetaResponse(const TreeMetaResponse& other) = default; + TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; + TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; + TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; }; struct AddDataResponse { index_t size; fr root; + + AddDataResponse() = default; + ~AddDataResponse() = default; + AddDataResponse(const AddDataResponse& other) = default; + AddDataResponse(AddDataResponse&& other) noexcept = default; + AddDataResponse& operator=(const AddDataResponse& other) = default; + AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; }; struct GetSiblingPathResponse { fr_sibling_path path; + + GetSiblingPathResponse() = default; + ~GetSiblingPathResponse() = default; + GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; + GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; }; template struct LeafUpdateWitnessData { @@ -32,6 +54,18 @@ template struct LeafUpdateWitnessData { index_t index; fr_sibling_path path; + LeafUpdateWitnessData(const IndexedLeaf& l, const index_t& i, fr_sibling_path p) + : leaf(l) + , index(i) + , path(std::move(p)) + {} + LeafUpdateWitnessData() = default; + ~LeafUpdateWitnessData() = default; + LeafUpdateWitnessData(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData(LeafUpdateWitnessData&& other) noexcept = default; + LeafUpdateWitnessData& operator=(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; + MSGPACK_FIELDS(leaf, index, path); }; @@ -40,20 +74,59 @@ template struct AddIndexedDataResponse { fr_sibling_path subtree_path; std::shared_ptr>> sorted_leaves; std::shared_ptr>> low_leaf_witness_data; + + AddIndexedDataResponse() = default; + ~AddIndexedDataResponse() = default; + AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; + AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; }; template struct AddIndexedDataSequentiallyResponse { AddDataResponse add_data_result; std::shared_ptr>> low_leaf_witness_data; std::shared_ptr>> insertion_witness_data; + + AddIndexedDataSequentiallyResponse() = default; + ~AddIndexedDataSequentiallyResponse() = default; + AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; + AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; +}; + +struct BlockForIndexResponse { + std::vector> blockNumbers; + + BlockForIndexResponse() = default; + ~BlockForIndexResponse() = default; + BlockForIndexResponse(const BlockForIndexResponse& other) = default; + BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; + BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; + BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; }; struct FindLeafIndexResponse { index_t leaf_index; + + FindLeafIndexResponse() = default; + ~FindLeafIndexResponse() = default; + FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; + FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; }; struct GetLeafResponse { std::optional leaf; + + GetLeafResponse() = default; + ~GetLeafResponse() = default; + GetLeafResponse(const GetLeafResponse& other) = default; + GetLeafResponse(GetLeafResponse&& other) noexcept = default; + GetLeafResponse& operator=(const GetLeafResponse& other) = default; + GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; }; template struct GetIndexedLeafResponse { @@ -66,6 +139,17 @@ struct GetLowIndexedLeafResponse { MSGPACK_FIELDS(is_already_present, index); + GetLowIndexedLeafResponse(bool p, const index_t& i) + : is_already_present(p) + , index(i) + {} + GetLowIndexedLeafResponse() = default; + ~GetLowIndexedLeafResponse() = default; + GetLowIndexedLeafResponse(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse(GetLowIndexedLeafResponse&& other) noexcept = default; + GetLowIndexedLeafResponse& operator=(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse& operator=(GetLowIndexedLeafResponse&& other) noexcept = default; + bool operator==(const GetLowIndexedLeafResponse& other) const { return is_already_present == other.is_already_present && index == other.index; @@ -75,27 +159,66 @@ struct GetLowIndexedLeafResponse { struct CommitResponse { TreeMeta meta; TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse& other) = default; + CommitResponse(CommitResponse&& other) noexcept = default; + CommitResponse& operator=(const CommitResponse& other) = default; + CommitResponse& operator=(CommitResponse&& other) noexcept = default; }; struct UnwindResponse { TreeMeta meta; TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse& other) = default; + UnwindResponse(UnwindResponse&& other) noexcept = default; + UnwindResponse& operator=(const UnwindResponse& other) = default; + UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; }; struct RemoveHistoricResponse { TreeMeta meta; TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; + RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; }; template struct TypedResponse { ResponseType inner; bool success{ true }; std::string message; + + TypedResponse() = default; + ~TypedResponse() = default; + TypedResponse(const TypedResponse& other) = default; + TypedResponse(TypedResponse&& other) noexcept = default; + TypedResponse& operator=(const TypedResponse& other) = default; + TypedResponse& operator=(TypedResponse&& other) noexcept = default; }; struct Response { bool success; std::string message; + + Response(bool s, std::string m) + : success(s) + , message(std::move(m)) + {} + Response() = default; + ~Response() = default; + Response(const Response& other) = default; + Response(Response&& other) noexcept = default; + Response& operator=(const Response& other) = default; + Response& operator=(Response&& other) noexcept = default; }; template @@ -116,8 +239,7 @@ void execute_and_report(const std::function&)>& } } -inline void execute_and_report(const std::function& f, - const std::function& on_completion) +inline void execute_and_report(const std::function& f, const std::function& on_completion) { Response response{ true, "" }; try { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp index 1aa333d061b..354ba949c20 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp @@ -10,7 +10,10 @@ namespace bb::crypto::merkle_tree { -void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess) +void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, + block_number_t blockNumber, + fr root, + bool expectedSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -25,7 +28,7 @@ void inline check_block_and_root_data(LMDBTreeStore::SharedPtr db, index_t block } void inline check_block_and_root_data( - LMDBTreeStore::SharedPtr db, index_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) + LMDBTreeStore::SharedPtr db, block_number_t blockNumber, fr root, bool expectedSuccess, bool expectedRootSuccess) { BlockPayload blockData; LMDBTreeStore::ReadTransaction::Ptr tx = db->create_read_transaction(); @@ -40,7 +43,7 @@ void inline check_block_and_root_data( } void inline check_block_and_size_data(LMDBTreeStore::SharedPtr db, - index_t blockNumber, + block_number_t blockNumber, index_t expectedSize, bool expectedSuccess) { diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 8d9c5de4933..c8ce520fb9c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -6,10 +6,11 @@ #include namespace bb::crypto::merkle_tree { using index_t = uint64_t; +using block_number_t = uint64_t; struct RequestContext { bool includeUncommitted; - std::optional blockNumber; + std::optional blockNumber; bb::fr root; }; @@ -17,6 +18,7 @@ const std::string BLOCKS_DB = "blocks"; const std::string NODES_DB = "nodes"; const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; +const std::string BLOCK_INDICES_DB = "block indices"; struct DBStats { std::string name; @@ -71,6 +73,7 @@ struct TreeDBStats { DBStats nodesDBStats; DBStats leafPreimagesDBStats; DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; TreeDBStats() = default; TreeDBStats(uint64_t mapSize) @@ -80,24 +83,27 @@ struct TreeDBStats { const DBStats& blockStats, const DBStats& nodesStats, const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats) + const DBStats& leafIndicesStats, + const DBStats& blockIndicesStats) : mapSize(mapSize) , blocksDBStats(blockStats) , nodesDBStats(nodesStats) , leafPreimagesDBStats(leafPreimagesDBStats) , leafIndicesDBStats(leafIndicesStats) + , blockIndicesDBStats(blockIndicesStats) {} TreeDBStats(const TreeDBStats& other) = default; TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } ~TreeDBStats() = default; - MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats) + MSGPACK_FIELDS(mapSize, blocksDBStats, nodesDBStats, leafPreimagesDBStats, leafIndicesDBStats, blockIndicesDBStats) bool operator==(const TreeDBStats& other) const { return mapSize == other.mapSize && blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats; + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; } TreeDBStats& operator=(TreeDBStats&& other) noexcept @@ -108,6 +114,7 @@ struct TreeDBStats { nodesDBStats = std::move(other.nodesDBStats); leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); } return *this; } @@ -118,7 +125,7 @@ struct TreeDBStats { { os << "Map Size: " << stats.mapSize << " Blocks DB " << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats << ", Leaf Indices DB " - << stats.leafIndicesDBStats; + << stats.leafIndicesDBStats << ", Block Indices DB " << stats.blockIndicesDBStats; return os; } }; diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/barretenberg/cpp/src/barretenberg/world_state/types.hpp index 34e74a0631c..f5fbbcbf257 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/types.hpp @@ -32,7 +32,7 @@ using StateReference = std::unordered_map; struct WorldStateRevision { index_t forkId{ 0 }; - index_t blockNumber{ 0 }; + block_number_t blockNumber{ 0 }; bool includeUncommitted{ false }; MSGPACK_FIELDS(forkId, blockNumber, includeUncommitted) diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp index 9cb3b36bf6c..7e74fe44896 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp @@ -142,7 +142,7 @@ Fork::SharedPtr WorldState::retrieve_fork(const uint64_t& forkId) const } uint64_t WorldState::create_fork(const std::optional& blockNumber) { - index_t blockNumberForFork = 0; + block_number_t blockNumberForFork = 0; if (!blockNumber.has_value()) { // we are forking at latest WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; @@ -159,7 +159,7 @@ uint64_t WorldState::create_fork(const std::optional& blockNumber) return forkId; } -void WorldState::remove_forks_for_block(const index_t& blockNumber) +void WorldState::remove_forks_for_block(const block_number_t& blockNumber) { // capture the shared pointers outside of the lock scope so we are not under the lock when the objects are destroyed std::vector forks; @@ -191,7 +191,7 @@ void WorldState::delete_fork(const uint64_t& forkId) } } -Fork::SharedPtr WorldState::create_new_fork(const index_t& blockNumber) +Fork::SharedPtr WorldState::create_new_fork(const block_number_t& blockNumber) { Fork::SharedPtr fork = std::make_shared(); fork->_blockNumber = blockNumber; @@ -241,10 +241,10 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M return std::visit( [=](auto&& wrapper) { Signal signal(1); - TreeMetaResponse response; + TypedResponse local; - auto callback = [&](const TypedResponse& meta) { - response = meta.inner; + auto callback = [&](TypedResponse& meta) { + local = std::move(meta); signal.signal_level(0); }; @@ -255,12 +255,15 @@ TreeMetaResponse WorldState::get_tree_info(const WorldStateRevision& revision, M } signal.wait_for_level(0); - return response; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner; }, fork->_trees.at(tree_id)); } -void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const +void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const { Fork::SharedPtr fork = retrieve_fork(revision.forkId); @@ -271,13 +274,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra Signal signal(static_cast(tree_ids.size())); std::mutex mutex; + std::unordered_map> local; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &responses, &mutex, id](const TypedResponse& meta) { + auto callback = [&signal, &local, &mutex, id](TypedResponse& meta) { { std::lock_guard lock(mutex); - responses[id] = meta.inner.meta; + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -293,6 +297,14 @@ void WorldState::get_all_tree_info(const WorldStateRevision& revision, std::arra } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + responses[tree_id] = std::move(m.inner.meta); + } } StateReference WorldState::get_state_reference(const WorldStateRevision& revision) const @@ -324,19 +336,15 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio Signal signal(static_cast(tree_ids.size())); StateReference state_reference; + std::unordered_map> local; std::mutex state_ref_mutex; for (auto id : tree_ids) { const auto& tree = fork->_trees.at(id); - auto callback = [&signal, &state_reference, &state_ref_mutex, initial_state, id]( - const TypedResponse& meta) { + auto callback = [&signal, &local, &state_ref_mutex, id](TypedResponse& meta) { { std::lock_guard lock(state_ref_mutex); - if (initial_state) { - state_reference.insert({ id, { meta.inner.meta.initialRoot, meta.inner.meta.initialSize } }); - } else { - state_reference.insert({ id, { meta.inner.meta.root, meta.inner.meta.size } }); - } + local[id] = std::move(meta); } signal.signal_decrement(); }; @@ -352,6 +360,19 @@ StateReference WorldState::get_state_reference(const WorldStateRevision& revisio } signal.wait_for_level(0); + + for (auto tree_id : tree_ids) { + auto& m = local[tree_id]; + if (!m.success) { + throw std::runtime_error(m.message); + } + if (initial_state) { + state_reference[tree_id] = std::make_pair(m.inner.meta.initialRoot, m.inner.meta.initialSize); + continue; + } + state_reference[tree_id] = std::make_pair(m.inner.meta.root, m.inner.meta.size); + } + return state_reference; } @@ -364,10 +385,10 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, return std::visit( [leaf_index, revision](auto&& wrapper) { Signal signal(1); - fr_sibling_path path; + TypedResponse local; - auto callback = [&signal, &path](const TypedResponse& response) { - path = response.inner.path; + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; @@ -378,7 +399,42 @@ fr_sibling_path WorldState::get_sibling_path(const WorldStateRevision& revision, } signal.wait_for_level(0); - return path; + if (!local.success) { + throw std::runtime_error(local.message); + } + return local.inner.path; + }, + fork->_trees.at(tree_id)); +} + +void WorldState::get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const +{ + Fork::SharedPtr fork = retrieve_fork(revision.forkId); + + std::visit( + [&leafIndices, revision, &blockNumbers](auto&& wrapper) { + Signal signal(1); + TypedResponse local; + + auto callback = [&signal, &local](TypedResponse& response) { + local = std::move(response); + signal.signal_level(); + }; + + if (revision.blockNumber) { + wrapper.tree->find_block_numbers(leafIndices, revision.blockNumber, callback); + } else { + wrapper.tree->find_block_numbers(leafIndices, callback); + } + signal.wait_for_level(0); + + if (!local.success) { + throw std::runtime_error(local.message); + } + blockNumbers = std::move(local.inner.blockNumbers); }, fork->_trees.at(tree_id)); } @@ -534,7 +590,15 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re { auto& wrapper = std::get>(fork->_trees.at(MerkleTreeId::PUBLIC_DATA_TREE)); - PublicDataTree::AddCompletionCallback completion = [&](const auto&) -> void { signal.signal_decrement(); }; + PublicDataTree::AddCompletionCallback completion = [&](const auto& resp) -> void { + // take the first error + bool expected = true; + if (!resp.success && success.compare_exchange_strong(expected, false)) { + err_message = resp.message; + } + + signal.signal_decrement(); + }; wrapper.tree->add_or_update_values_sequentially(public_writes, completion); } @@ -566,9 +630,9 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi { Fork::SharedPtr fork = retrieve_fork(revision.forkId); Signal signal; - GetLowIndexedLeafResponse low_leaf_info; - auto callback = [&signal, &low_leaf_info](const TypedResponse& response) { - low_leaf_info = response.inner; + TypedResponse low_leaf_info; + auto callback = [&signal, &low_leaf_info](TypedResponse& response) { + low_leaf_info = std::move(response); signal.signal_level(); }; @@ -591,7 +655,11 @@ GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevisi } signal.wait_for_level(); - return low_leaf_info; + + if (!low_leaf_info.success) { + throw std::runtime_error(low_leaf_info.message); + } + return low_leaf_info.inner; } WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockNumber) @@ -599,11 +667,13 @@ WorldStateStatusSummary WorldState::set_finalised_blocks(const index_t& toBlockN WorldStateRevision revision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 0, .includeUncommitted = false }; TreeMetaResponse archive_state = get_tree_info(revision, MerkleTreeId::ARCHIVE); if (toBlockNumber <= archive_state.meta.finalisedBlockHeight) { - throw std::runtime_error("Unable to finalise block, already finalised"); - } - if (!set_finalised_block(toBlockNumber)) { - throw std::runtime_error("Failed to set finalised block"); + throw std::runtime_error(format("Unable to finalise blocks to block number ", + toBlockNumber, + ", current finalised block: ", + archive_state.meta.finalisedBlockHeight)); } + // This will throw if it fails + set_finalised_block(toBlockNumber); WorldStateStatusSummary status; get_status_summary(status); return status; @@ -616,10 +686,10 @@ WorldStateStatusFull WorldState::unwind_blocks(const index_t& toBlockNumber) throw std::runtime_error("Unable to unwind block, block not found"); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; blockNumber--) { - if (!unwind_block(blockNumber, status)) { - throw std::runtime_error("Failed to unwind block"); - } + for (block_number_t blockNumber = archive_state.meta.unfinalisedBlockHeight; blockNumber > toBlockNumber; + blockNumber--) { + // This will throw if it fails + unwind_block(blockNumber, status); } populate_status_summary(status); return status; @@ -635,35 +705,43 @@ WorldStateStatusFull WorldState::remove_historical_blocks(const index_t& toBlock archive_state.meta.oldestHistoricBlock)); } WorldStateStatusFull status; - for (index_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; blockNumber++) { - if (!remove_historical_block(blockNumber, status)) { - throw std::runtime_error(format( - "Failed to remove historical block ", blockNumber, " when removing blocks up to ", toBlockNumber)); - } + for (block_number_t blockNumber = archive_state.meta.oldestHistoricBlock; blockNumber < toBlockNumber; + blockNumber++) { + // This will throw if it fails + remove_historical_block(blockNumber, status); } populate_status_summary(status); return status; } -bool WorldState::set_finalised_block(const index_t& blockNumber) +bool WorldState::set_finalised_block(const block_number_t& blockNumber) { - std::atomic_bool success = true; Fork::SharedPtr fork = retrieve_fork(CANONICAL_FORK_ID); Signal signal(static_cast(fork->_trees.size())); + std::array local; + std::mutex mtx; for (auto& [id, tree] : fork->_trees) { std::visit( - [&signal, &success, blockNumber](auto&& wrapper) { - wrapper.tree->finalise_block(blockNumber, [&signal, &success](const Response& resp) { - success = success && resp.success; + [&signal, &local, blockNumber, id, &mtx](auto&& wrapper) { + wrapper.tree->finalise_block(blockNumber, [&signal, &local, &mtx, id](Response& resp) { + { + std::lock_guard lock(mtx); + local[id] = std::move(resp); + } signal.signal_decrement(); }); }, tree); } signal.wait_for_level(); - return success; + for (auto& m : local) { + if (!m.success) { + throw std::runtime_error(m.message); + } + } + return true; } -bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -723,10 +801,13 @@ bool WorldState::unwind_block(const index_t& blockNumber, WorldStateStatusFull& blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } -bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status) +bool WorldState::remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status) { std::atomic_bool success = true; std::string message; @@ -786,8 +867,11 @@ bool WorldState::remove_historical_block(const index_t& blockNumber, WorldStateS blockNumber); } signal.wait_for_level(); + if (!success) { + throw std::runtime_error(message); + } remove_forks_for_block(blockNumber); - return success; + return true; } bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_ref, uint32_t generator_point) @@ -829,7 +913,12 @@ bb::fr WorldState::compute_initial_archive(const StateReference& initial_state_r bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const { - std::optional leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + std::optional leaf_index = std::nullopt; + + try { + leaf_index = find_leaf_index(revision, MerkleTreeId::ARCHIVE, block_header_hash); + } catch (std::runtime_error&) { + } if (!leaf_index.has_value()) { return false; @@ -887,7 +976,7 @@ void WorldState::validate_trees_are_equally_synched() bool WorldState::determine_if_synched(std::array& metaResponses) { - index_t blockNumber = metaResponses[0].unfinalisedBlockHeight; + block_number_t blockNumber = metaResponses[0].unfinalisedBlockHeight; for (size_t i = 1; i < metaResponses.size(); i++) { if (blockNumber != metaResponses[i].unfinalisedBlockHeight) { return false; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp index 5afa4a010ef..c66412aae77 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp @@ -107,6 +107,11 @@ class WorldState { MerkleTreeId tree_id, index_t leaf_index) const; + void get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const std::vector& leafIndices, + std::vector>& blockNumbers) const; + /** * @brief Get the leaf preimage object * @@ -259,12 +264,12 @@ class WorldState { uint64_t maxReaders); Fork::SharedPtr retrieve_fork(const uint64_t& forkId) const; - Fork::SharedPtr create_new_fork(const index_t& blockNumber); - void remove_forks_for_block(const index_t& blockNumber); + Fork::SharedPtr create_new_fork(const block_number_t& blockNumber); + void remove_forks_for_block(const block_number_t& blockNumber); - bool unwind_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool remove_historical_block(const index_t& blockNumber, WorldStateStatusFull& status); - bool set_finalised_block(const index_t& blockNumber); + bool unwind_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool remove_historical_block(const block_number_t& blockNumber, WorldStateStatusFull& status); + bool set_finalised_block(const block_number_t& blockNumber); void get_all_tree_info(const WorldStateRevision& revision, std::array& responses) const; @@ -304,7 +309,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); template void remove_historic_block_for_tree(TreeDBStats& dbStats, @@ -313,7 +318,7 @@ class WorldState { std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber); + const block_number_t& blockNumber); }; template @@ -342,7 +347,7 @@ void WorldState::unwind_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.unwind_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -362,7 +367,7 @@ void WorldState::remove_historic_block_for_tree(TreeDBStats& dbStats, std::atomic_bool& success, std::string& message, TreeMeta& meta, - const index_t& blockNumber) + const block_number_t& blockNumber) { tree.remove_historic_block(blockNumber, [&](TypedResponse& response) { bool expected = true; @@ -384,15 +389,13 @@ std::optional> WorldState::get_indexed_leaf( using Tree = ContentAddressedIndexedTree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse> local; if (auto* const wrapper = std::get_if>(&fork->_trees.at(id))) { - std::optional> value; - Signal signal; - auto callback = [&](const TypedResponse>& response) { - if (response.inner.indexed_leaf.has_value()) { - value = response.inner.indexed_leaf; - } + Signal signal; + auto callback = [&](TypedResponse>& response) { + local = std::move(response); signal.signal_level(0); }; @@ -403,7 +406,11 @@ std::optional> WorldState::get_indexed_leaf( } signal.wait_for_level(); - return value; + if (!local.success) { + throw std::runtime_error("Failed to find indexed leaf: " + local.message); + } + + return local.inner.indexed_leaf; } throw std::runtime_error("Invalid tree type"); @@ -419,12 +426,17 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, Fork::SharedPtr fork = retrieve_fork(revision.forkId); std::optional leaf; + bool success = true; + std::string error_msg; Signal signal; if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse& resp) { - if (resp.inner.leaf.has_value()) { - leaf = resp.inner.leaf.value(); + auto callback = [&signal, &leaf, &success, &error_msg](const TypedResponse& response) { + if (!response.success || !response.inner.leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.leaf; } signal.signal_level(); }; @@ -439,12 +451,16 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, using Tree = ContentAddressedIndexedTree; auto& wrapper = std::get>(fork->_trees.at(tree_id)); - auto callback = [&signal, &leaf](const TypedResponse>& resp) { - if (resp.inner.indexed_leaf.has_value()) { - leaf = resp.inner.indexed_leaf.value().value; - } - signal.signal_level(); - }; + auto callback = + [&signal, &leaf, &success, &error_msg](const TypedResponse>& response) { + if (!response.success || !response.inner.indexed_leaf.has_value()) { + success = false; + error_msg = response.message; + } else { + leaf = response.inner.indexed_leaf.value().value; + } + signal.signal_level(); + }; if (revision.blockNumber) { wrapper.tree->get_leaf(leaf_index, revision.blockNumber, revision.includeUncommitted, callback); @@ -454,6 +470,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, } signal.wait_for_level(); + return leaf; } @@ -464,15 +481,13 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev index_t start_index) const { using namespace crypto::merkle_tree; - std::optional index; Fork::SharedPtr fork = retrieve_fork(rev.forkId); + TypedResponse local; Signal signal; - auto callback = [&](const TypedResponse& response) { - if (response.success) { - index = response.inner.leaf_index; - } + auto callback = [&](TypedResponse& response) { + local = std::move(response); signal.signal_level(0); }; if constexpr (std::is_same_v) { @@ -496,7 +511,12 @@ std::optional WorldState::find_leaf_index(const WorldStateRevision& rev } signal.wait_for_level(0); - return index; + + if (!local.success) { + return std::nullopt; + } + + return local.inner.leaf_index; } template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) @@ -537,7 +557,7 @@ template void WorldState::append_leaves(MerkleTreeId id, const std: signal.wait_for_level(0); if (!success) { - throw std::runtime_error("Failed to append leaves: " + error_msg); + throw std::runtime_error(error_msg); } } @@ -576,7 +596,7 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to batch insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; @@ -615,7 +635,7 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, signal.wait_for_level(); if (!success) { - throw std::runtime_error("Failed to sequentially insert indexed leaves: " + error_msg); + throw std::runtime_error(error_msg); } return result; diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp index 024ffbf4ac0..a5ced2921ad 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp @@ -1,6 +1,7 @@ #include "barretenberg/world_state/world_state.hpp" #include "barretenberg/crypto/merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_read_transaction.hpp" #include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" @@ -11,6 +12,7 @@ #include #include #include +#include #include #include @@ -79,6 +81,10 @@ void assert_leaf_index( const WorldState& ws, WorldStateRevision revision, MerkleTreeId tree_id, const Leaf& value, index_t expected_index) { std::optional index = ws.find_leaf_index(revision, tree_id, value); + EXPECT_TRUE(index.has_value()); + if (!index.has_value()) { + return; + } EXPECT_EQ(index.value(), expected_index); } @@ -363,6 +369,7 @@ TEST_F(WorldStateTest, NullifierTree) auto test_leaf = ws.get_indexed_leaf(WorldStateRevision::committed(), tree_id, 128); // at this point 142 should be the biggest leaf so it wraps back to 0 + EXPECT_TRUE(test_leaf.has_value()); EXPECT_EQ(test_leaf.value(), IndexedLeaf(test_nullifier, 0, 0)); auto predecessor_of_142_again = @@ -524,6 +531,19 @@ TEST_F(WorldStateTest, SyncExternalBlockFromEmpty) for (const auto& [tree_id, snapshot] : block_state_ref) { EXPECT_EQ(state_ref.at(tree_id), snapshot); } + + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE, { 0 }, blockNumbers); + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_EQ(blockNumbers[0], 1); + + EXPECT_THROW(ws.get_block_numbers_for_leaf_indices( + WorldStateRevision{ .forkId = CANONICAL_FORK_ID, .blockNumber = 2, .includeUncommitted = false }, + MerkleTreeId::NOTE_HASH_TREE, + { 0 }, + blockNumbers), + std::runtime_error); } TEST_F(WorldStateTest, SyncBlockFromDirtyState) @@ -789,3 +809,43 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) EXPECT_EQ(fork_state_ref, ws.get_state_reference(WorldStateRevision::committed())); } + +TEST_F(WorldStateTest, GetBlockForIndex) +{ + WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); + // bb::fr block_hash(1); + StateReference block_state_ref = { + { MerkleTreeId::NULLIFIER_TREE, + { fr("0x187a19972150cd1e76d8201d720da7682fcf4d93ec6a3c7b0d84bbefde5bd927"), 129 } }, + { MerkleTreeId::NOTE_HASH_TREE, + { fr("0x2467e5f90736b4ea977e7d21cfb3714181e16b7d6cd867768b59e2ea90fa3eaf"), 1 } }, + { MerkleTreeId::PUBLIC_DATA_TREE, + { fr("0x0278dcf9ff541da255ee722aecfad849b66af0d42c2924d949b5a509f2e1aec9"), 129 } }, + { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + { fr("0x24ffd0fab86555ab2e86cffc706d4cfb4b8c405c3966af805de954504ffc27ac"), 1 } }, + }; + + WorldStateStatusFull status = ws.sync_block( + block_state_ref, fr(1), { 42 }, { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }); + WorldStateStatusSummary expected{ 1, 0, 1, true }; + EXPECT_EQ(status.summary, expected); + + StateReference state_ref = ws.get_state_reference(WorldStateRevision::committed()); + + std::vector tree_ids{ + MerkleTreeId::NULLIFIER_TREE, + MerkleTreeId::NOTE_HASH_TREE, + MerkleTreeId::PUBLIC_DATA_TREE, + MerkleTreeId::L1_TO_L2_MESSAGE_TREE, + }; + + for (const auto& id : tree_ids) { + std::vector> blockNumbers; + ws.get_block_numbers_for_leaf_indices( + WorldStateRevision::committed(), id, { state_ref[id].second - 1 }, blockNumbers); + + EXPECT_EQ(blockNumbers.size(), 1); + EXPECT_TRUE(blockNumbers[0].has_value()); + EXPECT_EQ(blockNumbers[0].value(), 1); + } +} diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp index d8ac68d0db1..f3290da5e9e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp @@ -150,6 +150,11 @@ WorldStateAddon::WorldStateAddon(const Napi::CallbackInfo& info) WorldStateMessageType::GET_SIBLING_PATH, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); + _dispatcher.registerTarget(WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, + [this](msgpack::object& obj, msgpack::sbuffer& buffer) { + return get_block_numbers_for_leaf_indices(obj, buffer); + }); + _dispatcher.registerTarget( WorldStateMessageType::FIND_LEAF_INDEX, [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_index(obj, buffer); }); @@ -387,6 +392,24 @@ bool WorldStateAddon::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& b return true; } +bool WorldStateAddon::get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const +{ + TypedMessage request; + obj.convert(request); + + GetBlockNumbersForLeafIndicesResponse response; + _ws->get_block_numbers_for_leaf_indices( + request.value.revision, request.value.treeId, request.value.leafIndices, response.blockNumbers); + + MsgHeader header(request.header.messageId); + messaging::TypedMessage resp_msg( + WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, header, response); + + msgpack::pack(buffer, resp_msg); + + return true; +} + bool WorldStateAddon::find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const { TypedMessage request; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp index 1a077a946e9..d0b33be2532 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/addon.hpp @@ -38,6 +38,7 @@ class WorldStateAddon : public Napi::ObjectWrap { bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; + bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_leaf_index(msgpack::object& obj, msgpack::sbuffer& buffer) const; bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; diff --git a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp index c876f5993bc..b98a8c6a69d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp +++ b/barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp @@ -1,5 +1,6 @@ #pragma once #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/messaging/header.hpp" #include "barretenberg/serialize/msgpack.hpp" @@ -20,6 +21,7 @@ enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -54,7 +56,7 @@ struct TreeIdOnlyRequest { struct CreateForkRequest { bool latest; - index_t blockNumber; + block_number_t blockNumber; MSGPACK_FIELDS(latest, blockNumber); }; @@ -129,6 +131,18 @@ struct GetSiblingPathRequest { MSGPACK_FIELDS(treeId, revision, leafIndex); }; +struct GetBlockNumbersForLeafIndicesRequest { + MerkleTreeId treeId; + WorldStateRevision revision; + std::vector leafIndices; + MSGPACK_FIELDS(treeId, revision, leafIndices); +}; + +struct GetBlockNumbersForLeafIndicesResponse { + std::vector> blockNumbers; + MSGPACK_FIELDS(blockNumbers); +}; + template struct FindLeafIndexRequest { MerkleTreeId treeId; WorldStateRevision revision; diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 57f576c55d3..4434785d41c 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -435,6 +435,22 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + /** + * Find the block numbers of the given leaf indices in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + public async findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const committedDb = await this.#getWorldState(blockNumber); + return await committedDb.getBlockNumbersForLeafIndices(treeId, leafIndices); + } + public async findNullifiersIndexesWithBlock( blockNumber: L2BlockNumber, nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 4e3f9bd4a8d..c2db85e5250 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -91,6 +91,11 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); + it('findBlockNumbersForIndexes', async () => { + const response = await context.client.findBlockNumbersForIndexes(1, MerkleTreeId.ARCHIVE, [5n, 58n]); + expect(response).toEqual([3n, 9n]); + }); + it('findNullifiersIndexesWithBlock', async () => { const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); expect(response).toEqual([ @@ -349,6 +354,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + + findBlockNumbersForIndexes( + _blockNumber: number | 'latest', + _treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + expect(leafIndices).toEqual([5n, 58n]); + return Promise.resolve([3n, 9n]); + } findNullifiersIndexesWithBlock( blockNumber: number | 'latest', nullifiers: Fr[], diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 8e5b2bda55f..457d188acf0 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -61,7 +61,7 @@ export interface AztecNode * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data * @param treeId - The tree to search in. - * @param leafValue - The values to search for + * @param leafValues - The values to search for * @returns The indexes of the given leaves in the given tree or undefined if not found. */ findLeavesIndexes( @@ -70,6 +70,19 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Find the indexes of the given leaves in the given tree. + * @param blockNumber - The block number at which to get the data or 'latest' for latest data + * @param treeId - The tree to search in. + * @param leafIndices - The values to search for + * @returns The indexes of the given leaves in the given tree or undefined if not found. + */ + findBlockNumbersForIndexes( + blockNumber: L2BlockNumber, + treeId: MerkleTreeId, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; + /** * Returns the indexes of the given nullifiers in the nullifier tree, * scoped to the block they were included in. @@ -430,6 +443,11 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findBlockNumbersForIndexes: z + .function() + .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.BigInt)) + .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z .function() .args(L2BlockNumberSchema, z.array(schemas.Fr)) diff --git a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts index 8520da7ca55..9017c1a6a84 100644 --- a/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts +++ b/yarn-project/circuit-types/src/interfaces/merkle_tree_operations.ts @@ -199,6 +199,16 @@ export interface MerkleTreeReadOperations { treeId: ID, index: bigint, ): Promise | undefined>; + + /** + * Get the block numbers for a set of leaf indices + * @param treeId - The tree for which the block numbers should be returned. + * @param leafIndices - The indices to be queried. + */ + getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]>; } export interface MerkleTreeWriteOperations extends MerkleTreeReadOperations { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index 01f3e483497..853ce0bb58f 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -173,6 +173,24 @@ export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state. export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.leaf_indices.message'; export const WORLD_STATE_LEAF_INDICES_DB_NUM_ITEMS_NOTE_HASH = 'aztec.world_state.db_num_items.leaf_indices.note_hash'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NULLIFIER = + 'aztec.world_state.db_used_size.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_PUBLIC_DATA = + 'aztec.world_state.db_used_size.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_ARCHIVE = 'aztec.world_state.db_used_size.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_MESSAGE = 'aztec.world_state.db_used_size.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_USED_SIZE_NOTE_HASH = + 'aztec.world_state.db_used_size.block_indices.note_hash'; + +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NULLIFIER = + 'aztec.world_state.db_num_items.block_indices.nullifier'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_PUBLIC_DATA = + 'aztec.world_state.db_num_items.block_indices.public_data'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_ARCHIVE = 'aztec.world_state.db_num_items.block_indices.archive'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_MESSAGE = 'aztec.world_state.db_num_items.block_indices.message'; +export const WORLD_STATE_BLOCK_INDICES_DB_NUM_ITEMS_NOTE_HASH = + 'aztec.world_state.db_num_items.block_indices.note_hash'; + export const PROOF_VERIFIER_COUNT = 'aztec.proof_verifier.count'; export const VALIDATOR_RE_EXECUTION_TIME = 'aztec.validator.re_execution_time'; diff --git a/yarn-project/world-state/src/native/merkle_trees_facade.ts b/yarn-project/world-state/src/native/merkle_trees_facade.ts index 49d53e8f3a5..9ce07806d0e 100644 --- a/yarn-project/world-state/src/native/merkle_trees_facade.ts +++ b/yarn-project/world-state/src/native/merkle_trees_facade.ts @@ -166,6 +166,19 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations { treeId, }; } + + async getBlockNumbersForLeafIndices( + treeId: ID, + leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + const response = await this.instance.call(WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, { + treeId, + revision: this.revision, + leafIndices, + }); + + return response.blockNumbers.map(x => (x === undefined || x === null ? undefined : BigInt(x))); + } } export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations { diff --git a/yarn-project/world-state/src/native/message.ts b/yarn-project/world-state/src/native/message.ts index 12c4c410edc..6f95755ce7c 100644 --- a/yarn-project/world-state/src/native/message.ts +++ b/yarn-project/world-state/src/native/message.ts @@ -54,6 +54,7 @@ export enum WorldStateMessageType { GET_LEAF_VALUE, GET_LEAF_PREIMAGE, GET_SIBLING_PATH, + GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, FIND_LEAF_INDEX, FIND_LOW_LEAF, @@ -139,6 +140,8 @@ export interface TreeDBStats { leafPreimagesDBStats: DBStats; /** Stats for the 'leaf indices' DB */ leafIndicesDBStats: DBStats; + /** Stats for the 'block indices' DB */ + blockIndicesDBStats: DBStats; } export interface WorldStateMeta { @@ -189,6 +192,7 @@ export function buildEmptyTreeDBStats() { leafIndicesDBStats: buildEmptyDBStats(), leafKeysDBStats: buildEmptyDBStats(), leafPreimagesDBStats: buildEmptyDBStats(), + blockIndicesDBStats: buildEmptyDBStats(), } as TreeDBStats; } @@ -271,6 +275,7 @@ export function sanitiseTreeDBStats(stats: TreeDBStats) { stats.blocksDBStats = sanitiseDBStats(stats.blocksDBStats); stats.leafIndicesDBStats = sanitiseDBStats(stats.leafIndicesDBStats); stats.leafPreimagesDBStats = sanitiseDBStats(stats.leafPreimagesDBStats); + stats.blockIndicesDBStats = sanitiseDBStats(stats.blockIndicesDBStats); stats.nodesDBStats = sanitiseDBStats(stats.nodesDBStats); stats.mapSize = BigInt(stats.mapSize); return stats; @@ -344,6 +349,14 @@ interface GetTreeInfoResponse { root: Buffer; } +interface GetBlockNumbersForLeafIndicesRequest extends WithTreeId, WithWorldStateRevision { + leafIndices: bigint[]; +} + +interface GetBlockNumbersForLeafIndicesResponse { + blockNumbers: bigint[]; +} + interface GetSiblingPathRequest extends WithTreeId, WithLeafIndex, WithWorldStateRevision {} type GetSiblingPathResponse = Buffer[]; @@ -446,6 +459,7 @@ export type WorldStateRequest = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafRequest; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageRequest; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathRequest; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesRequest; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexRequest; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafRequest; @@ -481,6 +495,7 @@ export type WorldStateResponse = { [WorldStateMessageType.GET_LEAF_VALUE]: GetLeafResponse; [WorldStateMessageType.GET_LEAF_PREIMAGE]: GetLeafPreImageResponse; [WorldStateMessageType.GET_SIBLING_PATH]: GetSiblingPathResponse; + [WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]: GetBlockNumbersForLeafIndicesResponse; [WorldStateMessageType.FIND_LEAF_INDEX]: FindLeafIndexResponse; [WorldStateMessageType.FIND_LOW_LEAF]: FindLowLeafResponse; diff --git a/yarn-project/world-state/src/native/native_world_state.test.ts b/yarn-project/world-state/src/native/native_world_state.test.ts index d18593fcbf0..91044fdef56 100644 --- a/yarn-project/world-state/src/native/native_world_state.test.ts +++ b/yarn-project/world-state/src/native/native_world_state.test.ts @@ -490,10 +490,75 @@ describe('NativeWorldState', () => { }); }); + describe('block numbers for indices', () => { + let block: L2Block; + let messages: Fr[]; + let noteHashes: number; + let nullifiers: number; + let publicTree: number; + + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + + it('correctly reports block numbers', async () => { + const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); + const statuses = []; + const numBlocks = 2; + const txsPerBlock = 2; + for (let i = 0; i < numBlocks; i++) { + const fork = await ws.fork(); + ({ block, messages } = await mockBlock(1, txsPerBlock, fork)); + noteHashes = block.body.txEffects[0].noteHashes.length; + nullifiers = block.body.txEffects[0].nullifiers.length; + publicTree = block.body.txEffects[0].publicDataWrites.length; + await fork.close(); + const status = await ws.handleL2BlockAndMessages(block, messages); + statuses.push(status); + } + + const checkTree = async ( + treeId: MerkleTreeId, + itemsLength: number, + blockNumber: number, + initialSize: number, + numPerBlock: number, + ) => { + const before = initialSize + itemsLength * blockNumber * numPerBlock - 2; + const on = before + 1; + const after = on + 1; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + treeId, + [before, on, after].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([blockNumber, blockNumber, blockNumber + 1].map(x => BigInt(x))); + }; + + for (let i = 0; i < numBlocks - 1; i++) { + await checkTree(MerkleTreeId.NOTE_HASH_TREE, noteHashes, i + 1, 0, 2); + await checkTree(MerkleTreeId.NULLIFIER_TREE, nullifiers, i + 1, 128, 2); + await checkTree(MerkleTreeId.PUBLIC_DATA_TREE, publicTree, i + 1, 128, 2); + await checkTree(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, messages.length, i + 1, 0, 1); + } + + const lastStatus = statuses[statuses.length - 1]; + const before = Number(lastStatus.meta.noteHashTreeMeta.committedSize) - 2; + const blockNumbers = await ws.getCommitted().getBlockNumbersForLeafIndices( + MerkleTreeId.NOTE_HASH_TREE, + [before, before + 1, before + 2].map(x => BigInt(x)), + ); + expect(blockNumbers).toEqual([2, 2, undefined].map(x => (x == undefined ? x : BigInt(x)))); + }); + }); + describe('status reporting', () => { let block: L2Block; let messages: Fr[]; + beforeAll(async () => { + await rm(dataDir, { recursive: true }); + }); + it('correctly reports status', async () => { const ws = await NativeWorldStateService.new(rollupAddress, dataDir, defaultDBMapSize); const statuses = []; diff --git a/yarn-project/world-state/src/synchronizer/instrumentation.ts b/yarn-project/world-state/src/synchronizer/instrumentation.ts index d52dca0aef4..9b4fb6a3480 100644 --- a/yarn-project/world-state/src/synchronizer/instrumentation.ts +++ b/yarn-project/world-state/src/synchronizer/instrumentation.ts @@ -5,7 +5,7 @@ import { type Gauge, type Meter, type TelemetryClient, ValueType } from '@aztec/ import { type DBStats, type TreeDBStats, type TreeMeta, type WorldStateStatusFull } from '../native/message.js'; type TreeTypeString = 'nullifier' | 'note_hash' | 'archive' | 'message' | 'public_data'; -type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks'; +type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices'; class TreeDBInstrumentation { private dbNumItems: Gauge; @@ -70,6 +70,7 @@ class TreeInstrumentation { this.treeDbInstrumentation.set('nodes', new TreeDBInstrumentation(meter, treeName, 'nodes')); this.treeDbInstrumentation.set('leaf_preimage', new TreeDBInstrumentation(meter, treeName, 'leaf_preimage')); this.treeDbInstrumentation.set('leaf_indices', new TreeDBInstrumentation(meter, treeName, 'leaf_indices')); + this.treeDbInstrumentation.set('block_indices', new TreeDBInstrumentation(meter, treeName, 'block_indices')); } private updateDBMetrics(dbName: DBTypeString, dbStats: DBStats) { @@ -92,6 +93,7 @@ class TreeInstrumentation { this.updateDBMetrics('leaf_preimage', treeDbStats.leafPreimagesDBStats); this.updateDBMetrics('blocks', treeDbStats.blocksDBStats); this.updateDBMetrics('nodes', treeDbStats.nodesDBStats); + this.updateDBMetrics('block_indices', treeDbStats.blockIndicesDBStats); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts index 82f5934fb18..a1b93999290 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_operations_facade.ts @@ -181,6 +181,13 @@ export class MerkleTreeReadOperationsFacade implements MerkleTreeWriteOperations throw new Error('Method not implemented in legacy merkle tree'); } + getBlockNumbersForLeafIndices( + _treeId: ID, + _leafIndices: bigint[], + ): Promise<(bigint | undefined)[]> { + throw new Error('Method not implemented in legacy merkle tree'); + } + close(): Promise { return Promise.resolve(); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts index 90560ef0960..7f4e4bc9d6a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_snapshot_operations_facade.ts @@ -101,6 +101,10 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeReadOperati }; } + getBlockNumbersForLeafIndices(_a: ID, _b: bigint[]): Promise<(bigint | undefined)[]> { + throw new Error('Not implemented'); + } + async getStateReference(): Promise { const snapshots = await Promise.all([ this.#getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE), From 4a38edfc1580aa1cb5113993ff8a2e5574076226 Mon Sep 17 00:00:00 2001 From: DanielKotov <159419107+DanielKotov@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:44:53 +0300 Subject: [PATCH 02/10] fix: witness changes in file sponge.hpp (#10345) Static analyzer found that initial values in cache and state arrays weren't properly constrained. Initially state fill in 0 + iv value, but these values weren't constrained as witnesses. We replaced witness_t constructor with function create_constant_witness from class witness_t in file sponge.hpp. All tests for poseidon2s passed after fix. --- .../barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp b/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp index 6d79834aa95..a812ec7811e 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/hash/poseidon2/sponge/sponge.hpp @@ -53,16 +53,16 @@ template (builder, 0); + state[i] = witness_t::create_constant_witness(builder, 0); } - state[rate] = witness_t(builder, domain_iv.get_value()); + state[rate] = witness_t::create_constant_witness(builder, domain_iv.get_value()); } std::array perform_duplex() { // zero-pad the cache for (size_t i = cache_size; i < rate; ++i) { - cache[i] = witness_t(builder, 0); + cache[i] = witness_t::create_constant_witness(builder, 0); } // add the cache into sponge state for (size_t i = 0; i < rate; ++i) { @@ -122,7 +122,7 @@ template (builder, 0); + cache[cache_size] = witness_t::create_constant_witness(builder, 0); return result; } From c53f4cf84c60b8d81cc62d5827ec4408da88cc4e Mon Sep 17 00:00:00 2001 From: Lucas Xia Date: Mon, 2 Dec 2024 12:53:55 -0500 Subject: [PATCH 03/10] feat: ultra rollup flows (#10162) Adds new flows to bb main for UltraRollupFlavor. Modifies honk recursion constraint to be able to extract IPA claims and call accumulate on them. closes https://github.com/AztecProtocol/barretenberg/issues/1153 --- barretenberg/cpp/src/barretenberg/bb/main.cpp | 22 ++++++++++++---- .../barretenberg/commitment_schemes/claim.hpp | 10 +++----- .../commitment_schemes/gemini/gemini_impl.hpp | 1 + .../commitment_schemes/ipa/ipa.hpp | 1 + .../commitment_schemes/ipa/ipa.test.cpp | 1 + .../commitment_schemes/shplonk/shplemini.hpp | 1 + .../commitment_schemes/shplonk/shplonk.hpp | 9 +++++-- .../zeromorph/zeromorph.hpp | 1 + .../types/aggregation_object_type.hpp | 2 +- .../ultra_recursive_verifier.cpp | 25 ++++++++++++------- .../stdlib/primitives/bigfield/bigfield.hpp | 2 +- .../ultra_honk/ultra_verifier.cpp | 23 +++++++++++------ 12 files changed, 67 insertions(+), 31 deletions(-) diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index 72c6c44fff0..a994b0b5929 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -1465,6 +1465,12 @@ int main(int argc, char* argv[]) } else if (command == "prove_ultra_honk_output_all") { std::string output_path = get_option(args, "-o", "./proofs"); prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + } else if (command == "prove_ultra_rollup_honk_output_all") { + std::string output_path = get_option(args, "-o", "./proofs/proof"); + prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + } else if (command == "prove_ultra_keccak_honk_output_all") { + std::string output_path = get_option(args, "-o", "./proofs/proof"); + prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); } else if (command == "prove_mega_honk_output_all") { std::string output_path = get_option(args, "-o", "./proofs"); prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); @@ -1526,9 +1532,9 @@ int main(int argc, char* argv[]) } else if (command == "prove_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); prove_honk(bytecode_path, witness_path, output_path, recursive); - } else if (command == "prove_ultra_keccak_honk_output_all") { + } else if (command == "prove_ultra_rollup_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); - prove_honk_output_all(bytecode_path, witness_path, output_path, recursive); + prove_honk(bytecode_path, witness_path, output_path, recursive); } else if (command == "verify_ultra_honk") { return verify_honk(proof_path, vk_path) ? 0 : 1; } else if (command == "verify_ultra_keccak_honk") { @@ -1539,6 +1545,9 @@ int main(int argc, char* argv[]) } else if (command == "write_vk_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", "./target/vk"); write_vk_honk(bytecode_path, output_path, recursive); + } else if (command == "write_vk_ultra_rollup_honk") { + std::string output_path = get_option(args, "-o", "./target/vk"); + write_vk_honk(bytecode_path, output_path, recursive); } else if (command == "prove_mega_honk") { std::string output_path = get_option(args, "-o", "./proofs/proof"); prove_honk(bytecode_path, witness_path, output_path, recursive); @@ -1553,12 +1562,15 @@ int main(int argc, char* argv[]) } else if (command == "vk_as_fields_ultra_honk") { std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); vk_as_fields_honk(vk_path, output_path); - } else if (command == "vk_as_fields_mega_honk") { - std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); - vk_as_fields_honk(vk_path, output_path); } else if (command == "vk_as_fields_ultra_keccak_honk") { std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); vk_as_fields_honk(vk_path, output_path); + } else if (command == "vk_as_fields_ultra_rollup_honk") { + std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); + vk_as_fields_honk(vk_path, output_path); + } else if (command == "vk_as_fields_mega_honk") { + std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); + vk_as_fields_honk(vk_path, output_path); } else { std::cerr << "Unknown command: " << command << "\n"; return 1; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp index 27917a072fa..1afb02a3c66 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/claim.hpp @@ -60,12 +60,10 @@ template class OpeningClaim { opening_pair.challenge.binary_basis_limbs[1].element.normalize().witness_index, opening_pair.challenge.binary_basis_limbs[2].element.normalize().witness_index, opening_pair.challenge.binary_basis_limbs[3].element.normalize().witness_index, - // TODO(https://github.com/AztecProtocol/barretenberg/issues/1153): Uncomment this when we turn the - // eval into witnesses. - // opening_pair.evaluation.binary_basis_limbs[0].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[1].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[2].element.normalize().witness_index, - // opening_pair.evaluation.binary_basis_limbs[3].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[0].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[1].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[2].element.normalize().witness_index, + opening_pair.evaluation.binary_basis_limbs[3].element.normalize().witness_index, commitment.x.normalize().witness_index, // no idea if we need these normalize() calls... commitment.y.normalize().witness_index }; } diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp index 0cb225dd7cf..a66387474d1 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/gemini/gemini_impl.hpp @@ -113,6 +113,7 @@ std::vector::Claim> GeminiProver_::prove( std::move(batched_to_be_shifted), std::move(batched_concatenated)); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t l = 0; l < CONST_PROOF_SIZE_LOG_N - 1; l++) { if (l < log_n - 1) { transcript->send_to_verifier("Gemini:FOLD_" + std::to_string(l + 1), diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp index c104f84a7d7..6c9aea63316 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp @@ -200,6 +200,7 @@ template class IPA { // Iterate for log(poly_degree) rounds to compute the round commitments. auto log_poly_length = static_cast(numeric::get_msb(poly_length)); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constant from IPA. if (log_poly_length > CONST_ECCVM_LOG_N) { throw_or_abort("IPA log_poly_length is too large"); } diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp index 3e40157661a..0abcb594b50 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.test.cpp @@ -159,6 +159,7 @@ TEST_F(IPATest, AIsZeroAfterOneRound) // initialize an empty mock transcript auto transcript = std::make_shared(); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constant from IPA. const size_t num_challenges = CONST_ECCVM_LOG_N + 1; std::vector random_vector(num_challenges); diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp index 83ced38c1f4..1f594bbd3eb 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplemini.hpp @@ -469,6 +469,7 @@ template class ShpleminiVerifier_ { // Initialize batching challenge as ν² Fr current_batching_challenge = shplonk_batching_challenge.sqr(); + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t j = 0; j < CONST_PROOF_SIZE_LOG_N - 1; ++j) { // Compute the scaling factor (ν²⁺ⁱ) / (z + r²⁽ⁱ⁺²⁾) for i = 0, … , d-2 Fr scaling_factor = current_batching_challenge * inverse_vanishing_evals[j + 2]; diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp index 527e1d0b73e..5e90d4a00bb 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/shplonk/shplonk.hpp @@ -2,6 +2,7 @@ #include "barretenberg/commitment_schemes/claim.hpp" #include "barretenberg/commitment_schemes/commitment_key.hpp" #include "barretenberg/commitment_schemes/verification_key.hpp" +#include "barretenberg/stdlib/primitives/curves/bn254.hpp" #include "barretenberg/transcript/transcript.hpp" /** @@ -224,7 +225,9 @@ template class ShplonkVerifier_ { // [G] = [Q] - ∑ⱼ (1/zⱼ(r))[Bⱼ] + ( ∑ⱼ (1/zⱼ(r)) Tⱼ(r) )[1] // = [Q] - ∑ⱼ (1/zⱼ(r))[Bⱼ] + G₀ [1] // G₀ = ∑ⱼ ρʲ ⋅ vⱼ / (z − xⱼ ) - auto G_commitment_constant = Fr(0); + Fr G_commitment_constant(0); + + Fr evaluation(0); // TODO(#673): The recursive and non-recursive (native) logic is completely separated via the following // conditional. Much of the logic could be shared, but I've chosen to do it this way since soon the "else" @@ -274,6 +277,8 @@ template class ShplonkVerifier_ { // [G] += G₀⋅[1] = [G] + (∑ⱼ νʲ ⋅ vⱼ / (z − xⱼ ))⋅[1] G_commitment = GroupElement::batch_mul(commitments, scalars); + // Set evaluation to constant witness + evaluation.convert_constant_to_fixed_witness(z_challenge.get_context()); } else { // [G] = [Q] - ∑ⱼ νʲ / (z − xⱼ )⋅[fⱼ] + G₀⋅[1] // = [Q] - [∑ⱼ νʲ ⋅ ( fⱼ(X) − vⱼ) / (z − xⱼ )] @@ -309,7 +314,7 @@ template class ShplonkVerifier_ { } // Return opening pair (z, 0) and commitment [G] - return { { z_challenge, Fr(0) }, G_commitment }; + return { { z_challenge, evaluation }, G_commitment }; }; /** * @brief Computes \f$ \frac{1}{z - r}, \frac{1}{z+r}, \ldots, \frac{1}{z+r^{2^{d-1}}} \f$. diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp index ff178e0267a..53f8a39c993 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/zeromorph/zeromorph.hpp @@ -421,6 +421,7 @@ template class ZeroMorphProver_ { transcript->send_to_verifier(label, q_k_commitment); } // Add buffer elements to remove log_N dependence in proof + // TODO(https://github.com/AztecProtocol/barretenberg/issues/1159): Decouple constants from primitives. for (size_t idx = log_N; idx < CONST_PROOF_SIZE_LOG_N; ++idx) { auto buffer_element = Commitment::one(); std::string label = "ZM:C_q_" + std::to_string(idx); diff --git a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp index 99608cc626f..3c44307c8e9 100644 --- a/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp +++ b/barretenberg/cpp/src/barretenberg/plonk_honk_shared/types/aggregation_object_type.hpp @@ -14,7 +14,7 @@ using PairingPointAccumulatorIndices = std::array; -static constexpr uint32_t IPA_CLAIM_SIZE = 6; +static constexpr uint32_t IPA_CLAIM_SIZE = 10; using IPAClaimIndices = std::array; using IPAClaimPubInputIndices = std::array; } // namespace bb \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp b/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp index 2cd6e8050d6..c8170f4373c 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.cpp @@ -133,7 +133,7 @@ UltraRecursiveVerifier_::Output UltraRecursiveVerifier_::verify_ // Extract the IPA claim from the public inputs // Parse out the nested IPA claim using key->ipa_claim_public_input_indices and runs the native IPA verifier. if constexpr (HasIPAAccumulator) { - const auto recover_fq_from_public_inputs = [](std::array& limbs) { + const auto recover_fq_from_public_inputs = [](std::array& limbs) { for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { limbs[k].create_range_constraint(Curve::BaseField::NUM_LIMB_BITS, "limb_" + std::to_string(k)); } @@ -142,18 +142,25 @@ UltraRecursiveVerifier_::Output UltraRecursiveVerifier_::verify_ if (verification_key->verification_key->contains_ipa_claim) { OpeningClaim> ipa_claim; - std::array bigfield_limbs; - for (size_t k = 0; k < 4; k++) { - bigfield_limbs[k] = + std::array challenge_bigfield_limbs; + for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { + challenge_bigfield_limbs[k] = verification_key ->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[k]]; } - ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(bigfield_limbs); - ipa_claim.opening_pair.evaluation = 0; + std::array evaluation_bigfield_limbs; + for (size_t k = 0; k < Curve::BaseField::NUM_LIMBS; k++) { + evaluation_bigfield_limbs[k] = + verification_key + ->public_inputs[verification_key->verification_key + ->ipa_claim_public_input_indices[Curve::BaseField::NUM_LIMBS + k]]; + } + ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(challenge_bigfield_limbs); + ipa_claim.opening_pair.evaluation = recover_fq_from_public_inputs(evaluation_bigfield_limbs); ipa_claim.commitment = { - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[4]], - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[5]], - false // WORKTODO: make this a witness? + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[8]], + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[9]], + false }; output.ipa_opening_claim = std::move(ipa_claim); } diff --git a/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp b/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp index dd3011149d8..06db1ba8d94 100644 --- a/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp +++ b/barretenberg/cpp/src/barretenberg/stdlib/primitives/bigfield/bigfield.hpp @@ -450,7 +450,7 @@ template class bigfield { void set_origin_tag(const bb::OriginTag& tag) const { - for (size_t i = 0; i < 4; i++) { + for (size_t i = 0; i < NUM_LIMBS; i++) { binary_basis_limbs[i].element.set_origin_tag(tag); } prime_basis_limb.set_origin_tag(tag); diff --git a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp index b14c4a6774d..3c4278c7439 100644 --- a/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp +++ b/barretenberg/cpp/src/barretenberg/ultra_honk/ultra_verifier.cpp @@ -34,18 +34,27 @@ template bool UltraVerifier_::verify_proof(const HonkP // Parse out the nested IPA claim using key->ipa_claim_public_input_indices and runs the native IPA verifier. if constexpr (HasIPAAccumulator) { if (verification_key->verification_key->contains_ipa_claim) { + + constexpr size_t NUM_LIMBS = 4; OpeningClaim ipa_claim; - std::array bigfield_limbs; - for (size_t k = 0; k < 4; k++) { - bigfield_limbs[k] = + + std::array challenge_bigfield_limbs; + std::array evaluation_bigfield_limbs; + for (size_t k = 0; k < NUM_LIMBS; k++) { + challenge_bigfield_limbs[k] = verification_key ->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[k]]; } - ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(bigfield_limbs); - ipa_claim.opening_pair.evaluation = 0; + for (size_t k = 0; k < NUM_LIMBS; k++) { + evaluation_bigfield_limbs[k] = + verification_key->public_inputs[verification_key->verification_key + ->ipa_claim_public_input_indices[NUM_LIMBS + k]]; + } + ipa_claim.opening_pair.challenge = recover_fq_from_public_inputs(challenge_bigfield_limbs); + ipa_claim.opening_pair.evaluation = recover_fq_from_public_inputs(evaluation_bigfield_limbs); ipa_claim.commitment = { - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[4]], - verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[5]] + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[8]], + verification_key->public_inputs[verification_key->verification_key->ipa_claim_public_input_indices[9]] }; // verify the ipa_proof with this claim From cae1203ec4263d3b64fbc3fba5cfa281922004bd Mon Sep 17 00:00:00 2001 From: just-mitch <68168980+just-mitch@users.noreply.github.com> Date: Mon, 2 Dec 2024 14:59:39 -0500 Subject: [PATCH 04/10] chore: boot node has fixed peer id private key (#10352) also update rough rhino scripts --- spartan/aztec-network/templates/boot-node.yaml | 2 ++ spartan/aztec-network/values.yaml | 1 + spartan/aztec-network/values/release.yaml | 1 + spartan/releases/rough-rhino/full-node.sh | 4 ++-- spartan/releases/rough-rhino/validator.sh | 4 ++-- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index f0ee82aabe0..00e585513a6 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -168,6 +168,8 @@ spec: value: "{{ .Values.aztec.epochDuration }}" - name: AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS value: "{{ .Values.aztec.epochProofClaimWindow }}" + - name: PEER_ID_PRIVATE_KEY + value: "{{ .Values.bootNode.peerIdPrivateKey }}" ports: - containerPort: {{ .Values.bootNode.service.nodePort }} - containerPort: {{ .Values.bootNode.service.p2pTcpPort }} diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index 245b51f9435..005d3061137 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -35,6 +35,7 @@ aztec: epochProofClaimWindow: 13 # in L2 slots bootNode: + peerIdPrivateKey: "" externalHost: "" replicas: 1 service: diff --git a/spartan/aztec-network/values/release.yaml b/spartan/aztec-network/values/release.yaml index e3e34dac3b8..b48f9cf2640 100644 --- a/spartan/aztec-network/values/release.yaml +++ b/spartan/aztec-network/values/release.yaml @@ -119,6 +119,7 @@ validator: bootNode: realProofs: true + peerIdPrivateKey: 080212200ba8451c6d62b03c4441f0a466c0bce7a3a595f2cf50a055ded3305c77aa3af0 validator: disabled: true diff --git a/spartan/releases/rough-rhino/full-node.sh b/spartan/releases/rough-rhino/full-node.sh index cf13a1d3a45..75a0d33bca1 100755 --- a/spartan/releases/rough-rhino/full-node.sh +++ b/spartan/releases/rough-rhino/full-node.sh @@ -22,8 +22,8 @@ docker run --rm --network=host \ -e AZTEC_SLOT_DURATION=36 \ -e AZTEC_EPOCH_DURATION=32 \ -e AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS=13 \ - -e ETHEREUM_HOST=http://35.221.3.35:8545 \ - -e BOOTSTRAP_NODES=enr:-Jq4QKIJisajcICBVMoMwFtbmPgmHt3KoonypbBIQCAMNjhMc6DKW0J4vJzDpGPFUX7T2fzyyjezHgKKzeZY_DbRz_kGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCPdAyOJc2VjcDI1NmsxoQK92C7GObzDvCt9uwzW0lhKJKGCvOWkmAZjd2E2w-svuoN0Y3CCndCDdWRwgp3Q \ + -e ETHEREUM_HOST=http://34.48.76.131:8545 \ + -e BOOTSTRAP_NODES=enr:-Jq4QO_3szmgtG2cbEdnFDIhpGAQkc1HwfNy4-M6sG9QmQbPTmp9PMOHR3xslfR23hORiU-GpA7uM9uXw49lFcnuuvYGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCIwTIOJc2VjcDI1NmsxoQKQTN17XKCwjYSSwmTc-6YzCMhd3v6Ofl8TS-WunX6LCoN0Y3CCndCDdWRwgp3Q \ -e REGISTRY_CONTRACT_ADDRESS=0x5fbdb2315678afecb367f032d93f642f64180aa3 \ -e GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ -e FEE_JUICE_CONTRACT_ADDRESS=0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 \ diff --git a/spartan/releases/rough-rhino/validator.sh b/spartan/releases/rough-rhino/validator.sh index 98e81124227..246e59527bb 100755 --- a/spartan/releases/rough-rhino/validator.sh +++ b/spartan/releases/rough-rhino/validator.sh @@ -25,8 +25,8 @@ docker run --rm --network=host \ -e AZTEC_SLOT_DURATION=36 \ -e AZTEC_EPOCH_DURATION=32 \ -e AZTEC_EPOCH_PROOF_CLAIM_WINDOW_IN_L2_SLOTS=13 \ - -e ETHEREUM_HOST=http://35.221.3.35:8545 \ - -e BOOTSTRAP_NODES=enr:-Jq4QKIJisajcICBVMoMwFtbmPgmHt3KoonypbBIQCAMNjhMc6DKW0J4vJzDpGPFUX7T2fzyyjezHgKKzeZY_DbRz_kGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCPdAyOJc2VjcDI1NmsxoQK92C7GObzDvCt9uwzW0lhKJKGCvOWkmAZjd2E2w-svuoN0Y3CCndCDdWRwgp3Q \ + -e ETHEREUM_HOST=http://34.48.76.131:8545 \ + -e BOOTSTRAP_NODES=enr:-Jq4QO_3szmgtG2cbEdnFDIhpGAQkc1HwfNy4-M6sG9QmQbPTmp9PMOHR3xslfR23hORiU-GpA7uM9uXw49lFcnuuvYGjWF6dGVjX25ldHdvcmsBgmlkgnY0gmlwhCIwTIOJc2VjcDI1NmsxoQKQTN17XKCwjYSSwmTc-6YzCMhd3v6Ofl8TS-WunX6LCoN0Y3CCndCDdWRwgp3Q \ -e REGISTRY_CONTRACT_ADDRESS=0x5fbdb2315678afecb367f032d93f642f64180aa3 \ -e GOVERNANCE_PROPOSER_CONTRACT_ADDRESS=0x9fe46736679d2d9a65f0992f2272de9f3c7fa6e0 \ -e FEE_JUICE_CONTRACT_ADDRESS=0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 \ From 7227b487f97e26a3f8f2aa8086fb7c2c7b0de557 Mon Sep 17 00:00:00 2001 From: just-mitch <68168980+just-mitch@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:44:57 -0500 Subject: [PATCH 05/10] chore: fix sassy-salamander chores v1 (#10218) Handful of chores. fix #10074 fix #10075 fix #10077 --- .../circuit-types/src/tx_execution_request.ts | 5 ++ yarn-project/end-to-end/src/spartan/utils.ts | 6 +- yarn-project/foundation/package.json | 2 +- yarn-project/prover-client/package.json | 2 +- .../pxe/src/pxe_service/pxe_service.ts | 36 ++++++++--- .../src/sequencer/sequencer.test.ts | 2 +- .../src/sequencer/sequencer.ts | 59 ++++++++----------- .../sequencer-client/src/sequencer/utils.ts | 2 +- 8 files changed, 66 insertions(+), 48 deletions(-) diff --git a/yarn-project/circuit-types/src/tx_execution_request.ts b/yarn-project/circuit-types/src/tx_execution_request.ts index 2a99636da2e..bdd90e28550 100644 --- a/yarn-project/circuit-types/src/tx_execution_request.ts +++ b/yarn-project/circuit-types/src/tx_execution_request.ts @@ -4,6 +4,7 @@ import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize'; import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; import { type FieldsOf } from '@aztec/foundation/types'; +import { inspect } from 'util'; import { z } from 'zod'; import { AuthWitness } from './auth_witness.js'; @@ -141,4 +142,8 @@ export class TxExecutionRequest { [AuthWitness.random()], ); } + + [inspect.custom]() { + return `TxExecutionRequest(${this.origin} called ${this.functionSelector})`; + } } diff --git a/yarn-project/end-to-end/src/spartan/utils.ts b/yarn-project/end-to-end/src/spartan/utils.ts index fe34c31534a..3152de9c53d 100644 --- a/yarn-project/end-to-end/src/spartan/utils.ts +++ b/yarn-project/end-to-end/src/spartan/utils.ts @@ -355,7 +355,11 @@ export async function awaitL2BlockNumber( await sleep(1000); tips = await rollupCheatCodes.getTips(); } - logger.info(`Reached L2 Block ${tips.pending}`); + if (tips.pending < blockNumber) { + throw new Error(`Timeout waiting for L2 Block ${blockNumber}, only reached ${tips.pending}`); + } else { + logger.info(`Reached L2 Block ${tips.pending}`); + } } export async function restartBot(namespace: string, logger: Logger) { diff --git a/yarn-project/foundation/package.json b/yarn-project/foundation/package.json index 149a879c05c..cdaaafa04e9 100644 --- a/yarn-project/foundation/package.json +++ b/yarn-project/foundation/package.json @@ -164,4 +164,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 65012f1a178..b8766542083 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -104,4 +104,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 4a5ab7a7576..201d69ba2b0 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -67,6 +67,8 @@ import { } from '@aztec/protocol-contracts'; import { type AcirSimulator } from '@aztec/simulator'; +import { inspect } from 'util'; + import { type PXEServiceConfig, getPackageInfo } from '../config/index.js'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; import { IncomingNoteDao } from '../database/incoming_note_dao.js'; @@ -519,8 +521,7 @@ export class PXEService implements PXE { return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!); }) .catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError(err, inspect(txRequest), inspect(privateExecutionResult)); }); } @@ -576,8 +577,15 @@ export class PXEService implements PXE { ); }) .catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError( + err, + inspect(txRequest), + `simulatePublic=${simulatePublic}`, + `msgSender=${msgSender?.toString() ?? 'undefined'}`, + `skipTxValidation=${skipTxValidation}`, + `profile=${profile}`, + `scopes=${scopes?.map(s => s.toString()).join(', ') ?? 'undefined'}`, + ); }); } @@ -588,8 +596,7 @@ export class PXEService implements PXE { } this.log.info(`Sending transaction ${txHash}`); await this.node.sendTx(tx).catch(err => { - this.log.error(err); - throw err; + throw this.contextualizeError(err, inspect(tx)); }); this.log.info(`Sent transaction ${txHash}`); return txHash; @@ -613,8 +620,12 @@ export class PXEService implements PXE { return executionResult; }) .catch(err => { - this.log.error(err); - throw err; + const stringifiedArgs = args.map(arg => arg.toString()).join(', '); + throw this.contextualizeError( + err, + `simulateUnconstrained ${to}:${functionName}(${stringifiedArgs})`, + `scopes=${scopes?.map(s => s.toString()).join(', ') ?? 'undefined'}`, + ); }); } @@ -986,4 +997,13 @@ export class PXEService implements PXE { async resetNoteSyncData() { return await this.db.resetNoteSyncData(); } + + private contextualizeError(err: Error, ...context: string[]): Error { + this.log.error(err.name, err); + this.log.debug('Context:'); + for (const c of context) { + this.log.debug(c); + } + return err; + } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index e5a55462b97..5acbbd261f6 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -843,7 +843,7 @@ class TestSubject extends Sequencer { } public override doRealWork() { - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); return super.doRealWork(); } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 8c9750ff043..325a2dd2d44 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -189,7 +189,7 @@ export class Sequencer { public start() { this.runningPromise = new RunningPromise(this.work.bind(this), this.pollingIntervalMs); this.runningPromise.start(); - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); this.log.info('Sequencer started'); return Promise.resolve(); } @@ -201,7 +201,7 @@ export class Sequencer { this.log.debug(`Stopping sequencer`); await this.runningPromise?.stop(); this.publisher.interrupt(); - this.setState(SequencerState.STOPPED, 0, true /** force */); + this.setState(SequencerState.STOPPED, 0n, true /** force */); this.log.info('Stopped sequencer'); } @@ -212,7 +212,7 @@ export class Sequencer { this.log.info('Restarting sequencer'); this.publisher.restart(); this.runningPromise!.start(); - this.setState(SequencerState.IDLE, 0, true /** force */); + this.setState(SequencerState.IDLE, 0n, true /** force */); } /** @@ -232,7 +232,7 @@ export class Sequencer { * - If our block for some reason is not included, revert the state */ protected async doRealWork() { - this.setState(SequencerState.SYNCHRONIZING, 0); + this.setState(SequencerState.SYNCHRONIZING, 0n); // Update state when the previous block has been synced const prevBlockSynced = await this.isBlockSynced(); // Do not go forward with new block if the previous one has not been mined and processed @@ -243,7 +243,7 @@ export class Sequencer { this.log.debug('Previous block has been mined and processed'); - this.setState(SequencerState.PROPOSER_CHECK, 0); + this.setState(SequencerState.PROPOSER_CHECK, 0n); const chainTip = await this.l2BlockSource.getBlock(-1); const historicalHeader = chainTip?.header; @@ -277,9 +277,8 @@ export class Sequencer { if (!this.shouldProposeBlock(historicalHeader, {})) { return; } - const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration, Number(slot)); - this.setState(SequencerState.WAITING_FOR_TXS, secondsIntoSlot); + this.setState(SequencerState.WAITING_FOR_TXS, slot); // Get txs to build the new block. const pendingTxs = this.p2pClient.getTxs('pending'); @@ -325,7 +324,7 @@ export class Sequencer { } catch (err) { this.log.error(`Error assembling block`, (err as any).stack); } - this.setState(SequencerState.IDLE, 0); + this.setState(SequencerState.IDLE, 0n); } protected async work() { @@ -339,7 +338,7 @@ export class Sequencer { throw err; } } finally { - this.setState(SequencerState.IDLE, 0); + this.setState(SequencerState.IDLE, 0n); } } @@ -398,13 +397,23 @@ export class Sequencer { return true; } - setState(proposedState: SequencerState, secondsIntoSlot: number, force: boolean = false) { + /** + * Sets the sequencer state and checks if we have enough time left in the slot to transition to the new state. + * @param proposedState - The new state to transition to. + * @param currentSlotNumber - The current slot number. + * @param force - Whether to force the transition even if the sequencer is stopped. + * + * @dev If the `currentSlotNumber` doesn't matter (e.g. transitioning to IDLE), pass in `0n`; + * it is only used to check if we have enough time left in the slot to transition to the new state. + */ + setState(proposedState: SequencerState, currentSlotNumber: bigint, force: boolean = false) { if (this.state === SequencerState.STOPPED && force !== true) { this.log.warn( `Cannot set sequencer from ${this.state} to ${proposedState} as it is stopped. Set force=true to override.`, ); return; } + const secondsIntoSlot = getSecondsIntoSlot(this.l1GenesisTime, this.aztecSlotDuration, Number(currentSlotNumber)); if (!this.doIHaveEnoughTimeLeft(proposedState, secondsIntoSlot)) { throw new SequencerTooSlowError(this.state, proposedState, this.timeTable[proposedState], secondsIntoSlot); } @@ -567,12 +576,7 @@ export class Sequencer { this.metrics.recordNewBlock(newGlobalVariables.blockNumber.toNumber(), validTxs.length); const workTimer = new Timer(); - const secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - newGlobalVariables.slotNumber.toNumber(), - ); - this.setState(SequencerState.CREATING_BLOCK, secondsIntoSlot); + this.setState(SequencerState.CREATING_BLOCK, newGlobalVariables.slotNumber.toBigInt()); this.log.info( `Building blockNumber=${newGlobalVariables.blockNumber.toNumber()} txCount=${ validTxs.length @@ -688,23 +692,13 @@ export class Sequencer { this.log.info('Creating block proposal'); const proposal = await this.validatorClient.createBlockProposal(block.header, block.archive.root, txHashes); - let secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); + const slotNumber = block.header.globalVariables.slotNumber.toBigInt(); - this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS, secondsIntoSlot); + this.setState(SequencerState.PUBLISHING_BLOCK_TO_PEERS, slotNumber); this.log.info('Broadcasting block proposal to validators'); this.validatorClient.broadcastBlockProposal(proposal); - secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); - - this.setState(SequencerState.WAITING_FOR_ATTESTATIONS, secondsIntoSlot); + this.setState(SequencerState.WAITING_FOR_ATTESTATIONS, slotNumber); const attestations = await this.validatorClient.collectAttestations(proposal, numberOfRequiredAttestations); this.log.info(`Collected attestations from validators, number of attestations: ${attestations.length}`); @@ -761,13 +755,8 @@ export class Sequencer { txHashes?: TxHash[], proofQuote?: EpochProofQuote, ) { - const secondsIntoSlot = getSecondsIntoSlot( - this.l1GenesisTime, - this.aztecSlotDuration, - block.header.globalVariables.slotNumber.toNumber(), - ); // Publishes new block to the network and awaits the tx to be mined - this.setState(SequencerState.PUBLISHING_BLOCK, secondsIntoSlot); + this.setState(SequencerState.PUBLISHING_BLOCK, block.header.globalVariables.slotNumber.toBigInt()); const publishedL2Block = await this.publisher.proposeL2Block(block, attestations, txHashes, proofQuote); if (!publishedL2Block) { diff --git a/yarn-project/sequencer-client/src/sequencer/utils.ts b/yarn-project/sequencer-client/src/sequencer/utils.ts index 4c16e8c8a9b..8bb4b440dc2 100644 --- a/yarn-project/sequencer-client/src/sequencer/utils.ts +++ b/yarn-project/sequencer-client/src/sequencer/utils.ts @@ -75,5 +75,5 @@ export function orderAttestations(attestations: BlockAttestation[], orderAddress export function getSecondsIntoSlot(l1GenesisTime: number, aztecSlotDuration: number, slotNumber: number): number { const slotStartTimestamp = l1GenesisTime + slotNumber * aztecSlotDuration; - return Date.now() / 1000 - slotStartTimestamp; + return Number((Date.now() / 1000 - slotStartTimestamp).toFixed(3)); } From da1470d074f4884e61b51e450a661432c6f0a10f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 2 Dec 2024 19:26:40 -0300 Subject: [PATCH 06/10] chore: Log manual contract class registrations (#10354) To track when a pxe manually registered a deployed contract class on a node. --- yarn-project/aztec-node/src/aztec-node/server.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 4434785d41c..587d97371a4 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -852,10 +852,12 @@ export class AztecNodeService implements AztecNode { // TODO(#10007): Remove this method public addContractClass(contractClass: ContractClassPublic): Promise { + this.log.info(`Adding contract class via API ${contractClass.id}`); return this.contractDataSource.addContractClass(contractClass); } public addContractArtifact(address: AztecAddress, artifact: ContractArtifact): Promise { + this.log.info(`Adding contract artifact ${artifact.name} for ${address.toString()} via API`); // TODO: Node should validate the artifact before accepting it return this.contractDataSource.addContractArtifact(address, artifact); } From c3d8e4882aee232ff09ff863bdeaa30808c09fa3 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Tue, 3 Dec 2024 02:29:17 +0000 Subject: [PATCH 07/10] git subrepo push --branch=master barretenberg subrepo: subdir: "barretenberg" merged: "0fe0a5d6df" upstream: origin: "https://github.com/AztecProtocol/barretenberg" branch: "master" commit: "0fe0a5d6df" git-subrepo: version: "0.4.6" origin: "???" commit: "???" [skip ci] --- barretenberg/.gitrepo | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/barretenberg/.gitrepo b/barretenberg/.gitrepo index 91492d1465b..4f192f1c1a9 100644 --- a/barretenberg/.gitrepo +++ b/barretenberg/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/barretenberg branch = master - commit = 5e751b81cd5e5f34d9286d5a19ea6d4853566480 - parent = 887c01103255ea4cbbb6cb33c8771d47123b3bff + commit = 0fe0a5d6dff43547aaec45256440982184e93bb0 + parent = da1470d074f4884e61b51e450a661432c6f0a10f method = merge cmdver = 0.4.6 From 2496118908db955d82222fe98514f4a55ff61e33 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Tue, 3 Dec 2024 02:29:51 +0000 Subject: [PATCH 08/10] chore: replace relative paths to noir-protocol-circuits --- noir-projects/aztec-nr/aztec/Nargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noir-projects/aztec-nr/aztec/Nargo.toml b/noir-projects/aztec-nr/aztec/Nargo.toml index 7a1f1af5863..5fbd827e148 100644 --- a/noir-projects/aztec-nr/aztec/Nargo.toml +++ b/noir-projects/aztec-nr/aztec/Nargo.toml @@ -5,4 +5,4 @@ compiler_version = ">=0.18.0" type = "lib" [dependencies] -protocol_types = { path = "../../noir-protocol-circuits/crates/types" } +protocol_types = { git="https://github.com/AztecProtocol/aztec-packages", tag="aztec-packages-v0.65.2", directory="noir-projects/noir-protocol-circuits/crates/types" } From d697142f2fc6d1f508680e02966d5b3bff346dca Mon Sep 17 00:00:00 2001 From: AztecBot Date: Tue, 3 Dec 2024 02:29:52 +0000 Subject: [PATCH 09/10] git_subrepo.sh: Fix parent in .gitrepo file. [skip ci] --- noir-projects/aztec-nr/.gitrepo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noir-projects/aztec-nr/.gitrepo b/noir-projects/aztec-nr/.gitrepo index 4696364255e..3a25b690cfc 100644 --- a/noir-projects/aztec-nr/.gitrepo +++ b/noir-projects/aztec-nr/.gitrepo @@ -9,4 +9,4 @@ remote = https://github.com/AztecProtocol/aztec-nr commit = 6344f8f2ef40886a0047d49c5685adec9ff1911f method = merge cmdver = 0.4.6 - parent = b535302b6f9b3ff8d27c1c104060ab7b19af0fb7 + parent = df0e0d9f50b79b7ae00df354323c1b23567414bb From fcbff0d2564c155b25805a8e13e11ba21eee3e26 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Tue, 3 Dec 2024 02:29:55 +0000 Subject: [PATCH 10/10] git subrepo push --branch=master noir-projects/aztec-nr subrepo: subdir: "noir-projects/aztec-nr" merged: "6e54884e4b" upstream: origin: "https://github.com/AztecProtocol/aztec-nr" branch: "master" commit: "6e54884e4b" git-subrepo: version: "0.4.6" origin: "???" commit: "???" [skip ci] --- noir-projects/aztec-nr/.gitrepo | 4 ++-- noir-projects/aztec-nr/aztec/Nargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/noir-projects/aztec-nr/.gitrepo b/noir-projects/aztec-nr/.gitrepo index 3a25b690cfc..e3be0061d71 100644 --- a/noir-projects/aztec-nr/.gitrepo +++ b/noir-projects/aztec-nr/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/AztecProtocol/aztec-nr branch = master - commit = 6344f8f2ef40886a0047d49c5685adec9ff1911f + commit = 6e54884e4b0ea5204966b98c490f6e93722b7ad9 method = merge cmdver = 0.4.6 - parent = df0e0d9f50b79b7ae00df354323c1b23567414bb + parent = 9d0a890cedaa29019e6312fbb24164b8adbd8bc4 diff --git a/noir-projects/aztec-nr/aztec/Nargo.toml b/noir-projects/aztec-nr/aztec/Nargo.toml index 5fbd827e148..7a1f1af5863 100644 --- a/noir-projects/aztec-nr/aztec/Nargo.toml +++ b/noir-projects/aztec-nr/aztec/Nargo.toml @@ -5,4 +5,4 @@ compiler_version = ">=0.18.0" type = "lib" [dependencies] -protocol_types = { git="https://github.com/AztecProtocol/aztec-packages", tag="aztec-packages-v0.65.2", directory="noir-projects/noir-protocol-circuits/crates/types" } +protocol_types = { path = "../../noir-protocol-circuits/crates/types" }