Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: World state tech debt cleanup 1 #9561

Merged
merged 28 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
aed5bce
WIP
PhilWindle Oct 29, 2024
0894c13
Updated logging
PhilWindle Oct 29, 2024
36f53d4
Merge branch 'master' into pw/world-state-cleanup
PhilWindle Oct 29, 2024
68f2c25
Removed erroneous code
PhilWindle Oct 29, 2024
f91d91e
Merge branch 'master' into pw/world-state-cleanup
ludamad Oct 30, 2024
4962d96
Detect if trees are out of sync
PhilWindle Oct 30, 2024
8d46f2c
Merge branch 'pw/world-state-cleanup' of github.com:AztecProtocol/azt…
PhilWindle Oct 31, 2024
6511d8a
WIP
PhilWindle Oct 31, 2024
e536a59
WIP
PhilWindle Nov 11, 2024
5019a35
WIP
PhilWindle Nov 11, 2024
22e1a14
Merge remote-tracking branch 'origin/master' into pw/world-state-cleanup
PhilWindle Nov 11, 2024
478296e
Merge remote-tracking branch 'origin/master' into pw/world-state-cleanup
PhilWindle Nov 11, 2024
0c62a94
Merge branch 'master' into pw/world-state-cleanup
PhilWindle Nov 11, 2024
70ec05a
Merge remote-tracking branch 'origin/master' into pw/world-state-cleanup
PhilWindle Nov 12, 2024
eb72f4f
WIP
PhilWindle Nov 12, 2024
c24b940
WIP
PhilWindle Nov 13, 2024
75e8472
WIP
PhilWindle Nov 13, 2024
14fa37b
WIP
PhilWindle Nov 13, 2024
21a5a1c
WIP
PhilWindle Nov 13, 2024
0ad1bdb
WIP
PhilWindle Nov 13, 2024
5cc95c5
Fixes
PhilWindle Nov 13, 2024
0ac899f
Formatting
PhilWindle Nov 13, 2024
ed8be9f
Additional test case
PhilWindle Nov 13, 2024
c7e64bb
Log formatting
PhilWindle Nov 13, 2024
b0b25bc
More log formatting
PhilWindle Nov 13, 2024
6224fa8
Formatting
PhilWindle Nov 13, 2024
4db7059
Merge branch 'master' into pw/world-state-cleanup
PhilWindle Nov 13, 2024
e9968b7
Merge branch 'master' into pw/world-state-cleanup
PhilWindle Nov 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param on_completion Callback to be called on completion
* @param includeUncommitted Whether to include uncommitted changes
*/
void get_subtree_sibling_path(index_t leaf_index,
void get_subtree_sibling_path(const index_t& leaf_index,
uint32_t subtree_depth,
const HashPathCallback& on_completion,
bool includeUncommitted) const;
Expand All @@ -131,7 +131,9 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param includeUncommitted Whether to include uncommitted changes
* @param on_completion Callback to be called on completion
*/
void get_meta_data(index_t blockNumber, bool includeUncommitted, const MetaDataCallback& on_completion) const;
void get_meta_data(const index_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const;

/**
* @brief Returns the leaf value at the provided index
Expand Down Expand Up @@ -226,12 +228,12 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
const RequestContext& requestContext,
ReadTransaction& tx) const;

std::optional<fr> find_leaf_hash(index_t leaf_index,
std::optional<fr> find_leaf_hash(const index_t& leaf_index,
const RequestContext& requestContext,
ReadTransaction& tx,
bool updateNodesByIndexCache = false) const;

index_t get_batch_insertion_size(index_t treeSize, index_t remainingAppendSize);
index_t get_batch_insertion_size(const index_t& treeSize, const index_t& remainingAppendSize);

void add_batch_internal(
std::vector<fr>& values, fr& new_root, index_t& new_size, bool update_index, ReadTransaction& tx);
Expand Down Expand Up @@ -323,7 +325,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(bool in
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(index_t blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(const index_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const
{
Expand All @@ -335,7 +337,9 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(index_t

BlockPayload blockData;
if (!store_->get_block_data(blockNumber, blockData, *tx)) {
throw std::runtime_error("Data for block unavailable");
throw std::runtime_error((std::stringstream() << "Unable to get meta data for block " << blockNumber
<< ", failed to get block data.")
.str());
}

response.inner.meta.size = blockData.size;
Expand Down Expand Up @@ -365,12 +369,15 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons
execute_and_report<GetSiblingPathResponse>(
[=, this](TypedResponse<GetSiblingPathResponse>& response) {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to get sibling path at block 0");
}
ReadTransactionPtr tx = store_->create_read_transaction();
BlockPayload blockData;
if (!store_->get_block_data(blockNumber, blockData, *tx)) {
throw std::runtime_error("Data for block unavailable");
throw std::runtime_error((std::stringstream()
<< "Unable to get sibling path for index " << index << " at block "
<< blockNumber << ", failed to get block data.")
.str());
}

RequestContext requestContext;
Expand All @@ -387,7 +394,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_subtree_sibling_path(
const uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const
uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const
{
auto job = [=, this]() {
execute_and_report<GetSiblingPathResponse>(
Expand All @@ -409,8 +416,8 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_subtree_sibling_p

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_subtree_sibling_path(
const index_t leaf_index,
const uint32_t subtree_depth,
const index_t& leaf_index,
uint32_t subtree_depth,
const HashPathCallback& on_completion,
bool includeUncommitted) const
{
Expand Down Expand Up @@ -450,7 +457,10 @@ fr_sibling_path ContentAddressedAppendOnlyTree<Store, HashingPolicy>::optional_s

template <typename Store, typename HashingPolicy>
std::optional<fr> ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_hash(
index_t leaf_index, const RequestContext& requestContext, ReadTransaction& tx, bool updateNodesByIndexCache) const
const index_t& leaf_index,
const RequestContext& requestContext,
ReadTransaction& tx,
bool updateNodesByIndexCache) const
{
fr hash = requestContext.root;
// std::cout << "Finding leaf hash for root " << hash << " at index " << leaf_index << std::endl;
Expand Down Expand Up @@ -510,7 +520,7 @@ template <typename Store, typename HashingPolicy>
ContentAddressedAppendOnlyTree<Store, HashingPolicy>::OptionalSiblingPath ContentAddressedAppendOnlyTree<
Store,
HashingPolicy>::get_subtree_sibling_path_internal(const index_t& leaf_index,
const uint32_t subtree_depth,
uint32_t subtree_depth,
const RequestContext& requestContext,
ReadTransaction& tx) const
{
Expand Down Expand Up @@ -586,15 +596,21 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_leaf(const index_
execute_and_report<GetLeafResponse>(
[=, this](TypedResponse<GetLeafResponse>& response) {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to get leaf at block 0");
}
ReadTransactionPtr tx = store_->create_read_transaction();
BlockPayload blockData;
if (!store_->get_block_data(blockNumber, blockData, *tx)) {
throw std::runtime_error("Data for block unavailable");
throw std::runtime_error((std::stringstream()
<< "Unable to get leaf at index " << leaf_index << " for block "
<< blockNumber << ", failed to get block data.")
.str());
}
if (blockData.size < leaf_index) {
response.message = "Data for block unavailable";
response.message =
(std::stringstream() << "Unable to get leaf at index " << leaf_index << " for block "
<< blockNumber << ", leaf index is too high.")
.str();
response.success = false;
return;
}
Expand Down Expand Up @@ -665,12 +681,15 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index_from(
execute_and_report<FindLeafIndexResponse>(
[=, this](TypedResponse<FindLeafIndexResponse>& response) {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to find leaf index for block number 0");
}
ReadTransactionPtr tx = store_->create_read_transaction();
BlockPayload blockData;
if (!store_->get_block_data(blockNumber, blockData, *tx)) {
throw std::runtime_error("Data for block unavailable");
throw std::runtime_error((std::stringstream()
<< "Unable to find leaf from index " << start_index << " for block "
<< blockNumber << ", failed to get block data.")
.str());
}
RequestContext requestContext;
requestContext.blockNumber = blockNumber;
Expand Down Expand Up @@ -739,7 +758,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::remove_historic_block
execute_and_report(
[=, this]() {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to remove historic block 0");
}
store_->remove_historical_block(blockNumber);
},
Expand All @@ -756,7 +775,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(
execute_and_report(
[=, this]() {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to unwind block 0");
}
store_->unwind_block(blockNumber);
},
Expand All @@ -773,7 +792,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const
execute_and_report(
[=, this]() {
if (blockNumber == 0) {
throw std::runtime_error("Invalid block number");
throw std::runtime_error("Unable to finalise block 0");
}
store_->advance_finalised_block(blockNumber);
},
Expand All @@ -783,8 +802,8 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const
}

template <typename Store, typename HashingPolicy>
index_t ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_batch_insertion_size(index_t treeSize,
index_t remainingAppendSize)
index_t ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_batch_insertion_size(
const index_t& treeSize, const index_t& remainingAppendSize)
{
index_t minPower2 = 1;
if (treeSize != 0U) {
Expand Down Expand Up @@ -845,7 +864,9 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::add_batch_internal(
}

if (new_size > max_size_) {
throw std::runtime_error("Tree is full");
throw std::runtime_error((std::stringstream() << "Unable to append leaves to tree " << meta.name
<< " new size: " << new_size << " max size: " << max_size_)
.str());
}

// Add the values at the leaf nodes of the tree
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -455,10 +455,13 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, reports_an_error_if_tree_is_
}
add_values(tree, values);

std::stringstream ss;
ss << "Unable to append leaves to tree " << name << " new size: 17 max size: 16";

Signal signal;
auto add_completion = [&](const TypedResponse<AddDataResponse>& response) {
EXPECT_EQ(response.success, false);
EXPECT_EQ(response.message, "Tree is full");
EXPECT_EQ(response.message, ss.str());
signal.signal_level();
};
tree.add_value(VALUES[16], add_completion);
Expand Down
Loading
Loading