Skip to content

Commit

Permalink
Frontier store break up (#3287)
Browse files Browse the repository at this point in the history 
* Start of moving frontier_store out of block_store class
* More changes for frontier_store
* Putting frontier_store_partial instance in to frontier_store and fixing some references.
* Add frontier_store class changes to the unit tests
* Clang-format fix
* Remove unnecessary comment
* Move macros to functions
* Remove reduntant implementation
* Removing function template arguments are they can be deduced.
* Removing unneeded argument parenthesis as this is no longer a macro.

Co-authored-by: clemahieu <[email protected]>
  • Loading branch information
@thsfs @clemahieu
thsfs and clemahieu authored May 24, 2021
1 parent 33a9741 commit 9683b6d
Show file tree
Hide file tree
Showing 9 changed files with 214 additions and 153 deletions.
10 changes: 5 additions & 5 deletions nano/core_test/block_store.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -817,11 +817,11 @@ TEST (block_store, frontier)
auto transaction (store->tx_begin_write ());
nano::block_hash hash (100);
nano::account account (200);
ASSERT_TRUE (store->frontier_get (transaction, hash).is_zero ());
store->frontier_put (transaction, hash, account);
ASSERT_EQ (account, store->frontier_get (transaction, hash));
store->frontier_del (transaction, hash);
ASSERT_TRUE (store->frontier_get (transaction, hash).is_zero ());
ASSERT_TRUE (store->frontier.get (transaction, hash).is_zero ());
store->frontier.put (transaction, hash, account);
ASSERT_EQ (account, store->frontier.get (transaction, hash));
store->frontier.del (transaction, hash);
ASSERT_TRUE (store->frontier.get (transaction, hash).is_zero ());
}

TEST (block_store, block_replace)
Expand Down
38 changes: 19 additions & 19 deletions nano/core_test/ledger.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,15 +109,15 @@ TEST (ledger, process_send)
nano::keypair key2;
nano::send_block send (info1.head, key2.pub, 50, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *pool.generate (info1.head));
nano::block_hash hash1 (send.hash ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, info1.head));
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, info1.head));
ASSERT_EQ (1, info1.block_count);
// This was a valid block, it should progress.
auto return1 (ledger.process (transaction, send));
ASSERT_EQ (nano::dev_genesis_key.pub, send.sideband ().account);
ASSERT_EQ (2, send.sideband ().height);
ASSERT_EQ (nano::genesis_amount - 50, ledger.amount (transaction, hash1));
ASSERT_TRUE (store->frontier_get (transaction, info1.head).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, hash1));
ASSERT_TRUE (store->frontier.get (transaction, info1.head).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, hash1));
ASSERT_EQ (nano::process_result::progress, return1.code);
ASSERT_EQ (nano::dev_genesis_key.pub, store->block_account_calculated (send));
ASSERT_EQ (50, ledger.account_balance (transaction, nano::dev_genesis_key.pub));
Expand All @@ -143,7 +143,7 @@ TEST (ledger, process_send)
ASSERT_EQ (nano::process_result::progress, return2.code);
ASSERT_EQ (key2.pub, store->block_account_calculated (open));
ASSERT_EQ (nano::genesis_amount - 50, ledger.amount (transaction, hash2));
ASSERT_EQ (key2.pub, store->frontier_get (transaction, hash2));
ASSERT_EQ (key2.pub, store->frontier.get (transaction, hash2));
ASSERT_EQ (nano::genesis_amount - 50, ledger.account_balance (transaction, key2.pub));
ASSERT_EQ (0, ledger.account_pending (transaction, key2.pub));
ASSERT_EQ (50, ledger.weight (nano::dev_genesis_key.pub));
Expand All @@ -163,7 +163,7 @@ TEST (ledger, process_send)
ASSERT_NE (nullptr, latest5);
ASSERT_EQ (open, *latest5);
ASSERT_FALSE (ledger.rollback (transaction, hash2));
ASSERT_TRUE (store->frontier_get (transaction, hash2).is_zero ());
ASSERT_TRUE (store->frontier.get (transaction, hash2).is_zero ());
nano::account_info info5;
ASSERT_TRUE (ledger.store.account_get (transaction, key2.pub, info5));
nano::pending_info pending1;
Expand All @@ -180,8 +180,8 @@ TEST (ledger, process_send)
ASSERT_EQ (hash1, info6.head);
ASSERT_FALSE (ledger.rollback (transaction, info6.head));
ASSERT_EQ (nano::genesis_amount, ledger.weight (nano::dev_genesis_key.pub));
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, info1.head));
ASSERT_TRUE (store->frontier_get (transaction, hash1).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, info1.head));
ASSERT_TRUE (store->frontier.get (transaction, hash1).is_zero ());
nano::account_info info7;
ASSERT_FALSE (ledger.store.account_get (transaction, nano::dev_genesis_key.pub, info7));
ASSERT_EQ (1, info7.block_count);
Expand Down Expand Up @@ -226,14 +226,14 @@ TEST (ledger, process_receive)
ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send2).code);
nano::receive_block receive (hash2, hash3, key2.prv, key2.pub, *pool.generate (hash2));
auto hash4 (receive.hash ());
ASSERT_EQ (key2.pub, store->frontier_get (transaction, hash2));
ASSERT_EQ (key2.pub, store->frontier.get (transaction, hash2));
auto return2 (ledger.process (transaction, receive));
ASSERT_EQ (key2.pub, receive.sideband ().account);
ASSERT_EQ (nano::genesis_amount - 25, receive.sideband ().balance.number ());
ASSERT_EQ (2, receive.sideband ().height);
ASSERT_EQ (25, ledger.amount (transaction, hash4));
ASSERT_TRUE (store->frontier_get (transaction, hash2).is_zero ());
ASSERT_EQ (key2.pub, store->frontier_get (transaction, hash4));
ASSERT_TRUE (store->frontier.get (transaction, hash2).is_zero ());
ASSERT_EQ (key2.pub, store->frontier.get (transaction, hash4));
ASSERT_EQ (nano::process_result::progress, return2.code);
ASSERT_EQ (key2.pub, store->block_account_calculated (receive));
ASSERT_EQ (hash4, ledger.latest (transaction, key2.pub));
Expand All @@ -243,8 +243,8 @@ TEST (ledger, process_receive)
ASSERT_EQ (nano::genesis_amount - 25, ledger.weight (key3.pub));
ASSERT_FALSE (ledger.rollback (transaction, hash4));
ASSERT_TRUE (store->block_successor (transaction, hash2).is_zero ());
ASSERT_EQ (key2.pub, store->frontier_get (transaction, hash2));
ASSERT_TRUE (store->frontier_get (transaction, hash4).is_zero ());
ASSERT_EQ (key2.pub, store->frontier.get (transaction, hash2));
ASSERT_TRUE (store->frontier.get (transaction, hash4).is_zero ());
ASSERT_EQ (25, ledger.account_balance (transaction, nano::dev_genesis_key.pub));
ASSERT_EQ (25, ledger.account_pending (transaction, key2.pub));
ASSERT_EQ (nano::genesis_amount - 50, ledger.account_balance (transaction, key2.pub));
Expand Down Expand Up @@ -437,11 +437,11 @@ TEST (ledger, representative_change)
nano::account_info info1;
ASSERT_FALSE (store->account_get (transaction, nano::dev_genesis_key.pub, info1));
nano::change_block block (info1.head, key2.pub, nano::dev_genesis_key.prv, nano::dev_genesis_key.pub, *pool.generate (info1.head));
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, info1.head));
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, info1.head));
auto return1 (ledger.process (transaction, block));
ASSERT_EQ (0, ledger.amount (transaction, block.hash ()));
ASSERT_TRUE (store->frontier_get (transaction, info1.head).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, block.hash ()));
ASSERT_TRUE (store->frontier.get (transaction, info1.head).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, block.hash ()));
ASSERT_EQ (nano::process_result::progress, return1.code);
ASSERT_EQ (nano::dev_genesis_key.pub, store->block_account_calculated (block));
ASSERT_EQ (0, ledger.weight (nano::dev_genesis_key.pub));
Expand All @@ -450,8 +450,8 @@ TEST (ledger, representative_change)
ASSERT_FALSE (store->account_get (transaction, nano::dev_genesis_key.pub, info2));
ASSERT_EQ (block.hash (), info2.head);
ASSERT_FALSE (ledger.rollback (transaction, info2.head));
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier_get (transaction, info1.head));
ASSERT_TRUE (store->frontier_get (transaction, block.hash ()).is_zero ());
ASSERT_EQ (nano::dev_genesis_key.pub, store->frontier.get (transaction, info1.head));
ASSERT_TRUE (store->frontier.get (transaction, block.hash ()).is_zero ());
nano::account_info info3;
ASSERT_FALSE (store->account_get (transaction, nano::dev_genesis_key.pub, info3));
ASSERT_EQ (info1.head, info3.head);
Expand Down Expand Up @@ -3863,7 +3863,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
store.confirmation_height_put (transaction, nano::genesis_account, { 2, send->hash () });

store.online_weight_put (transaction, 100, nano::amount (2));
store.frontier_put (transaction, nano::block_hash (2), nano::account (5));
store.frontier.put (transaction, nano::block_hash (2), nano::account (5));
store.peer_put (transaction, endpoint_key);

store.pending_put (transaction, nano::pending_key (nano::genesis_account, send->hash ()), nano::pending_info (nano::genesis_account, 100, nano::epoch::epoch_0));
Expand Down Expand Up @@ -3897,7 +3897,7 @@ TEST (ledger, migrate_lmdb_to_rocksdb)
ASSERT_EQ (*send, *block1);
ASSERT_TRUE (rocksdb_store.peer_exists (rocksdb_transaction, endpoint_key));
ASSERT_EQ (rocksdb_store.version_get (rocksdb_transaction), version);
ASSERT_EQ (rocksdb_store.frontier_get (rocksdb_transaction, 2), 5);
ASSERT_EQ (rocksdb_store.frontier.get (rocksdb_transaction, 2), 5);
nano::confirmation_height_info confirmation_height_info;
ASSERT_FALSE (rocksdb_store.confirmation_height_get (rocksdb_transaction, nano::genesis_account, confirmation_height_info));
ASSERT_EQ (confirmation_height_info.height, 2);
Expand Down
28 changes: 14 additions & 14 deletions nano/node/lmdb/lmdb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ void nano::mdb_store::upgrade_v14_to_v15 (nano::write_transaction & transaction_

nano::mdb_val value{ data.size (), (void *)data.data () };
auto s = mdb_put (env.tx (transaction_a), state_blocks_new, nano::mdb_val (hash), value, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);

// Every so often output to the log to indicate progress
constexpr auto output_cutoff = 1000000;
Expand Down Expand Up @@ -492,7 +492,7 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa

// Clear it then append
auto status (mdb_drop (env.tx (transaction_a), confirmation_height, 0));
release_assert_success (status);
release_assert_success (*this, status);

for (auto const & confirmation_height_info_pair : confirmation_height_infos)
{
Expand Down Expand Up @@ -548,7 +548,7 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa
}
nano::mdb_val value{ data.size (), (void *)data.data () };
auto s = mdb_cursor_put (state_i.cursor, state_i->first, value, MDB_CURRENT);
release_assert_success (s);
release_assert_success (*this, s);

// Every so often output to the log to indicate progress
constexpr auto output_cutoff = 1000000;
Expand Down Expand Up @@ -618,7 +618,7 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa

nano::mdb_val value{ data.size (), (void *)data.data () };
auto s = mdb_put (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, nano::mdb_val (legacy_block.first), value, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
}

Expand All @@ -642,7 +642,7 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa

nano::mdb_val value{ data.size (), (void *)data.data () };
auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_blocks, nano::mdb_val (i->first), value, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
}

Expand All @@ -660,7 +660,7 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa
for (; i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}

// Delete tables
Expand Down Expand Up @@ -707,7 +707,7 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa

nano::mdb_val value{ data.size (), (void *)data.data () };
auto s = mdb_put (env.tx (transaction_a), temp_state_blocks, nano::mdb_val (i->first), value, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
}

Expand All @@ -723,7 +723,7 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa
for (; i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}

// Delete tables
Expand Down Expand Up @@ -804,7 +804,7 @@ void nano::mdb_store::version_put (nano::write_transaction const & transaction_a
nano::uint256_union version_key (1);
nano::uint256_union version_value (version_a);
auto status (mdb_put (env.tx (transaction_a), meta, nano::mdb_val (version_key), nano::mdb_val (version_value), 0));
release_assert_success (status);
release_assert_success (*this, status);
}

bool nano::mdb_store::exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const
Expand Down Expand Up @@ -849,7 +849,7 @@ uint64_t nano::mdb_store::count (nano::transaction const & transaction_a, MDB_db
{
MDB_stat stats;
auto status (mdb_stat (env.tx (transaction_a), db_a, &stats));
release_assert_success (status);
release_assert_success (*this, status);
return (stats.ms_entries);
}

Expand Down Expand Up @@ -922,7 +922,7 @@ void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, table))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), i->second, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
release_assert (count (transaction_a, table) == count (transaction_a, temp));
// Clear existing table
Expand All @@ -931,7 +931,7 @@ void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
for (auto i (nano::store_iterator<nano::uint256_union, nano::mdb_val> (std::make_unique<nano::mdb_iterator<nano::uint256_union, nano::mdb_val>> (transaction_a, temp))), n (nano::store_iterator<nano::uint256_union, nano::mdb_val> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), table, nano::mdb_val (i->first), i->second, MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
release_assert (count (transaction_a, table) == count (transaction_a, temp));
// Remove temporary table
Expand All @@ -945,15 +945,15 @@ void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, pending))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), temp, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
release_assert (count (transaction_a, pending) == count (transaction_a, temp));
mdb_drop (env.tx (transaction_a), pending, 0);
// Put values from copy
for (auto i (nano::store_iterator<nano::pending_key, nano::pending_info> (std::make_unique<nano::mdb_iterator<nano::pending_key, nano::pending_info>> (transaction_a, temp))), n (nano::store_iterator<nano::pending_key, nano::pending_info> (nullptr)); i != n; ++i)
{
auto s = mdb_put (env.tx (transaction_a), pending, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND);
release_assert_success (s);
release_assert_success (*this, s);
}
release_assert (count (transaction_a, pending) == count (transaction_a, temp));
mdb_drop (env.tx (transaction_a), temp, 1);
Expand Down
3 changes: 2 additions & 1 deletion nano/secure/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ add_library(
utility.cpp
versioning.hpp
versioning.cpp
working.hpp)
working.hpp
store/frontier_store_partial.hpp)

target_link_libraries(
secure
Expand Down
5 changes: 5 additions & 0 deletions nano/secure/blockstore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,8 @@ bool nano::write_transaction::contains (nano::tables table_a) const
{
return impl->contains (table_a);
}

nano::block_store::block_store (nano::frontier_store & frontier_store_a) :
frontier (frontier_store_a)
{
}
21 changes: 14 additions & 7 deletions nano/secure/blockstore.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,12 +620,25 @@ class write_transaction final : public transaction

class ledger_cache;

class frontier_store
{
public:
virtual void put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) = 0;
virtual nano::account get (nano::transaction const &, nano::block_hash const &) const = 0;
virtual void del (nano::write_transaction const &, nano::block_hash const &) = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> begin (nano::transaction const &, nano::block_hash const &) const = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> end () const = 0;
virtual void for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::block_hash, nano::account>, nano::store_iterator<nano::block_hash, nano::account>)> const & action_a) const = 0;
};

/**
* Manages block storage and iteration
*/
class block_store
{
public:
explicit block_store (nano::frontier_store &);
virtual ~block_store () = default;
virtual void initialize (nano::write_transaction const &, nano::genesis const &, nano::ledger_cache &) = 0;
virtual void block_put (nano::write_transaction const &, nano::block_hash const &, nano::block const &) = 0;
Expand All @@ -645,12 +658,7 @@ class block_store
virtual nano::store_iterator<nano::block_hash, block_w_sideband> blocks_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<nano::block_hash, block_w_sideband> blocks_end () const = 0;

virtual void frontier_put (nano::write_transaction const &, nano::block_hash const &, nano::account const &) = 0;
virtual nano::account frontier_get (nano::transaction const &, nano::block_hash const &) const = 0;
virtual void frontier_del (nano::write_transaction const &, nano::block_hash const &) = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> frontiers_begin (nano::transaction const &) const = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> frontiers_begin (nano::transaction const &, nano::block_hash const &) const = 0;
virtual nano::store_iterator<nano::block_hash, nano::account> frontiers_end () const = 0;
frontier_store & frontier;

virtual void account_put (nano::write_transaction const &, nano::account const &, nano::account_info const &) = 0;
virtual bool account_get (nano::transaction const &, nano::account const &, nano::account_info &) = 0;
Expand Down Expand Up @@ -732,7 +740,6 @@ class block_store
virtual void unchecked_for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::unchecked_key, nano::unchecked_info>, nano::store_iterator<nano::unchecked_key, nano::unchecked_info>)> const & action_a) const = 0;
virtual void pruned_for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::block_hash, std::nullptr_t>, nano::store_iterator<nano::block_hash, std::nullptr_t>)> const & action_a) const = 0;
virtual void blocks_for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::block_hash, block_w_sideband>, nano::store_iterator<nano::block_hash, block_w_sideband>)> const & action_a) const = 0;
virtual void frontiers_for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::block_hash, nano::account>, nano::store_iterator<nano::block_hash, nano::account>)> const & action_a) const = 0;
virtual void final_vote_for_each_par (std::function<void (nano::read_transaction const &, nano::store_iterator<nano::qualified_root, nano::block_hash>, nano::store_iterator<nano::qualified_root, nano::block_hash>)> const & action_a) const = 0;

virtual uint64_t block_account_height (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
Expand Down
Loading

0 comments on commit 9683b6d

Please sign in to comment.