diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c893171a2..8112b72432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,9 @@ set (NANO_ROCKSDB OFF CACHE BOOL "") set (NANO_POW_SERVER OFF CACHE BOOL "") set (NANO_WARN_TO_ERR OFF CACHE BOOL "") set (NANO_TIMED_LOCKS 0 CACHE STRING "") +set (NANO_TIMED_LOCKS_IGNORE_BLOCKED OFF CACHE BOOL "") set (NANO_FUZZER_TEST OFF CACHE BOOL "") +set (NANO_ASIO_HANDLER_TRACKING 0 CACHE STRING "") option (NANO_STACKTRACE_BACKTRACE "Use BOOST_STACKTRACE_USE_BACKTRACE in stacktraces, for POSIX" OFF) if (NANO_STACKTRACE_BACKTRACE) @@ -75,6 +77,13 @@ endif () if (${NANO_TIMED_LOCKS} GREATER 0) add_definitions (-DNANO_TIMED_LOCKS=${NANO_TIMED_LOCKS}) + if (NANO_TIMED_LOCKS_IGNORE_BLOCKED) + add_definitions(-DNANO_TIMED_LOCKS_IGNORE_BLOCKED) + endif () +endif () + +if (${NANO_ASIO_HANDLER_TRACKING} GREATER 0) + add_definitions (-DNANO_ASIO_HANDLER_TRACKING=${NANO_ASIO_HANDLER_TRACKING} -DBOOST_ASIO_ENABLE_HANDLER_TRACKING) endif () add_definitions (-DNANO_ROCKSDB=$) diff --git a/lmdb b/lmdb index c6bb7edf9f..e931980823 160000 --- a/lmdb +++ b/lmdb @@ -1 +1 @@ -Subproject commit c6bb7edf9f8e99a78da8b7282287d75388734302 +Subproject commit e93198082347098e4222c28309c399bc17ac11a9 diff --git a/nano/core_test/block_store.cpp b/nano/core_test/block_store.cpp index 4fcffd04c1..52726c35ab 100644 --- a/nano/core_test/block_store.cpp +++ b/nano/core_test/block_store.cpp @@ -31,6 +31,7 @@ void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction cons void modify_confirmation_height_to_v15 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height); void write_sideband_v14 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a, MDB_dbi db_a); void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transaction_a, nano::block const & block_a); +void write_block_w_sideband_v18 (nano::mdb_store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a); } TEST (block_store, construction) @@ -126,7 +127,7 @@ TEST (block_store, add_item) ASSERT_EQ (block, *latest2); ASSERT_TRUE (store->block_exists (transaction, hash1)); ASSERT_FALSE (store->block_exists (transaction, hash1.number () - 1)); - store->block_del (transaction, hash1, block.type ()); + store->block_del (transaction, hash1); auto latest3 (store->block_get (transaction, hash1)); ASSERT_EQ (nullptr, latest3); } @@ -684,7 +685,9 @@ TEST (mdb_block_store, supported_version_upgrades) store.version_put (transaction, store.minimum_version); store.confirmation_height_del (transaction, nano::genesis_account); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); modify_account_info_to_v14 (store, transaction, nano::genesis_account, 1, nano::genesis_hash); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *nano::genesis ().open); } // Upgrade should work @@ -823,14 +826,14 @@ TEST (block_store, block_count) ASSERT_TRUE (!store->init_error ()); { auto transaction (store->tx_begin_write ()); - ASSERT_EQ (0, store->block_count (transaction).sum ()); + ASSERT_EQ (0, store->block_count (transaction)); nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0); block.sideband_set ({}); auto hash1 (block.hash ()); store->block_put (transaction, hash1, block); } auto transaction (store->tx_begin_read ()); - ASSERT_EQ (1, store->block_count (transaction).sum ()); + ASSERT_EQ (1, store->block_count (transaction)); } TEST (block_store, account_count) @@ -1012,13 +1015,13 @@ TEST (block_store, state_block) { auto transaction (store->tx_begin_write ()); auto count (store->block_count (transaction)); - ASSERT_EQ (1, count.state); - store->block_del (transaction, block1.hash (), block1.type ()); + ASSERT_EQ (2, count); + store->block_del (transaction, block1.hash ()); ASSERT_FALSE (store->block_exists (transaction, block1.hash ())); } auto transaction (store->tx_begin_read ()); auto count2 (store->block_count (transaction)); - ASSERT_EQ (0, count2.state); + ASSERT_EQ (1, count2); } TEST (mdb_block_store, sideband_height) @@ -1235,6 +1238,9 @@ TEST (mdb_block_store, upgrade_v14_v15) ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_v1", MDB_CREATE, &store.state_blocks_v1)); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "accounts_v1", MDB_CREATE, &store.accounts_v1)); ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "pending_v1", MDB_CREATE, &store.pending_v1)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, epoch).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send).code); @@ -1247,10 +1253,12 @@ TEST (mdb_block_store, upgrade_v14_v15) write_sideband_v14 (store, transaction, state_send, store.state_blocks_v1); write_sideband_v14 (store, transaction, epoch, store.state_blocks_v1); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send); - // Remove from state table - store.block_del (transaction, state_send.hash (), state_send.type ()); - store.block_del (transaction, epoch.hash (), epoch.type ()); + // Remove from blocks table + store.block_del (transaction, state_send.hash ()); + store.block_del (transaction, epoch.hash ()); // Turn pending into v14 ASSERT_FALSE (mdb_put (store.env.tx (transaction), store.pending_v0, nano::mdb_val (nano::pending_key (nano::test_genesis_key.pub, send.hash ())), nano::mdb_val (nano::pending_info_v14 (nano::genesis_account, nano::Gxrb_ratio, nano::epoch::epoch_0)), 0)); @@ -1328,6 +1336,8 @@ TEST (mdb_block_store, upgrade_v15_v16) ASSERT_FALSE (mdb_dbi_open (txn, "representation", MDB_CREATE, &store.representation)); auto weight = ledger.cache.rep_weights.representation_get (nano::genesis_account); ASSERT_EQ (MDB_SUCCESS, mdb_put (txn, store.representation, nano::mdb_val (nano::genesis_account), nano::mdb_val (nano::uint128_union (weight)), 0)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); // Lower the database to the previous version store.version_put (transaction, 15); // Confirm the rep weight exists in the database @@ -1374,6 +1384,13 @@ TEST (mdb_block_store, upgrade_v16_v17) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, block3).code); modify_confirmation_height_to_v15 (store, transaction, nano::genesis_account, confirmation_height); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block1); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block2); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, block3); + // Lower the database to the previous version store.version_put (transaction, 16); } @@ -1443,22 +1460,37 @@ TEST (mdb_block_store, upgrade_v17_v18) ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_open).code); ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state_send_epoch_link).code); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (store.env.tx (transaction), "state_blocks", MDB_CREATE, &store.state_blocks)); + // Downgrade the store store.version_put (transaction, 17); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_receive); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, epoch_first); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_send2); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state_send_epoch_link); + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send_zero); + // Replace with the previous sideband version for state blocks // The upgrade can resume after upgrading some blocks, test this by only downgrading some of them write_sideband_v15 (store, transaction, state_receive_zero); write_sideband_v15 (store, transaction, epoch); write_sideband_v15 (store, transaction, state_send); - // DISABLED write_sideband_v15 (store, transaction, state_receive); write_sideband_v15 (store, transaction, state_change); write_sideband_v15 (store, transaction, state_send_change); - // DISABLED write_sideband_v15 (store, transaction, epoch_first); write_sideband_v15 (store, transaction, state_receive2); - // DISABLED write_sideband_v15 (store, transaction, state_send2); write_sideband_v15 (store, transaction, state_open); - // DISABLED write_sideband_v15 (store, transaction, state_send_epoch_link); + + store.block_del (transaction, state_receive_zero.hash ()); + store.block_del (transaction, epoch.hash ()); + store.block_del (transaction, state_send.hash ()); + store.block_del (transaction, state_change.hash ()); + store.block_del (transaction, state_send_change.hash ()); + store.block_del (transaction, state_receive2.hash ()); + store.block_del (transaction, state_open.hash ()); } // Now do the upgrade @@ -1469,8 +1501,8 @@ TEST (mdb_block_store, upgrade_v17_v18) // Size of state block should equal that set in db (no change) nano::mdb_val value; - ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.state_blocks, nano::mdb_val (state_send.hash ()), value)); - ASSERT_EQ (value.size (), nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); + ASSERT_FALSE (mdb_get (store.env.tx (transaction), store.blocks, nano::mdb_val (state_send.hash ()), value)); + ASSERT_EQ (value.size (), sizeof (nano::block_type) + nano::state_block::size + nano::block_sideband::size (nano::block_type::state)); // Check that sidebands are correctly populated { @@ -1586,6 +1618,74 @@ TEST (mdb_block_store, upgrade_v17_v18) ASSERT_LT (17, store.version_get (transaction)); } +TEST (mdb_block_store, upgrade_v18_v19) +{ + auto path (nano::unique_path ()); + nano::keypair key1; + nano::work_pool pool (std::numeric_limits::max ()); + nano::send_block send (nano::genesis_hash, nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (nano::genesis_hash)); + nano::receive_block receive (send.hash (), send.hash (), nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send.hash ())); + nano::change_block change (receive.hash (), 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (receive.hash ())); + nano::state_block state (nano::test_genesis_key.pub, change.hash (), 0, nano::genesis_amount - nano::Gxrb_ratio, key1.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (change.hash ())); + + { + nano::genesis genesis; + nano::logger_mt logger; + nano::mdb_store store (logger, path); + nano::stat stats; + nano::ledger ledger (store, stats); + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, receive).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, change).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, state).code); + + // These tables need to be re-opened and populated so that an upgrade can be done + auto txn = store.env.tx (transaction); + ASSERT_FALSE (mdb_dbi_open (txn, "open", MDB_CREATE, &store.open_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "receive", MDB_CREATE, &store.receive_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "send", MDB_CREATE, &store.send_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "change", MDB_CREATE, &store.change_blocks)); + ASSERT_FALSE (mdb_dbi_open (txn, "state_blocks", MDB_CREATE, &store.state_blocks)); + + // Modify blocks back to the old tables + write_block_w_sideband_v18 (store, store.open_blocks, transaction, *genesis.open); + write_block_w_sideband_v18 (store, store.send_blocks, transaction, send); + write_block_w_sideband_v18 (store, store.receive_blocks, transaction, receive); + write_block_w_sideband_v18 (store, store.change_blocks, transaction, change); + write_block_w_sideband_v18 (store, store.state_blocks, transaction, state); + + store.version_put (transaction, 18); + } + + // Now do the upgrade + nano::logger_mt logger; + nano::mdb_store store (logger, path); + ASSERT_FALSE (store.init_error ()); + auto transaction (store.tx_begin_read ()); + + // These tables should be deleted + ASSERT_EQ (store.send_blocks, 0); + ASSERT_EQ (store.receive_blocks, 0); + ASSERT_EQ (store.change_blocks, 0); + ASSERT_EQ (store.open_blocks, 0); + ASSERT_EQ (store.state_blocks, 0); + + // Confirm these blocks all exist after the upgrade + ASSERT_TRUE (store.block_get (transaction, send.hash ())); + ASSERT_TRUE (store.block_get (transaction, receive.hash ())); + ASSERT_TRUE (store.block_get (transaction, change.hash ())); + ASSERT_TRUE (store.block_get (transaction, nano::genesis_hash)); + ASSERT_TRUE (store.block_get (transaction, state.hash ())); + + ASSERT_EQ (5, store.count (transaction, store.blocks)); + + // Version should be correct + ASSERT_LT (18, store.version_get (transaction)); +} + TEST (mdb_block_store, upgrade_backup) { auto dir (nano::unique_path ()); @@ -1791,6 +1891,23 @@ void write_sideband_v15 (nano::mdb_store & store_a, nano::transaction & transact ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), store_a.state_blocks, nano::mdb_val (block_a.hash ()), &val, 0)); } +void write_block_w_sideband_v18 (nano::mdb_store & store_a, MDB_dbi database, nano::write_transaction & transaction_a, nano::block const & block_a) +{ + auto block = store_a.block_get (transaction_a, block_a.hash ()); + ASSERT_NE (block, nullptr); + + std::vector data; + { + nano::vectorstream stream (data); + block->serialize (stream); + block->sideband ().serialize (stream, block->type ()); + } + + MDB_val val{ data.size (), data.data () }; + ASSERT_FALSE (mdb_put (store_a.env.tx (transaction_a), database, nano::mdb_val (block_a.hash ()), &val, 0)); + store_a.del (transaction_a, nano::tables::blocks, nano::mdb_val (block_a.hash ())); +} + void modify_account_info_to_v14 (nano::mdb_store & store, nano::transaction const & transaction, nano::account const & account, uint64_t confirmation_height, nano::block_hash const & rep_block) { nano::account_info info; diff --git a/nano/core_test/cli.cpp b/nano/core_test/cli.cpp index e5388fc3cf..27559d05f3 100644 --- a/nano/core_test/cli.cpp +++ b/nano/core_test/cli.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -43,6 +44,31 @@ TEST (cli, key_create) ASSERT_EQ (vals[2], public_key.to_account ()); } +TEST (cli, config_override_parsing) +{ + std::vector key_value_pairs; + auto config_overrides = nano::config_overrides (key_value_pairs); + ASSERT_TRUE (config_overrides.empty ()); + key_value_pairs.push_back ({ "key", "value" }); + config_overrides = nano::config_overrides (key_value_pairs); + ASSERT_EQ (config_overrides[0], "key=\"value\""); + key_value_pairs.push_back ({ "node.online_weight_minimum", "40000000000000000000000000000000000000" }); + config_overrides = nano::config_overrides (key_value_pairs); + ASSERT_EQ (config_overrides[1], "node.online_weight_minimum=\"40000000000000000000000000000000000000\""); + + // Should add this as it contains escaped quotes, and make sure these are not escaped again + key_value_pairs.push_back ({ "key", "\"value\"" }); + config_overrides = nano::config_overrides (key_value_pairs); + ASSERT_EQ (config_overrides[2], "key=\"value\""); + ASSERT_EQ (config_overrides.size (), 3); + + // Try it with arrays, with and without escaped quotes + key_value_pairs.push_back ({ "node.work_peers", "[127.0.0.1:7000,\"128.0.0.1:50000\"]" }); + config_overrides = nano::config_overrides (key_value_pairs); + ASSERT_EQ (config_overrides[3], "node.work_peers=[\"127.0.0.1:7000\",\"128.0.0.1:50000\"]"); + ASSERT_EQ (config_overrides.size (), 4); +} + namespace { std::string call_cli_command (boost::program_options::variables_map const & vm) diff --git a/nano/core_test/confirmation_height.cpp b/nano/core_test/confirmation_height.cpp index 9bae54bddc..44597b1491 100644 --- a/nano/core_test/confirmation_height.cpp +++ b/nano/core_test/confirmation_height.cpp @@ -1574,3 +1574,52 @@ TEST (confirmation_height, election_winner_details_clearing_node_process_confirm node->process_confirmed (election, 1000000); ASSERT_EQ (0, node->active.election_winner_details_size ()); } + +TEST (confirmation_height, unbounded_block_cache_iteration) +{ + nano::logger_mt logger; + auto path (nano::unique_path ()); + nano::mdb_store store (logger, path); + ASSERT_TRUE (!store.init_error ()); + nano::genesis genesis; + nano::stat stats; + nano::ledger ledger (store, stats); + nano::write_database_queue write_database_queue; + boost::latch initialized_latch{ 0 }; + nano::work_pool pool (std::numeric_limits::max ()); + nano::keypair key1; + auto send = std::make_shared (genesis.hash (), key1.pub, nano::genesis_amount - nano::Gxrb_ratio, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (genesis.hash ())); + auto send1 = std::make_shared (send->hash (), key1.pub, nano::genesis_amount - nano::Gxrb_ratio * 2, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *pool.generate (send->hash ())); + { + auto transaction (store.tx_begin_write ()); + store.initialize (transaction, genesis, ledger.cache); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send).code); + ASSERT_EQ (nano::process_result::progress, ledger.process (transaction, *send1).code); + } + + nano::confirmation_height_processor confirmation_height_processor (ledger, write_database_queue, 10ms, logger, initialized_latch, nano::confirmation_height_mode::unbounded); + nano::timer<> timer; + timer.start (); + { + // Prevent conf height processor doing any writes, so that we can query is_processing_block correctly + auto write_guard = write_database_queue.wait (nano::writer::testing); + // Add the frontier block + confirmation_height_processor.add (send1->hash ()); + + // The most uncemented block (previous block) should be seen as processing by the unbounded processor + while (!confirmation_height_processor.is_processing_block (send->hash ())) + { + ASSERT_LT (timer.since_start (), 10s); + } + } + + // Wait until the current block is finished processing + while (!confirmation_height_processor.current ().is_zero ()) + { + ASSERT_LT (timer.since_start (), 10s); + } + + ASSERT_EQ (2, stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed, nano::stat::dir::in)); + ASSERT_EQ (2, stats.count (nano::stat::type::confirmation_height, nano::stat::detail::blocks_confirmed_unbounded, nano::stat::dir::in)); + ASSERT_EQ (3, ledger.cache.cemented_count); +} diff --git a/nano/core_test/toml.cpp b/nano/core_test/toml.cpp index 5901759974..b15c2f366b 100644 --- a/nano/core_test/toml.cpp +++ b/nano/core_test/toml.cpp @@ -255,15 +255,8 @@ TEST (toml, daemon_config_deserialize_defaults) ASSERT_EQ (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size); ASSERT_EQ (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable); - ASSERT_EQ (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits); - ASSERT_EQ (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache); + ASSERT_EQ (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier); ASSERT_EQ (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads); - ASSERT_EQ (conf.node.rocksdb_config.enable_pipelined_write, defaults.node.rocksdb_config.enable_pipelined_write); - ASSERT_EQ (conf.node.rocksdb_config.cache_index_and_filter_blocks, defaults.node.rocksdb_config.cache_index_and_filter_blocks); - ASSERT_EQ (conf.node.rocksdb_config.block_size, defaults.node.rocksdb_config.block_size); - ASSERT_EQ (conf.node.rocksdb_config.memtable_size, defaults.node.rocksdb_config.memtable_size); - ASSERT_EQ (conf.node.rocksdb_config.num_memtables, defaults.node.rocksdb_config.num_memtables); - ASSERT_EQ (conf.node.rocksdb_config.total_memtable_size, defaults.node.rocksdb_config.total_memtable_size); } TEST (toml, optional_child) @@ -513,15 +506,8 @@ TEST (toml, daemon_config_deserialize_no_defaults) [node.rocksdb] enable = true - bloom_filter_bits = 10 - block_cache = 512 + memory_multiplier = 3 io_threads = 99 - enable_pipelined_write = true - cache_index_and_filter_blocks = true - block_size = 16 - memtable_size = 128 - num_memtables = 3 - total_memtable_size = 0 [node.experimental] secondary_work_peers = ["test.org:998"] @@ -668,15 +654,8 @@ TEST (toml, daemon_config_deserialize_no_defaults) ASSERT_NE (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size); ASSERT_NE (conf.node.rocksdb_config.enable, defaults.node.rocksdb_config.enable); - ASSERT_NE (conf.node.rocksdb_config.bloom_filter_bits, defaults.node.rocksdb_config.bloom_filter_bits); - ASSERT_NE (conf.node.rocksdb_config.block_cache, defaults.node.rocksdb_config.block_cache); + ASSERT_NE (conf.node.rocksdb_config.memory_multiplier, defaults.node.rocksdb_config.memory_multiplier); ASSERT_NE (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads); - ASSERT_NE (conf.node.rocksdb_config.enable_pipelined_write, defaults.node.rocksdb_config.enable_pipelined_write); - ASSERT_NE (conf.node.rocksdb_config.cache_index_and_filter_blocks, defaults.node.rocksdb_config.cache_index_and_filter_blocks); - ASSERT_NE (conf.node.rocksdb_config.block_size, defaults.node.rocksdb_config.block_size); - ASSERT_NE (conf.node.rocksdb_config.memtable_size, defaults.node.rocksdb_config.memtable_size); - ASSERT_NE (conf.node.rocksdb_config.num_memtables, defaults.node.rocksdb_config.num_memtables); - ASSERT_NE (conf.node.rocksdb_config.total_memtable_size, defaults.node.rocksdb_config.total_memtable_size); } /** There should be no required values **/ diff --git a/nano/core_test/work_pool.cpp b/nano/core_test/work_pool.cpp index a04b59210e..7d63204748 100644 --- a/nano/core_test/work_pool.cpp +++ b/nano/core_test/work_pool.cpp @@ -85,7 +85,7 @@ TEST (work, opencl) nano::logger_mt logger; bool error (false); nano::opencl_environment environment (error); - ASSERT_FALSE (error); + ASSERT_TRUE (!error || !nano::opencl_loaded); if (!environment.platforms.empty () && !environment.platforms.begin ()->devices.empty ()) { nano::opencl_config config (0, 0, 16 * 1024); diff --git a/nano/lib/CMakeLists.txt b/nano/lib/CMakeLists.txt index a741371e1b..5ae143fca2 100644 --- a/nano/lib/CMakeLists.txt +++ b/nano/lib/CMakeLists.txt @@ -20,6 +20,8 @@ add_library (nano_lib blockbuilders.cpp blocks.hpp blocks.cpp + cli.hpp + cli.cpp config.hpp config.cpp configbase.hpp diff --git a/nano/lib/cli.cpp b/nano/lib/cli.cpp new file mode 100644 index 0000000000..be1f038669 --- /dev/null +++ b/nano/lib/cli.cpp @@ -0,0 +1,66 @@ +#include + +#include +#include + +#include + +std::vector nano::config_overrides (std::vector const & key_value_pairs_a) +{ + std::vector overrides; + auto format (boost::format ("%1%=%2%")); + auto format_add_escaped_quotes (boost::format ("%1%=\"%2%\"")); + for (auto pair : key_value_pairs_a) + { + auto start = pair.value.find ('['); + + std::string value; + auto is_array = (start != std::string::npos); + if (is_array) + { + // Trim off the square brackets [] of the array + auto end = pair.value.find (']'); + auto array_values = pair.value.substr (start + 1, end - start - 1); + + // Split the string by comma + std::vector split_elements; + boost::split (split_elements, array_values, boost::is_any_of (",")); + + auto format (boost::format ("%1%")); + auto format_add_escaped_quotes (boost::format ("\"%1%\"")); + + // Rebuild the array string adding escaped quotes if necessary + std::ostringstream ss; + ss << "["; + for (auto i = 0; i < split_elements.size (); ++i) + { + auto & elem = split_elements[i]; + auto already_escaped = elem.find ('\"') != std::string::npos; + ss << ((!already_escaped ? format_add_escaped_quotes : format) % elem).str (); + if (i != split_elements.size () - 1) + { + ss << ","; + } + } + ss << "]"; + value = ss.str (); + } + else + { + value = pair.value; + } + auto already_escaped = value.find ('\"') != std::string::npos; + overrides.push_back (((!already_escaped ? format_add_escaped_quotes : format) % pair.key % value).str ()); + } + return overrides; +} + +std::istream & nano::operator>> (std::istream & is, nano::config_key_value_pair & into) +{ + char ch; + while (is >> ch && ch != '=') + { + into.key += ch; + } + return is >> into.value; +} diff --git a/nano/lib/cli.hpp b/nano/lib/cli.hpp new file mode 100644 index 0000000000..6e47a09175 --- /dev/null +++ b/nano/lib/cli.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include + +namespace nano +{ +class config_key_value_pair +{ +public: + std::string key; + std::string value; +}; + +std::vector config_overrides (std::vector const & key_value_pairs_a); + +std::istream & operator>> (std::istream & is, nano::config_key_value_pair & into); +} diff --git a/nano/lib/locks.cpp b/nano/lib/locks.cpp index e38a4e174a..8c784379dd 100644 --- a/nano/lib/locks.cpp +++ b/nano/lib/locks.cpp @@ -28,6 +28,7 @@ void output_if_held_long_enough (nano::timer & timer, timer.stop (); } +#ifndef NANO_TIMED_LOCKS_IGNORE_BLOCKED template void output_if_blocked_long_enough (nano::timer & timer, Mutex & mutex) { @@ -37,6 +38,7 @@ void output_if_blocked_long_enough (nano::timer & tim output ("blocked", time_blocked, mutex); } } +#endif } namespace nano @@ -47,7 +49,9 @@ mut (mutex) timer.start (); mut.lock (); +#ifndef NANO_TIMED_LOCKS_IGNORE_BLOCKED output_if_blocked_long_enough (timer, mut); +#endif } lock_guard::~lock_guard () noexcept @@ -73,8 +77,9 @@ void unique_lock::lock_impl () mut->lock (); owns = true; - +#ifndef NANO_TIMED_LOCKS_IGNORE_BLOCKED output_if_blocked_long_enough (timer, *mut); +#endif } template diff --git a/nano/lib/numbers.cpp b/nano/lib/numbers.cpp index bffd37e396..2b83972b0a 100644 --- a/nano/lib/numbers.cpp +++ b/nano/lib/numbers.cpp @@ -65,6 +65,11 @@ std::string nano::public_key::to_node_id () const return to_account ().replace (0, 4, "node"); } +bool nano::public_key::decode_node_id (std::string const & source_a) +{ + return decode_account (source_a); +} + bool nano::public_key::decode_account (std::string const & source_a) { auto error (source_a.size () < 5); @@ -72,10 +77,11 @@ bool nano::public_key::decode_account (std::string const & source_a) { auto xrb_prefix (source_a[0] == 'x' && source_a[1] == 'r' && source_a[2] == 'b' && (source_a[3] == '_' || source_a[3] == '-')); auto nano_prefix (source_a[0] == 'n' && source_a[1] == 'a' && source_a[2] == 'n' && source_a[3] == 'o' && (source_a[4] == '_' || source_a[4] == '-')); + auto node_id_prefix = (source_a[0] == 'n' && source_a[1] == 'o' && source_a[2] == 'd' && source_a[3] == 'e' && source_a[4] == '_'); error = (xrb_prefix && source_a.size () != 64) || (nano_prefix && source_a.size () != 65); if (!error) { - if (xrb_prefix || nano_prefix) + if (xrb_prefix || nano_prefix || node_id_prefix) { auto i (source_a.begin () + (xrb_prefix ? 4 : 5)); if (*i == '1' || *i == '3') diff --git a/nano/lib/numbers.hpp b/nano/lib/numbers.hpp index 1be15455c2..3e84d0221a 100644 --- a/nano/lib/numbers.hpp +++ b/nano/lib/numbers.hpp @@ -116,6 +116,7 @@ class public_key final : public uint256_union using uint256_union::uint256_union; std::string to_node_id () const; + bool decode_node_id (std::string const & source_a); void encode_account (std::string &) const; std::string to_account () const; bool decode_account (std::string const &); diff --git a/nano/lib/rocksdbconfig.cpp b/nano/lib/rocksdbconfig.cpp index c0f50955be..fb090ca41f 100644 --- a/nano/lib/rocksdbconfig.cpp +++ b/nano/lib/rocksdbconfig.cpp @@ -4,55 +4,25 @@ nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const { toml.put ("enable", enable, "Whether to use the RocksDB backend for the ledger database.\ntype:bool"); - toml.put ("enable_pipelined_write", enable_pipelined_write, "Whether to use 2 separate write queues for memtable/WAL, true is recommended.\ntype:bool"); - toml.put ("cache_index_and_filter_blocks", cache_index_and_filter_blocks, "Whether index and filter blocks are stored in block_cache, true is recommended.\ntype:bool"); - toml.put ("bloom_filter_bits", bloom_filter_bits, "Number of bits to use with a bloom filter. Helps with point reads but uses more memory. 0 disables the bloom filter, 10 is recommended.\ntype:uint32"); - toml.put ("block_cache", block_cache, "Size (MB) of the block cache; A larger number will increase performance of read operations. At least 512MB is recommended.\ntype:uint64"); + toml.put ("memory_multiplier", memory_multiplier, "This will modify how much memory is used represented by 1 (low), 2 (medium), 3 (high). Default is 2.\ntype:uint8"); toml.put ("io_threads", io_threads, "Number of threads to use with the background compaction and flushing. Number of hardware threads is recommended.\ntype:uint32"); - toml.put ("block_size", block_size, "Uncompressed data (KBs) per block. Increasing block size decreases memory usage and space amplification, but increases read amplification. 16 is recommended.\ntype:uint32"); - toml.put ("num_memtables", num_memtables, "Number of memtables to keep in memory per column family. 2 is the minimum, 3 is recommended.\ntype:uint32"); - toml.put ("memtable_size", memtable_size, "Amount of memory (MB) to build up before flushing to disk for an individual column family. Large values increase performance. 64 or 128 is recommended.\ntype:uint32"); - toml.put ("total_memtable_size", total_memtable_size, "Total memory (MB) which can be used across all memtables, set to 0 for unconstrained.\ntype:uint32"); return toml.get_error (); } nano::error nano::rocksdb_config::deserialize_toml (nano::tomlconfig & toml) { toml.get_optional ("enable", enable); - toml.get_optional ("enable_pipelined_write", enable_pipelined_write); - toml.get_optional ("cache_index_and_filter_blocks", cache_index_and_filter_blocks); - toml.get_optional ("bloom_filter_bits", bloom_filter_bits); - toml.get_optional ("block_cache", block_cache); + toml.get_optional ("memory_multiplier", memory_multiplier); toml.get_optional ("io_threads", io_threads); - toml.get_optional ("block_size", block_size); - toml.get_optional ("num_memtables", num_memtables); - toml.get_optional ("memtable_size", memtable_size); - toml.get_optional ("total_memtable_size", total_memtable_size); // Validate ranges - if (bloom_filter_bits > 100) - { - toml.get_error ().set ("bloom_filter_bits is too high"); - } - if (num_memtables < 2) - { - toml.get_error ().set ("num_memtables must be at least 2"); - } - if (memtable_size == 0) - { - toml.get_error ().set ("memtable_size must be non-zero"); - } - if ((total_memtable_size < memtable_size * 8) && (total_memtable_size != 0)) - { - toml.get_error ().set ("total_memtable_size should be at least 8 times greater than memtable_size or be set to 0"); - } if (io_threads == 0) { toml.get_error ().set ("io_threads must be non-zero"); } - if (block_size == 0) + if (memory_multiplier < 1 || memory_multiplier > 3) { - toml.get_error ().set ("block_size must be non-zero"); + toml.get_error ().set ("memory_multiplier must be either 1, 2 or 3"); } return toml.get_error (); diff --git a/nano/lib/rocksdbconfig.hpp b/nano/lib/rocksdbconfig.hpp index e47873b301..750eab9d5d 100644 --- a/nano/lib/rocksdbconfig.hpp +++ b/nano/lib/rocksdbconfig.hpp @@ -16,14 +16,7 @@ class rocksdb_config final nano::error deserialize_toml (nano::tomlconfig & toml_a); bool enable{ false }; - unsigned bloom_filter_bits{ 0 }; - uint64_t block_cache{ 64 }; // MB + uint8_t memory_multiplier{ 2 }; unsigned io_threads{ std::thread::hardware_concurrency () }; - bool enable_pipelined_write{ false }; - bool cache_index_and_filter_blocks{ false }; - unsigned block_size{ 4 }; // KB - unsigned memtable_size{ 32 }; // MB - unsigned num_memtables{ 2 }; // Need a minimum of 2 - unsigned total_memtable_size{ 512 }; // MB }; } diff --git a/nano/lib/threading.cpp b/nano/lib/threading.cpp index 0b0319bd3b..9d54240c1c 100644 --- a/nano/lib/threading.cpp +++ b/nano/lib/threading.cpp @@ -1,6 +1,9 @@ #include +#include + #include +#include namespace { @@ -124,7 +127,26 @@ io_guard (boost::asio::make_work_guard (io_ctx_a)) nano::thread_role::set (nano::thread_role::name::io); try { +#if NANO_ASIO_HANDLER_TRACKING == 0 io_ctx_a.run (); +#else + nano::timer<> timer; + timer.start (); + while (true) + { + timer.restart (); + // Run at most 1 completion handler and record the time it took to complete (non-blocking) + auto count = io_ctx_a.poll_one (); + if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING) + { + auto timestamp = std::chrono::duration_cast (std::chrono::system_clock::now ().time_since_epoch ()).count (); + std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl; + } + // Sleep for a bit to give more time slices to other threads + std::this_thread::sleep_for (std::chrono::milliseconds (5)); + std::this_thread::yield (); + } +#endif } catch (std::exception const & ex) { diff --git a/nano/nano_node/entry.cpp b/nano/nano_node/entry.cpp index 579273bf22..9f54f10292 100644 --- a/nano/nano_node/entry.cpp +++ b/nano/nano_node/entry.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -68,7 +69,7 @@ int main (int argc, char * const * argv) description.add_options () ("help", "Print out options") ("version", "Prints out version") - ("config", boost::program_options::value>()->multitoken(), "Pass node configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") + ("config", boost::program_options::value>()->multitoken(), "Pass node configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") ("daemon", "Start node daemon") ("compare_rep_weights", "Display a summarized comparison between the hardcoded bootstrap weights and representative weights from the ledger. Full comparison is output to logs") ("debug_block_count", "Display the number of block") @@ -312,7 +313,7 @@ int main (int argc, char * const * argv) auto inactive_node = nano::default_inactive_node (data_path, vm); auto node = inactive_node->node; auto transaction (node->store.tx_begin_read ()); - std::cout << boost::str (boost::format ("Block count: %1%\n") % node->store.block_count (transaction).sum ()); + std::cout << boost::str (boost::format ("Block count: %1%\n") % node->store.block_count (transaction)); } else if (vm.count ("debug_bootstrap_generate")) { @@ -1012,7 +1013,7 @@ int main (int argc, char * const * argv) { std::this_thread::sleep_for (std::chrono::milliseconds (10)); auto transaction (node->store.tx_begin_read ()); - block_count = node->store.block_count (transaction).sum (); + block_count = node->store.block_count (transaction); } auto end (std::chrono::high_resolution_clock::now ()); auto time (std::chrono::duration_cast (end - begin).count ()); @@ -1530,7 +1531,7 @@ int main (int argc, char * const * argv) { // State receive block_details_error = !sideband.details.is_receive || sideband.details.is_send || sideband.details.is_epoch; - block_details_error |= !node->store.source_exists (transaction, block->link ()); + block_details_error |= !node->store.block_exists (transaction, block->link ()); } } } @@ -1628,7 +1629,7 @@ int main (int argc, char * const * argv) } // Validate total block count - auto ledger_block_count (node->store.block_count (transaction).sum ()); + auto ledger_block_count (node->store.block_count (transaction)); if (block_count != ledger_block_count) { print_error_message (boost::str (boost::format ("Incorrect total block count. Blocks validated %1%. Block count in database: %2%\n") % block_count % ledger_block_count)); @@ -1787,7 +1788,7 @@ int main (int argc, char * const * argv) { std::this_thread::sleep_for (std::chrono::milliseconds (50)); auto transaction_2 (node.node->store.tx_begin_read ()); - block_count_2 = node.node->store.block_count (transaction_2).sum (); + block_count_2 = node.node->store.block_count (transaction_2); } auto end (std::chrono::high_resolution_clock::now ()); auto time (std::chrono::duration_cast (end - begin).count ()); diff --git a/nano/nano_rpc/entry.cpp b/nano/nano_rpc/entry.cpp index f6e31b47bf..45c46fc9d7 100644 --- a/nano/nano_rpc/entry.cpp +++ b/nano/nano_rpc/entry.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -90,7 +91,7 @@ int main (int argc, char * const * argv) // clang-format off description.add_options () ("help", "Print out options") - ("config", boost::program_options::value>()->multitoken(), "Pass RPC configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") + ("config", boost::program_options::value>()->multitoken(), "Pass RPC configuration values. This takes precedence over any values in the configuration file. This option can be repeated multiple times.") ("daemon", "Start RPC daemon") ("data_path", boost::program_options::value (), "Use the supplied path as the data directory") ("network", boost::program_options::value (), "Use the supplied network (live, beta or test)") @@ -139,7 +140,7 @@ int main (int argc, char * const * argv) auto config (vm.find ("config")); if (config != vm.end ()) { - config_overrides = config->second.as> (); + config_overrides = nano::config_overrides (config->second.as> ()); } run (data_path, config_overrides); } diff --git a/nano/node/active_transactions.cpp b/nano/node/active_transactions.cpp index 2227896282..fc3c340894 100644 --- a/nano/node/active_transactions.cpp +++ b/nano/node/active_transactions.cpp @@ -124,7 +124,7 @@ void nano::active_transactions::block_cemented_callback (std::shared_ptr election_status_type; - if (!confirmation_height_processor.is_processing_block (block_a->hash ())) + if (!confirmation_height_processor.is_processing_added_block (block_a->hash ())) { election_status_type = confirm_block (transaction, block_a); } diff --git a/nano/node/blockprocessor.cpp b/nano/node/blockprocessor.cpp index 75cfb6f13a..f1f2c22a10 100644 --- a/nano/node/blockprocessor.cpp +++ b/nano/node/blockprocessor.cpp @@ -236,7 +236,7 @@ void nano::block_processor::process_batch (nano::unique_lock & lock_ { auto scoped_write_guard = write_database_queue.wait (nano::writer::process_batch); block_post_events post_events ([& store = node.store] { return store.tx_begin_read (); }); - auto transaction (node.store.tx_begin_write ({ tables::accounts, nano::tables::cached_counts, nano::tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks, tables::unchecked }, { tables::confirmation_height })); + auto transaction (node.store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending, tables::unchecked }, { tables::confirmation_height })); nano::timer timer_l; lock_a.lock (); timer_l.start (); diff --git a/nano/node/cli.cpp b/nano/node/cli.cpp index 8df2281c14..8cc445e3b5 100644 --- a/nano/node/cli.cpp +++ b/nano/node/cli.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -170,7 +171,7 @@ std::error_code nano::update_flags (nano::node_flags & flags_a, boost::program_o auto config (vm.find ("config")); if (config != vm.end ()) { - flags_a.config_overrides = config->second.as> (); + flags_a.config_overrides = nano::config_overrides (config->second.as> ()); } return ec; } @@ -614,6 +615,14 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map nano::uint256_union junk2 (0); nano::kdf kdf; kdf.phs (junk1, "", junk2); + std::cout << "Testing time retrieval latency... " << std::flush; + nano::timer timer (nano::timer_state::started); + auto const iters = 2'000'000; + for (auto i (0); i < iters; ++i) + { + (void)std::chrono::steady_clock::now (); + } + std::cout << timer.stop ().count () / iters << " " << timer.unit () << std::endl; std::cout << "Dumping OpenCL information" << std::endl; bool error (false); nano::opencl_environment environment (error); diff --git a/nano/node/common.cpp b/nano/node/common.cpp index 3a4a7a6bce..dbb660326b 100644 --- a/nano/node/common.cpp +++ b/nano/node/common.cpp @@ -1252,7 +1252,7 @@ nano::error nano::telemetry_data::serialize_json (nano::jsonconfig & json, bool // Keep these last for UI purposes if (!ignore_identification_metrics_a) { - json.put ("node_id", node_id.to_string ()); + json.put ("node_id", node_id.to_node_id ()); json.put ("signature", signature.to_string ()); } return json.get_error (); @@ -1276,7 +1276,7 @@ nano::error nano::telemetry_data::deserialize_json (nano::jsonconfig & json, boo json.get ("node_id", node_id_l); if (!json.get_error ()) { - if (node_id.decode_hex (node_id_l)) + if (node_id.decode_node_id (node_id_l)) { json.get_error ().set ("Could not deserialize node id"); } diff --git a/nano/node/confirmation_height_bounded.cpp b/nano/node/confirmation_height_bounded.cpp index 05928da0ad..e2d4405f46 100644 --- a/nano/node/confirmation_height_bounded.cpp +++ b/nano/node/confirmation_height_bounded.cpp @@ -244,7 +244,7 @@ bool nano::confirmation_height_bounded::iterate (nano::read_transaction const & source = block->link (); } - if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.source_exists (transaction_a, source)) + if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.block_exists (transaction_a, source)) { hit_receive = true; reached_target = true; diff --git a/nano/node/confirmation_height_processor.cpp b/nano/node/confirmation_height_processor.cpp index d05ffeabc4..c37f663915 100644 --- a/nano/node/confirmation_height_processor.cpp +++ b/nano/node/confirmation_height_processor.cpp @@ -209,19 +209,24 @@ std::unique_ptr nano::collect_container_info (co return composite; } -size_t nano::confirmation_height_processor::awaiting_processing_size () +size_t nano::confirmation_height_processor::awaiting_processing_size () const { nano::lock_guard guard (mutex); return awaiting_processing.size (); } -bool nano::confirmation_height_processor::is_processing_block (nano::block_hash const & hash_a) +bool nano::confirmation_height_processor::is_processing_added_block (nano::block_hash const & hash_a) const { nano::lock_guard guard (mutex); return original_hashes_pending.count (hash_a) > 0 || awaiting_processing.get ().count (hash_a) > 0; } -nano::block_hash nano::confirmation_height_processor::current () +bool nano::confirmation_height_processor::is_processing_block (nano::block_hash const & hash_a) const +{ + return is_processing_added_block (hash_a) || unbounded_processor.has_iterated_over_block (hash_a); +} + +nano::block_hash nano::confirmation_height_processor::current () const { nano::lock_guard lk (mutex); return original_hash; diff --git a/nano/node/confirmation_height_processor.hpp b/nano/node/confirmation_height_processor.hpp index 60270a102a..a0cfb6ecb2 100644 --- a/nano/node/confirmation_height_processor.hpp +++ b/nano/node/confirmation_height_processor.hpp @@ -37,15 +37,16 @@ class confirmation_height_processor final void stop (); void add (nano::block_hash const & hash_a); void run (confirmation_height_mode); - size_t awaiting_processing_size (); - bool is_processing_block (nano::block_hash const &); - nano::block_hash current (); + size_t awaiting_processing_size () const; + bool is_processing_added_block (nano::block_hash const & hash_a) const; + bool is_processing_block (nano::block_hash const &) const; + nano::block_hash current () const; void add_cemented_observer (std::function)> const &); void add_block_already_cemented_observer (std::function const &); private: - std::mutex mutex; + mutable std::mutex mutex; // Hashes which have been added to the confirmation height processor, but not yet processed // clang-format off class tag_sequence {}; diff --git a/nano/node/confirmation_height_unbounded.cpp b/nano/node/confirmation_height_unbounded.cpp index 9cd29f02ea..8077ed1476 100644 --- a/nano/node/confirmation_height_unbounded.cpp +++ b/nano/node/confirmation_height_unbounded.cpp @@ -194,7 +194,7 @@ void nano::confirmation_height_unbounded::collect_unconfirmed_receive_and_source source = block->link (); } - if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.source_exists (transaction_a, source)) + if (!source.is_zero () && !ledger.is_epoch_link (source) && ledger.store.block_exists (transaction_a, source)) { if (!hit_receive && !block_callback_data_a.empty ()) { @@ -379,8 +379,9 @@ void nano::confirmation_height_unbounded::cement_blocks (nano::write_guard & sco // Reverse it so that the callbacks start from the lowest newly cemented block and move upwards std::reverse (pending.block_callback_data.begin (), pending.block_callback_data.end ()); + nano::lock_guard guard (block_cache_mutex); std::transform (pending.block_callback_data.begin (), pending.block_callback_data.end (), std::back_inserter (cemented_blocks), [& block_cache = block_cache](auto const & hash_a) { - debug_assert (block_cache.find (hash_a) != block_cache.end ()); + debug_assert (block_cache.count (hash_a) == 1); return block_cache.at (hash_a); }); } @@ -413,6 +414,7 @@ void nano::confirmation_height_unbounded::cement_blocks (nano::write_guard & sco std::shared_ptr nano::confirmation_height_unbounded::get_block_and_sideband (nano::block_hash const & hash_a, nano::transaction const & transaction_a) { + nano::lock_guard guard (block_cache_mutex); auto block_cache_it = block_cache.find (hash_a); if (block_cache_it != block_cache.cend ()) { @@ -422,7 +424,6 @@ std::shared_ptr nano::confirmation_height_unbounded::get_block_and_ { auto block (ledger.store.block_get (transaction_a, hash_a)); block_cache.emplace (hash_a, block); - ++block_cache_size; return block; } } @@ -440,8 +441,22 @@ void nano::confirmation_height_unbounded::clear_process_vars () confirmed_iterated_pairs_size = 0; implicit_receive_cemented_mapping.clear (); implicit_receive_cemented_mapping_size = 0; - block_cache.clear (); - block_cache_size = 0; + { + nano::lock_guard guard (block_cache_mutex); + block_cache.clear (); + } +} + +bool nano::confirmation_height_unbounded::has_iterated_over_block (nano::block_hash const & hash_a) const +{ + nano::lock_guard guard (block_cache_mutex); + return block_cache.count (hash_a) == 1; +} + +uint64_t nano::confirmation_height_unbounded::block_cache_size () const +{ + nano::lock_guard guard (block_cache_mutex); + return block_cache.size (); } nano::confirmation_height_unbounded::conf_height_details::conf_height_details (nano::account const & account_a, nano::block_hash const & hash_a, uint64_t height_a, uint64_t num_blocks_confirmed_a, std::vector const & block_callback_data_a) : @@ -471,6 +486,6 @@ std::unique_ptr nano::collect_container_info (co composite->add_component (std::make_unique (container_info{ "confirmed_iterated_pairs", confirmation_height_unbounded.confirmed_iterated_pairs_size, sizeof (decltype (confirmation_height_unbounded.confirmed_iterated_pairs)::value_type) })); composite->add_component (std::make_unique (container_info{ "pending_writes", confirmation_height_unbounded.pending_writes_size, sizeof (decltype (confirmation_height_unbounded.pending_writes)::value_type) })); composite->add_component (std::make_unique (container_info{ "implicit_receive_cemented_mapping", confirmation_height_unbounded.implicit_receive_cemented_mapping_size, sizeof (decltype (confirmation_height_unbounded.implicit_receive_cemented_mapping)::value_type) })); - composite->add_component (std::make_unique (container_info{ "block_cache", confirmation_height_unbounded.block_cache_size, sizeof (decltype (confirmation_height_unbounded.block_cache)::value_type) })); + composite->add_component (std::make_unique (container_info{ "block_cache", confirmation_height_unbounded.block_cache_size (), sizeof (decltype (confirmation_height_unbounded.block_cache)::value_type) })); return composite; } diff --git a/nano/node/confirmation_height_unbounded.hpp b/nano/node/confirmation_height_unbounded.hpp index 8a8d1aa197..20cfc5441d 100644 --- a/nano/node/confirmation_height_unbounded.hpp +++ b/nano/node/confirmation_height_unbounded.hpp @@ -23,6 +23,7 @@ class confirmation_height_unbounded final void clear_process_vars (); void process (); void cement_blocks (nano::write_guard &); + bool has_iterated_over_block (nano::block_hash const &) const; private: class confirmed_iterated_pair @@ -62,14 +63,16 @@ class confirmation_height_unbounded final // This allows the load and stores to use relaxed atomic memory ordering. std::unordered_map confirmed_iterated_pairs; nano::relaxed_atomic_integral confirmed_iterated_pairs_size{ 0 }; - std::unordered_map> block_cache; - nano::relaxed_atomic_integral block_cache_size{ 0 }; std::shared_ptr get_block_and_sideband (nano::block_hash const &, nano::transaction const &); std::deque pending_writes; nano::relaxed_atomic_integral pending_writes_size{ 0 }; std::unordered_map> implicit_receive_cemented_mapping; nano::relaxed_atomic_integral implicit_receive_cemented_mapping_size{ 0 }; + mutable std::mutex block_cache_mutex; + std::unordered_map> block_cache; + uint64_t block_cache_size () const; + nano::timer timer; class preparation_data final diff --git a/nano/node/json_handler.cpp b/nano/node/json_handler.cpp index bbec8f5ca5..419ceffd20 100644 --- a/nano/node/json_handler.cpp +++ b/nano/node/json_handler.cpp @@ -1247,18 +1247,6 @@ void nano::json_handler::block_count () response_errors (); } -void nano::json_handler::block_count_type () -{ - auto transaction (node.store.tx_begin_read ()); - nano::block_counts count (node.store.block_count (transaction)); - response_l.put ("send", std::to_string (count.send)); - response_l.put ("receive", std::to_string (count.receive)); - response_l.put ("open", std::to_string (count.open)); - response_l.put ("change", std::to_string (count.change)); - response_l.put ("state", std::to_string (count.state)); - response_errors (); -} - void nano::json_handler::block_create () { std::string type (request.get ("type")); @@ -5048,7 +5036,6 @@ ipc_json_handler_no_arg_func_map create_ipc_json_handler_no_arg_func_map () no_arg_funcs.emplace ("blocks_info", &nano::json_handler::blocks_info); no_arg_funcs.emplace ("block_account", &nano::json_handler::block_account); no_arg_funcs.emplace ("block_count", &nano::json_handler::block_count); - no_arg_funcs.emplace ("block_count_type", &nano::json_handler::block_count_type); no_arg_funcs.emplace ("block_create", &nano::json_handler::block_create); no_arg_funcs.emplace ("block_hash", &nano::json_handler::block_hash); no_arg_funcs.emplace ("bootstrap", &nano::json_handler::bootstrap); diff --git a/nano/node/json_handler.hpp b/nano/node/json_handler.hpp index f739ab6eca..bcfd85297b 100644 --- a/nano/node/json_handler.hpp +++ b/nano/node/json_handler.hpp @@ -51,7 +51,6 @@ class json_handler : public std::enable_shared_from_this void blocks_info (); void block_account (); void block_count (); - void block_count_type (); void block_create (); void block_hash (); void bootstrap (); diff --git a/nano/node/lmdb/lmdb.cpp b/nano/node/lmdb/lmdb.cpp index e4d06e673b..a49d878e8a 100644 --- a/nano/node/lmdb/lmdb.cpp +++ b/nano/node/lmdb/lmdb.cpp @@ -176,10 +176,6 @@ nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & transaction_a, unsigned flags) { error_a |= mdb_dbi_open (env.tx (transaction_a), "frontiers", flags, &frontiers) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; - error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "unchecked", flags, &unchecked) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "vote", flags, &vote) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0; @@ -191,13 +187,32 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "pending", flags, &pending_v0) != 0; pending = pending_v0; - if (version_get (transaction_a) < 16) + auto version_l = version_get (transaction_a); + if (version_l < 19) + { + // These legacy (and state) block databases are no longer used, but need opening so they can be deleted during an upgrade + error_a |= mdb_dbi_open (env.tx (transaction_a), "send", flags, &send_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "receive", flags, &receive_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "open", flags, &open_blocks) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "change", flags, &change_blocks) != 0; + if (version_l >= 15) + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &state_blocks) != 0; + state_blocks_v0 = state_blocks; + } + } + else + { + error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &blocks) != 0; + } + + if (version_l < 16) { // The representation database is no longer used, but needs opening so that it can be deleted during an upgrade error_a |= mdb_dbi_open (env.tx (transaction_a), "representation", flags, &representation) != 0; } - if (version_get (transaction_a) < 15) + if (version_l < 15) { // These databases are no longer used, but need opening so they can be deleted during an upgrade error_a |= mdb_dbi_open (env.tx (transaction_a), "state", flags, &state_blocks_v0) != 0; @@ -206,11 +221,6 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const & error_a |= mdb_dbi_open (env.tx (transaction_a), "pending_v1", flags, &pending_v1) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "state_v1", flags, &state_blocks_v1) != 0; } - else - { - error_a |= mdb_dbi_open (env.tx (transaction_a), "state_blocks", flags, &state_blocks) != 0; - state_blocks_v0 = state_blocks; - } } bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool & needs_vacuuming) @@ -247,6 +257,9 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool upgrade_v17_to_v18 (transaction_a); needs_vacuuming = true; case 18: + upgrade_v18_to_v19 (transaction_a); + needs_vacuuming = true; + case 19: break; default: logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l)); @@ -416,13 +429,13 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa if (account_info_i->second.block_count / 2 >= confirmation_height) { // The confirmation height of the account is closer to the bottom of the chain, so start there and work up - auto block = block_get (transaction_a, account_info.open_block); + auto block = block_get_v18 (transaction_a, account_info.open_block); debug_assert (block); auto height = 1; while (height != confirmation_height) { - block = block_get (transaction_a, block->sideband ().successor); + block = block_get_v18 (transaction_a, block->sideband ().successor); debug_assert (block); ++height; } @@ -433,11 +446,11 @@ void nano::mdb_store::upgrade_v16_to_v17 (nano::write_transaction const & transa else { // The confirmation height of the account is closer to the top of the chain so start there and work down - auto block = block_get (transaction_a, account_info.head); + auto block = block_get_v18 (transaction_a, account_info.head); auto height = block->sideband ().height; while (height != confirmation_height) { - block = block_get (transaction_a, block->previous ()); + block = block_get_v18 (transaction_a, block->previous ()); debug_assert (block); --height; } @@ -473,11 +486,11 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa auto count_pre (count (transaction_a, state_blocks)); auto num = 0u; - for (nano::mdb_iterator state_i (transaction_a, state_blocks), state_n{}; state_i != state_n; ++state_i, ++num) + for (nano::mdb_iterator> state_i (transaction_a, state_blocks), state_n{}; state_i != state_n; ++state_i, ++num) { - nano::state_block_w_sideband block_sideband (state_i->second); - auto & block (block_sideband.state_block); - auto & sideband (block_sideband.sideband); + nano::block_w_sideband_v18 block_w_sideband (state_i->second); + auto & block (block_w_sideband.block); + auto & sideband (block_w_sideband.sideband); bool is_send{ false }; bool is_receive{ false }; @@ -486,7 +499,7 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa nano::amount prev_balance (0); if (!block->hashables.previous.is_zero ()) { - prev_balance = block_balance (transaction_a, block->hashables.previous); + prev_balance = block_balance_v18 (transaction_a, block->hashables.previous); } if (block->hashables.balance == prev_balance && network_params.ledger.epochs.is_epoch_link (block->hashables.link)) { @@ -528,6 +541,153 @@ void nano::mdb_store::upgrade_v17_to_v18 (nano::write_transaction const & transa logger.always_log ("Finished upgrading the sideband"); } +void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transaction_a) +{ + logger.always_log ("Preparing v18 to v19 database upgrade..."); + auto count_pre (count (transaction_a, state_blocks) + count (transaction_a, send_blocks) + count (transaction_a, receive_blocks) + count (transaction_a, change_blocks) + count (transaction_a, open_blocks)); + + // Combine in order of likeliness of counts + std::map legacy_open_receive_change_blocks; + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, change_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; + } + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, open_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; + } + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, receive_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + legacy_open_receive_change_blocks[i->first] = { nano::block_w_sideband{ i->second.block, i->second.sideband } }; + } + + release_assert (!mdb_drop (env.tx (transaction_a), receive_blocks, 1)); + receive_blocks = 0; + release_assert (!mdb_drop (env.tx (transaction_a), open_blocks, 1)); + open_blocks = 0; + release_assert (!mdb_drop (env.tx (transaction_a), change_blocks, 1)); + change_blocks = 0; + + logger.always_log ("Write legacy open/receive/change to new format"); + + MDB_dbi temp_legacy_open_receive_change_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_open_receive_change_blocks", MDB_CREATE, &temp_legacy_open_receive_change_blocks); + + for (auto const & legacy_block : legacy_open_receive_change_blocks) + { + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *legacy_block.second.block); + legacy_block.second.sideband.serialize (stream, legacy_block.second.block->type ()); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, nano::mdb_val (legacy_block.first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + logger.always_log ("Write legacy send to new format"); + + // Write send blocks to a new table (this was not done in memory as it would push us above memory requirements) + MDB_dbi temp_legacy_send_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_blocks", MDB_CREATE, &temp_legacy_send_blocks); + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, send_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + auto const & block_w_sideband_v18 (i->second); + + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *block_w_sideband_v18.block); + block_w_sideband_v18.sideband.serialize (stream, nano::block_type::send); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_blocks, nano::mdb_val (i->first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + release_assert (!mdb_drop (env.tx (transaction_a), send_blocks, 1)); + send_blocks = 0; + + logger.always_log ("Merge legacy open/receive/change with legacy send blocks"); + + MDB_dbi temp_legacy_send_open_receive_change_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_legacy_send_open_receive_change_blocks", MDB_CREATE, &temp_legacy_send_open_receive_change_blocks); + + nano::mdb_merge_iterator i (transaction_a, temp_legacy_open_receive_change_blocks, temp_legacy_send_blocks); + nano::mdb_merge_iterator n{}; + for (; i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); + release_assert (success (s)); + } + + // Delete tables + mdb_drop (env.tx (transaction_a), temp_legacy_send_blocks, 1); + mdb_drop (env.tx (transaction_a), temp_legacy_open_receive_change_blocks, 1); + } + + logger.always_log ("Write state blocks to new format"); + + // Write state blocks to a new table (this was not done in memory as it would push us above memory requirements) + MDB_dbi temp_state_blocks; + { + mdb_dbi_open (env.tx (transaction_a), "temp_state_blocks", MDB_CREATE, &temp_state_blocks); + + for (auto i (nano::store_iterator> (std::make_unique>> (transaction_a, state_blocks))), n (nano::store_iterator> (nullptr)); i != n; ++i) + { + auto const & block_w_sideband_v18 (i->second); + + std::vector data; + { + nano::vectorstream stream (data); + nano::serialize_block (stream, *block_w_sideband_v18.block); + block_w_sideband_v18.sideband.serialize (stream, nano::block_type::state); + } + + nano::mdb_val value{ data.size (), (void *)data.data () }; + auto s = mdb_put (env.tx (transaction_a), temp_state_blocks, nano::mdb_val (i->first), value, MDB_APPEND); + release_assert (success (s)); + } + } + + release_assert (!mdb_drop (env.tx (transaction_a), state_blocks, 1)); + state_blocks = 0; + + logger.always_log ("Merging all legacy blocks with state blocks"); + + // Merge all legacy blocks with state blocks into the final table + nano::mdb_merge_iterator i (transaction_a, temp_legacy_send_open_receive_change_blocks, temp_state_blocks); + nano::mdb_merge_iterator n{}; + mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &blocks); + for (; i != n; ++i) + { + auto s = mdb_put (env.tx (transaction_a), blocks, nano::mdb_val (i->first), nano::mdb_val (i->second), MDB_APPEND); + release_assert (success (s)); + } + + // Delete tables + mdb_drop (env.tx (transaction_a), temp_legacy_send_open_receive_change_blocks, 1); + mdb_drop (env.tx (transaction_a), temp_state_blocks, 1); + + auto count_post (count (transaction_a, blocks)); + release_assert (count_pre == count_post); + + version_put (transaction_a, 19); + logger.always_log ("Finished upgrading all blocks to new blocks database"); +} + /** Takes a filepath, appends '_backup_' to the end (but before any extension) and saves that file in the same directory */ void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a) { @@ -621,16 +781,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const return frontiers; case tables::accounts: return accounts; - case tables::send_blocks: - return send_blocks; - case tables::receive_blocks: - return receive_blocks; - case tables::open_blocks: - return open_blocks; - case tables::change_blocks: - return change_blocks; - case tables::state_blocks: - return state_blocks; + case tables::blocks: + return blocks; case tables::pending: return pending; case tables::unchecked: @@ -674,7 +826,7 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file) void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a) { // Tables with uint256_union key - std::vector tables = { accounts, send_blocks, receive_blocks, open_blocks, change_blocks, state_blocks, vote, confirmation_height }; + std::vector tables = { accounts, blocks, vote, confirmation_height }; for (auto const & table : tables) { MDB_dbi temp; @@ -726,6 +878,99 @@ bool nano::mdb_store::init_error () const return error; } +std::shared_ptr nano::mdb_store::block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + nano::block_type type; + auto value (block_raw_get_v18 (transaction_a, hash_a, type)); + std::shared_ptr result; + if (value.size () != 0) + { + nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + result = nano::deserialize_block (stream, type); + release_assert (result != nullptr); + nano::block_sideband sideband; + auto error = (sideband.deserialize (stream, type)); + release_assert (!error); + result->sideband_set (sideband); + } + return result; +} + +nano::mdb_val nano::mdb_store::block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const +{ + nano::mdb_val result; + // Table lookups are ordered by match probability + nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; + for (auto current_type : block_types) + { + auto db_val (block_raw_get_by_type_v18 (transaction_a, hash_a, current_type)); + if (db_val.is_initialized ()) + { + type_a = current_type; + result = db_val.get (); + break; + } + } + + return result; +} + +boost::optional nano::mdb_store::block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const +{ + nano::mdb_val value; + nano::mdb_val hash (hash_a); + int status = status_code_not_found (); + switch (type_a) + { + case nano::block_type::send: + { + status = mdb_get (env.tx (transaction_a), send_blocks, hash, value); + break; + } + case nano::block_type::receive: + { + status = mdb_get (env.tx (transaction_a), receive_blocks, hash, value); + break; + } + case nano::block_type::open: + { + status = mdb_get (env.tx (transaction_a), open_blocks, hash, value); + break; + } + case nano::block_type::change: + { + status = mdb_get (env.tx (transaction_a), change_blocks, hash, value); + break; + } + case nano::block_type::state: + { + status = mdb_get (env.tx (transaction_a), state_blocks, hash, value); + break; + } + case nano::block_type::invalid: + case nano::block_type::not_a_block: + { + break; + } + } + + release_assert (success (status) || not_found (status)); + boost::optional result; + if (success (status)) + { + result = value; + } + return result; +} + +nano::uint128_t nano::mdb_store::block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const +{ + auto block (block_get_v18 (transaction_a, hash_a)); + release_assert (block); + nano::uint128_t result (block_balance_calculated (block)); + return result; +} + // All the v14 functions below are only needed during upgrades size_t nano::mdb_store::block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const { diff --git a/nano/node/lmdb/lmdb.hpp b/nano/node/lmdb/lmdb.hpp index 483e2af2e1..53d215550a 100644 --- a/nano/node/lmdb/lmdb.hpp +++ b/nano/node/lmdb/lmdb.hpp @@ -81,25 +81,25 @@ class mdb_store : public block_store_partial MDB_dbi accounts{ 0 }; /** - * Maps block hash to send block. + * Maps block hash to send block. (Removed) * nano::block_hash -> nano::send_block */ MDB_dbi send_blocks{ 0 }; /** - * Maps block hash to receive block. + * Maps block hash to receive block. (Removed) * nano::block_hash -> nano::receive_block */ MDB_dbi receive_blocks{ 0 }; /** - * Maps block hash to open block. + * Maps block hash to open block. (Removed) * nano::block_hash -> nano::open_block */ MDB_dbi open_blocks{ 0 }; /** - * Maps block hash to change block. + * Maps block hash to change block. (Removed) * nano::block_hash -> nano::change_block */ MDB_dbi change_blocks{ 0 }; @@ -117,7 +117,7 @@ class mdb_store : public block_store_partial MDB_dbi state_blocks_v1{ 0 }; /** - * Maps block hash to state block. + * Maps block hash to state block. (Removed) * nano::block_hash -> nano::state_block */ MDB_dbi state_blocks{ 0 }; @@ -182,6 +182,12 @@ class mdb_store : public block_store_partial */ MDB_dbi confirmation_height{ 0 }; + /* + * Contains block_sideband and block for all block types (legacy send/change/open/receive & state blocks) + * nano::block_hash -> nano::block_sideband, nano::block + */ + MDB_dbi blocks{ 0 }; + bool exists (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a) const; int get (nano::transaction const & transaction_a, tables table_a, nano::mdb_val const & key_a, nano::mdb_val & value_a) const; @@ -208,7 +214,7 @@ class mdb_store : public block_store_partial size_t count (nano::transaction const &, MDB_dbi) const; // These are only use in the upgrade process. - std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const override; + std::shared_ptr block_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_sideband_v14 * sideband_a = nullptr, bool * is_state_v1 = nullptr) const; size_t block_successor_offset_v14 (nano::transaction const & transaction_a, size_t entry_size_a, nano::block_type type_a) const; nano::block_hash block_successor_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; nano::mdb_val block_raw_get_v14 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a, bool * is_state_v1 = nullptr) const; @@ -220,6 +226,12 @@ class mdb_store : public block_store_partial void upgrade_v15_to_v16 (nano::write_transaction const &); void upgrade_v16_to_v17 (nano::write_transaction const &); void upgrade_v17_to_v18 (nano::write_transaction const &); + void upgrade_v18_to_v19 (nano::write_transaction const & transaction_a); + + std::shared_ptr block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; + nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; + boost::optional block_raw_get_by_type_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const; + nano::uint128_t block_balance_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const; void open_databases (bool &, nano::transaction const &, unsigned); diff --git a/nano/node/lmdb/lmdb_txn.cpp b/nano/node/lmdb/lmdb_txn.cpp index 405fd84e26..5e059ee1af 100644 --- a/nano/node/lmdb/lmdb_txn.cpp +++ b/nano/node/lmdb/lmdb_txn.cpp @@ -180,9 +180,9 @@ void nano::mdb_txn_tracker::serialize_json (boost::property_tree::ptree & json, } } -void nano::mdb_txn_tracker::output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const +void nano::mdb_txn_tracker::log_if_held_long_enough (nano::mdb_txn_stats const & mdb_txn_stats) const { - // Only output them if transactions were held for longer than a certain period of time + // Only log these transactions if they were held for longer than the min_read_txn_time/min_write_txn_time config values auto is_write = mdb_txn_stats.is_write (); auto time_open = mdb_txn_stats.timer.since_start (); @@ -212,13 +212,14 @@ void nano::mdb_txn_tracker::add (const nano::transaction_impl * transaction_impl /** Can be called without error if transaction does not exist */ void nano::mdb_txn_tracker::erase (const nano::transaction_impl * transaction_impl) { - nano::lock_guard guard (mutex); + nano::unique_lock lk (mutex); auto it = std::find_if (stats.begin (), stats.end (), matches_txn (transaction_impl)); if (it != stats.end ()) { - output_finished (*it); - it->timer.stop (); + auto tracker_stats_copy = *it; stats.erase (it); + lk.unlock (); + log_if_held_long_enough (tracker_stats_copy); } } diff --git a/nano/node/lmdb/lmdb_txn.hpp b/nano/node/lmdb/lmdb_txn.hpp index 0ec1767c26..63030a0347 100644 --- a/nano/node/lmdb/lmdb_txn.hpp +++ b/nano/node/lmdb/lmdb_txn.hpp @@ -78,6 +78,6 @@ class mdb_txn_tracker nano::txn_tracking_config txn_tracking_config; std::chrono::milliseconds block_processor_batch_max_time; - void output_finished (nano::mdb_txn_stats const & mdb_txn_stats) const; + void log_if_held_long_enough (nano::mdb_txn_stats const & mdb_txn_stats) const; }; } diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 7a069cce94..47e6fe41b1 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -115,10 +115,10 @@ rep_crawler (*this), warmed_up (0), block_processor (*this, write_database_queue), // clang-format off -block_processor_thread ([this]() { - nano::thread_role::set (nano::thread_role::name::block_processing); - this->block_processor.process_blocks (); -}), + block_processor_thread([this]() { + nano::thread_role::set(nano::thread_role::name::block_processing); + this->block_processor.process_blocks(); + }), // clang-format on online_reps (ledger, network_params, config.online_weight_minimum.number ()), votes_cache (wallets), @@ -365,7 +365,7 @@ node_seq (seq) if (!is_initialized) { release_assert (!flags.read_only); - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::confirmation_height, tables::frontiers, tables::open_blocks })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::confirmation_height, tables::frontiers })); // Store was empty meaning we just created it, add the genesis block store.initialize (transaction, genesis, ledger.cache); } @@ -521,20 +521,24 @@ bool nano::node::copy_with_compaction (boost::filesystem::path const & destinati void nano::node::process_fork (nano::transaction const & transaction_a, std::shared_ptr const & block_a, uint64_t const modified_a) { auto root (block_a->root ()); - if (!store.block_exists (transaction_a, block_a->type (), block_a->hash ()) && store.root_exists (transaction_a, block_a->root ())) + if (!store.block_exists (transaction_a, block_a->hash ()) && store.root_exists (transaction_a, block_a->root ())) { std::shared_ptr ledger_block (ledger.forked_block (transaction_a, *block_a)); if (ledger_block && !block_confirmed_or_being_confirmed (transaction_a, ledger_block->hash ()) && (ledger.can_vote (transaction_a, *ledger_block) || modified_a < nano::seconds_since_epoch () - 300 || !block_arrival.recent (block_a->hash ()))) { std::weak_ptr this_w (shared_from_this ()); - auto election = active.insert (ledger_block, boost::none, [this_w, root](std::shared_ptr) { + auto election = active.insert (ledger_block, boost::none, [this_w, root, root_block_type = block_a->type ()](std::shared_ptr) { if (auto this_l = this_w.lock ()) { auto attempt (this_l->bootstrap_initiator.current_attempt ()); if (attempt && attempt->mode == nano::bootstrap_mode::legacy) { auto transaction (this_l->store.tx_begin_read ()); - auto account (this_l->ledger.store.frontier_get (transaction, root)); + nano::account account{ 0 }; + if (root_block_type == nano::block_type::receive || root_block_type == nano::block_type::send || root_block_type == nano::block_type::change || root_block_type == nano::block_type::open) + { + account = this_l->ledger.store.frontier_get (transaction, root); + } if (!account.is_zero ()) { this_l->bootstrap_initiator.connections->requeue_pull (nano::pull_info (account, root, root, attempt->incremental_id)); @@ -596,7 +600,7 @@ void nano::node::process_active (std::shared_ptr const & incoming) nano::process_return nano::node::process (nano::block & block_a) { - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks }, { tables::confirmation_height })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending }, { tables::confirmation_height })); auto result (ledger.process (transaction, block_a)); return result; } @@ -611,7 +615,7 @@ nano::process_return nano::node::process_local (std::shared_ptr con block_processor.wait_write (); // Process block block_post_events post_events ([& store = store] { return store.tx_begin_read (); }); - auto transaction (store.tx_begin_write ({ tables::accounts, tables::cached_counts, tables::change_blocks, tables::frontiers, tables::open_blocks, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks }, { tables::confirmation_height })); + auto transaction (store.tx_begin_write ({ tables::accounts, tables::blocks, tables::cached_counts, tables::frontiers, tables::pending }, { tables::confirmation_height })); return block_processor.process_one (transaction, post_events, info, work_watcher_a, nano::block_origin::local); } @@ -1263,7 +1267,7 @@ void nano::node::process_confirmed (nano::election_status const & status_a, uint auto block_a (status_a.winner); auto hash (block_a->hash ()); const auto num_iters = (config.block_processor_batch_max_time / network_params.node.process_confirmed_interval) * 4; - if (ledger.block_exists (block_a->type (), hash)) + if (ledger.block_exists (hash)) { confirmation_height_processor.add (hash); } @@ -1397,14 +1401,14 @@ void nano::node::epoch_upgrader_impl (nano::private_key const & prv_a, nano::epo { }; // clang-format off - boost::multi_index_container, - boost::multi_index::member, - std::greater>, - boost::multi_index::hashed_unique, - boost::multi_index::member>>> - accounts_list; + boost::multi_index_container, + boost::multi_index::member, + std::greater>, + boost::multi_index::hashed_unique, + boost::multi_index::member>>> + accounts_list; // clang-format on bool finished_upgrade (false); @@ -1431,7 +1435,7 @@ void nano::node::epoch_upgrader_impl (nano::private_key const & prv_a, nano::epo } /* Upgrade accounts - Repeat until accounts with previous epoch exist in latest table */ + Repeat until accounts with previous epoch exist in latest table */ std::atomic upgraded_accounts (0); uint64_t workers (0); uint64_t attempts (0); @@ -1646,8 +1650,8 @@ work (1) boost::system::error_code error_chmod; /* - * @warning May throw a filesystem exception - */ + * @warning May throw a filesystem exception + */ boost::filesystem::create_directories (path_a); nano::set_secure_perm_directory (path_a, error_chmod); nano::daemon_config daemon_config (path_a); diff --git a/nano/node/openclwork.cpp b/nano/node/openclwork.cpp index 39c3d46667..e887fa8354 100644 --- a/nano/node/openclwork.cpp +++ b/nano/node/openclwork.cpp @@ -11,6 +11,8 @@ #include #include +bool nano::opencl_loaded{ false }; + namespace { std::string opencl_program = R"%%%( @@ -389,111 +391,125 @@ void printstate (blake2b_state * S) nano::opencl_environment::opencl_environment (bool & error_a) { - cl_uint platformIdCount = 0; - clGetPlatformIDs (0, nullptr, &platformIdCount); - std::vector platformIds (platformIdCount); - clGetPlatformIDs (platformIdCount, platformIds.data (), nullptr); - for (auto i (platformIds.begin ()), n (platformIds.end ()); i != n; ++i) + if (nano::opencl_loaded) { - nano::opencl_platform platform; - platform.platform = *i; - cl_uint deviceIdCount = 0; - clGetDeviceIDs (*i, CL_DEVICE_TYPE_ALL, 0, nullptr, &deviceIdCount); - std::vector deviceIds (deviceIdCount); - clGetDeviceIDs (*i, CL_DEVICE_TYPE_ALL, deviceIdCount, deviceIds.data (), nullptr); - for (auto j (deviceIds.begin ()), m (deviceIds.end ()); j != m; ++j) + cl_uint platformIdCount = 0; + clGetPlatformIDs (0, nullptr, &platformIdCount); + std::vector platformIds (platformIdCount); + clGetPlatformIDs (platformIdCount, platformIds.data (), nullptr); + for (auto i (platformIds.begin ()), n (platformIds.end ()); i != n; ++i) { - platform.devices.push_back (*j); + nano::opencl_platform platform; + platform.platform = *i; + cl_uint deviceIdCount = 0; + clGetDeviceIDs (*i, CL_DEVICE_TYPE_ALL, 0, nullptr, &deviceIdCount); + std::vector deviceIds (deviceIdCount); + clGetDeviceIDs (*i, CL_DEVICE_TYPE_ALL, deviceIdCount, deviceIds.data (), nullptr); + for (auto j (deviceIds.begin ()), m (deviceIds.end ()); j != m; ++j) + { + platform.devices.push_back (*j); + } + platforms.push_back (platform); } - platforms.push_back (platform); + } + else + { + error_a = true; } } void nano::opencl_environment::dump (std::ostream & stream) { - auto index (0); - size_t device_count (0); - for (auto & i : platforms) - { - device_count += i.devices.size (); - } - stream << boost::str (boost::format ("OpenCL found %1% platforms and %2% devices\n") % platforms.size () % device_count); - for (auto i (platforms.begin ()), n (platforms.end ()); i != n; ++i, ++index) + if (nano::opencl_loaded) { - std::vector queries = { CL_PLATFORM_PROFILE, CL_PLATFORM_VERSION, CL_PLATFORM_NAME, CL_PLATFORM_VENDOR, CL_PLATFORM_EXTENSIONS }; - stream << "Platform: " << index << std::endl; - for (auto j (queries.begin ()), m (queries.end ()); j != m; ++j) + auto index (0); + size_t device_count (0); + for (auto & i : platforms) { - size_t platformInfoCount = 0; - clGetPlatformInfo (i->platform, *j, 0, nullptr, &platformInfoCount); - std::vector info (platformInfoCount); - clGetPlatformInfo (i->platform, *j, info.size (), info.data (), nullptr); - stream << info.data () << std::endl; + device_count += i.devices.size (); } - for (auto j (i->devices.begin ()), m (i->devices.end ()); j != m; ++j) + stream << boost::str (boost::format ("OpenCL found %1% platforms and %2% devices\n") % platforms.size () % device_count); + for (auto i (platforms.begin ()), n (platforms.end ()); i != n; ++i, ++index) { - std::vector queries = { CL_DEVICE_NAME, CL_DEVICE_VENDOR, CL_DEVICE_PROFILE }; - stream << "Device: " << j - i->devices.begin () << std::endl; - for (auto k (queries.begin ()), o (queries.end ()); k != o; ++k) + std::vector queries = { CL_PLATFORM_PROFILE, CL_PLATFORM_VERSION, CL_PLATFORM_NAME, CL_PLATFORM_VENDOR, CL_PLATFORM_EXTENSIONS }; + stream << "Platform: " << index << std::endl; + for (auto j (queries.begin ()), m (queries.end ()); j != m; ++j) { size_t platformInfoCount = 0; - clGetDeviceInfo (*j, *k, 0, nullptr, &platformInfoCount); + clGetPlatformInfo (i->platform, *j, 0, nullptr, &platformInfoCount); std::vector info (platformInfoCount); - clGetDeviceInfo (*j, *k, info.size (), info.data (), nullptr); - stream << '\t' << info.data () << std::endl; + clGetPlatformInfo (i->platform, *j, info.size (), info.data (), nullptr); + stream << info.data () << std::endl; } - size_t deviceTypeCount = 0; - clGetDeviceInfo (*j, CL_DEVICE_TYPE, 0, nullptr, &deviceTypeCount); - std::vector deviceTypeInfo (deviceTypeCount); - clGetDeviceInfo (*j, CL_DEVICE_TYPE, deviceTypeCount, deviceTypeInfo.data (), 0); - std::string device_type_string; - switch (deviceTypeInfo[0]) + for (auto j (i->devices.begin ()), m (i->devices.end ()); j != m; ++j) { - case CL_DEVICE_TYPE_ACCELERATOR: - device_type_string = "ACCELERATOR"; - break; - case CL_DEVICE_TYPE_CPU: - device_type_string = "CPU"; - break; - case CL_DEVICE_TYPE_CUSTOM: - device_type_string = "CUSTOM"; - break; - case CL_DEVICE_TYPE_DEFAULT: - device_type_string = "DEFAULT"; - break; - case CL_DEVICE_TYPE_GPU: - device_type_string = "GPU"; - break; - default: - device_type_string = "Unknown"; - break; + std::vector queries = { CL_DEVICE_NAME, CL_DEVICE_VENDOR, CL_DEVICE_PROFILE }; + stream << "Device: " << j - i->devices.begin () << std::endl; + for (auto k (queries.begin ()), o (queries.end ()); k != o; ++k) + { + size_t platformInfoCount = 0; + clGetDeviceInfo (*j, *k, 0, nullptr, &platformInfoCount); + std::vector info (platformInfoCount); + clGetDeviceInfo (*j, *k, info.size (), info.data (), nullptr); + stream << '\t' << info.data () << std::endl; + } + size_t deviceTypeCount = 0; + clGetDeviceInfo (*j, CL_DEVICE_TYPE, 0, nullptr, &deviceTypeCount); + std::vector deviceTypeInfo (deviceTypeCount); + clGetDeviceInfo (*j, CL_DEVICE_TYPE, deviceTypeCount, deviceTypeInfo.data (), 0); + std::string device_type_string; + switch (deviceTypeInfo[0]) + { + case CL_DEVICE_TYPE_ACCELERATOR: + device_type_string = "ACCELERATOR"; + break; + case CL_DEVICE_TYPE_CPU: + device_type_string = "CPU"; + break; + case CL_DEVICE_TYPE_CUSTOM: + device_type_string = "CUSTOM"; + break; + case CL_DEVICE_TYPE_DEFAULT: + device_type_string = "DEFAULT"; + break; + case CL_DEVICE_TYPE_GPU: + device_type_string = "GPU"; + break; + default: + device_type_string = "Unknown"; + break; + } + stream << '\t' << device_type_string << std::endl; + size_t compilerAvailableCount = 0; + clGetDeviceInfo (*j, CL_DEVICE_COMPILER_AVAILABLE, 0, nullptr, &compilerAvailableCount); + std::vector compilerAvailableInfo (compilerAvailableCount); + clGetDeviceInfo (*j, CL_DEVICE_COMPILER_AVAILABLE, compilerAvailableCount, compilerAvailableInfo.data (), 0); + stream << "\tCompiler available: " << (compilerAvailableInfo[0] ? "true" : "false") << std::endl; + size_t computeUnitsAvailableCount = 0; + clGetDeviceInfo (*j, CL_DEVICE_MAX_COMPUTE_UNITS, 0, nullptr, &computeUnitsAvailableCount); + std::vector computeUnitsAvailableInfo (computeUnitsAvailableCount); + clGetDeviceInfo (*j, CL_DEVICE_MAX_COMPUTE_UNITS, computeUnitsAvailableCount, computeUnitsAvailableInfo.data (), 0); + uint64_t computeUnits (computeUnitsAvailableInfo[0] | (computeUnitsAvailableInfo[1] << 8) | (computeUnitsAvailableInfo[2] << 16) | (computeUnitsAvailableInfo[3] << 24)); + stream << "\tCompute units available: " << computeUnits << std::endl; + cl_ulong size{ 0 }; + clGetDeviceInfo (*j, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE, sizeof (cl_ulong), &size, 0); + stream << "\tMemory size" << std::endl; + stream << "\t\tConstant buffer: " << size << std::endl; + clGetDeviceInfo (*j, CL_DEVICE_LOCAL_MEM_SIZE, sizeof (cl_ulong), &size, 0); + stream << "\t\tLocal memory : " << size << std::endl; + clGetDeviceInfo (*j, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof (cl_ulong), &size, 0); + stream << "\t\tGlobal memory : " << size << std::endl; + clGetDeviceInfo (*j, CL_DEVICE_GLOBAL_MEM_CACHE_SIZE, sizeof (cl_ulong), &size, 0); + stream << "\t\tGlobal cache : " << size << std::endl; + clGetDeviceInfo (*j, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof (cl_ulong), &size, 0); + stream << "\t\tMax allocation : " << size << std::endl; } - stream << '\t' << device_type_string << std::endl; - size_t compilerAvailableCount = 0; - clGetDeviceInfo (*j, CL_DEVICE_COMPILER_AVAILABLE, 0, nullptr, &compilerAvailableCount); - std::vector compilerAvailableInfo (compilerAvailableCount); - clGetDeviceInfo (*j, CL_DEVICE_COMPILER_AVAILABLE, compilerAvailableCount, compilerAvailableInfo.data (), 0); - stream << "\tCompiler available: " << (compilerAvailableInfo[0] ? "true" : "false") << std::endl; - size_t computeUnitsAvailableCount = 0; - clGetDeviceInfo (*j, CL_DEVICE_MAX_COMPUTE_UNITS, 0, nullptr, &computeUnitsAvailableCount); - std::vector computeUnitsAvailableInfo (computeUnitsAvailableCount); - clGetDeviceInfo (*j, CL_DEVICE_MAX_COMPUTE_UNITS, computeUnitsAvailableCount, computeUnitsAvailableInfo.data (), 0); - uint64_t computeUnits (computeUnitsAvailableInfo[0] | (computeUnitsAvailableInfo[1] << 8) | (computeUnitsAvailableInfo[2] << 16) | (computeUnitsAvailableInfo[3] << 24)); - stream << "\tCompute units available: " << computeUnits << std::endl; - cl_ulong size{ 0 }; - clGetDeviceInfo (*j, CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE, sizeof (cl_ulong), &size, 0); - stream << "\tMemory size" << std::endl; - stream << "\t\tConstant buffer: " << size << std::endl; - clGetDeviceInfo (*j, CL_DEVICE_LOCAL_MEM_SIZE, sizeof (cl_ulong), &size, 0); - stream << "\t\tLocal memory : " << size << std::endl; - clGetDeviceInfo (*j, CL_DEVICE_GLOBAL_MEM_SIZE, sizeof (cl_ulong), &size, 0); - stream << "\t\tGlobal memory : " << size << std::endl; - clGetDeviceInfo (*j, CL_DEVICE_GLOBAL_MEM_CACHE_SIZE, sizeof (cl_ulong), &size, 0); - stream << "\t\tGlobal cache : " << size << std::endl; - clGetDeviceInfo (*j, CL_DEVICE_MAX_MEM_ALLOC_SIZE, sizeof (cl_ulong), &size, 0); - stream << "\t\tMax allocation : " << size << std::endl; } } + else + { + stream << boost::str (boost::format ("OpenCL library could not be found\n")); + } } nano::opencl_work::opencl_work (bool & error_a, nano::opencl_config const & config_a, nano::opencl_environment & environment_a, nano::logger_mt & logger_a) : @@ -534,23 +550,23 @@ logger (logger_a) if (!error_a) { cl_int attempt_error (0); - attempt_buffer = clCreateBuffer (context, 0, sizeof (uint64_t), nullptr, &attempt_error); + attempt_buffer = clCreateBuffer (context, CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY, sizeof (uint64_t), nullptr, &attempt_error); error_a |= attempt_error != CL_SUCCESS; if (!error_a) { cl_int result_error (0); - result_buffer = clCreateBuffer (context, 0, sizeof (uint64_t), nullptr, &result_error); + result_buffer = clCreateBuffer (context, CL_MEM_WRITE_ONLY | CL_MEM_HOST_READ_ONLY, sizeof (uint64_t), nullptr, &result_error); error_a |= result_error != CL_SUCCESS; if (!error_a) { cl_int item_error (0); size_t item_size (sizeof (nano::uint256_union)); - item_buffer = clCreateBuffer (context, 0, item_size, nullptr, &item_error); + item_buffer = clCreateBuffer (context, CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY, item_size, nullptr, &item_error); error_a |= item_error != CL_SUCCESS; if (!error_a) { cl_int difficulty_error (0); - difficulty_buffer = clCreateBuffer (context, 0, sizeof (uint64_t), nullptr, &difficulty_error); + difficulty_buffer = clCreateBuffer (context, CL_MEM_READ_ONLY | CL_MEM_HOST_WRITE_ONLY, sizeof (uint64_t), nullptr, &difficulty_error); error_a |= difficulty_error != CL_SUCCESS; if (!error_a) { diff --git a/nano/node/openclwork.hpp b/nano/node/openclwork.hpp index 2b1de6040c..c06f977caa 100644 --- a/nano/node/openclwork.hpp +++ b/nano/node/openclwork.hpp @@ -20,6 +20,7 @@ namespace nano { +extern bool opencl_loaded; class logger_mt; class opencl_platform { diff --git a/nano/node/plat/posix/openclapi.cpp b/nano/node/plat/posix/openclapi.cpp index 35f7886048..6429cd7385 100644 --- a/nano/node/plat/posix/openclapi.cpp +++ b/nano/node/plat/posix/openclapi.cpp @@ -31,12 +31,14 @@ class opencl_initializer clEnqueueNDRangeKernel = reinterpret_cast (dlsym (opencl_library, "clEnqueueNDRangeKernel")); clEnqueueReadBuffer = reinterpret_cast (dlsym (opencl_library, "clEnqueueReadBuffer")); clFinish = reinterpret_cast (dlsym (opencl_library, "clFinish")); + nano::opencl_loaded = true; } } ~opencl_initializer () { if (opencl_library != nullptr) { + nano::opencl_loaded = false; dlclose (opencl_library); } } diff --git a/nano/node/plat/windows/openclapi.cpp b/nano/node/plat/windows/openclapi.cpp index 8cb8098d62..8b71e2689b 100644 --- a/nano/node/plat/windows/openclapi.cpp +++ b/nano/node/plat/windows/openclapi.cpp @@ -31,12 +31,14 @@ class opencl_initializer clEnqueueNDRangeKernel = reinterpret_cast (GetProcAddress (opencl_library, "clEnqueueNDRangeKernel")); clEnqueueReadBuffer = reinterpret_cast (GetProcAddress (opencl_library, "clEnqueueReadBuffer")); clFinish = reinterpret_cast (GetProcAddress (opencl_library, "clFinish")); + nano::opencl_loaded = true; } } ~opencl_initializer () { if (opencl_library != nullptr) { + nano::opencl_loaded = false; FreeLibrary (opencl_library); } } diff --git a/nano/node/portmapping.cpp b/nano/node/portmapping.cpp index 07e947aa5a..5b646d0c7a 100644 --- a/nano/node/portmapping.cpp +++ b/nano/node/portmapping.cpp @@ -173,7 +173,7 @@ void nano::port_mapping::check_mapping_loop () } else { - if (check_count++ < 10) + if (check_count < 10) { node.logger.always_log (boost::str (boost::format ("UPnP No IGD devices found"))); } @@ -182,6 +182,7 @@ void nano::port_mapping::check_mapping_loop () node_l->port_mapping.check_mapping_loop (); }); } + ++check_count; } void nano::port_mapping::stop () diff --git a/nano/node/rocksdb/rocksdb.cpp b/nano/node/rocksdb/rocksdb.cpp index 2d77da3011..6f0ec6f021 100644 --- a/nano/node/rocksdb/rocksdb.cpp +++ b/nano/node/rocksdb/rocksdb.cpp @@ -74,7 +74,7 @@ nano::rocksdb_store::~rocksdb_store () void nano::rocksdb_store::open (bool & error_a, boost::filesystem::path const & path_a, bool open_read_only_a) { - std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "send", "receive", "open", "change", "state_blocks", "pending", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; + std::initializer_list names{ rocksdb::kDefaultColumnFamilyName.c_str (), "frontiers", "accounts", "blocks", "pending", "representation", "unchecked", "vote", "online_weight", "meta", "peers", "cached_counts", "confirmation_height" }; std::vector column_families; for (const auto & cf_name : names) { @@ -159,20 +159,10 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table return get_handle ("frontiers"); case tables::accounts: return get_handle ("accounts"); - case tables::send_blocks: - return get_handle ("send"); - case tables::receive_blocks: - return get_handle ("receive"); - case tables::open_blocks: - return get_handle ("open"); - case tables::change_blocks: - return get_handle ("change"); - case tables::state_blocks: - return get_handle ("state_blocks"); + case tables::blocks: + return get_handle ("blocks"); case tables::pending: return get_handle ("pending"); - case tables::representation: - return get_handle ("representation"); case tables::unchecked: return get_handle ("unchecked"); case tables::vote: @@ -269,11 +259,7 @@ bool nano::rocksdb_store::is_caching_counts (nano::tables table_a) const { switch (table_a) { - case tables::send_blocks: - case tables::receive_blocks: - case tables::open_blocks: - case tables::change_blocks: - case tables::state_blocks: + case tables::blocks: return true; default: return false; @@ -473,10 +459,7 @@ rocksdb::Options nano::rocksdb_store::get_db_options () const db_options.OptimizeLevelStyleCompaction (); // Adds a separate write queue for memtable/WAL - db_options.enable_pipelined_write = rocksdb_config.enable_pipelined_write; - - // Total size of memtables across column families. This can be used to manage the total memory used by memtables. - db_options.db_write_buffer_size = rocksdb_config.total_memtable_size * 1024 * 1024ULL; + db_options.enable_pipelined_write = true; return db_options; } @@ -486,21 +469,17 @@ rocksdb::BlockBasedTableOptions nano::rocksdb_store::get_table_options () const rocksdb::BlockBasedTableOptions table_options; // Block cache for reads - table_options.block_cache = rocksdb::NewLRUCache (rocksdb_config.block_cache * 1024 * 1024ULL); + table_options.block_cache = rocksdb::NewLRUCache (1024ULL * 1024 * base_block_cache_size * rocksdb_config.memory_multiplier); // Bloom filter to help with point reads - auto bloom_filter_bits = rocksdb_config.bloom_filter_bits; - if (bloom_filter_bits > 0) - { - table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (bloom_filter_bits, false)); - } + auto bloom_filter_bits = 10; + table_options.filter_policy.reset (rocksdb::NewBloomFilterPolicy (bloom_filter_bits, false)); // Increasing block_size decreases memory usage and space amplification, but increases read amplification. - table_options.block_size = rocksdb_config.block_size * 1024ULL; + table_options.block_size = 16 * 1024ULL; - // Whether index and filter blocks are stored in block_cache. These settings should be synced - table_options.cache_index_and_filter_blocks = rocksdb_config.cache_index_and_filter_blocks; - table_options.pin_l0_filter_and_index_blocks_in_cache = rocksdb_config.cache_index_and_filter_blocks; + // Whether index and filter blocks are stored in block_cache + table_options.pin_l0_filter_and_index_blocks_in_cache = true; return table_options; } @@ -514,7 +493,7 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const cf_options.level0_file_num_compaction_trigger = 4; // L1 size, compaction is triggered for L0 at this size (4 SST files in L1) - cf_options.max_bytes_for_level_base = 1024ULL * 1024 * 4 * rocksdb_config.memtable_size; + cf_options.max_bytes_for_level_base = 1024ULL * 1024 * 4 * rocksdb_config.memory_multiplier * base_memtable_size; // Each level is a multiple of the above. If L1 is 512MB. L2 will be 512 * 8 = 2GB. L3 will be 2GB * 8 = 16GB, and so on... cf_options.max_bytes_for_level_multiplier = 8; @@ -523,23 +502,23 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options () const cf_options.ttl = 1 * 24 * 60 * 60; // Size of level 1 sst files - cf_options.target_file_size_base = 1024ULL * 1024 * rocksdb_config.memtable_size; + cf_options.target_file_size_base = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_memtable_size; // Size of each memtable - cf_options.write_buffer_size = 1024ULL * 1024 * rocksdb_config.memtable_size; + cf_options.write_buffer_size = 1024ULL * 1024 * rocksdb_config.memory_multiplier * base_memtable_size; // Size target of levels are changed dynamically based on size of the last level cf_options.level_compaction_dynamic_level_bytes = true; - // Number of memtables to keep in memory (1 active, rest inactive/immutable) - cf_options.max_write_buffer_number = rocksdb_config.num_memtables; + // Number of memtables to keep in memory (1 active, 1 inactive) + cf_options.max_write_buffer_number = 2; return cf_options; } std::vector nano::rocksdb_store::all_tables () const { - return std::vector{ tables::accounts, tables::cached_counts, tables::change_blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::open_blocks, tables::peers, tables::pending, tables::receive_blocks, tables::representation, tables::send_blocks, tables::state_blocks, tables::unchecked, tables::vote }; + return std::vector{ tables::accounts, tables::blocks, tables::cached_counts, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::unchecked, tables::vote }; } bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path) diff --git a/nano/node/rocksdb/rocksdb.hpp b/nano/node/rocksdb/rocksdb.hpp index 723bfd54d0..c901dfbaa8 100644 --- a/nano/node/rocksdb/rocksdb.hpp +++ b/nano/node/rocksdb/rocksdb.hpp @@ -46,13 +46,6 @@ class rocksdb_store : public block_store_partial // Do nothing } - std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const override - { - // Should not be called as RocksDB has no such upgrade path - release_assert (false); - return nullptr; - } - bool copy_db (boost::filesystem::path const & destination) override; void rebuild_db (nano::write_transaction const & transaction_a) override; @@ -102,6 +95,9 @@ class rocksdb_store : public block_store_partial rocksdb::Options get_db_options () const; rocksdb::BlockBasedTableOptions get_table_options () const; nano::rocksdb_config rocksdb_config; + + constexpr static int base_memtable_size = 16; + constexpr static int base_block_cache_size = 16; }; extern template class block_store_partial; diff --git a/nano/node/testing.cpp b/nano/node/testing.cpp index d185779fe0..ec35731aa0 100644 --- a/nano/node/testing.cpp +++ b/nano/node/testing.cpp @@ -226,9 +226,24 @@ void nano::system::deadline_set (std::chrono::duration const std::error_code nano::system::poll (std::chrono::nanoseconds const & wait_time) { - std::error_code ec; +#if NANO_ASIO_HANDLER_TRACKING == 0 io_ctx.run_one_for (wait_time); +#else + nano::timer<> timer; + timer.start (); + auto count = io_ctx.poll_one (); + if (count == 0) + { + std::this_thread::sleep_for (wait_time); + } + else if (count == 1 && timer.since_start ().count () >= NANO_ASIO_HANDLER_TRACKING) + { + auto timestamp = std::chrono::duration_cast (std::chrono::system_clock::now ().time_since_epoch ()).count (); + std::cout << (boost::format ("[%1%] io_thread held for %2%ms") % timestamp % timer.since_start ().count ()).str () << std::endl; + } +#endif + std::error_code ec; if (std::chrono::steady_clock::now () > deadline) { ec = nano::error_system::deadline_expired; @@ -476,14 +491,11 @@ void nano::system::generate_mass_activity (uint32_t count_a, nano::node & node_a auto now (std::chrono::steady_clock::now ()); auto us (std::chrono::duration_cast (now - previous).count ()); uint64_t count (0); - uint64_t state (0); { auto transaction (node_a.store.tx_begin_read ()); - auto block_counts (node_a.store.block_count (transaction)); - count = block_counts.sum (); - state = block_counts.state; + count = node_a.store.block_count (transaction); } - std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% state: %4% old: %5%\n") % i % us % (us / 256) % state % (count - state)); + std::cerr << boost::str (boost::format ("Mass activity iteration %1% us %2% us/t %3% block count: %4%\n") % i % us % (us / 256) % count); previous = now; } generate_activity (node_a, accounts); diff --git a/nano/rpc_test/rpc.cpp b/nano/rpc_test/rpc.cpp index aa6e9ee858..a465f29b89 100644 --- a/nano/rpc_test/rpc.cpp +++ b/nano/rpc_test/rpc.cpp @@ -5100,40 +5100,6 @@ TEST (rpc, work_peers_all) ASSERT_EQ (0, peers_node.size ()); } -TEST (rpc, block_count_type) -{ - nano::system system; - auto node = add_ipc_enabled_node (system); - system.wallet (0)->insert_adhoc (nano::test_genesis_key.prv); - auto send (system.wallet (0)->send_action (nano::test_genesis_key.pub, nano::test_genesis_key.pub, node->config.receive_minimum.number ())); - ASSERT_NE (nullptr, send); - auto receive (system.wallet (0)->receive_action (*send, nano::test_genesis_key.pub, node->config.receive_minimum.number ())); - ASSERT_NE (nullptr, receive); - scoped_io_thread_name_change scoped_thread_name_io; - nano::node_rpc_config node_rpc_config; - nano::ipc::ipc_server ipc_server (*node, node_rpc_config); - nano::rpc_config rpc_config (nano::get_available_port (), true); - rpc_config.rpc_process.ipc_port = node->config.ipc_config.transport_tcp.port; - nano::ipc_rpc_processor ipc_rpc_processor (system.io_ctx, rpc_config); - nano::rpc rpc (system.io_ctx, rpc_config, ipc_rpc_processor); - rpc.start (); - boost::property_tree::ptree request; - request.put ("action", "block_count_type"); - test_response response (request, rpc.config.port, system.io_ctx); - ASSERT_TIMELY (5s, response.status != 0); - ASSERT_EQ (200, response.status); - std::string send_count (response.json.get ("send")); - ASSERT_EQ ("0", send_count); - std::string receive_count (response.json.get ("receive")); - ASSERT_EQ ("0", receive_count); - std::string open_count (response.json.get ("open")); - ASSERT_EQ ("1", open_count); - std::string change_count (response.json.get ("change")); - ASSERT_EQ ("0", change_count); - std::string state_count (response.json.get ("state")); - ASSERT_EQ ("2", state_count); -} - TEST (rpc, ledger) { nano::system system; @@ -6761,7 +6727,7 @@ TEST (rpc, block_confirmed) TEST (rpc, database_txn_tracker) { - // Don't test this in rocksdb mode + // Don't test this with the rocksdb backend auto use_rocksdb_str = std::getenv ("TEST_USE_ROCKSDB"); if (use_rocksdb_str && boost::lexical_cast (use_rocksdb_str) == 1) { @@ -7636,6 +7602,7 @@ TEST (rpc, telemetry_all) ASSERT_EQ (node->network.endpoint ().address ().to_string (), metrics.get ("address")); ASSERT_EQ (node->network.endpoint ().port (), metrics.get ("port")); + ASSERT_TRUE (node1.network.find_node_id (data.node_id)); } // Also tests all forms of ipv4/ipv6 diff --git a/nano/secure/blockstore.hpp b/nano/secure/blockstore.hpp index 623a1b728a..e783fe9796 100644 --- a/nano/secure/blockstore.hpp +++ b/nano/secure/blockstore.hpp @@ -18,10 +18,18 @@ namespace nano { // Move to versioning with a specific version if required for a future upgrade -class state_block_w_sideband +template +class block_w_sideband_v18 { public: - std::shared_ptr state_block; + std::shared_ptr block; + nano::block_sideband sideband; +}; + +class block_w_sideband +{ +public: + std::shared_ptr block; nano::block_sideband sideband; }; @@ -260,17 +268,28 @@ class db_val return result; } - explicit operator state_block_w_sideband () const + template + explicit operator block_w_sideband_v18 () const { nano::bufferstream stream (reinterpret_cast (data ()), size ()); auto error (false); - nano::state_block_w_sideband block_w_sideband; - block_w_sideband.state_block = std::make_shared (error, stream); - debug_assert (!error); + block_w_sideband_v18 block_w_sideband; + block_w_sideband.block = std::make_shared (error, stream); + release_assert (!error); - error = block_w_sideband.sideband.deserialize (stream, nano::block_type::state); - debug_assert (!error); + error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); + release_assert (!error); + + return block_w_sideband; + } + explicit operator block_w_sideband () const + { + nano::bufferstream stream (reinterpret_cast (data ()), size ()); + nano::block_w_sideband block_w_sideband; + block_w_sideband.block = (nano::deserialize_block (stream)); + auto error = block_w_sideband.sideband.deserialize (stream, block_w_sideband.block->type ()); + release_assert (!error); return block_w_sideband; } @@ -483,19 +502,14 @@ class store_iterator final enum class tables { accounts, + blocks, cached_counts, // RocksDB only - change_blocks, confirmation_height, frontiers, meta, online_weight, - open_blocks, peers, pending, - receive_blocks, - representation, - send_blocks, - state_blocks, unchecked, vote }; @@ -578,14 +592,11 @@ class block_store virtual void block_successor_clear (nano::write_transaction const &, nano::block_hash const &) = 0; virtual std::shared_ptr block_get (nano::transaction const &, nano::block_hash const &) const = 0; virtual std::shared_ptr block_get_no_sideband (nano::transaction const &, nano::block_hash const &) const = 0; - virtual std::shared_ptr block_get_v14 (nano::transaction const &, nano::block_hash const &, nano::block_sideband_v14 * = nullptr, bool * = nullptr) const = 0; virtual std::shared_ptr block_random (nano::transaction const &) = 0; - virtual void block_del (nano::write_transaction const &, nano::block_hash const &, nano::block_type) = 0; + virtual void block_del (nano::write_transaction const &, nano::block_hash const &) = 0; virtual bool block_exists (nano::transaction const &, nano::block_hash const &) = 0; - virtual bool block_exists (nano::transaction const &, nano::block_type, nano::block_hash const &) = 0; - virtual nano::block_counts block_count (nano::transaction const &) = 0; + virtual uint64_t block_count (nano::transaction const &) = 0; virtual bool root_exists (nano::transaction const &, nano::root const &) = 0; - virtual bool source_exists (nano::transaction const &, nano::block_hash const &) = 0; virtual nano::account block_account (nano::transaction const &, nano::block_hash const &) const = 0; virtual nano::account block_account_calculated (nano::block const &) const = 0; diff --git a/nano/secure/blockstore_partial.hpp b/nano/secure/blockstore_partial.hpp index d967842f1a..0f98bc0134 100644 --- a/nano/secure/blockstore_partial.hpp +++ b/nano/secure/blockstore_partial.hpp @@ -109,10 +109,10 @@ class block_store_partial : public block_store std::vector vector; { nano::vectorstream stream (vector); - block_a.serialize (stream); + nano::serialize_block (stream, block_a); block_a.sideband ().serialize (stream, block_a.type ()); } - block_raw_put (transaction_a, vector, block_a.type (), hash_a); + block_raw_put (transaction_a, vector, hash_a); nano::block_predecessor_set predecessor (transaction_a, *this); block_a.visit (predecessor); debug_assert (block_a.previous ().is_zero () || block_successor (transaction_a, block_a.previous ()) == hash_a); @@ -128,66 +128,48 @@ class block_store_partial : public block_store std::shared_ptr block_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); std::shared_ptr result; if (value.size () != 0) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); + nano::block_type type; + auto error (try_read (stream, type)); + release_assert (!error); result = nano::deserialize_block (stream, type); - debug_assert (result != nullptr); + release_assert (result != nullptr); nano::block_sideband sideband; - auto error (sideband.deserialize (stream, type)); - (void)error; - debug_assert (!error); + error = (sideband.deserialize (stream, type)); + release_assert (!error); result->sideband_set (sideband); } return result; } + bool block_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) override + { + auto junk = block_raw_get (transaction_a, hash_a); + return junk.size () != 0; + } + std::shared_ptr block_get_no_sideband (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); std::shared_ptr result; if (value.size () != 0) { nano::bufferstream stream (reinterpret_cast (value.data ()), value.size ()); - result = nano::deserialize_block (stream, type); + result = nano::deserialize_block (stream); debug_assert (result != nullptr); } return result; } - bool block_exists (nano::transaction const & transaction_a, nano::block_type type, nano::block_hash const & hash_a) override - { - auto junk = block_raw_get_by_type (transaction_a, hash_a, type); - return junk.is_initialized (); - } - - bool block_exists (nano::transaction const & tx_a, nano::block_hash const & hash_a) override - { - // Table lookups are ordered by match probability - // clang-format off - return - block_exists (tx_a, nano::block_type::state, hash_a) || - block_exists (tx_a, nano::block_type::send, hash_a) || - block_exists (tx_a, nano::block_type::receive, hash_a) || - block_exists (tx_a, nano::block_type::open, hash_a) || - block_exists (tx_a, nano::block_type::change, hash_a); - // clang-format on - } - bool root_exists (nano::transaction const & transaction_a, nano::root const & root_a) override { return block_exists (transaction_a, root_a) || account_exists (transaction_a, root_a); } - bool source_exists (nano::transaction const & transaction_a, nano::block_hash const & source_a) override - { - return block_exists (transaction_a, nano::block_type::state, source_a) || block_exists (transaction_a, nano::block_type::send, source_a); - } - nano::account block_account (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { auto block (block_get (transaction_a, hash_a)); @@ -233,12 +215,12 @@ class block_store_partial : public block_store nano::block_hash block_successor (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); nano::block_hash result; if (value.size () != 0) { debug_assert (value.size () >= result.bytes.size ()); + auto type = block_type_from_raw (value.data ()); nano::bufferstream stream (reinterpret_cast (value.data ()) + block_successor_offset (transaction_a, value.size (), type), result.bytes.size ()); auto error (nano::try_read (stream, result.bytes)); (void)error; @@ -253,12 +235,12 @@ class block_store_partial : public block_store void block_successor_clear (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { - nano::block_type type; - auto value (block_raw_get (transaction_a, hash_a, type)); + auto value (block_raw_get (transaction_a, hash_a)); debug_assert (value.size () != 0); + auto type = block_type_from_raw (value.data ()); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::fill_n (data.begin () + block_successor_offset (transaction_a, value.size (), type), sizeof (nano::block_hash), uint8_t{ 0 }); - block_raw_put (transaction_a, data, type, hash_a); + block_raw_put (transaction_a, data, hash_a); } void unchecked_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, std::shared_ptr const & block_a) override @@ -366,31 +348,9 @@ class block_store_partial : public block_store return cache_mutex; } - void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type block_type_a) override + void block_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) override { - auto table = tables::state_blocks; - switch (block_type_a) - { - case nano::block_type::open: - table = tables::open_blocks; - break; - case nano::block_type::receive: - table = tables::receive_blocks; - break; - case nano::block_type::send: - table = tables::send_blocks; - break; - case nano::block_type::change: - table = tables::change_blocks; - break; - case nano::block_type::state: - table = tables::state_blocks; - break; - default: - debug_assert (false); - } - - auto status = del (transaction_a, table, hash_a); + auto status = del (transaction_a, tables::blocks, hash_a); release_assert (success (status)); } @@ -421,11 +381,10 @@ class block_store_partial : public block_store return nano::epoch::epoch_0; } - void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_type block_type_a, nano::block_hash const & hash_a) + void block_raw_put (nano::write_transaction const & transaction_a, std::vector const & data, nano::block_hash const & hash_a) { - auto database_a = block_database (block_type_a); nano::db_val value{ data.size (), (void *)data.data () }; - auto status = put (transaction_a, database_a, hash_a, value); + auto status = put (transaction_a, tables::blocks, hash_a, value); release_assert (success (status)); } @@ -624,15 +583,9 @@ class block_store_partial : public block_store return static_cast (*this).exists (transaction_a, table_a, key_a); } - nano::block_counts block_count (nano::transaction const & transaction_a) override + uint64_t block_count (nano::transaction const & transaction_a) override { - nano::block_counts result; - result.send = count (transaction_a, tables::send_blocks); - result.receive = count (transaction_a, tables::receive_blocks); - result.open = count (transaction_a, tables::open_blocks); - result.change = count (transaction_a, tables::change_blocks); - result.state = count (transaction_a, tables::state_blocks); - return result; + return count (transaction_a, tables::blocks); } size_t account_count (nano::transaction const & transaction_a) override @@ -642,45 +595,16 @@ class block_store_partial : public block_store std::shared_ptr block_random (nano::transaction const & transaction_a) override { - auto count (block_count (transaction_a)); - release_assert (std::numeric_limits::max () > count.sum ()); - auto region = static_cast (nano::random_pool::generate_word32 (0, static_cast (count.sum () - 1))); - std::shared_ptr result; - auto & derived_store = static_cast (*this); - if (region < count.send) - { - result = derived_store.template block_random (transaction_a, tables::send_blocks); - } - else + nano::block_hash hash; + nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); + auto existing = make_iterator> (transaction_a, tables::blocks, nano::db_val (hash)); + auto end (nano::store_iterator> (nullptr)); + if (existing == end) { - region -= count.send; - if (region < count.receive) - { - result = derived_store.template block_random (transaction_a, tables::receive_blocks); - } - else - { - region -= count.receive; - if (region < count.open) - { - result = derived_store.template block_random (transaction_a, tables::open_blocks); - } - else - { - region -= count.open; - if (region < count.change) - { - result = derived_store.template block_random (transaction_a, tables::change_blocks); - } - else - { - result = derived_store.template block_random (transaction_a, tables::state_blocks); - } - } - } + existing = make_iterator> (transaction_a, tables::blocks); } - debug_assert (result != nullptr); - return result; + debug_assert (existing != end); + return existing->second; } uint64_t confirmation_height_count (nano::transaction const & transaction_a) override @@ -786,22 +710,7 @@ class block_store_partial : public block_store nano::network_params network_params; std::unordered_map> vote_cache_l1; std::unordered_map> vote_cache_l2; - int const version{ 18 }; - - template - std::shared_ptr block_random (nano::transaction const & transaction_a, tables table_a) - { - nano::block_hash hash; - nano::random_pool::generate_block (hash.bytes.data (), hash.bytes.size ()); - auto existing = make_iterator> (transaction_a, table_a, nano::db_val (hash)); - if (existing == nano::store_iterator> (nullptr)) - { - existing = make_iterator> (transaction_a, table_a); - } - auto end (nano::store_iterator> (nullptr)); - debug_assert (existing != end); - return block_get (transaction_a, nano::block_hash (existing->first)); - } + int const version{ 19 }; template nano::store_iterator make_iterator (nano::transaction const & transaction_a, tables table_a) const @@ -815,22 +724,11 @@ class block_store_partial : public block_store return static_cast (*this).template make_iterator (transaction_a, table_a, key); } - nano::db_val block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const + nano::db_val block_raw_get (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const { nano::db_val result; - // Table lookups are ordered by match probability - nano::block_type block_types[]{ nano::block_type::state, nano::block_type::send, nano::block_type::receive, nano::block_type::open, nano::block_type::change }; - for (auto current_type : block_types) - { - auto db_val (block_raw_get_by_type (transaction_a, hash_a, current_type)); - if (db_val.is_initialized ()) - { - type_a = current_type; - result = db_val.get (); - break; - } - } - + auto status = get (transaction_a, tables::blocks, hash_a, result); + release_assert (success (status) || not_found (status)); return result; } @@ -839,79 +737,10 @@ class block_store_partial : public block_store return entry_size_a - nano::block_sideband::size (type_a); } - boost::optional> block_raw_get_by_type (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const - { - nano::db_val value; - nano::db_val hash (hash_a); - int status = status_code_not_found (); - switch (type_a) - { - case nano::block_type::send: - { - status = get (transaction_a, tables::send_blocks, hash, value); - break; - } - case nano::block_type::receive: - { - status = get (transaction_a, tables::receive_blocks, hash, value); - break; - } - case nano::block_type::open: - { - status = get (transaction_a, tables::open_blocks, hash, value); - break; - } - case nano::block_type::change: - { - status = get (transaction_a, tables::change_blocks, hash, value); - break; - } - case nano::block_type::state: - { - status = get (transaction_a, tables::state_blocks, hash, value); - break; - } - case nano::block_type::invalid: - case nano::block_type::not_a_block: - { - break; - } - } - - release_assert (success (status) || not_found (status)); - boost::optional> result; - if (success (status)) - { - result = value; - } - return result; - } - - tables block_database (nano::block_type type_a) + static nano::block_type block_type_from_raw (void * data_a) { - tables result = tables::frontiers; - switch (type_a) - { - case nano::block_type::send: - result = tables::send_blocks; - break; - case nano::block_type::receive: - result = tables::receive_blocks; - break; - case nano::block_type::open: - result = tables::open_blocks; - break; - case nano::block_type::change: - result = tables::change_blocks; - break; - case nano::block_type::state: - result = tables::state_blocks; - break; - default: - debug_assert (false); - break; - } - return result; + // The block type is the first byte + return static_cast ((reinterpret_cast (data_a))[0]); } size_t count (nano::transaction const & transaction_a, std::initializer_list dbs_a) const @@ -962,12 +791,12 @@ class block_predecessor_set : public nano::block_visitor void fill_value (nano::block const & block_a) { auto hash (block_a.hash ()); - nano::block_type type; - auto value (store.block_raw_get (transaction, block_a.previous (), type)); + auto value (store.block_raw_get (transaction, block_a.previous ())); debug_assert (value.size () != 0); + auto type = store.block_type_from_raw (value.data ()); std::vector data (static_cast (value.data ()), static_cast (value.data ()) + value.size ()); std::copy (hash.bytes.begin (), hash.bytes.end (), data.begin () + store.block_successor_offset (transaction, value.size (), type)); - store.block_raw_put (transaction, data, type, block_a.previous ()); + store.block_raw_put (transaction, data, block_a.previous ()); } void send_block (nano::send_block const & block_a) override { diff --git a/nano/secure/common.cpp b/nano/secure/common.cpp index 69ec26d124..6f554b08b7 100644 --- a/nano/secure/common.cpp +++ b/nano/secure/common.cpp @@ -252,11 +252,6 @@ nano::epoch nano::account_info::epoch () const return epoch_m; } -size_t nano::block_counts::sum () const -{ - return send + receive + open + change + state; -} - nano::pending_info::pending_info (nano::account const & source_a, nano::amount const & amount_a, nano::epoch epoch_a) : source (source_a), amount (amount_a), diff --git a/nano/secure/common.hpp b/nano/secure/common.hpp index 96bfae445b..bbfe9db039 100644 --- a/nano/secure/common.hpp +++ b/nano/secure/common.hpp @@ -208,16 +208,6 @@ class block_info final nano::account account{ 0 }; nano::amount balance{ 0 }; }; -class block_counts final -{ -public: - size_t sum () const; - size_t send{ 0 }; - size_t receive{ 0 }; - size_t open{ 0 }; - size_t change{ 0 }; - size_t state{ 0 }; -}; class confirmation_height_info final { diff --git a/nano/secure/ledger.cpp b/nano/secure/ledger.cpp index bbf39361b2..73df183cbb 100644 --- a/nano/secure/ledger.cpp +++ b/nano/secure/ledger.cpp @@ -40,7 +40,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (info.representative, pending.amount.number ()); nano::account_info new_info (block_a.hashables.previous, info.representative, info.open_block, ledger.balance (transaction, block_a.hashables.previous), nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, pending.source, info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, pending.source); ledger.store.block_successor_clear (transaction, block_a.hashables.previous); @@ -60,7 +60,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (info.representative, 0 - amount); nano::account_info new_info (block_a.hashables.previous, info.representative, info.open_block, ledger.balance (transaction, block_a.hashables.previous), nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, destination_account, info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.pending_put (transaction, nano::pending_key (destination_account, block_a.hashables.source), { source_account, amount, nano::epoch::epoch_0 }); ledger.store.frontier_del (transaction, hash); ledger.store.frontier_put (transaction, block_a.hashables.previous, destination_account); @@ -76,7 +76,7 @@ class rollback_visitor : public nano::block_visitor ledger.cache.rep_weights.representation_add (block_a.representative (), 0 - amount); nano::account_info new_info; ledger.change_latest (transaction, destination_account, new_info, new_info); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); ledger.store.pending_put (transaction, nano::pending_key (destination_account, block_a.hashables.source), { source_account, amount, nano::epoch::epoch_0 }); ledger.store.frontier_del (transaction, hash); ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::open); @@ -96,7 +96,7 @@ class rollback_visitor : public nano::block_visitor auto representative = block->representative (); ledger.cache.rep_weights.representation_add (block_a.representative (), 0 - balance); ledger.cache.rep_weights.representation_add (representative, balance); - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); nano::account_info new_info (block_a.hashables.previous, representative, info.open_block, info.balance, nano::seconds_since_epoch (), info.block_count - 1, nano::epoch::epoch_0); ledger.change_latest (transaction, account, info, new_info); ledger.store.frontier_del (transaction, hash); @@ -165,7 +165,7 @@ class rollback_visitor : public nano::block_visitor { ledger.stats.inc (nano::stat::type::rollback, nano::stat::detail::open); } - ledger.store.block_del (transaction, hash, block_a.type ()); + ledger.store.block_del (transaction, hash); } nano::write_transaction const & transaction; nano::ledger & ledger; @@ -257,7 +257,7 @@ void ledger_processor::state_block (nano::state_block & block_a) void ledger_processor::state_block_impl (nano::state_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Unambiguous) if (result.code == nano::process_result::progress) { @@ -314,7 +314,7 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) { if (!block_a.hashables.link.is_zero ()) { - result.code = ledger.store.source_exists (transaction, block_a.hashables.link) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.link) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) if (result.code == nano::process_result::progress) { nano::pending_key key (block_a.hashables.account, block_a.hashables.link); @@ -381,7 +381,7 @@ void ledger_processor::state_block_impl (nano::state_block & block_a) void ledger_processor::epoch_block_impl (nano::state_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Unambiguous) if (result.code == nano::process_result::progress) { @@ -473,7 +473,7 @@ void ledger_processor::epoch_block_impl (nano::state_block & block_a) void ledger_processor::change_block (nano::change_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Harmless) if (result.code == nano::process_result::progress) { @@ -530,7 +530,7 @@ void ledger_processor::change_block (nano::change_block & block_a) void ledger_processor::send_block (nano::send_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block before? (Harmless) if (result.code == nano::process_result::progress) { @@ -592,7 +592,7 @@ void ledger_processor::send_block (nano::send_block & block_a) void ledger_processor::receive_block (nano::receive_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block already? (Harmless) if (result.code == nano::process_result::progress) { @@ -616,7 +616,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a) { debug_assert (!validate_message (account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - result.code = ledger.store.source_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block already? (Harmless) if (result.code == nano::process_result::progress) { nano::account_info info; @@ -672,7 +672,7 @@ void ledger_processor::receive_block (nano::receive_block & block_a) void ledger_processor::open_block (nano::open_block & block_a) { auto hash (block_a.hash ()); - auto existing (ledger.store.block_exists (transaction, block_a.type (), hash)); + auto existing (ledger.store.block_exists (transaction, hash)); result.code = existing ? nano::process_result::old : nano::process_result::progress; // Have we seen this block already? (Harmless) if (result.code == nano::process_result::progress) { @@ -685,7 +685,7 @@ void ledger_processor::open_block (nano::open_block & block_a) { debug_assert (!validate_message (block_a.hashables.account, hash, block_a.signature)); result.verified = nano::signature_verification::valid; - result.code = ledger.store.source_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block? (Harmless) + result.code = ledger.store.block_exists (transaction, block_a.hashables.source) ? nano::process_result::progress : nano::process_result::gap_source; // Have we seen the source block? (Harmless) if (result.code == nano::process_result::progress) { nano::account_info info; @@ -776,7 +776,7 @@ epoch_2_started_cb (epoch_2_started_cb_a) cache.unchecked_count = store.unchecked_count (transaction); } - cache.block_count = store.block_count (transaction).sum (); + cache.block_count = store.block_count (transaction); } } @@ -842,11 +842,6 @@ bool nano::ledger::block_exists (nano::block_hash const & hash_a) return store.block_exists (store.tx_begin_read (), hash_a); } -bool nano::ledger::block_exists (nano::block_type type, nano::block_hash const & hash_a) -{ - return store.block_exists (store.tx_begin_read (), type, hash_a); -} - std::string nano::ledger::block_text (char const * hash_a) { return block_text (nano::block_hash (hash_a)); @@ -1204,7 +1199,7 @@ std::shared_ptr nano::ledger::successor (nano::transaction const & std::shared_ptr nano::ledger::forked_block (nano::transaction const & transaction_a, nano::block const & block_a) { - debug_assert (!store.block_exists (transaction_a, block_a.type (), block_a.hash ())); + debug_assert (!store.block_exists (transaction_a, block_a.hash ())); auto root (block_a.root ()); debug_assert (store.block_exists (transaction_a, root) || store.account_exists (transaction_a, root)); auto result (store.block_get (transaction_a, store.block_successor (transaction_a, root))); @@ -1251,7 +1246,7 @@ bool nano::ledger::block_not_confirmed_or_not_exists (nano::block const & block_ bool result (true); auto hash (block_a.hash ()); auto transaction (store.tx_begin_read ()); - if (store.block_exists (transaction, block_a.type (), hash)) + if (store.block_exists (transaction, hash)) { result = !block_confirmed (transaction, hash); } diff --git a/nano/secure/ledger.hpp b/nano/secure/ledger.hpp index a246f0433c..cd7deeb9e9 100644 --- a/nano/secure/ledger.hpp +++ b/nano/secure/ledger.hpp @@ -33,7 +33,6 @@ class ledger final nano::block_hash representative (nano::transaction const &, nano::block_hash const &); nano::block_hash representative_calculated (nano::transaction const &, nano::block_hash const &); bool block_exists (nano::block_hash const &); - bool block_exists (nano::block_type, nano::block_hash const &); std::string block_text (char const *); std::string block_text (nano::block_hash const &); bool is_send (nano::transaction const &, nano::state_block const &) const; diff --git a/nano/slow_test/node.cpp b/nano/slow_test/node.cpp index a11604c961..fb717b3919 100644 --- a/nano/slow_test/node.cpp +++ b/nano/slow_test/node.cpp @@ -1623,7 +1623,7 @@ TEST (node, mass_epoch_upgrader) // Check upgrade { auto transaction (node.store.tx_begin_read ()); - ASSERT_EQ (expected_blocks, node.store.block_count (transaction).sum ()); + ASSERT_EQ (expected_blocks, node.store.block_count (transaction)); for (auto i (node.store.latest_begin (transaction)); i != node.store.latest_end (); ++i) { nano::account_info info (i->second);