Migrate LMDB database to RocksDB with new CLI command

nano/core_test/ledger.cpp

@@ -1,6 +1,7 @@
 #include <nano/lib/stats.hpp>
 #include <nano/lib/threading.hpp>
 #include <nano/node/election.hpp>
+#include <nano/node/rocksdb/rocksdb.hpp>
 #include <nano/node/testing.hpp>
 #include <nano/test_common/testutil.hpp>
 
@@ -3771,3 +3772,86 @@ TEST (ledger, hash_root_random)
 		ASSERT_LE (iteration, 1000);
 	}
 }
+
+TEST (ledger, migrate_lmdb_to_rocksdb)
+{
+	auto path (nano::unique_path ());
+	nano::genesis genesis;
+	nano::logger_mt logger;
+	boost::asio::ip::address_v6 address (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"));
+	uint16_t port = 100;
+	nano::mdb_store store (logger, path / "data.ldb");
+	nano::stat stats;
+	nano::ledger ledger (store, stats);
+	nano::work_pool pool (std::numeric_limits<unsigned>::max ());
+
+	auto send = nano::state_block_builder ()
+	            .account (nano::dev_genesis_key.pub)
+	            .previous (nano::genesis_hash)
+	            .representative (0)
+	            .link (nano::account (10))
+	            .balance (nano::genesis_amount - 100)
+	            .sign (nano::dev_genesis_key.prv, nano::dev_genesis_key.pub)
+	            .work (*pool.generate (nano::genesis_hash))
+	            .build_shared ();
+
+	auto vote (std::make_shared<nano::vote> (nano::dev_genesis_key.pub, nano::dev_genesis_key.prv, 0, std::vector<nano::block_hash> (1, send->hash ())));
+
+	nano::endpoint_key endpoint_key (address.to_bytes (), port);
+	auto version = 99;
+
+	{
+		auto transaction (store.tx_begin_write ());
+		store.initialize (transaction, genesis, ledger.cache);
+		ASSERT_FALSE (store.init_error ());
+
+		// Lower the database to the max version unsupported for upgrades
+		store.confirmation_height_put (transaction, nano::genesis_account, { 2, send->hash () });
+
+		store.online_weight_put (transaction, 100, nano::amount (2));
+		store.frontier_put (transaction, nano::block_hash (2), nano::account (5));
+		store.peer_put (transaction, endpoint_key);
+
+		store.pending_put (transaction, nano::pending_key (nano::genesis_account, send->hash ()), nano::pending_info (nano::genesis_account, 100, nano::epoch::epoch_0));
+		store.pruned_put (transaction, send->hash ());
+		store.unchecked_put (transaction, nano::genesis_hash, send);
+		store.version_put (transaction, version);
+		send->sideband_set ({});
+		store.block_put (transaction, send->hash (), *send);
+		store.vote_put (transaction, nano::account (5), vote);
+	}
+
+	auto error = ledger.migrate_lmdb_to_rocksdb (path);
+	ASSERT_FALSE (error);
+
+	nano::rocksdb_store rocksdb_store (logger, path / "rocksdb");
+	auto rocksdb_transaction (rocksdb_store.tx_begin_read ());
+
+	nano::pending_info pending_info;
+	ASSERT_FALSE (rocksdb_store.pending_get (rocksdb_transaction, nano::pending_key (nano::genesis_account, send->hash ()), pending_info));
+
+	for (auto i = rocksdb_store.online_weight_begin (rocksdb_transaction); i != rocksdb_store.online_weight_end (); ++i)
+	{
+		ASSERT_EQ (i->first, 100);
+		ASSERT_EQ (i->second, 2);
+	}
+
+	ASSERT_EQ (rocksdb_store.online_weight_count (rocksdb_transaction), 1);
+
+	auto block1 = rocksdb_store.block_get (rocksdb_transaction, send->hash ());
+
+	ASSERT_EQ (*send, *block1);
+	ASSERT_EQ (*vote, *rocksdb_store.vote_get (rocksdb_transaction, nano::account (5)));
+	ASSERT_TRUE (rocksdb_store.peer_exists (rocksdb_transaction, endpoint_key));
+	ASSERT_EQ (rocksdb_store.version_get (rocksdb_transaction), version);
+	ASSERT_EQ (rocksdb_store.frontier_get (rocksdb_transaction, 2), 5);
+	nano::confirmation_height_info confirmation_height_info;
+	ASSERT_FALSE (rocksdb_store.confirmation_height_get (rocksdb_transaction, nano::genesis_account, confirmation_height_info));
+	ASSERT_EQ (confirmation_height_info.height, 2);
+	ASSERT_EQ (confirmation_height_info.frontier, send->hash ());
+
+	auto unchecked_infos = rocksdb_store.unchecked_get (rocksdb_transaction, nano::genesis_hash);
+	ASSERT_EQ (unchecked_infos.size (), 1);
+	ASSERT_EQ (unchecked_infos.front ().account, nano::genesis_account);
+	ASSERT_EQ (*unchecked_infos.front ().block, *send);
+}

nano/node/cli.cpp

@@ -57,6 +57,7 @@ void nano::add_node_options (boost::program_options::options_description & descr
 	("unchecked_clear", "Clear unchecked blocks")
 	("confirmation_height_clear", "Clear confirmation height")
 	("rebuild_database", "Rebuild LMDB database with vacuum for best compaction")
+	("migrate_database_lmdb_to_rocksdb", "Migrates LMDB database to RocksDB")
 	("diagnostics", "Run internal diagnostics")
 	("generate_config", boost::program_options::value<std::string> (), "Write configuration to stdout, populated with defaults suitable for this system. Pass the configuration type node or rpc. See also use_defaults.")
 	("key_create", "Generates a adhoc random keypair and prints it to stdout")
@@ -446,6 +447,33 @@ std::error_code nano::handle_node_options (boost::program_options::variables_map
 			std::cerr << "Snapshot failed (unknown reason)" << std::endl;
 		}
 	}
+	else if (vm.count ("migrate_database_lmdb_to_rocksdb"))
+	{
+		auto data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();
+		auto node_flags = nano::inactive_node_flag_defaults ();
+		node_flags.config_overrides.push_back ("node.rocksdb.enable=false");
+		nano::update_flags (node_flags, vm);
+		nano::inactive_node node (data_path, node_flags);
+		auto error (false);
+		if (!node.node->init_error ())
+		{
+			std::cout << "Migrating LMDB database to RocksDB, might take a while..." << std::endl;
+			error = node.node->ledger.migrate_lmdb_to_rocksdb (data_path);
+		}
+		else
+		{
+			error = true;
+		}
+
+		if (!error)
+		{
+			std::cout << "Migration completed, after confirming it is correct the data.ldb file can be deleted if no longer required" << std::endl;
+		}
+		else
+		{
+			std::cerr << "There was an error migrating" << std::endl;
+		}
+	}
 	else if (vm.count ("unchecked_clear"))
 	{
 		boost::filesystem::path data_path = vm.count ("data_path") ? boost::filesystem::path (vm["data_path"].as<std::string> ()) : nano::working_path ();

nano/node/lmdb/lmdb.cpp

@@ -161,7 +161,7 @@ nano::write_transaction nano::mdb_store::tx_begin_write (std::vector<nano::table
 	return env.tx_begin_write (create_txn_callbacks ());
 }
 
-nano::read_transaction nano::mdb_store::tx_begin_read ()
+nano::read_transaction nano::mdb_store::tx_begin_read () const
 {
 	return env.tx_begin_read (create_txn_callbacks ());
 }
@@ -171,7 +171,7 @@ std::string nano::mdb_store::vendor_get () const
 	return boost::str (boost::format ("LMDB %1%.%2%.%3%") % MDB_VERSION_MAJOR % MDB_VERSION_MINOR % MDB_VERSION_PATCH);
 }
 
-nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks ()
+nano::mdb_txn_callbacks nano::mdb_store::create_txn_callbacks () const
 {
 	nano::mdb_txn_callbacks mdb_txn_callbacks;
 	if (txn_tracking_enabled)

nano/node/lmdb/lmdb.hpp

@@ -39,7 +39,7 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 
 	mdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
 	nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
-	nano::read_transaction tx_begin_read () override;
+	nano::read_transaction tx_begin_read () const override;
 
 	std::string vendor_get () const override;
 
@@ -256,8 +256,8 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 
 	MDB_dbi table_to_dbi (tables table_a) const;
 
-	nano::mdb_txn_tracker mdb_txn_tracker;
-	nano::mdb_txn_callbacks create_txn_callbacks ();
+	mutable nano::mdb_txn_tracker mdb_txn_tracker;
+	nano::mdb_txn_callbacks create_txn_callbacks () const;
 	bool txn_tracking_enabled;
 
 	uint64_t count (nano::transaction const & transaction_a, tables table_a) const override;

nano/node/lmdb/lmdb_txn.cpp

@@ -95,7 +95,7 @@ nano::write_mdb_txn::~write_mdb_txn ()
 	commit ();
 }
 
-void nano::write_mdb_txn::commit () const
+void nano::write_mdb_txn::commit ()
 {
 	auto status (mdb_txn_commit (handle));
 	release_assert (status == MDB_SUCCESS);

nano/node/lmdb/lmdb_txn.hpp

@@ -41,7 +41,7 @@ class write_mdb_txn final : public write_transaction_impl
 public:
 	write_mdb_txn (nano::mdb_env const &, mdb_txn_callbacks mdb_txn_callbacks);
 	~write_mdb_txn ();
-	void commit () const override;
+	void commit () override;
 	void renew () override;
 	void * get_handle () const override;
 	bool contains (nano::tables table_a) const override;

nano/node/node.cpp

@@ -95,7 +95,7 @@ alarm (alarm_a),
 work (work_a),
 distributed_work (*this),
 logger (config_a.logging.min_time_between_log_output),
-store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade, config_a.rocksdb_config.enable)),
+store_impl (nano::make_store (logger, application_path_a, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)),
 store (*store_impl),
 wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_config)),
 wallets_store (*wallets_store_impl),
@@ -1688,9 +1688,9 @@ nano::node_flags const & nano::inactive_node_flag_defaults ()
 	return node_flags;
 }
 
-std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade, bool use_rocksdb_backend)
+std::unique_ptr<nano::block_store> nano::make_store (nano::logger_mt & logger, boost::filesystem::path const & path, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade)
 {
-	if (use_rocksdb_backend || using_rocksdb_in_tests ())
+	if (rocksdb_config.enable || using_rocksdb_in_tests ())
 	{
 		return std::make_unique<nano::rocksdb_store> (logger, add_db_postfix ? path / "rocksdb" : path, rocksdb_config, read_only);
 	}

nano/node/rocksdb/rocksdb.cpp

@@ -310,7 +310,7 @@ nano::write_transaction nano::rocksdb_store::tx_begin_write (std::vector<nano::t
 	return nano::write_transaction{ std::move (txn) };
 }
 
-nano::read_transaction nano::rocksdb_store::tx_begin_read ()
+nano::read_transaction nano::rocksdb_store::tx_begin_read () const
 {
 	return nano::read_transaction{ std::make_unique<nano::read_rocksdb_txn> (db.get ()) };
 }
@@ -472,7 +472,7 @@ int nano::rocksdb_store::status_code_not_found () const
 uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, tables table_a) const
 {
 	uint64_t sum = 0;
-	// Some column families are small enough (except unchecked) that they can just be iterated, rather than doing extra io caching counts
+	// Peers/online weight are small enough that they can just be iterated to get accurate counts.
 	if (table_a == tables::peers)
 	{
 		for (auto i (peers_begin (transaction_a)), n (peers_end ()); i != n; ++i)

nano/node/rocksdb/rocksdb.hpp

@@ -28,7 +28,7 @@ class rocksdb_store : public block_store_partial<rocksdb::Slice, rocksdb_store>
 public:
 	rocksdb_store (nano::logger_mt &, boost::filesystem::path const &, nano::rocksdb_config const & = nano::rocksdb_config{}, bool open_read_only = false);
 	nano::write_transaction tx_begin_write (std::vector<nano::tables> const & tables_requiring_lock = {}, std::vector<nano::tables> const & tables_no_lock = {}) override;
-	nano::read_transaction tx_begin_read () override;
+	nano::read_transaction tx_begin_read () const override;
 
 	std::string vendor_get () const override;
 

nano/node/rocksdb/rocksdb_txn.cpp

@@ -51,7 +51,7 @@ nano::write_rocksdb_txn::~write_rocksdb_txn ()
 	unlock ();
 }
 
-void nano::write_rocksdb_txn::commit () const
+void nano::write_rocksdb_txn::commit ()
 {
 	auto status = txn->Commit ();
 

nano/node/rocksdb/rocksdb_txn.hpp

@@ -30,7 +30,7 @@ class write_rocksdb_txn final : public write_transaction_impl
 public:
 	write_rocksdb_txn (rocksdb::OptimisticTransactionDB * db_a, std::vector<nano::tables> const & tables_requiring_locks_a, std::vector<nano::tables> const & tables_no_locks_a, std::unordered_map<nano::tables, std::mutex> & mutexes_a);
 	~write_rocksdb_txn ();
-	void commit () const override;
+	void commit () override;
 	void renew () override;
 	void * get_handle () const override;
 	bool contains (nano::tables table_a) const override;

nano/secure/blockstore.cpp

@@ -84,7 +84,7 @@ void * nano::write_transaction::get_handle () const
 	return impl->get_handle ();
 }
 
-void nano::write_transaction::commit () const
+void nano::write_transaction::commit ()
 {
 	impl->commit ();
 }
@@ -94,6 +94,12 @@ void nano::write_transaction::renew ()
 	impl->renew ();
 }
 
+void nano::write_transaction::refresh ()
+{
+	impl->commit ();
+	impl->renew ();
+}
+
 bool nano::write_transaction::contains (nano::tables table_a) const
 {
 	return impl->contains (table_a);