[Pruning] Basic pruning blocks storage & database upgrade

nano/core_test/block_store.cpp

@@ -880,6 +880,24 @@ TEST (block_store, cemented_count_cache)
 	ASSERT_EQ (1, ledger_cache.cemented_count);
 }
 
+TEST (block_store, pruned_count)
+{
+	nano::logger_mt logger;
+	auto store = nano::make_store (logger, nano::unique_path ());
+	ASSERT_TRUE (!store->init_error ());
+	{
+		auto transaction (store->tx_begin_write ());
+		nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0);
+		block.sideband_set ({});
+		auto hash1 (block.hash ());
+		store->block_put (transaction, hash1, block);
+		store->pruned_put (transaction, hash1);
+	}
+	auto transaction (store->tx_begin_read ());
+	ASSERT_EQ (1, store->pruned_count (transaction));
+	ASSERT_EQ (1, store->block_count (transaction));
+}
+
 TEST (block_store, sequence_increment)
 {
 	nano::logger_mt logger;
@@ -926,6 +944,26 @@ TEST (block_store, block_random)
 	ASSERT_EQ (*block, *genesis.open);
 }
 
+TEST (block_store, pruned_random)
+{
+	nano::logger_mt logger;
+	auto store = nano::make_store (logger, nano::unique_path ());
+	ASSERT_TRUE (!store->init_error ());
+	nano::genesis genesis;
+	nano::open_block block (0, 1, 0, nano::keypair ().prv, 0, 0);
+	block.sideband_set ({});
+	auto hash1 (block.hash ());
+	{
+		nano::ledger_cache ledger_cache;
+		auto transaction (store->tx_begin_write ());
+		store->initialize (transaction, genesis, ledger_cache);
+		store->pruned_put (transaction, hash1);
+	}
+	auto transaction (store->tx_begin_read ());
+	auto random_hash (store->pruned_random (transaction));
+	ASSERT_EQ (hash1, random_hash);
+}
+
 // Databases need to be dropped in order to convert to dupsort compatible
 TEST (block_store, DISABLED_change_dupsort) // Unchecked is no longer dupsort table
 {
@@ -1232,6 +1270,69 @@ TEST (block_store, online_weight)
 	ASSERT_EQ (store->online_weight_end (), store->online_weight_begin (transaction));
 }
 
+TEST (block_store, pruned_blocks)
+{
+	nano::logger_mt logger;
+	auto store = nano::make_store (logger, nano::unique_path ());
+	ASSERT_TRUE (!store->init_error ());
+
+	nano::keypair key1;
+	nano::open_block block1 (0, 1, key1.pub, key1.prv, key1.pub, 0);
+	auto hash1 (block1.hash ());
+	{
+		auto transaction (store->tx_begin_write ());
+
+		// Confirm that the store is empty
+		ASSERT_FALSE (store->pruned_exists (transaction, hash1));
+		ASSERT_EQ (store->pruned_count (transaction), 0);
+
+		// Add one
+		store->pruned_put (transaction, hash1);
+		ASSERT_TRUE (store->pruned_exists (transaction, hash1));
+	}
+
+	// Confirm that it can be found
+	ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 1);
+
+	// Add another one and check that it (and the existing one) can be found
+	nano::open_block block2 (1, 2, key1.pub, key1.prv, key1.pub, 0);
+	block2.sideband_set ({});
+	auto hash2 (block2.hash ());
+	{
+		auto transaction (store->tx_begin_write ());
+		store->pruned_put (transaction, hash2);
+		ASSERT_TRUE (store->pruned_exists (transaction, hash2)); // Check new pruned hash is here
+		ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash2));
+		ASSERT_TRUE (store->pruned_exists (transaction, hash1)); // Check first pruned hash is still here
+		ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash1));
+	}
+
+	ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 2);
+
+	// Delete the first one
+	{
+		auto transaction (store->tx_begin_write ());
+		store->pruned_del (transaction, hash2);
+		ASSERT_FALSE (store->pruned_exists (transaction, hash2)); // Confirm it no longer exists
+		ASSERT_FALSE (store->block_or_pruned_exists (transaction, hash2));
+		store->block_put (transaction, hash2, block2); // Add corresponding block
+		ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash2));
+		ASSERT_TRUE (store->pruned_exists (transaction, hash1)); // Check first pruned hash is still here
+		ASSERT_TRUE (store->block_or_pruned_exists (transaction, hash1));
+	}
+
+	ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 1);
+
+	// Delete original one
+	{
+		auto transaction (store->tx_begin_write ());
+		store->pruned_del (transaction, hash1);
+		ASSERT_FALSE (store->pruned_exists (transaction, hash1));
+	}
+
+	ASSERT_EQ (store->pruned_count (store->tx_begin_read ()), 0);
+}
+
 TEST (mdb_block_store, upgrade_v14_v15)
 {
 	if (nano::using_rocksdb_in_tests ())
@@ -1752,6 +1853,36 @@ TEST (mdb_block_store, upgrade_v18_v19)
 	ASSERT_LT (18, store.version_get (transaction));
 }
 
+TEST (mdb_block_store, upgrade_v19_v20)
+{
+	if (nano::using_rocksdb_in_tests ())
+	{
+		// Don't test this in rocksdb mode
+		return;
+	}
+	auto path (nano::unique_path ());
+	nano::genesis genesis;
+	nano::logger_mt logger;
+	nano::stat stats;
+	{
+		nano::mdb_store store (logger, path);
+		nano::ledger ledger (store, stats);
+		auto transaction (store.tx_begin_write ());
+		store.initialize (transaction, genesis, ledger.cache);
+		// Delete pruned table
+		ASSERT_FALSE (mdb_drop (store.env.tx (transaction), store.pruned, 1));
+		store.version_put (transaction, 19);
+	}
+	// Upgrading should create the table
+	nano::mdb_store store (logger, path);
+	ASSERT_FALSE (store.init_error ());
+	ASSERT_NE (store.pruned, 0);
+
+	// Version should be correct
+	auto transaction (store.tx_begin_read ());
+	ASSERT_LT (19, store.version_get (transaction));
+}
+
 TEST (mdb_block_store, upgrade_backup)
 {
 	if (nano::using_rocksdb_in_tests ())

nano/core_test/ledger.cpp

@@ -3305,12 +3305,14 @@ TEST (ledger, cache)
 		auto block_count = 1 + 2 * (i + 1) - 2;
 		auto cemented_count = 1 + 2 * (i + 1) - 2;
 		auto genesis_weight = nano::genesis_amount - i;
+		auto pruned_count = i;
 
 		auto cache_check = [&, i](nano::ledger_cache const & cache_a) {
 			ASSERT_EQ (account_count, cache_a.account_count);
 			ASSERT_EQ (block_count, cache_a.block_count);
 			ASSERT_EQ (cemented_count, cache_a.cemented_count);
 			ASSERT_EQ (genesis_weight, cache_a.rep_weights.representation_get (nano::genesis_account));
+			ASSERT_EQ (pruned_count, cache_a.pruned_count);
 		};
 
 		nano::keypair key;
@@ -3382,5 +3384,14 @@ TEST (ledger, cache)
 		++cemented_count;
 		cache_check (ledger.cache);
 		cache_check (nano::ledger (*store, stats).cache);
+
+		{
+			auto transaction (store->tx_begin_write ());
+			ledger.store.pruned_put (transaction, open->hash ());
+			++ledger.cache.pruned_count;
+		}
+		++pruned_count;
+		cache_check (ledger.cache);
+		cache_check (nano::ledger (*store, stats).cache);
 	}
 }

nano/node/lmdb/lmdb.cpp

@@ -194,6 +194,7 @@ void nano::mdb_store::open_databases (bool & error_a, nano::transaction const &
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "online_weight", flags, &online_weight) != 0;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "meta", flags, &meta) != 0;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "peers", flags, &peers) != 0;
+	error_a |= mdb_dbi_open (env.tx (transaction_a), "pruned", flags, &pruned) != 0;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "confirmation_height", flags, &confirmation_height) != 0;
 	error_a |= mdb_dbi_open (env.tx (transaction_a), "accounts", flags, &accounts_v0) != 0;
 	accounts = accounts_v0;
@@ -273,6 +274,8 @@ bool nano::mdb_store::do_upgrades (nano::write_transaction & transaction_a, bool
 			upgrade_v18_to_v19 (transaction_a);
 			needs_vacuuming = true;
 		case 19:
+			upgrade_v19_to_v20 (transaction_a);
+		case 20:
 			break;
 		default:
 			logger.always_log (boost::str (boost::format ("The version of the ledger (%1%) is too high for this node") % version_l));
@@ -726,6 +729,14 @@ void nano::mdb_store::upgrade_v18_to_v19 (nano::write_transaction const & transa
 	logger.always_log ("Finished upgrading all blocks to new blocks database");
 }
 
+void nano::mdb_store::upgrade_v19_to_v20 (nano::write_transaction const & transaction_a)
+{
+	logger.always_log ("Preparing v19 to v20 database upgrade...");
+	mdb_dbi_open (env.tx (transaction_a), "pruned", MDB_CREATE, &pruned);
+	version_put (transaction_a, 20);
+	logger.always_log ("Finished creating new pruned table");
+}
+
 /** Takes a filepath, appends '_backup_<timestamp>' to the end (but before any extension) and saves that file in the same directory */
 void nano::mdb_store::create_backup_file (nano::mdb_env & env_a, boost::filesystem::path const & filepath_a, nano::logger_mt & logger_a)
 {
@@ -844,6 +855,8 @@ MDB_dbi nano::mdb_store::table_to_dbi (tables table_a) const
 			return meta;
 		case tables::peers:
 			return peers;
+		case tables::pruned:
+			return pruned;
 		case tables::confirmation_height:
 			return confirmation_height;
 		default:
@@ -875,7 +888,7 @@ bool nano::mdb_store::copy_db (boost::filesystem::path const & destination_file)
 void nano::mdb_store::rebuild_db (nano::write_transaction const & transaction_a)
 {
 	// Tables with uint256_union key
-	std::vector<MDB_dbi> tables = { accounts, blocks, vote, confirmation_height };
+	std::vector<MDB_dbi> tables = { accounts, blocks, vote, pruned, confirmation_height };
 	for (auto const & table : tables)
 	{
 		MDB_dbi temp;

nano/node/lmdb/lmdb.hpp

@@ -174,6 +174,12 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 	 */
 	MDB_dbi meta{ 0 };
 
+	/**
+	 * Pruned blocks hashes
+	 * nano::block_hash -> none
+	 */
+	MDB_dbi pruned{ 0 };
+
 	/*
 	 * Endpoints for peers
 	 * nano::endpoint_key -> no_value
@@ -231,7 +237,8 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 	void upgrade_v15_to_v16 (nano::write_transaction const &);
 	void upgrade_v16_to_v17 (nano::write_transaction const &);
 	void upgrade_v17_to_v18 (nano::write_transaction const &);
-	void upgrade_v18_to_v19 (nano::write_transaction const & transaction_a);
+	void upgrade_v18_to_v19 (nano::write_transaction const &);
+	void upgrade_v19_to_v20 (nano::write_transaction const &);
 
 	std::shared_ptr<nano::block> block_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const;
 	nano::mdb_val block_raw_get_v18 (nano::transaction const & transaction_a, nano::block_hash const & hash_a, nano::block_type & type_a) const;

nano/node/rocksdb/rocksdb.cpp

@@ -98,7 +98,8 @@ std::unordered_map<const char *, nano::tables> nano::rocksdb_store::create_cf_na
 		{ "online_weight", tables::online_weight },
 		{ "meta", tables::meta },
 		{ "peers", tables::peers },
-		{ "confirmation_height", tables::confirmation_height } };
+		{ "confirmation_height", tables::confirmation_height },
+		{ "pruned", tables::pruned } };
 
 	debug_assert (map.size () == all_tables ().size () + 1);
 	return map;
@@ -259,6 +260,11 @@ rocksdb::ColumnFamilyOptions nano::rocksdb_store::get_cf_options (std::string co
 		std::shared_ptr<rocksdb::TableFactory> table_factory (rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2)));
 		cf_options = get_active_cf_options (table_factory, memtable_size_bytes);
 	}
+	else if (cf_name_a == "pruned")
+	{
+		std::shared_ptr<rocksdb::TableFactory> table_factory (rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2)));
+		cf_options = get_active_cf_options (table_factory, memtable_size_bytes);
+	}
 	else if (cf_name_a == rocksdb::kDefaultColumnFamilyName)
 	{
 		// Do nothing.
@@ -343,6 +349,8 @@ rocksdb::ColumnFamilyHandle * nano::rocksdb_store::table_to_column_family (table
 			return get_handle ("meta");
 		case tables::peers:
 			return get_handle ("peers");
+		case tables::pruned:
+			return get_handle ("pruned");
 		case tables::confirmation_height:
 			return get_handle ("confirmation_height");
 		default:
@@ -482,6 +490,11 @@ uint64_t nano::rocksdb_store::count (nano::transaction const & transaction_a, ta
 	{
 		db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum);
 	}
+	// This should be correct at node start, later only cache should be used
+	else if (table_a == tables::pruned)
+	{
+		db->GetIntProperty (table_to_column_family (table_a), "rocksdb.estimate-num-keys", &sum);
+	}
 	// These should only be used in tests to check database consistency
 	else if (table_a == tables::accounts)
 	{
@@ -702,7 +715,7 @@ void nano::rocksdb_store::on_flush (rocksdb::FlushJobInfo const & flush_job_info
 
 std::vector<nano::tables> nano::rocksdb_store::all_tables () const
 {
-	return std::vector<nano::tables>{ tables::accounts, tables::blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::unchecked, tables::vote };
+	return std::vector<nano::tables>{ tables::accounts, tables::blocks, tables::confirmation_height, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::unchecked, tables::vote };
 }
 
 bool nano::rocksdb_store::copy_db (boost::filesystem::path const & destination_path)

nano/node/write_database_queue.hpp

@@ -14,6 +14,7 @@ enum class writer
 {
 	confirmation_height,
 	process_batch,
+	pruning,
 	testing // Used in tests to emulate a write lock
 };
 

nano/secure/blockstore.hpp

@@ -510,6 +510,7 @@ enum class tables
 	online_weight,
 	peers,
 	pending,
+	pruned,
 	unchecked,
 	vote
 };
@@ -662,6 +663,14 @@ class block_store
 	virtual void version_put (nano::write_transaction const &, int) = 0;
 	virtual int version_get (nano::transaction const &) const = 0;
 
+	virtual void pruned_put (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0;
+	virtual void pruned_del (nano::write_transaction const & transaction_a, nano::block_hash const & hash_a) = 0;
+	virtual bool pruned_exists (nano::transaction const & transaction_a, nano::block_hash const & hash_a) const = 0;
+	virtual bool block_or_pruned_exists (nano::transaction const &, nano::block_hash const &) = 0;
+	virtual nano::block_hash pruned_random (nano::transaction const & transaction_a) = 0;
+	virtual size_t pruned_count (nano::transaction const & transaction_a) const = 0;
+	virtual void pruned_clear (nano::write_transaction const &) = 0;
+
 	virtual void peer_put (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
 	virtual void peer_del (nano::write_transaction const & transaction_a, nano::endpoint_key const & endpoint_a) = 0;
 	virtual bool peer_exists (nano::transaction const & transaction_a, nano::endpoint_key const & endpoint_a) const = 0;