Stats RPC to print out rocksdb memory stats (#2852)

* Stats RPC to print out rocksdb memory stats

* Use "rocksdb" type instead, also add test (Colin Review)

* Change to database type in RPC and add LMDB memory stats

nano/node/json_handler.cpp

@@ -3686,6 +3686,10 @@ void nano::json_handler::stats ()
 		node.stats.log_samples (*sink);
 		use_sink = true;
 	}
+	else if (type == "database")
+	{
+		node.store.serialize_memory_stats (response_l);
+	}
 	else
 	{
 		ec = nano::error_rpc::invalid_missing_type;

nano/node/lmdb/lmdb.cpp

@@ -143,6 +143,19 @@ void nano::mdb_store::serialize_mdb_tracker (boost::property_tree::ptree & json,
 	mdb_txn_tracker.serialize_json (json, min_read_time, min_write_time);
 }
 
+void nano::mdb_store::serialize_memory_stats (boost::property_tree::ptree & json)
+{
+	MDB_stat stats;
+	auto status (mdb_env_stat (env.environment, &stats));
+	release_assert (status == 0);
+	json.put ("branch_pages", stats.ms_branch_pages);
+	json.put ("depth", stats.ms_depth);
+	json.put ("entries", stats.ms_entries);
+	json.put ("leaf_pages", stats.ms_leaf_pages);
+	json.put ("overflow_pages", stats.ms_overflow_pages);
+	json.put ("page_size", stats.ms_psize);
+}
+
 nano::write_transaction nano::mdb_store::tx_begin_write (std::vector<nano::tables> const &, std::vector<nano::tables> const &)
 {
 	return env.tx_begin_write (create_txn_callbacks ());

nano/node/lmdb/lmdb.hpp

@@ -49,6 +49,8 @@ class mdb_store : public block_store_partial<MDB_val, mdb_store>
 
 	static void create_backup_file (nano::mdb_env &, boost::filesystem::path const &, nano::logger_mt &);
 
+	void serialize_memory_stats (boost::property_tree::ptree &) override;
+
 private:
 	nano::logger_mt & logger;
 	bool error{ false };

nano/node/rocksdb/rocksdb.cpp

@@ -7,6 +7,7 @@
 #include <boost/endian/conversion.hpp>
 #include <boost/format.hpp>
 #include <boost/polymorphic_cast.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 #include <rocksdb/merge_operator.h>
 #include <rocksdb/slice.h>
@@ -526,5 +527,53 @@ bool nano::rocksdb_store::init_error () const
 {
 	return error;
 }
+
+void nano::rocksdb_store::serialize_memory_stats (boost::property_tree::ptree & json)
+{
+	uint64_t val;
+
+	// Approximate size of active and unflushed immutable memtables (bytes).
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kCurSizeAllMemTables, &val);
+	json.put ("cur-size-all-mem-tables", val);
+
+	// Approximate size of active, unflushed immutable, and pinned immutable memtables (bytes).
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kSizeAllMemTables, &val);
+	json.put ("size-all-mem-tables", val);
+
+	// Estimated memory used for reading SST tables, excluding memory used in block cache (e.g. filter and index blocks).
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kEstimateTableReadersMem, &val);
+	json.put ("estimate-table-readers-mem", val);
+
+	//  An estimate of the amount of live data in bytes.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kEstimateLiveDataSize, &val);
+	json.put ("estimate-live-data-size", val);
+
+	//  Returns 1 if at least one compaction is pending; otherwise, returns 0.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kCompactionPending, &val);
+	json.put ("compaction-pending", val);
+
+	// Estimated number of total keys in the active and unflushed immutable memtables and storage.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kEstimateNumKeys, &val);
+	json.put ("estimate-num-keys", val);
+
+	// Estimated total number of bytes compaction needs to rewrite to get all levels down
+	// to under target size. Not valid for other compactions than level-based.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kEstimatePendingCompactionBytes, &val);
+	json.put ("estimate-pending-compaction-bytes", val);
+
+	//  Total size (bytes) of all SST files.
+	//  WARNING: may slow down online queries if there are too many files.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kTotalSstFilesSize, &val);
+	json.put ("total-sst-files-size", val);
+
+	// Block cache capacity.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kBlockCacheCapacity, &val);
+	json.put ("block-cache-capacity", val);
+
+	// Memory size for the entries residing in block cache.
+	db->GetAggregatedIntProperty (rocksdb::DB::Properties::kBlockCacheUsage, &val);
+	json.put ("block-cache-usage", val);
+}
+
 // Explicitly instantiate
 template class nano::block_store_partial<rocksdb::Slice, nano::rocksdb_store>;

nano/node/rocksdb/rocksdb.hpp

@@ -40,10 +40,7 @@ class rocksdb_store : public block_store_partial<rocksdb::Slice, rocksdb_store>
 	int put (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a, nano::rocksdb_val const & value_a);
 	int del (nano::write_transaction const & transaction_a, tables table_a, nano::rocksdb_val const & key_a);
 
-	void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) override
-	{
-		// Do nothing
-	}
+	void serialize_memory_stats (boost::property_tree::ptree &) override;
 
 	bool copy_db (boost::filesystem::path const & destination) override;
 	void rebuild_db (nano::write_transaction const & transaction_a) override;

nano/rpc_test/rpc.cpp

@@ -6650,11 +6650,21 @@ TEST (rpc, memory_stats)
 	boost::property_tree::ptree request;
 	request.put ("action", "stats");
 	request.put ("type", "objects");
-	test_response response (request, rpc.config.port, system.io_ctx);
-	ASSERT_TIMELY (5s, response.status != 0);
-	ASSERT_EQ (200, response.status);
+	{
+		test_response response (request, rpc.config.port, system.io_ctx);
+		ASSERT_TIMELY (5s, response.status != 0);
+		ASSERT_EQ (200, response.status);
 
-	ASSERT_EQ (response.json.get_child ("node").get_child ("vote_uniquer").get_child ("votes").get<std::string> ("count"), "1");
+		ASSERT_EQ (response.json.get_child ("node").get_child ("vote_uniquer").get_child ("votes").get<std::string> ("count"), "1");
+	}
+
+	request.put ("type", "database");
+	{
+		test_response response (request, rpc.config.port, system.io_ctx);
+		ASSERT_TIMELY (5s, response.status != 0);
+		ASSERT_EQ (200, response.status);
+		ASSERT_TRUE (!response.json.empty ());
+	}
 }
 
 TEST (rpc, block_confirmed)

nano/secure/blockstore.hpp

@@ -688,7 +688,8 @@ class block_store
 	virtual void rebuild_db (nano::write_transaction const & transaction_a) = 0;
 
 	/** Not applicable to all sub-classes */
-	virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds) = 0;
+	virtual void serialize_mdb_tracker (boost::property_tree::ptree &, std::chrono::milliseconds, std::chrono::milliseconds){};
+	virtual void serialize_memory_stats (boost::property_tree::ptree &) = 0;
 
 	virtual bool init_error () const = 0;