WIP- Storing unconfirmed blocks in memory.

# Conflicts:
#	nano/secure/ledger_view_unconfirmed.cpp

nano/core_test/bootstrap.cpp

@@ -56,7 +56,9 @@ TEST (bulk_pull, end_not_owned)
 	nano::test::system system (1);
 	nano::keypair key2;
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
-	ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, 100));
+	auto block = system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, 100);
+	ASSERT_NE (nullptr, block);
+	system.nodes[0]->ledger.confirm (system.nodes[0]->store.tx_begin_write (), block->hash ());
 	nano::block_hash latest (system.nodes[0]->latest (nano::dev::genesis_key.pub));
 	nano::block_builder builder;
 	auto open = builder
@@ -127,6 +129,7 @@ TEST (bulk_pull, ascending_one_hash)
 				  .build ();
 	node.work_generate_blocking (*block1);
 	ASSERT_EQ (nano::block_status::progress, node.process (block1));
+	node.ledger.confirm (node.store.tx_begin_write (), block1->hash ());
 	auto socket = std::make_shared<nano::transport::socket> (node, nano::transport::socket::endpoint_type_t::server);
 	auto connection = std::make_shared<nano::transport::tcp_server> (socket, system.nodes[0]);
 	auto req = std::make_unique<nano::bulk_pull> (nano::dev::network_params.network);
@@ -159,6 +162,7 @@ TEST (bulk_pull, ascending_two_account)
 				  .build ();
 	node.work_generate_blocking (*block1);
 	ASSERT_EQ (nano::block_status::progress, node.process (block1));
+	node.ledger.confirm (node.store.tx_begin_write (), block1->hash ());
 	auto socket = std::make_shared<nano::transport::socket> (node, nano::transport::socket::endpoint_type_t::server);
 	auto connection = std::make_shared<nano::transport::tcp_server> (socket, system.nodes[0]);
 	auto req = std::make_unique<nano::bulk_pull> (nano::dev::network_params.network);
@@ -262,7 +266,7 @@ TEST (bulk_pull, count_limit)
 					.work (*system.work.generate (send1->hash ()))
 					.build ();
 	ASSERT_EQ (nano::block_status::progress, node0->process (receive1));
-
+	node0->ledger.confirm (node0->store.tx_begin_write (), receive1->hash ());
 	auto connection (std::make_shared<nano::transport::tcp_server> (std::make_shared<nano::transport::socket> (*node0, nano::transport::socket::endpoint_type_t::server), node0));
 	auto req = std::make_unique<nano::bulk_pull> (nano::dev::network_params.network);
 	req->start = receive1->hash ();

nano/core_test/confirming_set.cpp

@@ -276,6 +276,6 @@ TEST (confirmation_callback, election_winner_details_clearing_node_process_confi
 	node->active.add_election_winner_details (send->hash (), nullptr);
 	nano::election_status election;
 	election.winner = send;
-	node->process_confirmed (election, 1000000);
-	ASSERT_EQ (0, node->active.election_winner_details_size ());
+	node->process_confirmed (election);
+	ASSERT_TIMELY_EQ (5s, 0, node->active.election_winner_details_size ());
 }

nano/core_test/ledger_confirm.cpp

@@ -44,8 +44,8 @@ TEST (ledger_confirm, single)
 	ASSERT_FALSE (confirmed.exists (transaction, send1->hash ()));
 	node->ledger.confirm (transaction, send1->hash ());
 	ASSERT_TRUE (confirmed.exists (transaction, send1->hash ()));
-	ASSERT_EQ (2, node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().height);
-	ASSERT_EQ (send1->hash (), node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().frontier);
+	ASSERT_EQ (2, node->ledger.confirmed.height (transaction, nano::dev::genesis_key.pub));
+	ASSERT_EQ (send1->hash (), confirmed.head (transaction, nano::dev::genesis_key.pub));
 
 	// Rollbacks should fail as these blocks have been cemented
 	ASSERT_TRUE (node->ledger.rollback (transaction, latest1));
@@ -200,17 +200,17 @@ TEST (ledger_confirm, multiple_accounts)
 
 	ASSERT_TRUE (node->ledger.confirmed.exists (transaction, receive3->hash ()));
 	ASSERT_EQ (4, node->ledger.any.get (transaction, nano::dev::genesis_key.pub).value ().block_count);
-	ASSERT_EQ (4, node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().height);
-	ASSERT_EQ (send3->hash (), node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().frontier);
+	ASSERT_EQ (4, node->ledger.confirmed.height (transaction, nano::dev::genesis_key.pub));
+	ASSERT_EQ (send3->hash (), node->ledger.confirmed.head (transaction, nano::dev::genesis_key.pub));
 	ASSERT_EQ (3, node->ledger.any.get (transaction, key1.pub).value ().block_count);
-	ASSERT_EQ (2, node->store.confirmation_height.get (transaction, key1.pub).value ().height);
-	ASSERT_EQ (send4->hash (), node->store.confirmation_height.get (transaction, key1.pub).value ().frontier);
+	ASSERT_EQ (2, node->ledger.confirmed.height (transaction, key1.pub));
+	ASSERT_EQ (send4->hash (), node->ledger.confirmed.head (transaction, key1.pub));
 	ASSERT_EQ (4, node->ledger.any.get (transaction, key2.pub).value ().block_count);
-	ASSERT_EQ (3, node->store.confirmation_height.get (transaction, key2.pub).value ().height);
-	ASSERT_EQ (send6->hash (), node->store.confirmation_height.get (transaction, key2.pub).value ().frontier);
+	ASSERT_EQ (3, node->ledger.confirmed.height (transaction, key2.pub));
+	ASSERT_EQ (send6->hash (), node->ledger.confirmed.head (transaction, key2.pub));
 	ASSERT_EQ (2, node->ledger.any.get (transaction, key3.pub).value ().block_count);
-	ASSERT_EQ (2, node->store.confirmation_height.get (transaction, key3.pub).value ().height);
-	ASSERT_EQ (receive3->hash (), node->store.confirmation_height.get (transaction, key3.pub).value ().frontier);
+	ASSERT_EQ (2, node->ledger.confirmed.height (transaction, key3.pub));
+	ASSERT_EQ (receive3->hash (), node->ledger.confirmed.head (transaction, key3.pub));
 
 	// The accounts for key1 and key2 have 1 more block in the chain than is confirmed.
 	// So this can be rolled back, but the one before that cannot. Check that this is the case
@@ -350,12 +350,12 @@ TEST (ledger_confirm, send_receive_between_2_accounts)
 
 	ASSERT_TRUE (node->ledger.confirmed.exists (transaction, receive4->hash ()));
 	ASSERT_EQ (7, node->ledger.any.get (transaction, nano::dev::genesis_key.pub).value ().block_count);
-	ASSERT_EQ (6, node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().height);
-	ASSERT_EQ (send5->hash (), node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().frontier);
+	ASSERT_EQ (6, node->ledger.confirmed.height (transaction, nano::dev::genesis_key.pub));
+	ASSERT_EQ (send5->hash (), node->ledger.confirmed.head (transaction, nano::dev::genesis_key.pub));
 
 	ASSERT_EQ (5, node->ledger.any.get (transaction, key1.pub).value ().block_count);
-	ASSERT_EQ (5, node->store.confirmation_height.get (transaction, key1.pub).value ().height);
-	ASSERT_EQ (receive4->hash (), node->store.confirmation_height.get (transaction, key1.pub).value ().frontier);
+	ASSERT_EQ (5, node->ledger.confirmed.height (transaction, key1.pub));
+	ASSERT_EQ (receive4->hash (), node->ledger.confirmed.head (transaction, key1.pub));
 }
 
 TEST (ledger_confirm, send_receive_self)
@@ -441,8 +441,8 @@ TEST (ledger_confirm, send_receive_self)
 
 	ASSERT_TRUE (node->ledger.confirmed.exists (transaction, receive3->hash ()));
 	ASSERT_EQ (8, node->ledger.any.get (transaction, nano::dev::genesis_key.pub).value ().block_count);
-	ASSERT_EQ (7, node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().height);
-	ASSERT_EQ (receive3->hash (), node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().frontier);
+	ASSERT_EQ (7, node->ledger.confirmed.height (transaction, nano::dev::genesis_key.pub));
+	ASSERT_EQ (receive3->hash (), node->ledger.confirmed.head (transaction, nano::dev::genesis_key.pub));
 	ASSERT_EQ (7, node->ledger.cemented_count ());
 }
 
@@ -668,16 +668,16 @@ TEST (ledger_confirm, all_block_types)
 	ASSERT_TRUE (node->ledger.confirmed.exists (transaction, state_send2->hash ()));
 	nano::confirmation_height_info confirmation_height_info;
 	ASSERT_LE (4, node->ledger.any.get (transaction, nano::dev::genesis_key.pub).value ().block_count);
-	ASSERT_EQ (3, node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().height);
-	ASSERT_EQ (send1->hash (), node->store.confirmation_height.get (transaction, nano::dev::genesis_key.pub).value ().frontier);
+	ASSERT_EQ (3, node->ledger.confirmed.height (transaction, nano::dev::genesis_key.pub));
+	ASSERT_EQ (send1->hash (), node->ledger.confirmed.head (transaction, nano::dev::genesis_key.pub));
 
 	ASSERT_LE (7, node->ledger.any.get (transaction, key1.pub).value ().block_count);
-	ASSERT_EQ (6, node->store.confirmation_height.get (transaction, key1.pub).value ().height);
-	ASSERT_EQ (state_send1->hash (), node->store.confirmation_height.get (transaction, key1.pub).value ().frontier);
+	ASSERT_EQ (6, node->ledger.confirmed.height (transaction, key1.pub));
+	ASSERT_EQ (state_send1->hash (), node->ledger.confirmed.head (transaction, key1.pub));
 
 	ASSERT_EQ (8, node->ledger.any.get (transaction, key2.pub).value ().block_count);
-	ASSERT_EQ (7, node->store.confirmation_height.get (transaction, key2.pub).value ().height);
-	ASSERT_EQ (state_send2->hash (), node->store.confirmation_height.get (transaction, key2.pub).value ().frontier);
+	ASSERT_EQ (7, node->ledger.confirmed.height (transaction, key2.pub));
+	ASSERT_EQ (state_send2->hash (), node->ledger.confirmed.head (transaction, key2.pub));
 }
 
 // This test ensures a block that's cemented cannot be rolled back by the node
@@ -755,7 +755,7 @@ TEST (ledger_confirm, observers)
 	ASSERT_EQ (2, node1->ledger.cemented_count ());
 }
 
-TEST (ledger_confirm, election_winner_details_clearing_node_process_confirmed)
+TEST (ledger_confirm, DISABLED_election_winner_details_clearing_node_process_confirmed)
 {
 	// Make sure election_winner_details is also cleared if the block never enters the confirmation height processor from node::process_confirmed
 	nano::test::system system (1);
@@ -774,7 +774,7 @@ TEST (ledger_confirm, election_winner_details_clearing_node_process_confirmed)
 	node->active.add_election_winner_details (send->hash (), nullptr);
 	nano::election_status election;
 	election.winner = send;
-	node->process_confirmed (election, 1000000);
+	node->process_confirmed (election);
 	ASSERT_EQ (0, node->active.election_winner_details_size ());
 }
 
@@ -846,6 +846,7 @@ TEST (ledger_confirm, pruned_source)
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send1));
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, open1));
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send2));
+	ledger.confirm (transaction, send2->hash ());
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, send3));
 	ASSERT_EQ (nano::block_status::progress, ledger.process (transaction, open2));
 	ledger.confirm (transaction, send2->hash ());

nano/core_test/node.cpp

@@ -147,7 +147,7 @@ TEST (node, send_single)
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	system.wallet (1)->insert_adhoc (key2.prv);
 	ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ()));
-	ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max () - system.nodes[0]->config.receive_minimum.number (), system.nodes[0]->balance (nano::dev::genesis_key.pub));
+	ASSERT_TIMELY_EQ (5s, std::numeric_limits<nano::uint128_t>::max () - system.nodes[0]->config.receive_minimum.number (), system.nodes[0]->balance (nano::dev::genesis_key.pub));
 	ASSERT_TRUE (system.nodes[0]->balance (key2.pub).is_zero ());
 	ASSERT_TIMELY (10s, !system.nodes[0]->balance (key2.pub).is_zero ());
 }
@@ -159,7 +159,7 @@ TEST (node, send_single_observing_peer)
 	system.wallet (0)->insert_adhoc (nano::dev::genesis_key.prv);
 	system.wallet (1)->insert_adhoc (key2.prv);
 	ASSERT_NE (nullptr, system.wallet (0)->send_action (nano::dev::genesis_key.pub, key2.pub, system.nodes[0]->config.receive_minimum.number ()));
-	ASSERT_EQ (std::numeric_limits<nano::uint128_t>::max () - system.nodes[0]->config.receive_minimum.number (), system.nodes[0]->balance (nano::dev::genesis_key.pub));
+	ASSERT_TIMELY_EQ (5s, std::numeric_limits<nano::uint128_t>::max () - system.nodes[0]->config.receive_minimum.number (), system.nodes[0]->balance (nano::dev::genesis_key.pub));
 	ASSERT_TRUE (system.nodes[0]->balance (key2.pub).is_zero ());
 	ASSERT_TIMELY (10s, std::all_of (system.nodes.begin (), system.nodes.end (), [&] (std::shared_ptr<nano::node> const & node_a) { return !node_a->balance (key2.pub).is_zero (); }));
 }
@@ -1574,6 +1574,7 @@ TEST (node, unconfirmed_send)
 	ASSERT_TIMELY_EQ (5s, node2.get_confirmation_height (node2.store.tx_begin_read (), key2.pub), 1);
 	ASSERT_EQ (node2.balance (key2.pub), 2 * nano::Mxrb_ratio);
 	auto recv1 = node2.ledger.find_receive_block_by_send_hash (node2.store.tx_begin_read (), key2.pub, send1->hash ());
+	ASSERT_NE (nullptr, recv1);
 
 	// create send2 to send from node2 to node1 and save it to node2's ledger without triggering an election (node1 does not hear about it)
 	auto send2 = nano::state_block_builder{}

nano/node/active_transactions.cpp

@@ -97,7 +97,7 @@ void nano::active_transactions::block_cemented_callback (std::shared_ptr<nano::b
 		status = election->get_status ();
 		votes = election->votes_with_weight ();
 	}
-	if (confirming_set.exists (block->hash ()))
+	if (node.confirming_set.exists (block->hash ()))
 	{
 		status.type = nano::election_status_type::active_confirmed_quorum;
 	}

nano/node/active_transactions.hpp

@@ -171,7 +171,7 @@ class active_transactions final
 	bool empty () const;
 	std::size_t size () const;
 	bool publish (std::shared_ptr<nano::block> const &);
-	void block_cemented_callback (std::shared_ptr<nano::block> const &);
+	void block_cemented_callback (std::shared_ptr<nano::block> const & block);
 	void block_already_cemented_callback (nano::block_hash const &);
 
 	/**

nano/node/backlog_population.cpp

@@ -2,13 +2,15 @@
 #include <nano/node/backlog_population.hpp>
 #include <nano/node/nodeconfig.hpp>
 #include <nano/node/scheduler/priority.hpp>
+#include <nano/secure/ledger.hpp>
+#include <nano/secure/ledger_set_unconfirmed.hpp>
 #include <nano/store/account.hpp>
 #include <nano/store/component.hpp>
 #include <nano/store/confirmation_height.hpp>
 
-nano::backlog_population::backlog_population (const config & config_a, nano::store::component & store_a, nano::stats & stats_a) :
+nano::backlog_population::backlog_population (const config & config_a, nano::ledger & ledger, nano::stats & stats_a) :
 	config_m{ config_a },
-	store{ store_a },
+	ledger{ ledger },
 	stats{ stats_a }
 {
 }
@@ -90,22 +92,22 @@ void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex>
 		lock.unlock ();
 
 		{
-			auto transaction = store.tx_begin_read ();
+			auto transaction = ledger.store.tx_begin_read ();
 
 			auto count = 0u;
-			auto i = store.account.begin (transaction, next);
-			auto const end = store.account.end ();
+			auto i = ledger.unconfirmed.account_upper_bound (transaction, next);
+			auto const end = ledger.unconfirmed.account_end ();
 			for (; i != end && count < chunk_size; ++i, ++count, ++total)
 			{
 				transaction.refresh_if_needed ();
 
 				stats.inc (nano::stat::type::backlog, nano::stat::detail::total);
 
-				auto const & account = i->first;
-				activate (transaction, account);
-				next = account.number () + 1;
+				next = i->first;
+				stats.inc (nano::stat::type::backlog, nano::stat::detail::activated);
+				activate_callback.notify (transaction, next);
 			}
-			done = store.account.begin (transaction, next) == end;
+			done = i == end;
 		}
 
 		lock.lock ();
@@ -114,26 +116,3 @@ void nano::backlog_population::populate_backlog (nano::unique_lock<nano::mutex>
 		condition.wait_for (lock, std::chrono::milliseconds{ 1000 / config_m.frequency });
 	}
 }
-
-void nano::backlog_population::activate (store::transaction const & transaction, nano::account const & account)
-{
-	debug_assert (!activate_callback.empty ());
-
-	auto const maybe_account_info = store.account.get (transaction, account);
-	if (!maybe_account_info)
-	{
-		return;
-	}
-	auto const account_info = *maybe_account_info;
-
-	auto const maybe_conf_info = store.confirmation_height.get (transaction, account);
-	auto const conf_info = maybe_conf_info.value_or (nano::confirmation_height_info{});
-
-	// If conf info is empty then it means then it means nothing is confirmed yet
-	if (conf_info.height < account_info.block_count)
-	{
-		stats.inc (nano::stat::type::backlog, nano::stat::detail::activated);
-
-		activate_callback.notify (transaction, account);
-	}
-}

nano/node/backlog_population.hpp

@@ -10,13 +10,13 @@
 
 namespace nano::store
 {
-class component;
 class transaction;
 }
 namespace nano
 {
 class account_info;
 class election_scheduler;
+class ledger;
 class stats;
 
 class backlog_population final
@@ -34,7 +34,7 @@ class backlog_population final
 		unsigned frequency;
 	};
 
-	backlog_population (const config &, store::component &, nano::stats &);
+	backlog_population (const config &, nano::ledger & ledger, nano::stats &);
 	~backlog_population ();
 
 	void start ();
@@ -54,7 +54,7 @@ class backlog_population final
 	callback_t activate_callback;
 
 private: // Dependencies
-	nano::store::component & store;
+	nano::ledger & ledger;
 	nano::stats & stats;
 
 	config config_m;
@@ -64,7 +64,6 @@ class backlog_population final
 	bool predicate () const;
 
 	void populate_backlog (nano::unique_lock<nano::mutex> & lock);
-	void activate (store::transaction const &, nano::account const &);
 
 	/** This is a manual trigger, the ongoing backlog population does not use this.
 	 *  It can be triggered even when backlog population (frontiers confirmation) is disabled. */

nano/node/bootstrap/bootstrap_frontier.cpp

@@ -5,6 +5,7 @@
 #include <nano/node/transport/tcp.hpp>
 #include <nano/secure/ledger.hpp>
 #include <nano/secure/ledger_set_any.hpp>
+#include <nano/secure/ledger_set_confirmed.hpp>
 
 #include <boost/format.hpp>
 
@@ -392,15 +393,9 @@ void nano::frontier_req_server::next ()
 		}
 		else
 		{
-			for (auto i (node->store.confirmation_height.begin (transaction, current.number () + 1)), n (node->store.confirmation_height.end ()); i != n && accounts.size () != max_size; ++i)
+			for (auto i (node->ledger.confirmed.account_upper_bound (transaction, current.number ())), n (node->ledger.confirmed.account_end ()); i != n && accounts.size () != max_size; ++i)
 			{
-				nano::confirmation_height_info const & info (i->second);
-				nano::block_hash const & confirmed_frontier (info.frontier);
-				if (!confirmed_frontier.is_zero ())
-				{
-					nano::account const & account (i->first);
-					accounts.emplace_back (account, confirmed_frontier);
-				}
+				accounts.emplace_back (i->first, i->second.head);
 			}
 		}
 

nano/node/bootstrap/bootstrap_server.cpp

@@ -5,6 +5,7 @@
 #include <nano/node/transport/transport.hpp>
 #include <nano/secure/ledger.hpp>
 #include <nano/secure/ledger_set_any.hpp>
+#include <nano/secure/ledger_set_confirmed.hpp>
 #include <nano/store/account.hpp>
 #include <nano/store/block.hpp>
 #include <nano/store/component.hpp>
@@ -303,11 +304,11 @@ nano::asc_pull_ack nano::bootstrap_server::process (const store::transaction & t
 		response_payload.account_head = account_info->head;
 		response_payload.account_block_count = account_info->block_count;
 
-		auto conf_info = store.confirmation_height.get (transaction, target);
+		auto conf_info = ledger.confirmed.get (transaction, target);
 		if (conf_info)
 		{
-			response_payload.account_conf_frontier = conf_info->frontier;
-			response_payload.account_conf_height = conf_info->height;
+			response_payload.account_conf_frontier = conf_info.value ().head;
+			response_payload.account_conf_height = conf_info.value ().block_count;
 		}
 	}
 	// If account is missing the response payload will contain all 0 fields, except for the target