Fix nano::test::blocks_confirm (#4117)

* Fix iace in nano::test::blocks_confirm, use start_election

Refactor nano::test::blocks_confirm to use nano::test::start_election
and fix the race conition that arises when process_active is followed
by blocks_confirm.

* Rename nano::test::blocks_confirm to nano::test::start_elections

* Move nano::test::start_elections to testutil and provide 2 ways to call it

We have many similar functions in testutil.cpp so it makes sense to move
it there. Also, provide a way to call it with hashes only for maximum
reusability.

nano/core_test/active_transactions.cpp

@@ -556,7 +556,7 @@ TEST (active_transactions, vote_replays)
 	ASSERT_NE (nullptr, open1);
 	node.process_active (send1);
 	node.process_active (open1);
-	nano::test::blocks_confirm (node, { send1, open1 });
+	nano::test::start_elections (system, node, { send1, open1 });
 	ASSERT_EQ (2, node.active.size ());
 	// First vote is not a replay and confirms the election, second vote should be a replay since the election has confirmed but not yet removed
 	auto vote_send1 (std::make_shared<nano::vote> (nano::dev::genesis_key.pub, nano::dev::genesis_key.prv, nano::vote::timestamp_max, nano::vote::duration_max, std::vector<nano::block_hash>{ send1->hash () }));
@@ -585,7 +585,7 @@ TEST (active_transactions, vote_replays)
 				 .build_shared ();
 	ASSERT_NE (nullptr, send2);
 	node.process_active (send2);
-	nano::test::blocks_confirm (node, { send2 });
+	nano::test::start_elections (system, node, { send2 });
 	ASSERT_EQ (1, node.active.size ());
 	auto vote1_send2 (std::make_shared<nano::vote> (nano::dev::genesis_key.pub, nano::dev::genesis_key.prv, nano::vote::timestamp_max, nano::vote::duration_max, std::vector<nano::block_hash>{ send2->hash () }));
 	auto vote2_send2 (std::make_shared<nano::vote> (key.pub, key.prv, 0, 0, std::vector<nano::block_hash>{ send2->hash () }));
@@ -1278,7 +1278,7 @@ TEST (active_transactions, list_active)
 
 	ASSERT_EQ (nano::process_result::progress, node.process (*open).code);
 
-	nano::test::blocks_confirm (node, { send, send2, open });
+	nano::test::start_elections (system, node, { send, send2, open });
 	ASSERT_EQ (3, node.active.size ());
 	ASSERT_EQ (1, node.active.list_active (1).size ());
 	ASSERT_EQ (2, node.active.list_active (2).size ());

nano/core_test/bootstrap.cpp

@@ -939,7 +939,7 @@ TEST (bootstrap_processor, lazy_hash_pruning)
 	node1->process_active (change2);
 	ASSERT_TIMELY (5s, node1->block (change2->hash ()) != nullptr);
 	// Confirm last block to prune previous
-	nano::test::blocks_confirm (*node1, { send1, receive1, change1, change2 }, true);
+	nano::test::start_elections (system, *node1, { send1, receive1, change1, change2 }, true);
 	ASSERT_TIMELY (5s, node1->block_confirmed (send1->hash ()) && node1->block_confirmed (receive1->hash ()) && node1->block_confirmed (change1->hash ()) && node1->block_confirmed (change2->hash ()) && node1->active.empty ());
 	ASSERT_EQ (5, node1->ledger.cache.block_count);
 	ASSERT_EQ (5, node1->ledger.cache.cemented_count);
@@ -1321,7 +1321,7 @@ TEST (bootstrap_processor, lazy_pruning_missing_block)
 	node1->process_active (state_open);
 	ASSERT_TIMELY (5s, node1->block (state_open->hash ()) != nullptr);
 	// Confirm last block to prune previous
-	nano::test::blocks_confirm (*node1, { send1, send2, open, state_open }, true);
+	nano::test::start_elections (system, *node1, { send1, send2, open, state_open }, true);
 	ASSERT_TIMELY (5s, node1->block_confirmed (send1->hash ()) && node1->block_confirmed (send2->hash ()) && node1->block_confirmed (open->hash ()) && node1->block_confirmed (state_open->hash ()) && node1->active.empty ());
 	ASSERT_EQ (5, node1->ledger.cache.block_count);
 	ASSERT_EQ (5, node1->ledger.cache.cemented_count);
@@ -1878,7 +1878,7 @@ TEST (frontier_req, confirmed_frontier)
 	ASSERT_EQ (receive2->hash (), request5->frontier);
 
 	// Confirm account before genesis (confirmed only)
-	nano::test::blocks_confirm (*node1, { send1, receive1 }, true);
+	nano::test::start_elections (system, *node1, { send1, receive1 }, true);
 	ASSERT_TIMELY (5s, node1->block_confirmed (send1->hash ()) && node1->block_confirmed (receive1->hash ()));
 	auto connection6 (std::make_shared<nano::transport::tcp_server> (std::make_shared<nano::socket> (*node1, nano::socket::endpoint_type_t::server), node1));
 	auto req6 = std::make_unique<nano::frontier_req> (nano::dev::network_params.network);
@@ -1893,7 +1893,7 @@ TEST (frontier_req, confirmed_frontier)
 	ASSERT_EQ (receive1->hash (), request6->frontier);
 
 	// Confirm account after genesis (confirmed only)
-	nano::test::blocks_confirm (*node1, { send2, receive2 }, true);
+	nano::test::start_elections (system, *node1, { send2, receive2 }, true);
 	ASSERT_TIMELY (5s, node1->block_confirmed (send2->hash ()) && node1->block_confirmed (receive2->hash ()));
 	auto connection7 (std::make_shared<nano::transport::tcp_server> (std::make_shared<nano::socket> (*node1, nano::socket::endpoint_type_t::server), node1));
 	auto req7 = std::make_unique<nano::frontier_req> (nano::dev::network_params.network);

nano/core_test/ledger.cpp

@@ -1101,7 +1101,7 @@ TEST (votes, DISABLED_add_old_different_account)
 	node1.work_generate_blocking (*send2);
 	ASSERT_EQ (nano::process_result::progress, node1.process (*send1).code);
 	ASSERT_EQ (nano::process_result::progress, node1.process (*send2).code);
-	nano::test::blocks_confirm (node1, { send1, send2 });
+	nano::test::start_elections (system, node1, { send1, send2 });
 	auto election1 = node1.active.election (send1->qualified_root ());
 	ASSERT_NE (nullptr, election1);
 	auto election2 = node1.active.election (send2->qualified_root ());
@@ -4090,7 +4090,7 @@ TEST (ledger, block_hash_account_conflict)
 	ASSERT_EQ (nano::process_result::progress, node1.process (*receive1).code);
 	ASSERT_EQ (nano::process_result::progress, node1.process (*send2).code);
 	ASSERT_EQ (nano::process_result::progress, node1.process (*open_epoch1).code);
-	nano::test::blocks_confirm (node1, { send1, receive1, send2, open_epoch1 });
+	nano::test::start_elections (system, node1, { send1, receive1, send2, open_epoch1 });
 	auto election1 = node1.active.election (send1->qualified_root ());
 	ASSERT_NE (nullptr, election1);
 	auto election2 = node1.active.election (receive1->qualified_root ());

nano/core_test/node.cpp

@@ -3157,7 +3157,7 @@ TEST (node, confirm_back)
 	node.process_active (open);
 	node.process_active (send2);
 	ASSERT_TIMELY (5s, node.block (send2->hash ()) != nullptr);
-	nano::test::blocks_confirm (node, { send1, open, send2 });
+	nano::test::start_elections (system, node, { send1, open, send2 });
 	ASSERT_EQ (3, node.active.size ());
 	std::vector<nano::block_hash> vote_blocks;
 	vote_blocks.push_back (send2->hash ());
@@ -3486,9 +3486,9 @@ TEST (node, aggressive_flooding)
 		ASSERT_EQ (node1.latest (nano::dev::genesis_key.pub), node_wallet.first->latest (nano::dev::genesis_key.pub));
 		ASSERT_EQ (genesis_blocks.back ()->hash (), node_wallet.first->latest (nano::dev::genesis_key.pub));
 		// Confirm blocks for rep crawler & receiving
-		nano::test::blocks_confirm (*node_wallet.first, { genesis_blocks.back () }, true);
+		nano::test::start_elections (system, *node_wallet.first, { genesis_blocks.back () }, true);
 	}
-	nano::test::blocks_confirm (node1, { genesis_blocks.back () }, true);
+	nano::test::start_elections (system, node1, { genesis_blocks.back () }, true);
 
 	// Wait until all genesis blocks are received
 	auto all_received = [&nodes_wallets] () {
@@ -3697,7 +3697,7 @@ TEST (node, rollback_gap_source)
 	// Node has fork & doesn't have source for correct block open (send2)
 	ASSERT_EQ (nullptr, node.block (send2->hash ()));
 	// Start election for fork
-	nano::test::blocks_confirm (node, { fork });
+	nano::test::start_elections (system, node, { fork });
 	{
 		auto election = node.active.election (fork->qualified_root ());
 		ASSERT_NE (nullptr, election);
@@ -3725,7 +3725,7 @@ TEST (node, rollback_gap_source)
 	ASSERT_NE (nullptr, node.block (fork->hash ()));
 	// With send2 block in ledger election can start again to remove fork block
 	ASSERT_EQ (nano::process_result::progress, node.process (*send2).code);
-	nano::test::blocks_confirm (node, { fork });
+	nano::test::start_elections (system, node, { fork });
 	{
 		auto election = node.active.election (fork->qualified_root ());
 		ASSERT_NE (nullptr, election);

nano/rpc_test/rpc.cpp

@@ -7525,7 +7525,7 @@ TEST (rpc, confirmation_active)
 				 .build_shared ();
 	node1->process_active (send1);
 	node1->process_active (send2);
-	nano::test::blocks_confirm (*node1, { send1, send2 });
+	nano::test::start_elections (system, *node1, { send1, send2 });
 	ASSERT_EQ (2, node1->active.size ());
 	auto election (node1->active.election (send1->qualified_root ()));
 	ASSERT_NE (nullptr, election);

nano/test_common/system.cpp

@@ -263,26 +263,6 @@ std::unique_ptr<nano::state_block> nano::test::upgrade_epoch (nano::work_pool &
 	return !error ? std::move (epoch) : nullptr;
 }
 
-void nano::test::blocks_confirm (nano::node & node_a, std::vector<std::shared_ptr<nano::block>> const & blocks_a, bool const forced_a)
-{
-	// Finish processing all blocks
-	node_a.block_processor.flush ();
-	for (auto const & block : blocks_a)
-	{
-		auto disk_block (node_a.block (block->hash ()));
-		// A sideband is required to start an election
-		debug_assert (disk_block != nullptr);
-		debug_assert (disk_block->has_sideband ());
-		node_a.block_confirm (disk_block);
-		if (forced_a)
-		{
-			auto election = node_a.active.election (disk_block->qualified_root ());
-			debug_assert (election != nullptr);
-			election->force_confirm ();
-		}
-	}
-}
-
 std::unique_ptr<nano::state_block> nano::test::system::upgrade_genesis_epoch (nano::node & node_a, nano::epoch const epoch_a)
 {
 	return upgrade_epoch (work, node_a.ledger, epoch_a);

nano/test_common/system.hpp

@@ -75,7 +75,6 @@ namespace test
 	};
 
 	std::unique_ptr<nano::state_block> upgrade_epoch (nano::work_pool &, nano::ledger &, nano::epoch);
-	void blocks_confirm (nano::node &, std::vector<std::shared_ptr<nano::block>> const &, bool const = false);
 	uint16_t get_available_port ();
 	void cleanup_dev_directories_on_exit ();
 }

nano/test_common/testutil.cpp

@@ -250,4 +250,22 @@ std::shared_ptr<nano::election> nano::test::start_election (nano::test::system &
 
 	election->transition_active ();
 	return election;
+}
+
+void nano::test::start_elections (nano::test::system & system_a, nano::node & node_a, std::vector<nano::block_hash> const & hashes_a, bool const forced_a)
+{
+	for (auto const & hash_l : hashes_a)
+	{
+		auto election = nano::test::start_election (system_a, node_a, hash_l);
+		release_assert (election);
+		if (forced_a)
+		{
+			election->force_confirm ();
+		}
+	}
+}
+
+void nano::test::start_elections (nano::test::system & system_a, nano::node & node_a, std::vector<std::shared_ptr<nano::block>> const & blocks_a, bool const forced_a)
+{
+	nano::test::start_elections (system_a, node_a, blocks_to_hashes (blocks_a), forced_a);
 }

nano/test_common/testutil.hpp

@@ -415,5 +415,17 @@ namespace test
 	 * Returns nullptr if the election did not start within the timeframe.
 	 */
 	std::shared_ptr<nano::election> start_election (nano::test::system & system_a, nano::node & node_a, const nano::block_hash & hash_a);
+	/*
+	 * Call start_election for every block identified in the hash vector.
+	 * Optionally, force confirm the election if forced_a is set.
+	 * NOTE: Each election is given 5 seconds to complete, if it does not complete in 5 seconds, it will assert.
+	 */
+	void start_elections (nano::test::system &, nano::node &, std::vector<nano::block_hash> const &, bool const forced_a = false);
+	/*
+	 * Call start_election for every block in the vector.
+	 * Optionally, force confirm the election if forced_a is set.
+	 * NOTE: Each election is given 5 seconds to complete, if it does not complete in 5 seconds, it will assert.
+	 */
+	void start_elections (nano::test::system &, nano::node &, std::vector<std::shared_ptr<nano::block>> const &, bool const forced_a = false);
 }
 }