Attach sideband to block (#2596)

* Move epoch from secure to nano_lib

* Attach sideband to block and always (de)serialize along with it

Using the sideband is only valid for blocks that were processed with code `progress`, otherwise they may not be set (important examples: old, fork).

* Make sideband optional to ensure usage correctness

* Interim

* Use new nano::optional_ptr to hold the sideband

* Adjust some tests to ensure two nodes don't simultaneously process the same block object

* Update comment

nano/core_test/epochs.cpp

@@ -1,5 +1,5 @@
+#include <nano/lib/epoch.hpp>
 #include <nano/secure/common.hpp>
-#include <nano/secure/epoch.hpp>
 
 #include <gtest/gtest.h>
 

nano/core_test/node.cpp

@@ -1506,8 +1506,10 @@ TEST (node, fork_open_flip)
 	nano::keypair rep1;
 	nano::keypair rep2;
 	auto send1 (std::make_shared<nano::send_block> (genesis.hash (), key1.pub, nano::genesis_amount - 1, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *system.work.generate (genesis.hash ())));
+	// A copy is necessary to avoid data races during ledger processing, which sets the sideband
+	auto send1_copy (std::make_shared<nano::send_block> (*send1));
 	node1.process_active (send1);
-	node2.process_active (send1);
+	node2.process_active (send1_copy);
 	// We should be keeping this block
 	auto open1 (std::make_shared<nano::open_block> (send1->hash (), rep1.pub, key1.pub, key1.prv, key1.pub, *system.work.generate (key1.pub)));
 	// This block should be evicted
@@ -1741,9 +1743,6 @@ TEST (node, broadcast_elected)
 		nano::keypair rep_big;
 		nano::keypair rep_small;
 		nano::keypair rep_other;
-		//std::cerr << "Big: " << rep_big.pub.to_account () << std::endl;
-		//std::cerr << "Small: " << rep_small.pub.to_account () << std::endl;
-		//std::cerr << "Other: " << rep_other.pub.to_account () << std::endl;
 		{
 			auto transaction0 (node0->store.tx_begin_write ());
 			auto transaction1 (node1->store.tx_begin_write ());
@@ -1784,14 +1783,14 @@ TEST (node, broadcast_elected)
 		system.wallet (2)->insert_adhoc (rep_other.prv);
 		auto fork0 (std::make_shared<nano::send_block> (node2->latest (nano::test_genesis_key.pub), rep_small.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0));
 		node0->work_generate_blocking (*fork0);
+		// A copy is necessary to avoid data races during ledger processing, which sets the sideband
+		auto fork0_copy (std::make_shared<nano::send_block> (*fork0));
 		node0->process_active (fork0);
-		node1->process_active (fork0);
+		node1->process_active (fork0_copy);
 		auto fork1 (std::make_shared<nano::send_block> (node2->latest (nano::test_genesis_key.pub), rep_big.pub, 0, nano::test_genesis_key.prv, nano::test_genesis_key.pub, 0));
 		node0->work_generate_blocking (*fork1);
 		system.wallet (2)->insert_adhoc (rep_small.prv);
 		node2->process_active (fork1);
-		//std::cerr << "fork0: " << fork_hash.to_string () << std::endl;
-		//std::cerr << "fork1: " << fork1.hash ().to_string () << std::endl;
 		system.deadline_set (10s);
 		while (!node0->ledger.block_exists (fork0->hash ()) || !node1->ledger.block_exists (fork0->hash ()))
 		{
@@ -2486,15 +2485,17 @@ TEST (node, block_confirm)
 		nano::keypair key;
 		system.wallet (1)->insert_adhoc (nano::test_genesis_key.prv);
 		auto send1 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, genesis.hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *node1.work_generate_blocking (genesis.hash ())));
+		// A copy is necessary to avoid data races during ledger processing, which sets the sideband
+		auto send1_copy (std::make_shared<nano::state_block> (*send1));
 		node1.block_processor.add (send1, nano::seconds_since_epoch ());
-		node2.block_processor.add (send1, nano::seconds_since_epoch ());
+		node2.block_processor.add (send1_copy, nano::seconds_since_epoch ());
 		system.deadline_set (std::chrono::seconds (5));
-		while (!node1.ledger.block_exists (send1->hash ()) || !node2.ledger.block_exists (send1->hash ()))
+		while (!node1.ledger.block_exists (send1->hash ()) || !node2.ledger.block_exists (send1_copy->hash ()))
 		{
 			ASSERT_NO_ERROR (system.poll ());
 		}
 		ASSERT_TRUE (node1.ledger.block_exists (send1->hash ()));
-		ASSERT_TRUE (node2.ledger.block_exists (send1->hash ()));
+		ASSERT_TRUE (node2.ledger.block_exists (send1_copy->hash ()));
 		auto send2 (std::make_shared<nano::state_block> (nano::test_genesis_key.pub, send1->hash (), nano::test_genesis_key.pub, nano::genesis_amount - nano::Gxrb_ratio * 2, key.pub, nano::test_genesis_key.prv, nano::test_genesis_key.pub, *node1.work_generate_blocking (send1->hash ())));
 		{
 			auto transaction (node1.store.tx_begin_write ());

nano/core_test/versioning.cpp

@@ -11,14 +11,14 @@ TEST (versioning, account_info_v1)
 	auto file (nano::unique_path ());
 	nano::account account (1);
 	nano::open_block open (1, 2, 3, nullptr);
+	open.sideband_set ({});
 	nano::account_info_v1 v1 (open.hash (), open.hash (), 3, 4);
 	{
 		nano::logger_mt logger;
 		nano::mdb_store store (logger, file);
 		ASSERT_FALSE (store.init_error ());
 		auto transaction (store.tx_begin_write ());
-		nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false);
-		store.block_put (transaction, open.hash (), open, sideband);
+		store.block_put (transaction, open.hash (), open);
 		auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v1), &v1), 0));
 		ASSERT_EQ (0, status);
 		store.version_put (transaction, 1);
@@ -47,14 +47,14 @@ TEST (versioning, account_info_v5)
 	auto file (nano::unique_path ());
 	nano::account account (1);
 	nano::open_block open (1, 2, 3, nullptr);
+	open.sideband_set ({});
 	nano::account_info_v5 v5 (open.hash (), open.hash (), open.hash (), 3, 4);
 	{
 		nano::logger_mt logger;
 		nano::mdb_store store (logger, file);
 		ASSERT_FALSE (store.init_error ());
 		auto transaction (store.tx_begin_write ());
-		nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false);
-		store.block_put (transaction, open.hash (), open, sideband);
+		store.block_put (transaction, open.hash (), open);
 		auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (sizeof (v5), &v5), 0));
 		ASSERT_EQ (0, status);
 		store.version_put (transaction, 5);
@@ -83,14 +83,14 @@ TEST (versioning, account_info_v13)
 	auto file (nano::unique_path ());
 	nano::account account (1);
 	nano::open_block open (1, 2, 3, nullptr);
+	open.sideband_set ({});
 	nano::account_info_v13 v13 (open.hash (), open.hash (), open.hash (), 3, 4, 10, nano::epoch::epoch_0);
 	{
 		nano::logger_mt logger;
 		nano::mdb_store store (logger, file);
 		ASSERT_FALSE (store.init_error ());
 		auto transaction (store.tx_begin_write ());
-		nano::block_sideband sideband (nano::block_type::open, 0, 0, 0, 0, 0, nano::epoch::epoch_0, false, false, false);
-		store.block_put (transaction, open.hash (), open, sideband);
+		store.block_put (transaction, open.hash (), open);
 		auto status (mdb_put (store.env.tx (transaction), store.accounts_v0, nano::mdb_val (account), nano::mdb_val (v13), 0));
 		ASSERT_EQ (0, status);
 		store.version_put (transaction, 13);

nano/lib/CMakeLists.txt

@@ -25,6 +25,8 @@ add_library (nano_lib
 	configbase.hpp
 	diagnosticsconfig.hpp
 	diagnosticsconfig.cpp
+	epoch.hpp
+	epoch.cpp
 	errors.hpp
 	errors.cpp
 	ipc.hpp

nano/lib/blocks.cpp

@@ -9,6 +9,8 @@
 #include <boost/endian/conversion.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
+#include <bitset>
+
 /** Compare blocks, first by type, then content. This is an optimization over dynamic_cast, which is very slow on some platforms. */
 namespace
 {
@@ -136,6 +138,17 @@ nano::block_hash nano::block::full_hash () const
 	return result;
 }
 
+nano::block_sideband const & nano::block::sideband () const
+{
+	debug_assert (sideband_m.is_initialized ());
+	return *sideband_m;
+}
+
+void nano::block::sideband_set (nano::block_sideband const & sideband_a)
+{
+	sideband_m = sideband_a;
+}
+
 nano::account const & nano::block::representative () const
 {
 	static nano::account rep{ 0 };
@@ -176,6 +189,11 @@ void nano::send_block::visit (nano::block_visitor & visitor_a) const
 	visitor_a.send_block (*this);
 }
 
+void nano::send_block::visit (nano::mutable_block_visitor & visitor_a)
+{
+	visitor_a.send_block (*this);
+}
+
 void nano::send_block::hash (blake2b_state & hash_a) const
 {
 	hashables.hash (hash_a);
@@ -660,6 +678,11 @@ void nano::open_block::visit (nano::block_visitor & visitor_a) const
 	visitor_a.open_block (*this);
 }
 
+void nano::open_block::visit (nano::mutable_block_visitor & visitor_a)
+{
+	visitor_a.open_block (*this);
+}
+
 nano::block_type nano::open_block::type () const
 {
 	return nano::block_type::open;
@@ -896,6 +919,11 @@ void nano::change_block::visit (nano::block_visitor & visitor_a) const
 	visitor_a.change_block (*this);
 }
 
+void nano::change_block::visit (nano::mutable_block_visitor & visitor_a)
+{
+	visitor_a.change_block (*this);
+}
+
 nano::block_type nano::change_block::type () const
 {
 	return nano::block_type::change;
@@ -1203,6 +1231,11 @@ void nano::state_block::visit (nano::block_visitor & visitor_a) const
 	visitor_a.state_block (*this);
 }
 
+void nano::state_block::visit (nano::mutable_block_visitor & visitor_a)
+{
+	visitor_a.state_block (*this);
+}
+
 nano::block_type nano::state_block::type () const
 {
 	return nano::block_type::state;
@@ -1364,6 +1397,11 @@ void nano::receive_block::visit (nano::block_visitor & visitor_a) const
 	visitor_a.receive_block (*this);
 }
 
+void nano::receive_block::visit (nano::mutable_block_visitor & visitor_a)
+{
+	visitor_a.receive_block (*this);
+}
+
 bool nano::receive_block::operator== (nano::receive_block const & other_a) const
 {
 	auto result (hashables.previous == other_a.hashables.previous && hashables.source == other_a.hashables.source && work == other_a.work && signature == other_a.signature);
@@ -1608,6 +1646,157 @@ void nano::receive_hashables::hash (blake2b_state & hash_a) const
 	blake2b_update (&hash_a, source.bytes.data (), sizeof (source.bytes));
 }
 
+nano::block_details::block_details (nano::epoch const epoch_a, bool const is_send_a, bool const is_receive_a, bool const is_epoch_a) :
+epoch (epoch_a), is_send (is_send_a), is_receive (is_receive_a), is_epoch (is_epoch_a)
+{
+}
+
+constexpr size_t nano::block_details::size ()
+{
+	return 1;
+}
+
+bool nano::block_details::operator== (nano::block_details const & other_a) const
+{
+	return epoch == other_a.epoch && is_send == other_a.is_send && is_receive == other_a.is_receive && is_epoch == other_a.is_epoch;
+}
+
+uint8_t nano::block_details::packed () const
+{
+	std::bitset<8> result (static_cast<uint8_t> (epoch));
+	result.set (7, is_send);
+	result.set (6, is_receive);
+	result.set (5, is_epoch);
+	return static_cast<uint8_t> (result.to_ulong ());
+}
+
+void nano::block_details::unpack (uint8_t details_a)
+{
+	constexpr std::bitset<8> epoch_mask{ 0b00011111 };
+	auto as_bitset = static_cast<std::bitset<8>> (details_a);
+	is_send = as_bitset.test (7);
+	is_receive = as_bitset.test (6);
+	is_epoch = as_bitset.test (5);
+	epoch = static_cast<nano::epoch> ((as_bitset & epoch_mask).to_ulong ());
+}
+
+void nano::block_details::serialize (nano::stream & stream_a) const
+{
+	nano::write (stream_a, packed ());
+}
+
+bool nano::block_details::deserialize (nano::stream & stream_a)
+{
+	bool result (false);
+	try
+	{
+		uint8_t packed{ 0 };
+		nano::read (stream_a, packed);
+		unpack (packed);
+	}
+	catch (std::runtime_error &)
+	{
+		result = true;
+	}
+
+	return result;
+}
+
+nano::block_sideband::block_sideband (nano::account const & account_a, nano::block_hash const & successor_a, nano::amount const & balance_a, uint64_t height_a, uint64_t timestamp_a, nano::epoch epoch_a, bool is_send, bool is_receive, bool is_epoch) :
+successor (successor_a),
+account (account_a),
+balance (balance_a),
+height (height_a),
+timestamp (timestamp_a),
+details (epoch_a, is_send, is_receive, is_epoch)
+{
+}
+
+size_t nano::block_sideband::size (nano::block_type type_a)
+{
+	size_t result (0);
+	result += sizeof (successor);
+	if (type_a != nano::block_type::state && type_a != nano::block_type::open)
+	{
+		result += sizeof (account);
+	}
+	if (type_a != nano::block_type::open)
+	{
+		result += sizeof (height);
+	}
+	if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
+	{
+		result += sizeof (balance);
+	}
+	result += sizeof (timestamp);
+	if (type_a == nano::block_type::state)
+	{
+		static_assert (sizeof (nano::epoch) == nano::block_details::size (), "block_details is larger than the epoch enum");
+		result += nano::block_details::size ();
+	}
+	return result;
+}
+
+void nano::block_sideband::serialize (nano::stream & stream_a, nano::block_type type_a) const
+{
+	nano::write (stream_a, successor.bytes);
+	if (type_a != nano::block_type::state && type_a != nano::block_type::open)
+	{
+		nano::write (stream_a, account.bytes);
+	}
+	if (type_a != nano::block_type::open)
+	{
+		nano::write (stream_a, boost::endian::native_to_big (height));
+	}
+	if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
+	{
+		nano::write (stream_a, balance.bytes);
+	}
+	nano::write (stream_a, boost::endian::native_to_big (timestamp));
+	if (type_a == nano::block_type::state)
+	{
+		details.serialize (stream_a);
+	}
+}
+
+bool nano::block_sideband::deserialize (nano::stream & stream_a, nano::block_type type_a)
+{
+	bool result (false);
+	try
+	{
+		nano::read (stream_a, successor.bytes);
+		if (type_a != nano::block_type::state && type_a != nano::block_type::open)
+		{
+			nano::read (stream_a, account.bytes);
+		}
+		if (type_a != nano::block_type::open)
+		{
+			nano::read (stream_a, height);
+			boost::endian::big_to_native_inplace (height);
+		}
+		else
+		{
+			height = 1;
+		}
+		if (type_a == nano::block_type::receive || type_a == nano::block_type::change || type_a == nano::block_type::open)
+		{
+			nano::read (stream_a, balance.bytes);
+		}
+		nano::read (stream_a, timestamp);
+		boost::endian::big_to_native_inplace (timestamp);
+		if (type_a == nano::block_type::state)
+		{
+			result = details.deserialize (stream_a);
+		}
+	}
+	catch (std::runtime_error &)
+	{
+		result = true;
+	}
+
+	return result;
+}
+
 std::shared_ptr<nano::block> nano::block_uniquer::unique (std::shared_ptr<nano::block> block_a)
 {
 	auto result (block_a);