feat!: change how sha3 difficulty is calculated (#4528)

Description
---
Change how the sha3 hash for the difficulty is calculated. 
Before this pr the difficulty was calculated as the Hash(header.part || header.part || ... || header.part), which is then hashed again. 

The merge mining is calculated as the hash of all the parts bar the nonce and pow data. This hash is then concatenated with the nonce and pow data. This makes it much easier to send over data to hash as only this hash of data is required to mine the header and not the entire header. 

This updates the sha3 mining to also use this same hash rather than the concatenated header part. 

How Has This Been Tested?
---
unit tests

applications/tari_base_node/src/grpc/base_node_grpc_server.rs

@@ -542,7 +542,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
         };
         // construct response
         let block_hash = new_block.hash();
-        let mining_hash = new_block.header.merged_mining_hash().to_vec();
+        let mining_hash = new_block.header.mining_hash().to_vec();
         let block: Option<tari_rpc::Block> = Some(
             new_block
                 .try_into()
@@ -587,7 +587,7 @@ impl tari_rpc::base_node_server::BaseNode for BaseNodeGrpcServer {
         };
         // construct response
         let block_hash = new_block.hash();
-        let mining_hash = new_block.header.merged_mining_hash().to_vec();
+        let mining_hash = new_block.header.mining_hash().to_vec();
 
         let (header, block_body) = new_block.into_header_body();
         let mut header_bytes = Vec::new();

applications/tari_miner/src/difficulty.rs

@@ -20,108 +20,80 @@
 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+use std::convert::TryInto;
+
 use sha3::{Digest, Sha3_256};
-use tari_app_grpc::tari_rpc::BlockHeader;
-use tari_core::large_ints::U256;
-use tari_utilities::ByteArray;
+use tari_app_grpc::tari_rpc::BlockHeader as grpc_header;
+use tari_core::{blocks::BlockHeader, large_ints::U256};
+use tari_utilities::epoch_time::EpochTime;
 
-use crate::errors::{err_empty, MinerError};
+use crate::errors::MinerError;
 
 pub type Difficulty = u64;
 
 #[derive(Clone)]
 pub struct BlockHeaderSha3 {
-    header: BlockHeader,
-    pow_bytes: Vec<u8>,
-    hash_before_timestamp: Sha3_256,
-    pub timestamp: u64,
-    pub nonce: u64,
+    pub header: BlockHeader,
+    hash_merge_mining: Sha3_256,
     pub hashes: u64,
 }
 
 impl BlockHeaderSha3 {
     #[allow(clippy::cast_possible_truncation)]
     #[allow(clippy::cast_sign_loss)]
-    pub fn new(header: BlockHeader) -> Result<Self, MinerError> {
-        use std::convert::TryFrom;
-
-        use tari_core::proof_of_work::ProofOfWork; // this is only dep left on tari_code
+    pub fn new(header: grpc_header) -> Result<Self, MinerError> {
+        let header: BlockHeader = header.try_into().map_err(MinerError::BlockHeader)?;
 
-        // Not stressing about efficiency here as it will change soon
-        let pow = ProofOfWork::try_from(header.pow.clone().ok_or_else(|| err_empty("header.pow"))?)
-            .map_err(MinerError::BlockHeader)?;
-        let timestamp = header.timestamp.as_ref().ok_or_else(|| err_empty("header.timestamp"))?;
-        let hash_before_timestamp = Sha3_256::new()
-            .chain((header.version as u16).to_le_bytes())
-            .chain(header.height.to_le_bytes())
-            .chain(header.prev_hash.as_bytes());
+        let hash_merge_mining = Sha3_256::new().chain(header.mining_hash());
 
         Ok(Self {
-            pow_bytes: pow.to_bytes(),
-            hash_before_timestamp,
-            timestamp: timestamp.seconds as u64,
-            nonce: header.nonce,
+            hash_merge_mining,
             header,
             hashes: 0,
         })
     }
 
     #[inline]
     fn get_hash_before_nonce(&self) -> Sha3_256 {
-        self.hash_before_timestamp
-            .clone()
-            .chain(self.timestamp.to_le_bytes())
-            .chain(self.header.input_mr.as_bytes())
-            .chain(self.header.output_mr.as_bytes())
-            .chain(self.header.output_mmr_size.to_le_bytes())
-            .chain(self.header.witness_mr.as_bytes())
-            .chain(self.header.kernel_mr.as_bytes())
-            .chain(self.header.kernel_mmr_size.to_le_bytes())
-            .chain(self.header.total_kernel_offset.as_bytes())
-            .chain(self.header.total_script_offset.as_bytes())
+        self.hash_merge_mining.clone()
     }
 
     /// This function will update the timestamp of the header, but only if the new timestamp is greater than the current
     /// one.
     pub fn set_forward_timestamp(&mut self, timestamp: u64) {
         // if the timestamp has been advanced by the base_node due to the median time we should not reverse it but we
         // should only change the timestamp if we move it forward.
-        if timestamp > self.timestamp {
-            self.timestamp = timestamp;
+        if timestamp > self.header.timestamp.as_u64() {
+            self.header.timestamp = EpochTime::from(timestamp);
+            self.hash_merge_mining = Sha3_256::new().chain(self.header.mining_hash());
         }
     }
 
     pub fn random_nonce(&mut self) {
         use rand::{rngs::OsRng, RngCore};
-        self.nonce = OsRng.next_u64();
+        self.header.nonce = OsRng.next_u64();
     }
 
     #[inline]
     pub fn inc_nonce(&mut self) {
-        self.nonce = self.nonce.wrapping_add(1);
+        self.header.nonce = self.header.nonce.wrapping_add(1);
     }
 
     #[inline]
     pub fn difficulty(&mut self) -> Difficulty {
         self.hashes = self.hashes.saturating_add(1);
         let hash = self
             .get_hash_before_nonce()
-            .chain(self.nonce.to_le_bytes())
-            .chain(&self.pow_bytes)
+            .chain(self.header.nonce.to_le_bytes())
+            .chain(self.header.pow.to_bytes())
             .finalize();
         let hash = Sha3_256::digest(&hash);
         big_endian_difficulty(&hash)
     }
 
     #[allow(clippy::cast_possible_wrap)]
-    pub fn create_header(&self) -> BlockHeader {
-        let mut header = self.header.clone();
-        header.timestamp = Some(prost_types::Timestamp {
-            seconds: self.timestamp as i64,
-            nanos: 0,
-        });
-        header.nonce = self.nonce;
-        header
+    pub fn create_header(&self) -> grpc_header {
+        self.header.clone().into()
     }
 
     #[inline]
@@ -140,13 +112,13 @@ fn big_endian_difficulty(hash: &[u8]) -> Difficulty {
 #[cfg(test)]
 pub mod test {
     use chrono::{DateTime, NaiveDate, Utc};
-    use tari_core::{blocks::BlockHeader as CoreBlockHeader, proof_of_work::sha3_difficulty as core_sha3_difficulty};
+    use tari_core::proof_of_work::sha3_difficulty as core_sha3_difficulty;
 
     use super::*;
 
     #[allow(clippy::cast_sign_loss)]
-    pub fn get_header() -> (BlockHeader, CoreBlockHeader) {
-        let mut header = CoreBlockHeader::new(0);
+    pub fn get_header() -> (grpc_header, BlockHeader) {
+        let mut header = BlockHeader::new(0);
         header.timestamp =
             (DateTime::<Utc>::from_utc(NaiveDate::from_ymd(2000, 1, 1).and_hms(1, 1, 1), Utc).timestamp() as u64)
                 .into();
@@ -165,7 +137,7 @@ pub mod test {
                 hasher.difficulty(),
                 core_sha3_difficulty(&core_header).as_u64(),
                 "with nonces = {}:{}",
-                hasher.nonce,
+                hasher.header.nonce,
                 core_header.nonce
             );
             core_header.nonce += 1;

applications/tari_miner/src/miner.rs

@@ -188,15 +188,15 @@ pub fn mining_task(
         if difficulty >= target_difficulty {
             debug!(
                 target: LOG_TARGET,
-                "Miner {} found nonce {} with matching difficulty {}", miner, hasher.nonce, difficulty
+                "Miner {} found nonce {} with matching difficulty {}", miner, hasher.header.nonce, difficulty
             );
             if let Err(err) = sender.try_send(MiningReport {
                 miner,
                 difficulty,
                 hashes: hasher.hashes,
                 elapsed: start.elapsed(),
                 height: hasher.height(),
-                last_nonce: hasher.nonce,
+                last_nonce: hasher.header.nonce,
                 header: Some(hasher.create_header()),
                 target_difficulty,
             }) {
@@ -212,14 +212,14 @@ pub fn mining_task(
                 return;
             }
         }
-        if hasher.nonce % REPORTING_FREQUENCY == 0 {
+        if hasher.header.nonce % REPORTING_FREQUENCY == 0 {
             let res = sender.try_send(MiningReport {
                 miner,
                 difficulty,
                 hashes: hasher.hashes,
                 elapsed: start.elapsed(),
                 header: None,
-                last_nonce: hasher.nonce,
+                last_nonce: hasher.header.nonce,
                 height: hasher.height(),
                 target_difficulty,
             });

base_layer/core/src/blocks/block_header.rs

@@ -215,9 +215,9 @@ impl BlockHeader {
         }
     }
 
-    /// Provides a hash of the header, used for the merge mining.
+    /// Provides a mining hash of the header, used for the mining.
     /// This differs from the normal hash by not hashing the nonce and kernel pow.
-    pub fn merged_mining_hash(&self) -> FixedHash {
+    pub fn mining_hash(&self) -> FixedHash {
         DomainSeparatedConsensusHasher::<BlocksHashDomain>::new("block_header")
             .chain(&self.version)
             .chain(&self.height)
@@ -277,7 +277,7 @@ impl From<NewBlockHeaderTemplate> for BlockHeader {
 impl Hashable for BlockHeader {
     fn hash(&self) -> Vec<u8> {
         DomainSeparatedConsensusHasher::<BlocksHashDomain>::new("block_header")
-            .chain(&self.merged_mining_hash().as_slice())
+            .chain(&self.mining_hash().as_slice())
             .chain(&self.pow)
             .chain(&self.nonce)
             .finalize()

base_layer/core/src/proof_of_work/monero_rx/helpers.rs

@@ -66,7 +66,7 @@ fn get_random_x_difficulty(input: &[u8], vm: &RandomXVMInstance) -> Result<(Diff
 /// If these assertions pass, a valid `MoneroPowData` instance is returned
 fn verify_header(header: &BlockHeader) -> Result<MoneroPowData, MergeMineError> {
     let monero_data = MoneroPowData::from_header(header)?;
-    let expected_merge_mining_hash = header.merged_mining_hash();
+    let expected_merge_mining_hash = header.mining_hash();
 
     // Check that the Tari MM hash is found in the monero coinbase transaction
     let is_found = monero_data.coinbase_tx.prefix.extra.0.iter().any(|item| match item {
@@ -308,7 +308,7 @@ mod test {
             nonce: 0,
             pow: ProofOfWork::default(),
         };
-        let hash = block_header.merged_mining_hash();
+        let hash = block_header.mining_hash();
         append_merge_mining_tag(&mut block, hash).unwrap();
         let hashes = create_ordered_transaction_hashes_from_block(&block);
         assert_eq!(hashes.len(), block.tx_hashes.len() + 1);
@@ -364,7 +364,7 @@ mod test {
             nonce: 0,
             pow: ProofOfWork::default(),
         };
-        let hash = block_header.merged_mining_hash();
+        let hash = block_header.mining_hash();
         append_merge_mining_tag(&mut block, hash).unwrap();
         let count = 1 + (u16::try_from(block.tx_hashes.len()).unwrap());
         let mut hashes = Vec::with_capacity(count as usize);
@@ -522,7 +522,7 @@ mod test {
             nonce: 0,
             pow: ProofOfWork::default(),
         };
-        let hash = block_header.merged_mining_hash();
+        let hash = block_header.mining_hash();
         append_merge_mining_tag(&mut block, hash).unwrap();
         let count = 1 + (u16::try_from(block.tx_hashes.len()).unwrap());
         let mut hashes = Vec::with_capacity(count as usize);
@@ -615,7 +615,7 @@ mod test {
             nonce: 0,
             pow: ProofOfWork::default(),
         };
-        let hash = block_header.merged_mining_hash();
+        let hash = block_header.mining_hash();
         append_merge_mining_tag(&mut block, hash).unwrap();
         let count = 1 + (u16::try_from(block.tx_hashes.len()).unwrap());
         let mut hashes = Vec::with_capacity(count as usize);

base_layer/core/src/proof_of_work/sha3_pow.rs

@@ -21,7 +21,6 @@
 // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 use sha3::{Digest, Sha3_256};
-use tari_utilities::ByteArray;
 
 use crate::{
     blocks::BlockHeader,
@@ -39,18 +38,7 @@ pub fn sha3_difficulty(header: &BlockHeader) -> Difficulty {
 
 pub fn sha3_hash(header: &BlockHeader) -> Vec<u8> {
     Sha3_256::new()
-        .chain(header.version.to_le_bytes())
-        .chain(header.height.to_le_bytes())
-        .chain(header.prev_hash.as_bytes())
-        .chain(header.timestamp.as_u64().to_le_bytes())
-        .chain(header.input_mr.as_bytes())
-        .chain(header.output_mr.as_bytes())
-        .chain(header.output_mmr_size.to_le_bytes())
-        .chain(header.witness_mr.as_bytes())
-        .chain(header.kernel_mr.as_bytes())
-        .chain(header.kernel_mmr_size.to_le_bytes())
-        .chain(header.total_kernel_offset.as_bytes())
-        .chain(header.total_script_offset.as_bytes())
+        .chain(header.mining_hash())
         .chain(header.nonce.to_le_bytes())
         .chain(header.pow.to_bytes())
         .finalize()
@@ -101,6 +89,6 @@ pub mod test {
     fn validate_max_target() {
         let mut header = get_header();
         header.nonce = 1;
-        assert_eq!(sha3_difficulty(&header), Difficulty::from(1));
+        assert_eq!(sha3_difficulty(&header), Difficulty::from(3));
     }
 }

base_layer/core/tests/block_validation.rs

@@ -179,7 +179,7 @@ fn add_monero_data(tblock: &mut Block, seed_key: &str) {
 .to_string();
     let bytes = hex::decode(blocktemplate_blob).unwrap();
     let mut mblock = monero_rx::deserialize::<MoneroBlock>(&bytes[..]).unwrap();
-    let hash = tblock.header.merged_mining_hash();
+    let hash = tblock.header.mining_hash();
     monero_rx::append_merge_mining_tag(&mut mblock, hash).unwrap();
     let hashes = monero_rx::create_ordered_transaction_hashes_from_block(&mblock);
     let merkle_root = monero_rx::tree_hash(&hashes).unwrap();

base_layer/tari_mining_helper_ffi/src/lib.rs

@@ -371,8 +371,8 @@ mod tests {
 
     #[test]
     fn detect_change_in_consensus_encoding() {
-        const NONCE: u64 = 17497411907229199779;
-        const DIFFICULTY: Difficulty = Difficulty::from_u64(1984);
+        const NONCE: u64 = 5714152803266684615;
+        const DIFFICULTY: Difficulty = Difficulty::from_u64(1565);
         unsafe {
             let mut error = -1;
             let error_ptr = &mut error as *mut c_int;

base_layer/wallet_ffi/wallet.h

@@ -2921,6 +2921,8 @@ TariPublicKey *wallet_get_public_key(struct TariWallet *wallet,
  * `script_private_key` - Tari script private key, k_S, is used to create the script signature
  * `covenant` - The covenant that will be executed when spending this output
  * `message` - The message that the transaction will have
+ * `encrypted_value` - Encrypted value.
+ * `minimum_value_promise` - The minimum value of the commitment that is proven by the range proof
  * `error_out` - Pointer to an int which will be modified to an error code should one occur, may not be null. Functions
  * as an out parameter.
  *