Merge branch 'development' into sw_fix_naming_witnessmr

applications/tari_base_node/src/command_handler.rs

@@ -304,7 +304,7 @@ impl CommandHandler {
                 Ok(mut data) => match data.pop() {
                     Some(v) => println!("{}", v.block()),
                     _ => println!(
-                        "Pruned node: utxo found, but lock not found for utxo commitment {}",
+                        "Pruned node: utxo found, but block not found for utxo commitment {}",
                         commitment.to_hex()
                     ),
                 },

applications/tari_console_wallet/src/grpc/wallet_grpc_server.rs

@@ -208,7 +208,6 @@ impl wallet_server::Wallet for WalletGrpcServer {
         let queries = message.transaction_ids.into_iter().map(|tx_id| {
             let mut transaction_service = self.get_transaction_service();
             async move {
-                error!(target: LOG_TARGET, "TX_ID: {}", tx_id);
                 transaction_service
                     .get_any_transaction(tx_id)
                     .await
@@ -404,7 +403,6 @@ fn convert_wallet_transaction_into_transaction_info(
     wallet_pk: &CommsPublicKey,
 ) -> TransactionInfo {
     use models::WalletTransaction::*;
-    error!(target: LOG_TARGET, "FOUND WALLET: {:?}", tx);
     match tx {
         PendingInbound(tx) => TransactionInfo {
             tx_id: tx.tx_id,

base_layer/core/src/base_node/comms_interface/inbound_handlers.rs

@@ -224,7 +224,7 @@ where T: BlockchainBackend + 'static
             },
             NodeCommsRequest::FetchMatchingUtxos(utxo_hashes) => {
                 let mut res = Vec::with_capacity(utxo_hashes.len());
-                for (output, spent) in (self.blockchain_db.fetch_utxos(utxo_hashes, None).await?)
+                for (output, spent) in (self.blockchain_db.fetch_utxos(utxo_hashes).await?)
                     .into_iter()
                     .flatten()
                 {
@@ -237,7 +237,7 @@ where T: BlockchainBackend + 'static
             NodeCommsRequest::FetchMatchingTxos(hashes) => {
                 let res = self
                     .blockchain_db
-                    .fetch_utxos(hashes, None)
+                    .fetch_utxos(hashes)
                     .await?
                     .into_iter()
                     .filter_map(|opt| opt.map(|(output, _)| output))
@@ -362,7 +362,13 @@ where T: BlockchainBackend + 'static
                     .await?
                     .into_iter()
                     .map(|tx| Arc::try_unwrap(tx).unwrap_or_else(|tx| (*tx).clone()))
-                    .collect();
+                    .collect::<Vec<_>>();
+
+                debug!(
+                    target: LOG_TARGET,
+                    "Adding {} transaction(s) to new block template",
+                    transactions.len()
+                );
 
                 let prev_hash = header.prev_hash.clone();
                 let height = header.height;

base_layer/core/src/base_node/rpc/service.rs

@@ -289,7 +289,7 @@ impl<B: BlockchainBackend + 'static> BaseNodeWalletService for BaseNodeWalletRpc
         let db = self.db();
         let mut res = Vec::with_capacity(message.output_hashes.len());
         for (output, spent) in (db
-            .fetch_utxos(message.output_hashes, None)
+            .fetch_utxos(message.output_hashes)
             .await
             .map_err(RpcStatus::log_internal_error(LOG_TARGET))?)
         .into_iter()

base_layer/core/src/base_node/state_machine_service/states/horizon_state_sync/horizon_state_synchronization.rs

@@ -47,7 +47,10 @@ use crate::{
 use croaring::Bitmap;
 use futures::StreamExt;
 use log::*;
-use std::convert::{TryFrom, TryInto};
+use std::{
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};
 use tari_comms::PeerConnection;
 use tari_crypto::{
     commitment::HomomorphicCommitment,
@@ -206,9 +209,7 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
                 let kernel_pruned_set = block_data.dissolve().0;
                 let mut kernel_mmr = MerkleMountainRange::<HashDigest, _>::new(kernel_pruned_set);
 
-                // let mut kernel_sum = HomomorphicCommitment::default();
                 for kernel in kernels.drain(..) {
-                    // kernel_sum = &kernel.excess + &kernel_sum;
                     kernel_mmr.push(kernel.hash())?;
                 }
 
@@ -531,6 +532,12 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
 
         let mut prev_mmr = 0;
         let mut prev_kernel_mmr = 0;
+        let bitmap = Arc::new(
+            self.db()
+                .fetch_complete_deleted_bitmap_at(header.hash().clone())
+                .await?
+                .into_bitmap(),
+        );
         for h in 0..=header.height() {
             let curr_header = self.db().fetch_chain_header(h).await?;
 
@@ -544,11 +551,7 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
             );
             let (utxos, _) = self
                 .db()
-                .fetch_utxos_by_mmr_position(
-                    prev_mmr,
-                    curr_header.header().output_mmr_size - 1,
-                    header.hash().clone(),
-                )
+                .fetch_utxos_by_mmr_position(prev_mmr, curr_header.header().output_mmr_size - 1, bitmap.clone())
                 .await?;
             trace!(
                 target: LOG_TARGET,

base_layer/core/src/base_node/sync/rpc/service.rs

@@ -39,7 +39,7 @@ use crate::{
 };
 use futures::{channel::mpsc, stream, SinkExt};
 use log::*;
-use std::{cmp, time::Instant};
+use std::{cmp, sync::Arc, time::Instant};
 use tari_comms::protocol::rpc::{Request, Response, RpcStatus, Streaming};
 use tari_crypto::tari_utilities::hex::Hex;
 use tokio::task;
@@ -448,7 +448,6 @@ impl<B: BlockchainBackend + 'static> BaseNodeSyncService for BaseNodeSyncRpcServ
                             self.request.end_header_hash.to_hex()
                         ))
                     })?;
-                let end_header_hash = end_header.hash();
 
                 if self.request.start > end_header.output_mmr_size - 1 {
                     return Err(RpcStatus::bad_request(format!(
@@ -467,7 +466,15 @@ impl<B: BlockchainBackend + 'static> BaseNodeSyncService for BaseNodeSyncRpcServ
                 if prev_header.height > end_header.height {
                     return Err(RpcStatus::bad_request("start index is greater than end index"));
                 }
+                // we need to construct a temp bitmap for the height the client requested
+                let bitmap = self
+                    .db
+                    .fetch_complete_deleted_bitmap_at(end_header.hash())
+                    .await
+                    .map_err(|_| RpcStatus::not_found("Could not get tip deleted bitmap"))?
+                    .into_bitmap();
 
+                let bitmap = Arc::new(bitmap);
                 loop {
                     let timer = Instant::now();
                     if prev_header.height == end_header.height {
@@ -513,7 +520,7 @@ impl<B: BlockchainBackend + 'static> BaseNodeSyncService for BaseNodeSyncRpcServ
                     );
                     let (utxos, deleted_diff) = self
                         .db
-                        .fetch_utxos_by_mmr_position(start, end, end_header_hash.clone())
+                        .fetch_utxos_by_mmr_position(start, end, bitmap.clone())
                         .await
                         .map_err(RpcStatus::log_internal_error(LOG_TARGET))?;
                     trace!(

base_layer/core/src/chain_storage/accumulated_data.rs

@@ -205,6 +205,32 @@ impl<'de> Visitor<'de> for DeletedBitmapVisitor {
     }
 }
 
+/// Wrapper struct to get a completed bitmap with the height it was created at
+#[derive(Debug, Clone)]
+pub struct CompleteDeletedBitmap {
+    deleted: Bitmap,
+    height: u64,
+    hash: HashOutput,
+}
+
+impl CompleteDeletedBitmap {
+    pub fn new(deleted: Bitmap, height: u64, hash: HashOutput) -> CompleteDeletedBitmap {
+        CompleteDeletedBitmap { deleted, height, hash }
+    }
+
+    pub fn into_bitmap(self) -> Bitmap {
+        self.deleted
+    }
+
+    pub fn bitmap(&self) -> &Bitmap {
+        &self.deleted
+    }
+
+    pub fn dissolve(self) -> (Bitmap, u64, HashOutput) {
+        (self.deleted, self.height, self.hash)
+    }
+}
+
 pub struct BlockHeaderAccumulatedDataBuilder<'a> {
     previous_accum: &'a BlockHeaderAccumulatedData,
     hash: Option<HashOutput>,

base_layer/core/src/chain_storage/async_db.rs

@@ -31,6 +31,7 @@ use crate::{
         ChainBlock,
         ChainHeader,
         ChainStorageError,
+        CompleteDeletedBitmap,
         DbTransaction,
         HistoricalBlock,
         HorizonData,
@@ -140,9 +141,9 @@ impl<B: BlockchainBackend + 'static> AsyncBlockchainDb<B> {
     //---------------------------------- TXO --------------------------------------------//
     make_async_fn!(fetch_utxo(hash: HashOutput) -> Option<TransactionOutput>, "fetch_utxo");
 
-    make_async_fn!(fetch_utxos(hashes: Vec<HashOutput>, is_spent_as_of: Option<HashOutput>) -> Vec<Option<(TransactionOutput, bool)>>, "fetch_utxos");
+    make_async_fn!(fetch_utxos(hashes: Vec<HashOutput>) -> Vec<Option<(TransactionOutput, bool)>>, "fetch_utxos");
 
-    make_async_fn!(fetch_utxos_by_mmr_position(start: u64, end: u64, end_header_hash: HashOutput) -> (Vec<PrunedOutput>, Bitmap), "fetch_utxos_by_mmr_position");
+    make_async_fn!(fetch_utxos_by_mmr_position(start: u64, end: u64, deleted: Arc<Bitmap>) -> (Vec<PrunedOutput>, Bitmap), "fetch_utxos_by_mmr_position");
 
     //---------------------------------- Kernel --------------------------------------------//
     make_async_fn!(fetch_kernel_by_excess_sig(excess_sig: Signature) -> Option<(TransactionKernel, HashOutput)>, "fetch_kernel_by_excess_sig");
@@ -215,13 +216,16 @@ impl<B: BlockchainBackend + 'static> AsyncBlockchainDb<B> {
     make_async_fn!(fetch_block_accumulated_data_by_height(height: u64) -> BlockAccumulatedData, "fetch_block_accumulated_data_by_height");
 
     //---------------------------------- Misc. --------------------------------------------//
+
     make_async_fn!(fetch_block_timestamps(start_hash: HashOutput) -> RollingVec<EpochTime>, "fetch_block_timestamps");
 
     make_async_fn!(fetch_target_difficulty_for_next_block(pow_algo: PowAlgorithm, current_block_hash: HashOutput) -> TargetDifficultyWindow, "fetch_target_difficulty");
 
     make_async_fn!(fetch_target_difficulties_for_next_block(current_block_hash: HashOutput) -> TargetDifficulties, "fetch_target_difficulties_for_next_block");
 
     make_async_fn!(fetch_block_hashes_from_header_tip(n: usize, offset: usize) -> Vec<HashOutput>, "fetch_block_hashes_from_header_tip");
+
+    make_async_fn!(fetch_complete_deleted_bitmap_at(hash: HashOutput) -> CompleteDeletedBitmap, "fetch_deleted_bitmap");
 }
 
 impl<B: BlockchainBackend + 'static> From<BlockchainDatabase<B>> for AsyncBlockchainDb<B> {