Add block cache trait

zcash_client_backend/CHANGELOG.md

@@ -25,6 +25,7 @@ and this library adheres to Rust's notion of
   - `BlockMetadata::orchard_tree_size`
   - `WalletSummary::next_orchard_subtree_index`
   - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}`
+  - `chain::BlockCache` trait
 - `zcash_client_backend::fees`:
   - `orchard`
   - `ChangeValue::orchard`
@@ -38,6 +39,8 @@ and this library adheres to Rust's notion of
   - `Nullifiers::{orchard, extend_orchard, retain_orchard}`
   - `TaggedOrchardBatch`
   - `TaggedOrchardBatchRunner`
+  - `testing` module
+  - `testing::{'fake_compact_block`, `random_compact_tx`} (moved from `tests` module).
 - `zcash_client_backend::wallet`:
   - `Note::Orchard`
   - `WalletOrchardSpend`
@@ -81,6 +84,8 @@ and this library adheres to Rust's notion of
     constraint on its `<AccountId>` parameter has been strengthened to `Copy`.
 - `zcash_client_backend::fees`:
   - Arguments to `ChangeStrategy::compute_balance` have changed.
+- `zcash_client_backend::scanning`:
+  - `testing::fake_compact_block` is now public.
   - `ChangeError::DustInputs` now has an `orchard` field behind the `orchard`
     feature flag.
 - `zcash_client_backend::proto`:

zcash_client_backend/Cargo.toml

@@ -78,12 +78,16 @@ group.workspace = true
 orchard = { workspace = true, optional = true }
 sapling.workspace = true
 
+# - Sync engine
+tokio = { version = "1.21.0", optional = true, features = ["fs", "macros"] }
+
 # - Note commitment trees
 incrementalmerkletree.workspace = true
 shardtree.workspace = true
 
 # - Test dependencies
 proptest = { workspace = true, optional = true }
+jubjub = { workspace = true, optional = true }
 
 # - ZIP 321
 nom = "7"
@@ -116,6 +120,7 @@ shardtree = { workspace = true, features = ["test-dependencies"] }
 zcash_proofs.workspace = true
 zcash_address = { workspace = true, features = ["test-dependencies"] }
 zcash_keys = { workspace = true, features = ["test-dependencies"] }
+tokio = { version = "1.21.0", features = ["rt-multi-thread"] }
 
 time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67
 
@@ -133,9 +138,13 @@ transparent-inputs = [
 ## Enables receiving and spending Orchard funds.
 orchard = ["dep:orchard", "zcash_keys/orchard"]
 
+## Exposes a wallet synchronization function that implements the necessary state machine.
+sync = ["dep:tokio"]
+
 ## Exposes APIs that are useful for testing, such as `proptest` strategies.
 test-dependencies = [
     "dep:proptest",
+    "dep:jubjub",
     "orchard?/test-dependencies",
     "zcash_keys/test-dependencies",
     "zcash_primitives/test-dependencies",

zcash_client_backend/src/data_api/chain.rs

@@ -168,6 +168,9 @@ use error::Error;
 
 use super::WalletRead;
 
+#[cfg(feature = "sync")]
+use {crate::data_api::scanning::ScanRange, tokio::task::JoinHandle};
+
 /// A struct containing metadata about a subtree root of the note commitment tree.
 ///
 /// This stores the block height at which the leaf that completed the subtree was
@@ -220,6 +223,198 @@ pub trait BlockSource {
         F: FnMut(CompactBlock) -> Result<(), error::Error<WalletErrT, Self::Error>>;
 }
 
+/// `BlockCache` is a trait that extends `BlockSource` and defines methods for managing
+/// a cache of compact blocks.
+///
+/// # Examples
+///
+/// ```
+///    use std::sync::{Arc, Mutex};
+///    use tokio::task::JoinHandle;
+///    use zcash_client_backend::data_api::{
+///        chain::{error, BlockCache, BlockSource},
+///        scanning::{ScanPriority, ScanRange},
+///    };
+///    use zcash_client_backend::proto::compact_formats::CompactBlock;
+///    use zcash_primitives::consensus::BlockHeight;
+///
+///    struct ExampleBlockCache {
+///        cached_blocks: Arc<Mutex<Vec<CompactBlock>>>,
+///    }
+///
+/// #    impl BlockSource for ExampleBlockCache {
+/// #        type Error = ();
+/// #
+/// #        fn with_blocks<F, WalletErrT>(
+/// #            &self,
+/// #            _from_height: Option<BlockHeight>,
+/// #            _limit: Option<usize>,
+/// #            _with_block: F,
+/// #        ) -> Result<(), error::Error<WalletErrT, Self::Error>>
+/// #        where
+/// #            F: FnMut(CompactBlock) -> Result<(), error::Error<WalletErrT, Self::Error>>,
+/// #        {
+/// #            Ok(())
+/// #        }
+/// #    }
+/// #
+///    impl BlockCache for ExampleBlockCache {
+///        fn read(&self, range: &ScanRange) -> Result<Vec<CompactBlock>, Self::Error> {
+///            Ok(self
+///                .cached_blocks
+///                .lock()
+///                .unwrap()
+///                .iter()
+///                .filter(|block| {
+///                    let block_height = BlockHeight::from_u32(block.height as u32);
+///                    range.block_range().contains(&block_height)
+///                })
+///                .cloned()
+///                .collect())
+///        }
+///
+///        fn get_tip_height(&self, range: Option<&ScanRange>) -> Result<Option<BlockHeight>, Self::Error> {
+///            let cached_blocks = self.cached_blocks.lock().unwrap();
+///            let blocks: Vec<&CompactBlock> = match range {
+///                Some(range) => cached_blocks
+///                    .iter()
+///                    .filter(|&block| {
+///                        let block_height = BlockHeight::from_u32(block.height as u32);
+///                        range.block_range().contains(&block_height)
+///                    })
+///                    .collect(),
+///                None => cached_blocks.iter().collect(),
+///            };
+///            let highest_block = blocks.iter().max_by_key(|&&block| block.height);
+///            Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32)))
+///        }
+///
+///        fn insert(&self, mut compact_blocks: Vec<CompactBlock>) -> Result<(), Self::Error> {
+///            self.cached_blocks
+///                .lock()
+///                .unwrap()
+///                .append(&mut compact_blocks);
+///            Ok(())
+///        }
+///
+///        fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> {
+///            self.cached_blocks
+///                .lock()
+///                .unwrap()
+///                .retain(|block| block.height <= block_height.into());
+///            Ok(())
+///        }
+///
+///        fn delete(&self, range: &ScanRange) -> JoinHandle<Result<(), Self::Error>> {
+///            let cached_blocks = Arc::clone(&self.cached_blocks);
+///            let range = range.block_range().clone();
+///            tokio::spawn(async move {
+///                cached_blocks
+///                    .lock()
+///                    .unwrap()
+///                    .retain(|block| !range.contains(&BlockHeight::from_u32(block.height as u32)));
+///                Ok(())
+///            })
+///        }
+///    }
+///
+///    // Example usage
+///    let mut block_cache = ExampleBlockCache {
+///        cached_blocks: Arc::new(Mutex::new(Vec::new())),
+///    };
+///    let range = ScanRange::from_parts(
+///        BlockHeight::from_u32(1)..BlockHeight::from_u32(3),
+///        ScanPriority::Historic,
+///    );
+/// #    let extsk = sapling::zip32::ExtendedSpendingKey::master(&[]);
+/// #    let dfvk = extsk.to_diversifiable_full_viewing_key();
+/// #    let compact_block1 = zcash_client_backend::scanning::testing::fake_compact_block(
+/// #        1u32.into(),
+/// #        zcash_primitives::block::BlockHash([0; 32]),
+/// #        sapling::Nullifier([0; 32]),
+/// #        &dfvk,
+/// #        zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5),
+/// #        false,
+/// #        None,
+/// #    );
+/// #    let compact_block2 = zcash_client_backend::scanning::testing::fake_compact_block(
+/// #        2u32.into(),
+/// #        zcash_primitives::block::BlockHash([0; 32]),
+/// #        sapling::Nullifier([0; 32]),
+/// #        &dfvk,
+/// #        zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5),
+/// #        false,
+/// #        None,
+/// #    );
+///    let compact_blocks = vec![compact_block1, compact_block2];
+///
+///    // Insert blocks into the block cache
+///    block_cache.insert(compact_blocks.clone()).unwrap();
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2);
+///
+///    // Find highest block in the block cache
+///    let get_tip_height = block_cache.get_tip_height(None).unwrap();
+///    assert_eq!(get_tip_height, Some(BlockHeight::from_u32(2)));
+///
+///    // Read from the block cache
+///    let blocks_from_cache = block_cache.read(&range).unwrap();
+///    assert_eq!(blocks_from_cache, compact_blocks);
+///
+///    // Truncate the block cache
+///    block_cache.truncate(BlockHeight::from_u32(1)).unwrap();
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1);
+///    assert_eq!(
+///        block_cache.get_tip_height(None).unwrap(),
+///        Some(BlockHeight::from_u32(1))
+///    );
+///
+///    // Delete blocks from the block cache
+///    let rt = tokio::runtime::Runtime::new().unwrap();
+///    rt.block_on(async {
+///        block_cache.delete(&range).await.unwrap();
+///    });
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0);
+///    assert_eq!(block_cache.get_tip_height(None).unwrap(), None);
+/// ```
+#[cfg(feature = "sync")]
+pub trait BlockCache: BlockSource + Send + Sync {
+    /// Retrieves contiguous compact blocks specified by the given `range` from the block cache.
+    ///
+    /// Returns `Ok(Vec<CompactBlock>)` on success, otherwise returns an error.
+    ///
+    /// Short reads are allowed, meaning that returning fewer blocks than requested should not
+    /// return an error as long as all blocks are sequentially continuous in height.
+    ///
+    /// # Errors
+    ///
+    /// This method should return an error if there are gaps in the requested range of blocks,
+    /// indicating there are blocks missing from the cache.
+    fn read(&self, range: &ScanRange) -> Result<Vec<CompactBlock>, Self::Error>;
+
+    /// Finds the height of the highest block known to the block cache within a specified range.
+    /// If `range` is `None`, returns the tip of the entire cache.
+    ///
+    /// Returns `Ok(Some(BlockHeight))` on success, otherwise returns an error.
+    /// If no blocks are found in the cache, returns Ok(`None`).
+    fn get_tip_height(&self, range: Option<&ScanRange>)
+        -> Result<Option<BlockHeight>, Self::Error>;
+
+    /// Inserts a vec of compact blocks into the block cache.
+    ///
+    /// Returns `Ok(())` on success, otherwise returns an error.
+    fn insert(&self, compact_blocks: Vec<CompactBlock>) -> Result<(), Self::Error>;
+
+    /// Removes all cached blocks above a specified block height.
+    ///
+    /// Returns `Ok(())` on success, otherwise returns an error.
+    fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error>;
+
+    /// Deletes a range of compact blocks from the block cache.
+    ///
+    /// Returns a `JoinHandle` from a `tokio::spawn` task.
+    fn delete(&self, range: &ScanRange) -> JoinHandle<Result<(), Self::Error>>;
+}
+
 /// Metadata about modifications to the wallet state made in the course of scanning a set of
 /// blocks.
 #[derive(Clone, Debug)]

zcash_client_backend/src/scanning.rs

@@ -1125,16 +1125,12 @@ fn find_received<
     (shielded_outputs, note_commitments)
 }
 
-#[cfg(test)]
-mod tests {
-
-    use std::convert::Infallible;
-
+#[cfg(any(test, feature = "test-dependencies"))]
+pub mod testing {
     use group::{
         ff::{Field, PrimeField},
         GroupEncoding,
     };
-    use incrementalmerkletree::{Position, Retention};
     use rand_core::{OsRng, RngCore};
     use sapling::{
         constants::SPENDING_KEY_GENERATOR,
@@ -1144,26 +1140,18 @@ mod tests {
         zip32::DiversifiableFullViewingKey,
         Nullifier,
     };
-    use zcash_keys::keys::UnifiedSpendingKey;
     use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE};
     use zcash_primitives::{
         block::BlockHash,
         consensus::{BlockHeight, Network},
         memo::MemoBytes,
         transaction::components::{amount::NonNegativeAmount, sapling::zip212_enforcement},
-        zip32::AccountId,
     };
 
-    use crate::{
-        data_api::BlockMetadata,
-        proto::compact_formats::{
-            self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
-        },
-        scanning::{BatchRunners, ScanningKeys},
+    use crate::proto::compact_formats::{
+        self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
     };
 
-    use super::{scan_block, scan_block_with_runners, Nullifiers};
-
     fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
         let fake_nf = {
             let mut nf = vec![0; 32];
@@ -1202,7 +1190,7 @@ mod tests {
     ///
     /// Set `initial_tree_sizes` to `None` to simulate a `CompactBlock` retrieved
     /// from a `lightwalletd` that is not currently tracking note commitment tree sizes.
-    fn fake_compact_block(
+    pub fn fake_compact_block(
         height: BlockHeight,
         prev_hash: BlockHash,
         nf: Nullifier,
@@ -1281,6 +1269,29 @@ mod tests {
 
         cb
     }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use std::convert::Infallible;
+
+    use incrementalmerkletree::{Position, Retention};
+    use sapling::Nullifier;
+    use zcash_keys::keys::UnifiedSpendingKey;
+    use zcash_primitives::{
+        block::BlockHash,
+        consensus::{BlockHeight, Network},
+        transaction::components::amount::NonNegativeAmount,
+        zip32::AccountId,
+    };
+
+    use crate::{
+        data_api::BlockMetadata,
+        scanning::{BatchRunners, ScanningKeys},
+    };
+
+    use super::{scan_block, scan_block_with_runners, testing::fake_compact_block, Nullifiers};
 
     #[test]
     fn scan_block_with_my_tx() {