From 1b4e5fae741b7d7e9d44691e54e43c1462b8a501 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Wed, 21 Feb 2024 10:33:22 +0000 Subject: [PATCH 1/7] zcash_client_backend: Added `BlockCache` trait extending `BlockSource` with methods for managing a block cache. --- Cargo.lock | 2 + Cargo.toml | 3 + zcash_client_backend/Cargo.toml | 5 + zcash_client_backend/src/data_api/chain.rs | 175 ++++++++++++++++++++- zcash_client_backend/src/scanning.rs | 45 ++++-- 5 files changed, 212 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2f8224bb9..a22bc0aaa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2486,6 +2486,7 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", "pin-project-lite", "socket2 0.5.5", "windows-sys", @@ -3036,6 +3037,7 @@ dependencies = [ "shardtree", "subtle", "time", + "tokio", "tonic", "tonic-build", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 3049022446..ca10636cf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,9 @@ aes = "0.8" fpe = "0.6" zip32 = "0.1" +# Runtime +tokio = { version = "1.21.0", features = ["rt-multi-thread"] } + [profile.release] lto = true panic = 'abort' diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 43605960a5..a58a24c9aa 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -74,10 +74,14 @@ shardtree.workspace = true # - Test dependencies proptest = { workspace = true, optional = true } +jubjub = { workspace = true, optional = true } # - ZIP 321 nom = "7" +# - Runtime +tokio.workspace = true + # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Documentation @@ -126,6 +130,7 @@ orchard = ["dep:orchard", "zcash_keys/orchard"] ## Exposes APIs that are useful for testing, such as `proptest` strategies. test-dependencies = [ "dep:proptest", + "dep:jubjub", "orchard?/test-dependencies", "zcash_keys/test-dependencies", "zcash_primitives/test-dependencies", diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 5cd911c522..3d2e7c5ea4 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -146,10 +146,11 @@ use std::ops::Range; use subtle::ConditionallySelectable; +use tokio::task::JoinHandle; use zcash_primitives::consensus::{self, BlockHeight}; use crate::{ - data_api::{NullifierQuery, WalletWrite}, + data_api::{scanning::ScanRange, NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scanning::{scan_block_with_runners, BatchRunners, Nullifiers, ScanningKeys}, }; @@ -211,6 +212,178 @@ pub trait BlockSource { F: FnMut(CompactBlock) -> Result<(), error::Error>; } +/// `BlockCache` is a trait that extends `BlockSource` and defines methods for managing +/// a cache of compact blocks. +/// +/// # Examples +/// +/// ``` +/// use std::sync::{Arc, Mutex}; +/// use tokio::task::JoinHandle; +/// use zcash_client_backend::data_api::{ +/// chain::{error, BlockCache, BlockSource}, +/// scanning::{ScanPriority, ScanRange}, +/// }; +/// use zcash_client_backend::proto::compact_formats::CompactBlock; +/// use zcash_primitives::consensus::BlockHeight; +/// +/// struct ExampleBlockCache { +/// cached_blocks: Arc>>, +/// } +/// +/// # impl BlockSource for ExampleBlockCache { +/// # type Error = (); +/// # +/// # fn with_blocks( +/// # &self, +/// # _from_height: Option, +/// # _limit: Option, +/// # _with_block: F, +/// # ) -> Result<(), error::Error> +/// # where +/// # F: FnMut(CompactBlock) -> Result<(), error::Error>, +/// # { +/// # Ok(()) +/// # } +/// # } +/// # +/// impl BlockCache for ExampleBlockCache { +/// fn read(&self, range: &ScanRange) -> Result, Self::Error> { +/// Ok(self +/// .cached_blocks +/// .lock() +/// .unwrap() +/// .iter() +/// .filter(|block| { +/// let block_height = BlockHeight::from_u32(block.height as u32); +/// range.block_range().contains(&block_height) +/// }) +/// .cloned() +/// .collect()) +/// } +/// +/// fn cache_tip(&self, range: Option<&ScanRange>) -> Result, Self::Error> { +/// let cached_blocks = self.cached_blocks.lock().unwrap(); +/// let blocks: Vec<&CompactBlock> = match range { +/// Some(range) => cached_blocks +/// .iter() +/// .filter(|&block| { +/// let block_height = BlockHeight::from_u32(block.height as u32); +/// range.block_range().contains(&block_height) +/// }) +/// .collect(), +/// None => cached_blocks.iter().collect(), +/// }; +/// let highest_block = blocks.iter().max_by_key(|&&block| block.height); +/// Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32))) +/// } +/// +/// fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { +/// self.cached_blocks +/// .lock() +/// .unwrap() +/// .append(&mut compact_blocks); +/// Ok(()) +/// } +/// +/// fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> { +/// self.cached_blocks +/// .lock() +/// .unwrap() +/// .retain(|block| block.height <= block_height.into()); +/// Ok(()) +/// } +/// +/// fn delete(&self, range: &ScanRange) -> JoinHandle> { +/// let cached_blocks = Arc::clone(&self.cached_blocks); +/// let range = range.block_range().clone(); +/// tokio::spawn(async move { +/// cached_blocks +/// .lock() +/// .unwrap() +/// .retain(|block| !range.contains(&BlockHeight::from_u32(block.height as u32))); +/// Ok(()) +/// }) +/// } +/// } +/// +/// // Example usage +/// let mut block_cache = ExampleBlockCache { +/// cached_blocks: Arc::new(Mutex::new(Vec::new())), +/// }; +/// let range = ScanRange::from_parts( +/// BlockHeight::from_u32(1)..BlockHeight::from_u32(3), +/// ScanPriority::Historic, +/// ); +/// # let extsk = sapling::zip32::ExtendedSpendingKey::master(&[]); +/// # let dfvk = extsk.to_diversifiable_full_viewing_key(); +/// # let compact_block1 = zcash_client_backend::scanning::testing::fake_compact_block( +/// # 1u32.into(), +/// # zcash_primitives::block::BlockHash([0; 32]), +/// # sapling::Nullifier([0; 32]), +/// # &dfvk, +/// # zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5), +/// # false, +/// # None, +/// # ); +/// # let compact_block2 = zcash_client_backend::scanning::testing::fake_compact_block( +/// # 2u32.into(), +/// # zcash_primitives::block::BlockHash([0; 32]), +/// # sapling::Nullifier([0; 32]), +/// # &dfvk, +/// # zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5), +/// # false, +/// # None, +/// # ); +/// let compact_blocks = vec![compact_block1, compact_block2]; +/// +/// // Insert blocks into the block cache +/// block_cache.insert(compact_blocks.clone()).unwrap(); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); +/// +/// // Find highest block in the block cache +/// let cache_tip = block_cache.cache_tip(None).unwrap(); +/// assert_eq!(cache_tip, Some(BlockHeight::from_u32(2))); +/// +/// // Read from the block cache +/// let blocks_from_cache = block_cache.read(&range).unwrap(); +/// assert_eq!(blocks_from_cache, compact_blocks); +/// +/// // Truncate the block cache +/// block_cache.truncate(BlockHeight::from_u32(1)).unwrap(); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1); +/// assert_eq!( +/// block_cache.cache_tip(None).unwrap(), +/// Some(BlockHeight::from_u32(1)) +/// ); +/// +/// // Delete blocks from the block cache +/// let rt = tokio::runtime::Runtime::new().unwrap(); +/// rt.block_on(async { +/// block_cache.delete(&range).await.unwrap(); +/// }); +/// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0); +/// assert_eq!(block_cache.cache_tip(None).unwrap(), None); +/// ``` +pub trait BlockCache: BlockSource + Send + Sync { + /// Returns a range of compact blocks from the cache. + fn read(&self, range: &ScanRange) -> Result, Self::Error>; + + /// Returns the height of highest block known to the block cache within a specified range. + /// If `range` is `None`, returns the tip of the entire cache. + fn cache_tip(&self, range: Option<&ScanRange>) -> Result, Self::Error>; + + /// Inserts a set of compact blocks into the block cache. + fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; + + /// Removes all cached blocks above a specified block height. + fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error>; + + /// Deletes a range of compact blocks from the block cache. + /// Returns a `JoinHandle` from a `tokio::spawn` task. + fn delete(&self, range: &ScanRange) -> JoinHandle>; +} + /// Metadata about modifications to the wallet state made in the course of scanning a set of /// blocks. #[derive(Clone, Debug)] diff --git a/zcash_client_backend/src/scanning.rs b/zcash_client_backend/src/scanning.rs index 8f132055ea..2ffdfe6d70 100644 --- a/zcash_client_backend/src/scanning.rs +++ b/zcash_client_backend/src/scanning.rs @@ -1124,16 +1124,12 @@ fn find_received< (shielded_outputs, note_commitments) } -#[cfg(test)] -mod tests { - - use std::convert::Infallible; - +#[cfg(any(test, feature = "test-dependencies"))] +pub mod testing { use group::{ ff::{Field, PrimeField}, GroupEncoding, }; - use incrementalmerkletree::{Position, Retention}; use rand_core::{OsRng, RngCore}; use sapling::{ constants::SPENDING_KEY_GENERATOR, @@ -1143,26 +1139,18 @@ mod tests { zip32::DiversifiableFullViewingKey, Nullifier, }; - use zcash_keys::keys::UnifiedSpendingKey; use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE}; use zcash_primitives::{ block::BlockHash, consensus::{sapling_zip212_enforcement, BlockHeight, Network}, memo::MemoBytes, transaction::components::amount::NonNegativeAmount, - zip32::AccountId, }; - use crate::{ - data_api::BlockMetadata, - proto::compact_formats::{ - self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, - }, - scanning::{BatchRunners, ScanningKeys}, + use crate::proto::compact_formats::{ + self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx, }; - use super::{scan_block, scan_block_with_runners, Nullifiers}; - fn random_compact_tx(mut rng: impl RngCore) -> CompactTx { let fake_nf = { let mut nf = vec![0; 32]; @@ -1201,7 +1189,7 @@ mod tests { /// /// Set `initial_tree_sizes` to `None` to simulate a `CompactBlock` retrieved /// from a `lightwalletd` that is not currently tracking note commitment tree sizes. - fn fake_compact_block( + pub fn fake_compact_block( height: BlockHeight, prev_hash: BlockHash, nf: Nullifier, @@ -1280,6 +1268,29 @@ mod tests { cb } +} + +#[cfg(test)] +mod tests { + + use std::convert::Infallible; + + use incrementalmerkletree::{Position, Retention}; + use sapling::Nullifier; + use zcash_keys::keys::UnifiedSpendingKey; + use zcash_primitives::{ + block::BlockHash, + consensus::{BlockHeight, Network}, + transaction::components::amount::NonNegativeAmount, + zip32::AccountId, + }; + + use crate::{ + data_api::BlockMetadata, + scanning::{BatchRunners, ScanningKeys}, + }; + + use super::{scan_block, scan_block_with_runners, testing::fake_compact_block, Nullifiers}; #[test] fn scan_block_with_my_tx() { From 3852ba0f63a32a3c1f29d29b655ed897d6e230fd Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Tue, 5 Mar 2024 18:45:58 +0000 Subject: [PATCH 2/7] zcash_client_backend: Updated CHANGELOG.md --- zcash_client_backend/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 10ee7c5ea7..88bd9a7e39 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -20,6 +20,7 @@ and this library adheres to Rust's notion of - `ORCHARD_SHARD_HEIGHT` - `BlockMetadata::orchard_tree_size` - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}` + - `chain::BlockCache` trait - `zcash_client_backend::fees`: - `orchard` - `ChangeValue::orchard` @@ -33,6 +34,8 @@ and this library adheres to Rust's notion of - `Nullifiers::{orchard, extend_orchard, retain_orchard}` - `TaggedOrchardBatch` - `TaggedOrchardBatchRunner` + - `testing` module + - `testing::{'fake_compact_block`, `random_compact_tx`} (moved from `tests` module). - `zcash_client_backend::wallet`: - `Note::Orchard` - `WalletOrchardSpend` @@ -52,6 +55,8 @@ and this library adheres to Rust's notion of - `fn put_orchard_subtree_roots` - `zcash_client_backend::fees`: - Arguments to `ChangeStrategy::compute_balance` have changed. +- `zcash_client_backend::scanning`: + - `testing::fake_compact_block` is now public. ## [0.11.0] - 2024-03-01 From 773af0866b73f9754cced3b459e6e88a976eff48 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Fri, 8 Mar 2024 17:50:33 +0000 Subject: [PATCH 3/7] zcash_client_backend: Added a sync feature to BlockCache for optional tokio deps and moved tokio rt-multi-thread feature to dev-dependencies --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 3 --- zcash_client_backend/Cargo.toml | 10 +++++++--- zcash_client_backend/src/data_api/chain.rs | 7 +++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1cfa2314c..6a9331103d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2489,6 +2489,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2 0.5.5", + "tokio-macros", "windows-sys", ] @@ -2502,6 +2503,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "tokio-stream" version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index 7fad5f236c..4ef3e13064 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,9 +114,6 @@ aes = "0.8" fpe = "0.6" zip32 = "0.1" -# Runtime -tokio = { version = "1.21.0", features = ["rt-multi-thread"] } - [profile.release] lto = true panic = 'abort' diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 540f76c6de..3dcb118516 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -69,6 +69,9 @@ group.workspace = true orchard = { workspace = true, optional = true } sapling.workspace = true +# - Sync engine +tokio = { version = "1.21.0", optional = true, features = ["fs", "macros"] } + # - Note commitment trees incrementalmerkletree.workspace = true shardtree.workspace = true @@ -80,9 +83,6 @@ jubjub = { workspace = true, optional = true } # - ZIP 321 nom = "7" -# - Runtime -tokio.workspace = true - # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Documentation @@ -111,6 +111,7 @@ shardtree = { workspace = true, features = ["test-dependencies"] } zcash_proofs.workspace = true zcash_address = { workspace = true, features = ["test-dependencies"] } zcash_keys = { workspace = true, features = ["test-dependencies"] } +tokio = { version = "1.21.0", features = ["rt-multi-thread"] } time = ">=0.3.22, <0.3.24" # time 0.3.24 has MSRV 1.67 @@ -128,6 +129,9 @@ transparent-inputs = [ ## Enables receiving and spending Orchard funds. orchard = ["dep:orchard", "zcash_keys/orchard"] +## Exposes a wallet synchronization function that implements the necessary state machine. +sync = ["dep:tokio"] + ## Exposes APIs that are useful for testing, such as `proptest` strategies. test-dependencies = [ "dep:proptest", diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 3d2e7c5ea4..12671e2ac7 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -146,11 +146,10 @@ use std::ops::Range; use subtle::ConditionallySelectable; -use tokio::task::JoinHandle; use zcash_primitives::consensus::{self, BlockHeight}; use crate::{ - data_api::{scanning::ScanRange, NullifierQuery, WalletWrite}, + data_api::{NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scanning::{scan_block_with_runners, BatchRunners, Nullifiers, ScanningKeys}, }; @@ -160,6 +159,9 @@ use error::Error; use super::WalletRead; +#[cfg(feature = "sync")] +use {crate::data_api::scanning::ScanRange, tokio::task::JoinHandle}; + /// A struct containing metadata about a subtree root of the note commitment tree. /// /// This stores the block height at which the leaf that completed the subtree was @@ -365,6 +367,7 @@ pub trait BlockSource { /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0); /// assert_eq!(block_cache.cache_tip(None).unwrap(), None); /// ``` +#[cfg(feature = "sync")] pub trait BlockCache: BlockSource + Send + Sync { /// Returns a range of compact blocks from the cache. fn read(&self, range: &ScanRange) -> Result, Self::Error>; From ea991b7559dda279e37ccbe8cc58c7ebdce28b59 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Tue, 12 Mar 2024 13:55:05 +0000 Subject: [PATCH 4/7] requested changes: - Added read error documentation - Changed name of `cache_tip` method to `get_tip_height` - Improved doc comments --- zcash_client_backend/src/data_api/chain.rs | 37 ++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 12671e2ac7..4f2adbb7e4 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -264,7 +264,7 @@ pub trait BlockSource { /// .collect()) /// } /// -/// fn cache_tip(&self, range: Option<&ScanRange>) -> Result, Self::Error> { +/// fn get_tip_height(&self, range: Option<&ScanRange>) -> Result, Self::Error> { /// let cached_blocks = self.cached_blocks.lock().unwrap(); /// let blocks: Vec<&CompactBlock> = match range { /// Some(range) => cached_blocks @@ -344,8 +344,8 @@ pub trait BlockSource { /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); /// /// // Find highest block in the block cache -/// let cache_tip = block_cache.cache_tip(None).unwrap(); -/// assert_eq!(cache_tip, Some(BlockHeight::from_u32(2))); +/// let get_tip_height = block_cache.get_tip_height(None).unwrap(); +/// assert_eq!(get_tip_height, Some(BlockHeight::from_u32(2))); /// /// // Read from the block cache /// let blocks_from_cache = block_cache.read(&range).unwrap(); @@ -355,7 +355,7 @@ pub trait BlockSource { /// block_cache.truncate(BlockHeight::from_u32(1)).unwrap(); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1); /// assert_eq!( -/// block_cache.cache_tip(None).unwrap(), +/// block_cache.get_tip_height(None).unwrap(), /// Some(BlockHeight::from_u32(1)) /// ); /// @@ -365,24 +365,43 @@ pub trait BlockSource { /// block_cache.delete(&range).await.unwrap(); /// }); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0); -/// assert_eq!(block_cache.cache_tip(None).unwrap(), None); +/// assert_eq!(block_cache.get_tip_height(None).unwrap(), None); /// ``` #[cfg(feature = "sync")] pub trait BlockCache: BlockSource + Send + Sync { - /// Returns a range of compact blocks from the cache. + /// Retrieves contiguous compact blocks specified by the given `range` from the block cache. + /// + /// Returns `Ok(Vec)` on success, otherwise returns an error. + /// + /// Short reads are allowed, meaning that returning fewer blocks than requested should not + /// return an error as long as all blocks are sequentially continuous in height. + /// + /// # Errors + /// + /// This method should return an error if there are gaps in the requested range of blocks, + /// indicating there are blocks missing from the cache. fn read(&self, range: &ScanRange) -> Result, Self::Error>; - /// Returns the height of highest block known to the block cache within a specified range. + /// Finds the height of the highest block known to the block cache within a specified range. /// If `range` is `None`, returns the tip of the entire cache. - fn cache_tip(&self, range: Option<&ScanRange>) -> Result, Self::Error>; + /// + /// Returns `Ok(Some(BlockHeight))` on success, otherwise returns an error. + /// If no blocks are found in the cache, returns Ok(`None`). + fn get_tip_height(&self, range: Option<&ScanRange>) + -> Result, Self::Error>; - /// Inserts a set of compact blocks into the block cache. + /// Inserts a vec of compact blocks into the block cache. + /// + /// Returns `Ok(())` on success, otherwise returns an error. fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; /// Removes all cached blocks above a specified block height. + /// + /// Returns `Ok(())` on success, otherwise returns an error. fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error>; /// Deletes a range of compact blocks from the block cache. + /// /// Returns a `JoinHandle` from a `tokio::spawn` task. fn delete(&self, range: &ScanRange) -> JoinHandle>; } From 9c095676e30ec7b8f68b49bdcc44d8ebd3bd4d11 Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Wed, 20 Mar 2024 14:10:17 +0000 Subject: [PATCH 5/7] Requested changes: remove tokio specific code from BlockCache trait and replace with async trait --- Cargo.lock | 57 +++++++++------------- zcash_client_backend/CHANGELOG.md | 3 -- zcash_client_backend/Cargo.toml | 9 ++-- zcash_client_backend/src/data_api/chain.rs | 42 ++++++++-------- 4 files changed, 47 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc6535da77..6a8cc6f28c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,18 +115,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1609,7 +1609,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1704,7 +1704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1719,9 +1719,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -1773,7 +1773,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.39", + "syn 2.0.53", "tempfile", "which", ] @@ -1788,7 +1788,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -1817,9 +1817,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2216,7 +2216,7 @@ checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2361,9 +2361,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" dependencies = [ "proc-macro2", "quote", @@ -2418,7 +2418,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2486,7 +2486,6 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2 0.5.5", - "tokio-macros", "windows-sys", ] @@ -2500,17 +2499,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.39", -] - [[package]] name = "tokio-stream" version = "0.1.14" @@ -2573,7 +2561,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2627,7 +2615,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -2829,7 +2817,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-shared", ] @@ -2851,7 +2839,7 @@ checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3020,6 +3008,7 @@ name = "zcash_client_backend" version = "0.11.1" dependencies = [ "assert_matches", + "async-trait", "base64", "bech32", "bls12_381", @@ -3286,7 +3275,7 @@ checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] @@ -3306,7 +3295,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.53", ] [[package]] diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index fb299b7d1b..c9697a70de 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -40,7 +40,6 @@ and this library adheres to Rust's notion of - `TaggedOrchardBatch` - `TaggedOrchardBatchRunner` - `testing` module - - `testing::{'fake_compact_block`, `random_compact_tx`} (moved from `tests` module). - `zcash_client_backend::wallet`: - `Note::Orchard` - `WalletOrchardSpend` @@ -84,8 +83,6 @@ and this library adheres to Rust's notion of constraint on its `` parameter has been strengthened to `Copy`. - `zcash_client_backend::fees`: - Arguments to `ChangeStrategy::compute_balance` have changed. -- `zcash_client_backend::scanning`: - - `testing::fake_compact_block` is now public. - `ChangeError::DustInputs` now has an `orchard` field behind the `orchard` feature flag. - `zcash_client_backend::proto`: diff --git a/zcash_client_backend/Cargo.toml b/zcash_client_backend/Cargo.toml index 9d45e4b472..2fb8a1d231 100644 --- a/zcash_client_backend/Cargo.toml +++ b/zcash_client_backend/Cargo.toml @@ -78,9 +78,6 @@ group.workspace = true orchard = { workspace = true, optional = true } sapling.workspace = true -# - Sync engine -tokio = { version = "1.21.0", optional = true, features = ["fs", "macros"] } - # - Note commitment trees incrementalmerkletree.workspace = true shardtree.workspace = true @@ -92,6 +89,9 @@ jubjub = { workspace = true, optional = true } # - ZIP 321 nom = "7" +# - Asychronous +async-trait = "0.1.78" + # Dependencies used internally: # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.) # - Documentation @@ -138,9 +138,6 @@ transparent-inputs = [ ## Enables receiving and spending Orchard funds. orchard = ["dep:orchard", "zcash_keys/orchard"] -## Exposes a wallet synchronization function that implements the necessary state machine. -sync = ["dep:tokio"] - ## Exposes APIs that are useful for testing, such as `proptest` strategies. test-dependencies = [ "dep:proptest", diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index d5f7b1fcdf..0c84989acc 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -153,12 +153,13 @@ use std::ops::Range; +use async_trait::async_trait; use incrementalmerkletree::frontier::Frontier; use subtle::ConditionallySelectable; use zcash_primitives::consensus::{self, BlockHeight}; use crate::{ - data_api::{NullifierQuery, WalletWrite}, + data_api::{scanning::ScanRange, NullifierQuery, WalletWrite}, proto::compact_formats::CompactBlock, scanning::{scan_block_with_runners, BatchRunners, Nullifiers, ScanningKeys}, }; @@ -168,9 +169,6 @@ use error::Error; use super::WalletRead; -#[cfg(feature = "sync")] -use {crate::data_api::scanning::ScanRange, tokio::task::JoinHandle}; - /// A struct containing metadata about a subtree root of the note commitment tree. /// /// This stores the block height at which the leaf that completed the subtree was @@ -229,6 +227,7 @@ pub trait BlockSource { /// # Examples /// /// ``` +/// use async_trait::async_trait; /// use std::sync::{Arc, Mutex}; /// use tokio::task::JoinHandle; /// use zcash_client_backend::data_api::{ @@ -258,6 +257,7 @@ pub trait BlockSource { /// # } /// # } /// # +/// #[async_trait] /// impl BlockCache for ExampleBlockCache { /// fn read(&self, range: &ScanRange) -> Result, Self::Error> { /// Ok(self @@ -289,7 +289,7 @@ pub trait BlockSource { /// Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32))) /// } /// -/// fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { +/// async fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { /// self.cached_blocks /// .lock() /// .unwrap() @@ -305,16 +305,12 @@ pub trait BlockSource { /// Ok(()) /// } /// -/// fn delete(&self, range: &ScanRange) -> JoinHandle> { -/// let cached_blocks = Arc::clone(&self.cached_blocks); -/// let range = range.block_range().clone(); -/// tokio::spawn(async move { -/// cached_blocks -/// .lock() -/// .unwrap() -/// .retain(|block| !range.contains(&BlockHeight::from_u32(block.height as u32))); -/// Ok(()) -/// }) +/// async fn delete(&self, range: &ScanRange) -> Result<(), Self::Error> { +/// self.cached_blocks +/// .lock() +/// .unwrap() +/// .retain(|block| !range.block_range().contains(&BlockHeight::from_u32(block.height as u32))); +/// Ok(()) /// } /// } /// @@ -348,8 +344,13 @@ pub trait BlockSource { /// # ); /// let compact_blocks = vec![compact_block1, compact_block2]; /// +/// // Create a runtime +/// let rt = tokio::runtime::Runtime::new().unwrap(); +/// /// // Insert blocks into the block cache -/// block_cache.insert(compact_blocks.clone()).unwrap(); +/// rt.block_on(async { +/// block_cache.insert(compact_blocks.clone()).await.unwrap(); +/// }); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); /// /// // Find highest block in the block cache @@ -369,14 +370,13 @@ pub trait BlockSource { /// ); /// /// // Delete blocks from the block cache -/// let rt = tokio::runtime::Runtime::new().unwrap(); /// rt.block_on(async { /// block_cache.delete(&range).await.unwrap(); /// }); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0); /// assert_eq!(block_cache.get_tip_height(None).unwrap(), None); /// ``` -#[cfg(feature = "sync")] +#[async_trait] pub trait BlockCache: BlockSource + Send + Sync { /// Retrieves contiguous compact blocks specified by the given `range` from the block cache. /// @@ -402,7 +402,7 @@ pub trait BlockCache: BlockSource + Send + Sync { /// Inserts a vec of compact blocks into the block cache. /// /// Returns `Ok(())` on success, otherwise returns an error. - fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; + async fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; /// Removes all cached blocks above a specified block height. /// @@ -411,8 +411,8 @@ pub trait BlockCache: BlockSource + Send + Sync { /// Deletes a range of compact blocks from the block cache. /// - /// Returns a `JoinHandle` from a `tokio::spawn` task. - fn delete(&self, range: &ScanRange) -> JoinHandle>; + /// Returns `Ok(())` on success, otherwise returns an error. + async fn delete(&self, range: &ScanRange) -> Result<(), Self::Error>; } /// Metadata about modifications to the wallet state made in the course of scanning a set of From 7301c0c76e8dddd965b4705097b7a97144aebe9b Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Wed, 20 Mar 2024 14:33:50 +0000 Subject: [PATCH 6/7] zcash_client_backend: Removed async from insert method --- zcash_client_backend/src/data_api/chain.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index 1ab9aab1ca..ced27c0a3f 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -292,7 +292,7 @@ pub trait BlockSource { /// Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32))) /// } /// -/// async fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { +/// fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { /// self.cached_blocks /// .lock() /// .unwrap() @@ -347,13 +347,8 @@ pub trait BlockSource { /// # ); /// let compact_blocks = vec![compact_block1, compact_block2]; /// -/// // Create a runtime -/// let rt = tokio::runtime::Runtime::new().unwrap(); -/// /// // Insert blocks into the block cache -/// rt.block_on(async { -/// block_cache.insert(compact_blocks.clone()).await.unwrap(); -/// }); +/// block_cache.insert(compact_blocks.clone()).unwrap(); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); /// /// // Find highest block in the block cache @@ -373,6 +368,7 @@ pub trait BlockSource { /// ); /// /// // Delete blocks from the block cache +/// let rt = tokio::runtime::Runtime::new().unwrap(); /// rt.block_on(async { /// block_cache.delete(&range).await.unwrap(); /// }); @@ -405,7 +401,7 @@ pub trait BlockCache: BlockSource + Send + Sync { /// Inserts a vec of compact blocks into the block cache. /// /// Returns `Ok(())` on success, otherwise returns an error. - async fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; + fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; /// Removes all cached blocks above a specified block height. /// From 48e3ff13c8ebc20c180deb7d181857d6ea01f6db Mon Sep 17 00:00:00 2001 From: Oscar Pepper Date: Sat, 30 Mar 2024 11:57:18 +0000 Subject: [PATCH 7/7] Made requested changes to str4d's review of 7301c0c --- zcash_client_backend/src/data_api/chain.rs | 108 +++++++++++---------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/zcash_client_backend/src/data_api/chain.rs b/zcash_client_backend/src/data_api/chain.rs index ced27c0a3f..b48d2c30cd 100644 --- a/zcash_client_backend/src/data_api/chain.rs +++ b/zcash_client_backend/src/data_api/chain.rs @@ -151,7 +151,7 @@ //! # } //! ``` -use std::ops::Range; +use std::ops::{Add, Range}; use async_trait::async_trait; use incrementalmerkletree::frontier::Frontier; @@ -170,7 +170,7 @@ use crate::{ pub mod error; use error::Error; -use super::WalletRead; +use super::{scanning::ScanPriority, WalletRead}; /// A struct containing metadata about a subtree root of the note commitment tree. /// @@ -262,20 +262,6 @@ pub trait BlockSource { /// # /// #[async_trait] /// impl BlockCache for ExampleBlockCache { -/// fn read(&self, range: &ScanRange) -> Result, Self::Error> { -/// Ok(self -/// .cached_blocks -/// .lock() -/// .unwrap() -/// .iter() -/// .filter(|block| { -/// let block_height = BlockHeight::from_u32(block.height as u32); -/// range.block_range().contains(&block_height) -/// }) -/// .cloned() -/// .collect()) -/// } -/// /// fn get_tip_height(&self, range: Option<&ScanRange>) -> Result, Self::Error> { /// let cached_blocks = self.cached_blocks.lock().unwrap(); /// let blocks: Vec<&CompactBlock> = match range { @@ -292,19 +278,25 @@ pub trait BlockSource { /// Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32))) /// } /// -/// fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { -/// self.cached_blocks +/// async fn read(&self, range: &ScanRange) -> Result, Self::Error> { +/// Ok(self +/// .cached_blocks /// .lock() /// .unwrap() -/// .append(&mut compact_blocks); -/// Ok(()) +/// .iter() +/// .filter(|block| { +/// let block_height = BlockHeight::from_u32(block.height as u32); +/// range.block_range().contains(&block_height) +/// }) +/// .cloned() +/// .collect()) /// } /// -/// fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> { +/// async fn insert(&self, mut compact_blocks: Vec) -> Result<(), Self::Error> { /// self.cached_blocks /// .lock() /// .unwrap() -/// .retain(|block| block.height <= block_height.into()); +/// .append(&mut compact_blocks); /// Ok(()) /// } /// @@ -318,6 +310,7 @@ pub trait BlockSource { /// } /// /// // Example usage +/// let rt = tokio::runtime::Runtime::new().unwrap(); /// let mut block_cache = ExampleBlockCache { /// cached_blocks: Arc::new(Mutex::new(Vec::new())), /// }; @@ -348,7 +341,9 @@ pub trait BlockSource { /// let compact_blocks = vec![compact_block1, compact_block2]; /// /// // Insert blocks into the block cache -/// block_cache.insert(compact_blocks.clone()).unwrap(); +/// rt.block_on(async { +/// block_cache.insert(compact_blocks.clone()).await.unwrap(); +/// }); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2); /// /// // Find highest block in the block cache @@ -356,11 +351,15 @@ pub trait BlockSource { /// assert_eq!(get_tip_height, Some(BlockHeight::from_u32(2))); /// /// // Read from the block cache -/// let blocks_from_cache = block_cache.read(&range).unwrap(); -/// assert_eq!(blocks_from_cache, compact_blocks); +/// rt.block_on(async { +/// let blocks_from_cache = block_cache.read(&range).await.unwrap(); +/// assert_eq!(blocks_from_cache, compact_blocks); +/// }); /// /// // Truncate the block cache -/// block_cache.truncate(BlockHeight::from_u32(1)).unwrap(); +/// rt.block_on(async { +/// block_cache.truncate(BlockHeight::from_u32(1)).await.unwrap(); +/// }); /// assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1); /// assert_eq!( /// block_cache.get_tip_height(None).unwrap(), @@ -368,7 +367,6 @@ pub trait BlockSource { /// ); /// /// // Delete blocks from the block cache -/// let rt = tokio::runtime::Runtime::new().unwrap(); /// rt.block_on(async { /// block_cache.delete(&range).await.unwrap(); /// }); @@ -376,41 +374,53 @@ pub trait BlockSource { /// assert_eq!(block_cache.get_tip_height(None).unwrap(), None); /// ``` #[async_trait] -pub trait BlockCache: BlockSource + Send + Sync { - /// Retrieves contiguous compact blocks specified by the given `range` from the block cache. - /// - /// Returns `Ok(Vec)` on success, otherwise returns an error. - /// - /// Short reads are allowed, meaning that returning fewer blocks than requested should not - /// return an error as long as all blocks are sequentially continuous in height. - /// - /// # Errors - /// - /// This method should return an error if there are gaps in the requested range of blocks, - /// indicating there are blocks missing from the cache. - fn read(&self, range: &ScanRange) -> Result, Self::Error>; - +pub trait BlockCache: BlockSource + Send + Sync +where + Self::Error: Send, +{ /// Finds the height of the highest block known to the block cache within a specified range. - /// If `range` is `None`, returns the tip of the entire cache. /// - /// Returns `Ok(Some(BlockHeight))` on success, otherwise returns an error. + /// If `range` is `None`, returns the tip of the entire cache. /// If no blocks are found in the cache, returns Ok(`None`). fn get_tip_height(&self, range: Option<&ScanRange>) -> Result, Self::Error>; + /// Retrieves contiguous compact blocks specified by the given `range` from the block cache. + /// + /// Short reads are allowed, meaning that this method may return fewer blocks than requested + /// provided that all returned blocks are contiguous and start from `range.block_range().start`. + /// + /// # Errors + /// + /// This method should return an error if contiguous blocks cannot be read from the cache, + /// indicating there are blocks missing. + async fn read(&self, range: &ScanRange) -> Result, Self::Error>; + /// Inserts a vec of compact blocks into the block cache. /// - /// Returns `Ok(())` on success, otherwise returns an error. - fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; + /// This method permits insertion of non-contiguous compact blocks. + async fn insert(&self, compact_blocks: Vec) -> Result<(), Self::Error>; /// Removes all cached blocks above a specified block height. - /// - /// Returns `Ok(())` on success, otherwise returns an error. - fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error>; + async fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> { + if let Some(latest) = self.get_tip_height(None)? { + self.delete(&ScanRange::from_parts( + Range { + start: block_height.add(1), + end: latest.add(1), + }, + ScanPriority::Ignored, + )) + .await?; + } + Ok(()) + } /// Deletes a range of compact blocks from the block cache. /// - /// Returns `Ok(())` on success, otherwise returns an error. + /// # Errors + /// + /// In the case of an error, some blocks requested for deletion may remain in the block cache. async fn delete(&self, range: &ScanRange) -> Result<(), Self::Error>; }