From eb00e6d7898059c40b81a1005011b25c9f81a597 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:35:11 +0000 Subject: [PATCH 01/49] add snapshots_path to ChainPath --- bin/reth/src/dirs.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index c93e8234ba4d..0bb05125717c 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -264,6 +264,11 @@ impl ChainPath { self.0.join("db").into() } + /// Returns the path to the snapshots directory for this chain. + pub fn snapshots_path(&self) -> PathBuf { + self.0.join("snapshots").into() + } + /// Returns the path to the reth p2p secret key for this chain. pub fn p2p_secret_path(&self) -> PathBuf { self.0.join("discovery-secret").into() From a8ef221e63ee8141a20d4b04ddc91c348df5224a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:36:02 +0000 Subject: [PATCH 02/49] move HighestSnapshots to primitives crate --- crates/primitives/src/snapshot/mod.rs | 15 +++++++++++++++ crates/snapshot/src/lib.rs | 3 +-- crates/snapshot/src/snapshotter.rs | 20 +++----------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index d8fc8db53624..28c21432d1b3 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -4,9 +4,24 @@ mod compression; mod filters; mod segment; +use alloy_primitives::BlockNumber; pub use compression::Compression; pub use filters::{Filters, InclusionFilter, PerfectHashingFunction}; pub use segment::{SegmentHeader, SnapshotSegment}; /// Default snapshot block count. pub const BLOCKS_PER_SNAPSHOT: u64 = 500_000; + +/// Highest snapshotted block numbers, per data part. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct HighestSnapshots { + /// Highest snapshotted block of headers, inclusive. + /// If [`None`], no snapshot is available. + pub headers: Option, + /// Highest snapshotted block of receipts, inclusive. + /// If [`None`], no snapshot is available. + pub receipts: Option, + /// Highest snapshotted block of transactions, inclusive. + /// If [`None`], no snapshot is available. + pub transactions: Option, +} diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index 18b22bdb54a6..82f42b2d4c05 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -15,6 +15,5 @@ mod snapshotter; pub use error::SnapshotterError; pub use snapshotter::{ - HighestSnapshots, HighestSnapshotsTracker, SnapshotTargets, Snapshotter, SnapshotterResult, - SnapshotterWithResult, + HighestSnapshotsTracker, SnapshotTargets, Snapshotter, SnapshotterResult, SnapshotterWithResult, }; diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index b850f68a1c74..51c9940120b8 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -3,7 +3,7 @@ use crate::SnapshotterError; use reth_db::database::Database; use reth_interfaces::{RethError, RethResult}; -use reth_primitives::{BlockNumber, ChainSpec, TxNumber}; +use reth_primitives::{snapshot::HighestSnapshots, BlockNumber, ChainSpec, TxNumber}; use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory}; use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; use tokio::sync::watch; @@ -28,20 +28,6 @@ pub struct Snapshotter { /// Tracker for the latest [`HighestSnapshots`] value. pub type HighestSnapshotsTracker = watch::Receiver>; -/// Highest snapshotted block numbers, per data part. -#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] -pub struct HighestSnapshots { - /// Highest snapshotted block of headers, inclusive. - /// If [`None`], no snapshot is available. - pub headers: Option, - /// Highest snapshotted block of receipts, inclusive. - /// If [`None`], no snapshot is available. - pub receipts: Option, - /// Highest snapshotted block of transactions, inclusive. - /// If [`None`], no snapshot is available. - pub transactions: Option, -} - /// Snapshot targets, per data part, measured in [`BlockNumber`] and [`TxNumber`], if applicable. #[derive(Debug, Clone, Eq, PartialEq)] pub struct SnapshotTargets { @@ -240,13 +226,13 @@ impl Snapshotter { #[cfg(test)] mod tests { - use crate::{snapshotter::SnapshotTargets, HighestSnapshots, Snapshotter}; + use crate::{snapshotter::SnapshotTargets, Snapshotter}; use assert_matches::assert_matches; use reth_interfaces::{ test_utils::{generators, generators::random_block_range}, RethError, }; - use reth_primitives::{B256, MAINNET}; + use reth_primitives::{snapshot::HighestSnapshots, B256, MAINNET}; use reth_stages::test_utils::TestTransaction; use tokio::sync::watch; From f1f32d77f878c9a1303171a34d223b5268bf98ff Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:36:30 +0000 Subject: [PATCH 03/49] add with_highest_tracker to SnapshotProvider --- .../provider/src/providers/snapshot/manager.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index a065f6d1db48..59d93af44e46 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -4,11 +4,13 @@ use dashmap::DashMap; use reth_interfaces::RethResult; use reth_nippy_jar::NippyJar; use reth_primitives::{ - snapshot::BLOCKS_PER_SNAPSHOT, Address, BlockHash, BlockHashOrNumber, BlockNumber, ChainInfo, - Header, SealedHeader, SnapshotSegment, TransactionMeta, TransactionSigned, - TransactionSignedNoHash, TxHash, TxNumber, B256, U256, + snapshot::{HighestSnapshots, BLOCKS_PER_SNAPSHOT}, + Address, BlockHash, BlockHashOrNumber, BlockNumber, ChainInfo, Header, SealedHeader, + SnapshotSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, + B256, U256, }; use std::{ops::RangeBounds, path::PathBuf}; +use tokio::sync::watch; /// SnapshotProvider #[derive(Debug, Default)] @@ -16,9 +18,19 @@ pub struct SnapshotProvider { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, + highest_tracker: Option>>, } impl SnapshotProvider { + /// Adds a highest snapshot tracker to the provider + pub fn with_highest_tracker( + mut self, + highest_tracker: Option>>, + ) -> Self { + self.highest_tracker = highest_tracker; + self + } + /// Gets the provider of the requested segment and range. pub fn get_segment_provider( &self, From f5cadf89ecf02d47166b15fe390124b6efa529ba Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:37:05 +0000 Subject: [PATCH 04/49] add a shared snapshot provvider to providerfactory and dbprovider --- crates/blockchain-tree/src/blockchain_tree.rs | 2 + .../provider/src/providers/database/mod.rs | 41 ++++++++++++++++--- .../src/providers/database/provider.rs | 19 +++++++-- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index bd1d72c971e4..4085f649b8e9 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -1071,6 +1071,7 @@ impl BlockchainTree { let provider = DatabaseProvider::new_rw( self.externals.db.tx_mut()?, self.externals.chain_spec.clone(), + None, ); let (blocks, state) = chain.into_inner(); @@ -1116,6 +1117,7 @@ impl BlockchainTree { let provider = DatabaseProvider::new_rw( self.externals.db.tx_mut()?, self.externals.chain_spec.clone(), + None, ); let tip = provider.last_block_number()?; diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 140de2a2ff43..71fe7e64073a 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -1,5 +1,8 @@ use crate::{ - providers::state::{historical::HistoricalStateProvider, latest::LatestStateProvider}, + providers::{ + state::{historical::HistoricalStateProvider, latest::LatestStateProvider}, + SnapshotProvider, + }, traits::{BlockSource, ReceiptProvider}, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, EvmEnvProvider, HeaderProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProviderBox, @@ -8,6 +11,7 @@ use crate::{ use reth_db::{database::Database, init_db, models::StoredBlockBodyIndices, DatabaseEnv}; use reth_interfaces::{db::LogLevel, RethError, RethResult}; use reth_primitives::{ + snapshot::HighestSnapshots, stage::{StageCheckpoint, StageId}, Address, Block, BlockHash, BlockHashOrNumber, BlockNumber, BlockWithSenders, ChainInfo, ChainSpec, Header, PruneCheckpoint, PruneSegment, Receipt, SealedBlock, SealedHeader, @@ -19,6 +23,7 @@ use std::{ ops::{RangeBounds, RangeInclusive}, sync::Arc, }; +use tokio::sync::watch; use tracing::trace; mod provider; @@ -33,6 +38,8 @@ pub struct ProviderFactory { db: DB, /// Chain spec chain_spec: Arc, + /// Snapshot Provider + snapshot_provider: Option>, } impl ProviderFactory { @@ -40,7 +47,11 @@ impl ProviderFactory { /// database using different types of providers. Example: [`HeaderProvider`] /// [`BlockHashReader`]. This may fail if the inner read database transaction fails to open. pub fn provider(&self) -> RethResult> { - Ok(DatabaseProvider::new(self.db.tx()?, self.chain_spec.clone())) + Ok(DatabaseProvider::new( + self.db.tx()?, + self.chain_spec.clone(), + self.snapshot_provider.clone(), + )) } /// Returns a provider with a created `DbTxMut` inside, which allows fetching and updating @@ -48,14 +59,29 @@ impl ProviderFactory { /// [`BlockHashReader`]. This may fail if the inner read/write database transaction fails to /// open. pub fn provider_rw(&self) -> RethResult> { - Ok(DatabaseProviderRW(DatabaseProvider::new_rw(self.db.tx_mut()?, self.chain_spec.clone()))) + Ok(DatabaseProviderRW(DatabaseProvider::new_rw( + self.db.tx_mut()?, + self.chain_spec.clone(), + self.snapshot_provider.clone(), + ))) } } impl ProviderFactory { /// create new database provider pub fn new(db: DB, chain_spec: Arc) -> Self { - Self { db, chain_spec } + Self { db, chain_spec, snapshot_provider: None } + } + + /// database provider comes with a shared snapshot provider + pub fn with_snapshots( + mut self, + highest_snapshot_tracker: watch::Receiver>, + ) -> Self { + self.snapshot_provider = Some(Arc::new( + SnapshotProvider::default().with_highest_tracker(Some(highest_snapshot_tracker)), + )); + self } } @@ -70,13 +96,18 @@ impl ProviderFactory { Ok(ProviderFactory:: { db: init_db(path, log_level).map_err(|e| RethError::Custom(e.to_string()))?, chain_spec, + snapshot_provider: None, }) } } impl Clone for ProviderFactory { fn clone(&self) -> Self { - Self { db: self.db.clone(), chain_spec: Arc::clone(&self.chain_spec) } + Self { + db: self.db.clone(), + chain_spec: Arc::clone(&self.chain_spec), + snapshot_provider: self.snapshot_provider.clone(), + } } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index f92a37c4be1c..4a946455466b 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,5 +1,6 @@ use crate::{ bundle_state::{BundleStateInit, BundleStateWithReceipts, RevertsInit}, + providers::SnapshotProvider, traits::{ AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, }, @@ -96,12 +97,18 @@ pub struct DatabaseProvider { tx: TX, /// Chain spec chain_spec: Arc, + /// Snapshot provider + snapshot_provider: Option>, } impl DatabaseProvider { /// Creates a provider with an inner read-write transaction. - pub fn new_rw(tx: TX, chain_spec: Arc) -> Self { - Self { tx, chain_spec } + pub fn new_rw( + tx: TX, + chain_spec: Arc, + snapshot_provider: Option>, + ) -> Self { + Self { tx, chain_spec, snapshot_provider } } } @@ -155,8 +162,12 @@ where impl DatabaseProvider { /// Creates a provider with an inner read-only transaction. - pub fn new(tx: TX, chain_spec: Arc) -> Self { - Self { tx, chain_spec } + pub fn new( + tx: TX, + chain_spec: Arc, + snapshot_provider: Option>, + ) -> Self { + Self { tx, chain_spec, snapshot_provider } } /// Consume `DbTx` or `DbTxMut`. From 1acd28f2c8e3b88ffeded04b27f9774a037032fa Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:45:10 +0000 Subject: [PATCH 05/49] add snapshot provider to shared blockchain_db --- bin/reth/src/node/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 5aa011bce06a..9061448d338d 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -295,7 +295,9 @@ impl NodeCommand { let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; // setup the blockchain provider - let factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)); + let (highest_snapshots_tx, highest_snapshots_rx) = watch::channel(None); + let factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)) + .with_snapshots(highest_snapshots_rx.clone()); let blockchain_db = BlockchainProvider::new(factory, blockchain_tree.clone())?; let blob_store = InMemoryBlobStore::default(); let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) @@ -450,8 +452,6 @@ impl NodeCommand { None }; - let (highest_snapshots_tx, highest_snapshots_rx) = watch::channel(None); - let mut hooks = EngineHooks::new(); let pruner_events = if let Some(prune_config) = prune_config { From 470a6b6a651b7e210e8836e47963ea2f18b2475b Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 11:48:10 +0000 Subject: [PATCH 06/49] allow unused for now --- crates/storage/provider/src/providers/database/provider.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 4a946455466b..fe63caac61d6 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -98,6 +98,7 @@ pub struct DatabaseProvider { /// Chain spec chain_spec: Arc, /// Snapshot provider + #[allow(unused)] snapshot_provider: Option>, } From 113ef23de1c09c008ced21718c514369d5e0a17a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 31 Oct 2023 12:40:52 +0000 Subject: [PATCH 07/49] add get_highest_snapshot to SnapshotProvider --- crates/primitives/src/snapshot/mod.rs | 11 +++++++++++ .../provider/src/providers/snapshot/manager.rs | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index 28c21432d1b3..f73ba9874d77 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -25,3 +25,14 @@ pub struct HighestSnapshots { /// If [`None`], no snapshot is available. pub transactions: Option, } + +impl HighestSnapshots { + /// Returns the highest snapshot if it exists for a segment + pub fn highest(&self, segment: SnapshotSegment) -> Option { + match segment { + SnapshotSegment::Headers => self.headers, + SnapshotSegment::Transactions => self.transactions, + SnapshotSegment::Receipts => self.receipts, + } + } +} diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 59d93af44e46..946e8349a7cb 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -18,6 +18,7 @@ pub struct SnapshotProvider { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, + /// Tracks the latest and highest snapshot of every segment. highest_tracker: Option>>, } @@ -56,6 +57,13 @@ impl SnapshotProvider { self.get_segment_provider(segment, block, path) } + + /// Gets the highest snapshot if it exists for a snapshot segment. + pub fn get_highest_snapshot(&self, segment: SnapshotSegment) -> Option { + self.highest_tracker + .as_ref() + .and_then(|tracker| tracker.borrow().and_then(|highest| highest.highest(segment))) + } } impl HeaderProvider for SnapshotProvider { From 73fd00fc40dafd597e0e44bcc0d2fc12f2c4d69d Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 2 Nov 2023 16:29:11 +0000 Subject: [PATCH 08/49] move hihgest snapshot channel inside snapshotter --- bin/reth/src/node/mod.rs | 23 +++++++++++++---------- crates/snapshot/src/snapshotter.rs | 27 ++++++++++++++------------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 9061448d338d..da4fe6a4ea76 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -294,10 +294,16 @@ impl NodeCommand { // fetch the head block from the database let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; + // configure snapshotter + let snapshotter = reth_snapshot::Snapshotter::new( + db.clone(), + self.chain.clone(), + self.chain.snapshot_block_interval, + ); + // setup the blockchain provider - let (highest_snapshots_tx, highest_snapshots_rx) = watch::channel(None); let factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)) - .with_snapshots(highest_snapshots_rx.clone()); + .with_snapshots(snapshotter.highest_snapshot_receiver()); let blockchain_db = BlockchainProvider::new(factory, blockchain_tree.clone())?; let blob_store = InMemoryBlobStore::default(); let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) @@ -455,7 +461,11 @@ impl NodeCommand { let mut hooks = EngineHooks::new(); let pruner_events = if let Some(prune_config) = prune_config { - let mut pruner = self.build_pruner(&prune_config, db.clone(), highest_snapshots_rx); + let mut pruner = self.build_pruner( + &prune_config, + db.clone(), + snapshotter.highest_snapshot_receiver(), + ); let events = pruner.events(); hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); @@ -466,13 +476,6 @@ impl NodeCommand { Either::Right(stream::empty()) }; - let _snapshotter = reth_snapshot::Snapshotter::new( - db, - self.chain.clone(), - self.chain.snapshot_block_interval, - highest_snapshots_tx, - ); - // Configure the consensus engine let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( client, diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 51c9940120b8..d47f4f69f902 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -74,12 +74,9 @@ impl SnapshotTargets { impl Snapshotter { /// Creates a new [Snapshotter]. - pub fn new( - db: DB, - chain_spec: Arc, - block_interval: u64, - highest_snapshots_tracker: watch::Sender>, - ) -> Self { + pub fn new(db: DB, chain_spec: Arc, block_interval: u64) -> Self { + let (highest_snapshots_tracker, _) = watch::channel(None); + let snapshotter = Self { provider_factory: ProviderFactory::new(db, chain_spec), // TODO(alexey): fill from on-disk snapshot data @@ -112,6 +109,11 @@ impl Snapshotter { }); } + /// Returns a new [`HighestSnapshotsTracker`]. + pub fn highest_snapshot_receiver(&self) -> HighestSnapshotsTracker { + self.highest_snapshots_tracker.subscribe() + } + /// Run the snapshotter pub fn run(&mut self, targets: SnapshotTargets) -> SnapshotterResult { debug_assert!(targets.is_multiple_of_block_interval(self.block_interval)); @@ -234,17 +236,17 @@ mod tests { }; use reth_primitives::{snapshot::HighestSnapshots, B256, MAINNET}; use reth_stages::test_utils::TestTransaction; - use tokio::sync::watch; #[test] fn new() { let tx = TestTransaction::default(); - let (highest_snapshots_tx, highest_snapshots_rx) = watch::channel(None); - assert_eq!(*highest_snapshots_rx.borrow(), None); + let snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2); - Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2, highest_snapshots_tx); - assert_eq!(*highest_snapshots_rx.borrow(), Some(HighestSnapshots::default())); + assert_eq!( + *snapshotter.highest_snapshot_receiver().borrow(), + Some(HighestSnapshots::default()) + ); } #[test] @@ -255,8 +257,7 @@ mod tests { let blocks = random_block_range(&mut rng, 0..=3, B256::ZERO, 2..3); tx.insert_blocks(blocks.iter(), None).expect("insert blocks"); - let mut snapshotter = - Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2, watch::channel(None).0); + let mut snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2); // Snapshot targets has data per part up to the passed finalized block number, // respecting the block interval From 2fef51f286f9c660d74e92f8a96bc39fe91b6164 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 2 Nov 2023 16:53:04 +0000 Subject: [PATCH 09/49] add default receiver to snapshotter --- crates/snapshot/src/snapshotter.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index d47f4f69f902..3ba37d5f1316 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -18,9 +18,14 @@ pub type SnapshotterWithResult = (Snapshotter, SnapshotterResult); /// Snapshotting routine. Main snapshotting logic happens in [Snapshotter::run]. #[derive(Debug)] pub struct Snapshotter { + /// Provider factory provider_factory: ProviderFactory, + /// Highest snapshot block number for each highest_snapshots: HighestSnapshots, - highest_snapshots_tracker: watch::Sender>, + /// Channel sender to notify other components of the new highest snapshot values + highest_snapshots_notifier: watch::Sender>, + /// Channel receiver to be cloned and shared that already comes with the newest value + highest_snapshots_tracker: HighestSnapshotsTracker, /// Block interval after which the snapshot is taken. block_interval: u64, } @@ -75,12 +80,13 @@ impl SnapshotTargets { impl Snapshotter { /// Creates a new [Snapshotter]. pub fn new(db: DB, chain_spec: Arc, block_interval: u64) -> Self { - let (highest_snapshots_tracker, _) = watch::channel(None); + let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None); let snapshotter = Self { provider_factory: ProviderFactory::new(db, chain_spec), // TODO(alexey): fill from on-disk snapshot data highest_snapshots: HighestSnapshots::default(), + highest_snapshots_notifier, highest_snapshots_tracker, block_interval, }; @@ -104,14 +110,14 @@ impl Snapshotter { } fn update_highest_snapshots_tracker(&self) { - let _ = self.highest_snapshots_tracker.send(Some(self.highest_snapshots)).map_err(|_| { + let _ = self.highest_snapshots_notifier.send(Some(self.highest_snapshots)).map_err(|_| { warn!(target: "snapshot", "Highest snapshots channel closed"); }); } /// Returns a new [`HighestSnapshotsTracker`]. pub fn highest_snapshot_receiver(&self) -> HighestSnapshotsTracker { - self.highest_snapshots_tracker.subscribe() + self.highest_snapshots_tracker.clone() } /// Run the snapshotter From 33dc284d7f263bd8ce0a7021c48fa71cbd09e2c6 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 2 Nov 2023 17:07:23 +0000 Subject: [PATCH 10/49] replace with with_snapshot_provider on db provider --- crates/blockchain-tree/src/blockchain_tree.rs | 2 -- .../provider/src/providers/database/mod.rs | 24 +++++++++++-------- .../src/providers/database/provider.rs | 22 ++++++++--------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/blockchain-tree/src/blockchain_tree.rs b/crates/blockchain-tree/src/blockchain_tree.rs index 4085f649b8e9..bd1d72c971e4 100644 --- a/crates/blockchain-tree/src/blockchain_tree.rs +++ b/crates/blockchain-tree/src/blockchain_tree.rs @@ -1071,7 +1071,6 @@ impl BlockchainTree { let provider = DatabaseProvider::new_rw( self.externals.db.tx_mut()?, self.externals.chain_spec.clone(), - None, ); let (blocks, state) = chain.into_inner(); @@ -1117,7 +1116,6 @@ impl BlockchainTree { let provider = DatabaseProvider::new_rw( self.externals.db.tx_mut()?, self.externals.chain_spec.clone(), - None, ); let tip = provider.last_block_number()?; diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 71fe7e64073a..8c3cce805e55 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -47,11 +47,13 @@ impl ProviderFactory { /// database using different types of providers. Example: [`HeaderProvider`] /// [`BlockHashReader`]. This may fail if the inner read database transaction fails to open. pub fn provider(&self) -> RethResult> { - Ok(DatabaseProvider::new( - self.db.tx()?, - self.chain_spec.clone(), - self.snapshot_provider.clone(), - )) + let mut provider = DatabaseProvider::new(self.db.tx()?, self.chain_spec.clone()); + + if let Some(snapshot_provider) = &self.snapshot_provider { + provider = provider.with_snapshot_provider(snapshot_provider.clone()); + } + + Ok(provider) } /// Returns a provider with a created `DbTxMut` inside, which allows fetching and updating @@ -59,11 +61,13 @@ impl ProviderFactory { /// [`BlockHashReader`]. This may fail if the inner read/write database transaction fails to /// open. pub fn provider_rw(&self) -> RethResult> { - Ok(DatabaseProviderRW(DatabaseProvider::new_rw( - self.db.tx_mut()?, - self.chain_spec.clone(), - self.snapshot_provider.clone(), - ))) + let mut provider = DatabaseProvider::new_rw(self.db.tx_mut()?, self.chain_spec.clone()); + + if let Some(snapshot_provider) = &self.snapshot_provider { + provider = provider.with_snapshot_provider(snapshot_provider.clone()); + } + + Ok(DatabaseProviderRW(provider)) } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index fe63caac61d6..452ac4247cf6 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -104,12 +104,8 @@ pub struct DatabaseProvider { impl DatabaseProvider { /// Creates a provider with an inner read-write transaction. - pub fn new_rw( - tx: TX, - chain_spec: Arc, - snapshot_provider: Option>, - ) -> Self { - Self { tx, chain_spec, snapshot_provider } + pub fn new_rw(tx: TX, chain_spec: Arc) -> Self { + Self { tx, chain_spec, snapshot_provider: None } } } @@ -163,12 +159,14 @@ where impl DatabaseProvider { /// Creates a provider with an inner read-only transaction. - pub fn new( - tx: TX, - chain_spec: Arc, - snapshot_provider: Option>, - ) -> Self { - Self { tx, chain_spec, snapshot_provider } + pub fn new(tx: TX, chain_spec: Arc) -> Self { + Self { tx, chain_spec, snapshot_provider: None } + } + + /// Creates a new [`Self`] with access to a [`SnapshotProvider`]. + pub fn with_snapshot_provider(mut self, snapshot_provider: Arc) -> Self { + self.snapshot_provider = Some(snapshot_provider); + self } /// Consume `DbTx` or `DbTxMut`. From cda1ca79dcc202bd26e010f611b155206ca7df1a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 13:34:56 +0000 Subject: [PATCH 11/49] use strum for SnapshotSegment, compression and filters --- crates/primitives/src/snapshot/compression.rs | 8 ++- crates/primitives/src/snapshot/filters.rs | 9 ++- crates/primitives/src/snapshot/segment.rs | 56 ++++++++++--------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/crates/primitives/src/snapshot/compression.rs b/crates/primitives/src/snapshot/compression.rs index c67e3f63bc9a..69fe4b2a4328 100644 --- a/crates/primitives/src/snapshot/compression.rs +++ b/crates/primitives/src/snapshot/compression.rs @@ -1,11 +1,17 @@ -#[derive(Debug, Copy, Clone, Default)] +use strum::AsRefStr; + +#[derive(Debug, Copy, Clone, Default, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] #[allow(missing_docs)] /// Snapshot compression pub enum Compression { + #[strum(serialize = "lz4")] Lz4, + #[strum(serialize = "zstd")] Zstd, + #[strum(serialize = "zstd-dict")] ZstdWithDictionary, + #[strum(serialize = "uncompressed")] #[default] Uncompressed, } diff --git a/crates/primitives/src/snapshot/filters.rs b/crates/primitives/src/snapshot/filters.rs index e9716ac707da..3443d474706e 100644 --- a/crates/primitives/src/snapshot/filters.rs +++ b/crates/primitives/src/snapshot/filters.rs @@ -1,3 +1,5 @@ +use strum::AsRefStr; + #[derive(Debug, Copy, Clone)] /// Snapshot filters. pub enum Filters { @@ -14,20 +16,23 @@ impl Filters { } } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] /// Snapshot inclusion filter. Also see [Filters]. pub enum InclusionFilter { + #[strum(serialize = "cuckoo")] /// Cuckoo filter Cuckoo, } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, AsRefStr)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] /// Snapshot perfect hashing function. Also see [Filters]. pub enum PerfectHashingFunction { + #[strum(serialize = "fmph")] /// Fingerprint-Based Minimal Perfect Hash Function Fmph, + #[strum(serialize = "gofmph")] /// Fingerprint-Based Minimal Perfect Hash Function with Group Optimization GoFmph, } diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 8a86768ede68..318d7501b1bb 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -1,17 +1,35 @@ -use crate::{snapshot::PerfectHashingFunction, BlockNumber, TxNumber}; +use crate::{ + snapshot::{Compression, Filters, InclusionFilter}, + BlockNumber, TxNumber, +}; use serde::{Deserialize, Serialize}; -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{ops::RangeInclusive, path::PathBuf, str::FromStr}; +use strum::{AsRefStr, EnumString}; -use super::{Compression, Filters, InclusionFilter}; - -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +#[derive( + Debug, + Copy, + Clone, + Eq, + PartialEq, + Hash, + Ord, + PartialOrd, + Deserialize, + Serialize, + EnumString, + AsRefStr, +)] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] /// Segment of the data that can be snapshotted. pub enum SnapshotSegment { + #[strum(serialize = "headers")] /// Snapshot segment responsible for the `CanonicalHeaders`, `Headers`, `HeaderTD` tables. Headers, + #[strum(serialize = "transactions")] /// Snapshot segment responsible for the `Transactions` table. Transactions, + #[strum(serialize = "receipts")] /// Snapshot segment responsible for the `Receipts` table. Receipts, } @@ -44,35 +62,23 @@ impl SnapshotSegment { compression: Compression, range: &RangeInclusive, ) -> PathBuf { - let segment_name = match self { - SnapshotSegment::Headers => "headers", - SnapshotSegment::Transactions => "transactions", - SnapshotSegment::Receipts => "receipts", - }; + let segment_name = self.as_ref(); + let filters_name = match filters { Filters::WithFilters(inclusion_filter, phf) => { - let inclusion_filter = match inclusion_filter { - InclusionFilter::Cuckoo => "cuckoo", - }; - let phf = match phf { - PerfectHashingFunction::Fmph => "fmph", - PerfectHashingFunction::GoFmph => "gofmph", - }; - format!("{inclusion_filter}-{phf}") + format!("{}-{}", inclusion_filter.as_ref(), phf.as_ref()) } Filters::WithoutFilters => "none".to_string(), }; - let compression_name = match compression { - Compression::Lz4 => "lz4", - Compression::Zstd => "zstd", - Compression::ZstdWithDictionary => "zstd-dict", - Compression::Uncompressed => "uncompressed", - }; + // ATTENTION: if changing the name format, be sure to reflect those changes in + // [`Self::parse_filename`.] format!( - "snapshot_{segment_name}_{}_{}_{filters_name}_{compression_name}", + "snapshot_{segment_name}_{}_{}_{}_{}", range.start(), range.end(), + filters_name, + compression.as_ref() ) .into() } From ec89704e1f99425ae9a866feaf7c37de2f259922 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 13:36:06 +0000 Subject: [PATCH 12/49] snapshotter takes a directory and reads from it --- Cargo.lock | 1 + bin/reth/src/node/mod.rs | 5 +- crates/primitives/src/snapshot/segment.rs | 16 +++++ crates/snapshot/Cargo.toml | 2 +- crates/snapshot/src/snapshotter.rs | 65 +++++++++++++++---- .../provider/src/providers/database/mod.rs | 5 +- .../src/providers/snapshot/manager.rs | 11 +++- 7 files changed, 88 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05c28fd0dc68..166cd3a4abbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6543,6 +6543,7 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-stages", + "tempfile", "thiserror", "tokio", "tracing", diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index da4fe6a4ea76..108031f556a7 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -297,13 +297,14 @@ impl NodeCommand { // configure snapshotter let snapshotter = reth_snapshot::Snapshotter::new( db.clone(), + data_dir.snapshots_path(), self.chain.clone(), self.chain.snapshot_block_interval, - ); + )?; // setup the blockchain provider let factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)) - .with_snapshots(snapshotter.highest_snapshot_receiver()); + .with_snapshots(data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver()); let blockchain_db = BlockchainProvider::new(factory, blockchain_tree.clone())?; let blob_store = InMemoryBlobStore::default(); let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 318d7501b1bb..e59268473cb1 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -82,6 +82,22 @@ impl SnapshotSegment { ) .into() } + + /// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range. + pub fn parse_filename(name: &str) -> Option<(Self, RangeInclusive)> { + let parts: Vec<&str> = name.split('_').collect(); + if let (Ok(segment), true) = (Self::from_str(parts[1]), parts.len() >= 4) { + let start: u64 = parts[2].parse().unwrap_or(0); + let end: u64 = parts[3].parse().unwrap_or(0); + + if start <= end || parts[0] != "snapshot" { + return None + } + + return Some((segment, start..=end)) + } + None + } } /// A segment header that contains information common to all segments. Used for storage. diff --git a/crates/snapshot/Cargo.toml b/crates/snapshot/Cargo.toml index 5597331fd2d2..6455863348c4 100644 --- a/crates/snapshot/Cargo.toml +++ b/crates/snapshot/Cargo.toml @@ -32,7 +32,7 @@ reth-db = { workspace = true, features = ["test-utils"] } reth-stages = { path = "../stages", features = ["test-utils"] } # misc - +tempfile.workspace = true assert_matches.workspace = true [features] diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 3ba37d5f1316..e818b894c2fa 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -3,9 +3,11 @@ use crate::SnapshotterError; use reth_db::database::Database; use reth_interfaces::{RethError, RethResult}; -use reth_primitives::{snapshot::HighestSnapshots, BlockNumber, ChainSpec, TxNumber}; +use reth_primitives::{ + snapshot::HighestSnapshots, BlockNumber, ChainSpec, SnapshotSegment, TxNumber, +}; use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory}; -use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; +use std::{collections::HashMap, ops::RangeInclusive, path::PathBuf, sync::Arc}; use tokio::sync::watch; use tracing::warn; @@ -20,6 +22,8 @@ pub type SnapshotterWithResult = (Snapshotter, SnapshotterResult); pub struct Snapshotter { /// Provider factory provider_factory: ProviderFactory, + /// Directory where snapshots are located + snapshots_path: PathBuf, /// Highest snapshot block number for each highest_snapshots: HighestSnapshots, /// Channel sender to notify other components of the new highest snapshot values @@ -79,11 +83,17 @@ impl SnapshotTargets { impl Snapshotter { /// Creates a new [Snapshotter]. - pub fn new(db: DB, chain_spec: Arc, block_interval: u64) -> Self { + pub fn new( + db: DB, + snapshots_path: PathBuf, + chain_spec: Arc, + block_interval: u64, + ) -> RethResult { let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None); - let snapshotter = Self { + let mut snapshotter = Self { provider_factory: ProviderFactory::new(db, chain_spec), + snapshots_path, // TODO(alexey): fill from on-disk snapshot data highest_snapshots: HighestSnapshots::default(), highest_snapshots_notifier, @@ -91,9 +101,9 @@ impl Snapshotter { block_interval, }; - snapshotter.update_highest_snapshots_tracker(); + snapshotter.update_highest_snapshots_tracker()?; - snapshotter + Ok(snapshotter) } #[cfg(test)] @@ -109,10 +119,38 @@ impl Snapshotter { } } - fn update_highest_snapshots_tracker(&self) { + /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and + /// notifies every tracker. + fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> { + // It walks over the directory and parses the snapshot filenames extracing `SnapshotSegment` + // and their inclusive range. It then takes the maximum block number for each specific + // segment. + for (segment, range) in std::fs::read_dir(&self.snapshots_path) + .map_err(|err| RethError::Custom(err.to_string()))? + .filter_map(Result::ok) + .filter_map(|entry| { + if let Ok(true) = entry.metadata().map(|metadata| metadata.is_file()) { + return SnapshotSegment::parse_filename(&entry.file_name().to_string_lossy()) + } + None + }) + { + let max_segment_block = match segment { + SnapshotSegment::Headers => &mut self.highest_snapshots.headers, + SnapshotSegment::Transactions => &mut self.highest_snapshots.transactions, + SnapshotSegment::Receipts => &mut self.highest_snapshots.receipts, + }; + + if max_segment_block.map_or(true, |block| block < *range.end()) { + *max_segment_block = Some(*range.end()); + } + } + let _ = self.highest_snapshots_notifier.send(Some(self.highest_snapshots)).map_err(|_| { warn!(target: "snapshot", "Highest snapshots channel closed"); }); + + Ok(()) } /// Returns a new [`HighestSnapshotsTracker`]. @@ -127,7 +165,7 @@ impl Snapshotter { // TODO(alexey): snapshot logic - self.update_highest_snapshots_tracker(); + self.update_highest_snapshots_tracker()?; Ok(targets) } @@ -246,8 +284,10 @@ mod tests { #[test] fn new() { let tx = TestTransaction::default(); - - let snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2); + let snapshots_dir = tempfile::TempDir::new().unwrap(); + let snapshotter = + Snapshotter::new(tx.inner_raw(), snapshots_dir.into_path(), MAINNET.clone(), 2) + .unwrap(); assert_eq!( *snapshotter.highest_snapshot_receiver().borrow(), @@ -258,12 +298,15 @@ mod tests { #[test] fn get_snapshot_targets() { let tx = TestTransaction::default(); + let snapshots_dir = tempfile::TempDir::new().unwrap(); let mut rng = generators::rng(); let blocks = random_block_range(&mut rng, 0..=3, B256::ZERO, 2..3); tx.insert_blocks(blocks.iter(), None).expect("insert blocks"); - let mut snapshotter = Snapshotter::new(tx.inner_raw(), MAINNET.clone(), 2); + let mut snapshotter = + Snapshotter::new(tx.inner_raw(), snapshots_dir.into_path(), MAINNET.clone(), 2) + .unwrap(); // Snapshot targets has data per part up to the passed finalized block number, // respecting the block interval diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 8c3cce805e55..ccbe0a97b42a 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -21,6 +21,7 @@ use reth_primitives::{ use revm::primitives::{BlockEnv, CfgEnv}; use std::{ ops::{RangeBounds, RangeInclusive}, + path::PathBuf, sync::Arc, }; use tokio::sync::watch; @@ -80,10 +81,12 @@ impl ProviderFactory { /// database provider comes with a shared snapshot provider pub fn with_snapshots( mut self, + snapshots_path: PathBuf, highest_snapshot_tracker: watch::Receiver>, ) -> Self { self.snapshot_provider = Some(Arc::new( - SnapshotProvider::default().with_highest_tracker(Some(highest_snapshot_tracker)), + SnapshotProvider::new(snapshots_path) + .with_highest_tracker(Some(highest_snapshot_tracker)), )); self } diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 946e8349a7cb..f4c19d24f1e4 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -20,9 +20,16 @@ pub struct SnapshotProvider { map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, /// Tracks the latest and highest snapshot of every segment. highest_tracker: Option>>, + /// Directory where snapshots are located + path: PathBuf, } impl SnapshotProvider { + /// Creates a new [`SnapshotProvider`]. + pub fn new(path: PathBuf) -> Self { + Self { map: Default::default(), highest_tracker: None, path } + } + /// Adds a highest snapshot tracker to the provider pub fn with_highest_tracker( mut self, @@ -50,9 +57,9 @@ impl SnapshotProvider { if let Some(path) = &path { self.map.insert(key, LoadedJar::new(NippyJar::load(path)?)?); } else { - path = Some(segment.filename( + path = Some(self.path.join(segment.filename( &((snapshot * BLOCKS_PER_SNAPSHOT)..=((snapshot + 1) * BLOCKS_PER_SNAPSHOT - 1)), - )); + ))); } self.get_segment_provider(segment, block, path) From e87c64d6c3679dfc8aa989254f9d1047c2d8e3a7 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 14:10:57 +0000 Subject: [PATCH 13/49] default snapshot filename doesnt have configuration --- bin/reth/src/db/snapshots/headers.rs | 39 ++++++++++++++------- bin/reth/src/db/snapshots/receipts.rs | 41 ++++++++++++++--------- bin/reth/src/db/snapshots/transactions.rs | 40 +++++++++++++--------- crates/primitives/src/snapshot/segment.rs | 26 ++++++-------- crates/snapshot/src/segments/mod.rs | 4 +-- 5 files changed, 89 insertions(+), 61 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 6533dd881759..cc3bd0a0b152 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -12,8 +12,11 @@ use reth_primitives::{ use reth_provider::{ providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory, }; -use reth_snapshot::segments::{Headers, Segment}; -use std::{path::Path, sync::Arc}; +use reth_snapshot::{segments, segments::Segment}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_headers_snapshot( @@ -23,15 +26,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = Headers::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Headers::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Headers + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -55,8 +67,9 @@ impl Command { let mut row_indexes = range.clone().collect::>(); let mut rng = rand::thread_rng(); - let path = - SnapshotSegment::Headers.filename_with_configuration(filters, compression, &range); + let path = SnapshotSegment::Headers + .filename_with_configuration(filters, compression, &range) + .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Headers, self.from, Some(path))?; diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index ffe09814e2aa..70a95f31b9b8 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -14,7 +14,10 @@ use reth_provider::{ ReceiptProvider, TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_receipts_snapshot( @@ -24,15 +27,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = segments::Receipts::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Receipts::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Receipts + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -62,11 +74,10 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Receipts.filename_with_configuration( - filters, - compression, - &block_range, - ); + let path = SnapshotSegment::Receipts + .filename_with_configuration(filters, compression, &block_range) + .into(); + let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Receipts, self.from, Some(path))?; diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index a52c33ddb7f9..a2ced80a3f6a 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -14,7 +14,10 @@ use reth_provider::{ TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_transactions_snapshot( @@ -24,15 +27,24 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let segment = segments::Transactions::new( - compression, - if self.with_filters { - Filters::WithFilters(inclusion_filter, phf) - } else { - Filters::WithoutFilters - }, - ); - segment.snapshot::(provider, self.from..=(self.from + self.block_interval - 1))?; + let range = self.from..=(self.from + self.block_interval - 1); + let filters = if self.with_filters { + Filters::WithFilters(inclusion_filter, phf) + } else { + Filters::WithoutFilters + }; + + let segment = segments::Transactions::new(compression, filters); + + segment.snapshot::(provider, range.clone())?; + + // Default name doesn't have any configuration + let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); + let new_name: PathBuf = SnapshotSegment::Transactions + .filename_with_configuration(filters, compression, &range) + .into(); + + std::fs::rename(default_name, new_name)?; Ok(()) } @@ -62,11 +74,9 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Transactions.filename_with_configuration( - filters, - compression, - &block_range, - ); + let path = SnapshotSegment::Transactions + .filename_with_configuration(filters, compression, &block_range) + .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider(SnapshotSegment::Transactions, self.from, Some(path))?; diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index e59268473cb1..1854f10921b2 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -3,7 +3,7 @@ use crate::{ BlockNumber, TxNumber, }; use serde::{Deserialize, Serialize}; -use std::{ops::RangeInclusive, path::PathBuf, str::FromStr}; +use std::{ops::RangeInclusive, str::FromStr}; use strum::{AsRefStr, EnumString}; #[derive( @@ -36,7 +36,7 @@ pub enum SnapshotSegment { impl SnapshotSegment { /// Returns the default configuration of the segment. - const fn config(&self) -> (Filters, Compression) { + pub const fn config(&self) -> (Filters, Compression) { let default_config = ( Filters::WithFilters(InclusionFilter::Cuckoo, super::PerfectHashingFunction::Fmph), Compression::Lz4, @@ -50,19 +50,20 @@ impl SnapshotSegment { } /// Returns the default file name for the provided segment and range. - pub fn filename(&self, range: &RangeInclusive) -> PathBuf { - let (filters, compression) = self.config(); - self.filename_with_configuration(filters, compression, range) + pub fn filename(&self, range: &RangeInclusive) -> String { + // ATTENTION: if changing the name format, be sure to reflect those changes in + // [`Self::parse_filename`.] + format!("snapshot_{}_{}_{}", self.as_ref(), range.start(), range.end(),) } - /// Returns file name for the provided segment, filters, compression and range. + /// Returns file name for the provided segment and range, alongisde filters, compression. pub fn filename_with_configuration( &self, filters: Filters, compression: Compression, range: &RangeInclusive, - ) -> PathBuf { - let segment_name = self.as_ref(); + ) -> String { + let prefix = self.filename(range); let filters_name = match filters { Filters::WithFilters(inclusion_filter, phf) => { @@ -73,14 +74,7 @@ impl SnapshotSegment { // ATTENTION: if changing the name format, be sure to reflect those changes in // [`Self::parse_filename`.] - format!( - "snapshot_{segment_name}_{}_{}_{}_{}", - range.start(), - range.end(), - filters_name, - compression.as_ref() - ) - .into() + format!("{prefix}_{}_{}", filters_name, compression.as_ref()) } /// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range. diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 9a8bb462789f..6293c3896c10 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -19,7 +19,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::ops::RangeInclusive; +use std::{ops::RangeInclusive, path::Path}; pub(crate) type Rows = [Vec>; COLUMNS]; @@ -61,7 +61,7 @@ pub(crate) fn prepare_jar( let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; let mut nippy_jar = NippyJar::new( COLUMNS, - &segment.filename_with_configuration(filters, compression, &block_range), + Path::new(segment.filename(&block_range).as_str()), SegmentHeader::new(block_range, tx_range, segment), ); From 99c1fa505f9eedd8a64cd712c6af4027de040581 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 17:08:58 +0000 Subject: [PATCH 14/49] create snapshots directory if it doesnt exist --- crates/snapshot/src/snapshotter.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index e818b894c2fa..c3ad10710588 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -89,6 +89,16 @@ impl Snapshotter { chain_spec: Arc, block_interval: u64, ) -> RethResult { + // Create directory for snapshots if it doesn't exist. + if !snapshots_path.exists() { + std::fs::create_dir_all(&snapshots_path).map_err(|e| { + RethError::Custom(format!( + "Could not create snapshots directory {}: {e}", + snapshots_path.display() + )) + })?; + } + let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None); let mut snapshotter = Self { From d4b5f61a8168ba53d272491291ba3c1d853f0082 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 18:35:50 +0000 Subject: [PATCH 15/49] add run_segment --- bin/reth/src/db/snapshots/headers.rs | 2 +- bin/reth/src/db/snapshots/receipts.rs | 2 +- bin/reth/src/db/snapshots/transactions.rs | 2 +- crates/snapshot/src/segments/headers.rs | 17 ++++- crates/snapshot/src/segments/mod.rs | 9 ++- crates/snapshot/src/segments/receipts.rs | 17 ++++- crates/snapshot/src/segments/transactions.rs | 17 ++++- crates/snapshot/src/snapshotter.rs | 78 +++++++++++++++++--- 8 files changed, 121 insertions(+), 23 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index cc3bd0a0b152..d97a52b92eaf 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -35,7 +35,7 @@ impl Command { let segment = segments::Headers::new(compression, filters); - segment.snapshot::(provider, range.clone())?; + segment.snapshot::(provider, "".into(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index 70a95f31b9b8..4ded466082c6 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -36,7 +36,7 @@ impl Command { let segment = segments::Receipts::new(compression, filters); - segment.snapshot::(provider, range.clone())?; + segment.snapshot::(provider, "".into(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index a2ced80a3f6a..220e3cfd7a05 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -36,7 +36,7 @@ impl Command { let segment = segments::Transactions::new(compression, filters); - segment.snapshot::(provider, range.clone())?; + segment.snapshot::(provider, "".into(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); diff --git a/crates/snapshot/src/segments/headers.rs b/crates/snapshot/src/segments/headers.rs index 4cc3ced20470..874bf98da4ce 100644 --- a/crates/snapshot/src/segments/headers.rs +++ b/crates/snapshot/src/segments/headers.rs @@ -9,7 +9,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::DatabaseProviderRO; -use std::ops::RangeInclusive; +use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Headers] part of data. #[derive(Debug)] @@ -25,16 +25,29 @@ impl Headers { } } +impl Default for Headers { + fn default() -> Self { + let (filters, compression) = SnapshotSegment::Headers.config(); + Self { compression, filters } + } +} + impl Segment for Headers { + fn segment() -> SnapshotSegment { + SnapshotSegment::Headers + } + fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, + directory: PathBuf, range: RangeInclusive, ) -> RethResult<()> { let range_len = range.clone().count(); let mut jar = prepare_jar::( provider, - SnapshotSegment::Headers, + directory, + Self::segment(), self.filters, self.compression, range.clone(), diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 6293c3896c10..98a733430360 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -19,7 +19,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::{ops::RangeInclusive, path::Path}; +use std::{ops::RangeInclusive, path::PathBuf}; pub(crate) type Rows = [Vec>; COLUMNS]; @@ -29,9 +29,13 @@ pub trait Segment { fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, + directory: PathBuf, range: RangeInclusive, ) -> RethResult<()>; + /// Returns this struct's [`SnapshotSegment`]. + fn segment() -> SnapshotSegment; + /// Generates the dataset to train a zstd dictionary with the most recent rows (at most 1000). fn dataset_for_compression>( &self, @@ -51,6 +55,7 @@ pub trait Segment { /// Returns a [`NippyJar`] according to the desired configuration. pub(crate) fn prepare_jar( provider: &DatabaseProviderRO<'_, DB>, + directory: PathBuf, segment: SnapshotSegment, filters: Filters, compression: Compression, @@ -61,7 +66,7 @@ pub(crate) fn prepare_jar( let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; let mut nippy_jar = NippyJar::new( COLUMNS, - Path::new(segment.filename(&block_range).as_str()), + &directory.join(segment.filename(&block_range).as_str()), SegmentHeader::new(block_range, tx_range, segment), ); diff --git a/crates/snapshot/src/segments/receipts.rs b/crates/snapshot/src/segments/receipts.rs index 4fb2e399d115..d9b00afc3bd3 100644 --- a/crates/snapshot/src/segments/receipts.rs +++ b/crates/snapshot/src/segments/receipts.rs @@ -6,7 +6,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::ops::RangeInclusive; +use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Receipts] part of data. #[derive(Debug)] @@ -15,6 +15,13 @@ pub struct Receipts { filters: Filters, } +impl Default for Receipts { + fn default() -> Self { + let (filters, compression) = SnapshotSegment::Receipts.config(); + Self { compression, filters } + } +} + impl Receipts { /// Creates new instance of [Receipts] snapshot segment. pub fn new(compression: Compression, filters: Filters) -> Self { @@ -23,9 +30,14 @@ impl Receipts { } impl Segment for Receipts { + fn segment() -> SnapshotSegment { + SnapshotSegment::Receipts + } + fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, + directory: PathBuf, block_range: RangeInclusive, ) -> RethResult<()> { let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; @@ -33,7 +45,8 @@ impl Segment for Receipts { let mut jar = prepare_jar::( provider, - SnapshotSegment::Receipts, + directory, + Self::segment(), self.filters, self.compression, block_range, diff --git a/crates/snapshot/src/segments/transactions.rs b/crates/snapshot/src/segments/transactions.rs index 09d120c09db4..a18f73e727b4 100644 --- a/crates/snapshot/src/segments/transactions.rs +++ b/crates/snapshot/src/segments/transactions.rs @@ -6,7 +6,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::ops::RangeInclusive; +use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Transactions] part of data. #[derive(Debug)] @@ -22,10 +22,22 @@ impl Transactions { } } +impl Default for Transactions { + fn default() -> Self { + let (filters, compression) = SnapshotSegment::Transactions.config(); + Self { compression, filters } + } +} + impl Segment for Transactions { + fn segment() -> SnapshotSegment { + SnapshotSegment::Transactions + } + fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, + directory: PathBuf, block_range: RangeInclusive, ) -> RethResult<()> { let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; @@ -33,7 +45,8 @@ impl Segment for Transactions { let mut jar = prepare_jar::( provider, - SnapshotSegment::Transactions, + directory, + Self::segment(), self.filters, self.compression, block_range, diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index c3ad10710588..634e7ef4b45b 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -1,6 +1,6 @@ //! Support for snapshotting. -use crate::SnapshotterError; +use crate::{segments, segments::Segment, SnapshotterError}; use reth_db::database::Database; use reth_interfaces::{RethError, RethResult}; use reth_primitives::{ @@ -17,6 +17,10 @@ pub type SnapshotterResult = Result; /// The snapshotter type itself with the result of [Snapshotter::run] pub type SnapshotterWithResult = (Snapshotter, SnapshotterResult); +/// Snapshots are initially created in `{...}/datadir/snapshots/temp` and moved once finished. This +/// directory is cleaned up on every booting up of the node. +const TEMPORARY_SUBDIRECTORY: &str = "temp"; + /// Snapshotting routine. Main snapshotting logic happens in [Snapshotter::run]. #[derive(Debug)] pub struct Snapshotter { @@ -89,16 +93,6 @@ impl Snapshotter { chain_spec: Arc, block_interval: u64, ) -> RethResult { - // Create directory for snapshots if it doesn't exist. - if !snapshots_path.exists() { - std::fs::create_dir_all(&snapshots_path).map_err(|e| { - RethError::Custom(format!( - "Could not create snapshots directory {}: {e}", - snapshots_path.display() - )) - })?; - } - let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None); let mut snapshotter = Self { @@ -111,11 +105,35 @@ impl Snapshotter { block_interval, }; + snapshotter.create_directory()?; snapshotter.update_highest_snapshots_tracker()?; Ok(snapshotter) } + /// Ensures the snapshots directory and its temporary subdirectory are properly set up. + /// + /// This function performs the following actions: + /// 1. If `datadir/snapshots` does not exist, it creates it. + /// 2. Ensures `datadir/snapshots/temp` exists and is empty. + /// + /// The `temp` subdirectory is where snapshots are initially created before being + /// moved to their final location within `datadir/snapshots`. + fn create_directory(&self) -> RethResult<()> { + let temporary_path = self.snapshots_path.join(TEMPORARY_SUBDIRECTORY); + let err = |e: std::io::Error| RethError::Custom(e.to_string()); + + if !self.snapshots_path.exists() { + std::fs::create_dir_all(&self.snapshots_path).map_err(err)?; + } else if temporary_path.exists() { + std::fs::remove_dir_all(&temporary_path).map_err(err)?; + } + + std::fs::create_dir_all(temporary_path).map_err(err)?; + + Ok(()) + } + #[cfg(test)] fn set_highest_snapshots_from_targets(&mut self, targets: &SnapshotTargets) { if let Some(block_number) = &targets.headers { @@ -173,13 +191,49 @@ impl Snapshotter { debug_assert!(targets.is_multiple_of_block_interval(self.block_interval)); debug_assert!(targets.is_contiguous_to_highest_snapshots(self.highest_snapshots)); - // TODO(alexey): snapshot logic + self.run_segment::( + targets.receipts.as_ref().map(|(range, _)| range.clone()), + )?; + + self.run_segment::( + targets.transactions.as_ref().map(|(range, _)| range.clone()), + )?; + + self.run_segment::(targets.headers.clone())?; self.update_highest_snapshots_tracker()?; Ok(targets) } + /// Run the snapshotter for one segment. + /// + /// It first builds the snapshot in a **temporary directory** inside the snapshots directory. If + /// for some reason the node is terminated during the snapshot process, it will be cleaned + /// up on boot (on [`Snapshotter::new`]) and the snapshot process restarted from scratch for + /// this block range and segment. + /// + /// If it succeeds, then we move the snapshot file from the temporary directory to its main one. + fn run_segment( + &self, + block_range: Option>, + ) -> RethResult<()> { + if let Some(block_range) = block_range { + let temp = self.snapshots_path.join(TEMPORARY_SUBDIRECTORY); + let filename = S::segment().filename(&block_range); + + S::default().snapshot::( + &self.provider_factory.provider()?, + temp.clone(), + block_range.clone(), + )?; + + std::fs::rename(temp.join(&filename), self.snapshots_path.join(filename)) + .map_err(|err| RethError::Custom(err.to_string()))?; + } + Ok(()) + } + /// Returns a snapshot targets at the provided finalized block number, respecting the block /// interval. The target is determined by the check against last snapshots. pub fn get_snapshot_targets( From dd338e50597ff035239455e1c8c07ab0dffb7c25 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 3 Nov 2023 19:03:00 +0000 Subject: [PATCH 16/49] add type SegmentConfig --- crates/primitives/src/snapshot/mod.rs | 2 +- crates/primitives/src/snapshot/segment.rs | 22 +++++++++++++---- crates/snapshot/src/segments/headers.rs | 15 +++++------- crates/snapshot/src/segments/mod.rs | 11 +++++---- crates/snapshot/src/segments/receipts.rs | 25 +++++++++----------- crates/snapshot/src/segments/transactions.rs | 15 +++++------- 6 files changed, 47 insertions(+), 43 deletions(-) diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index f73ba9874d77..8c595c75c1c7 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -7,7 +7,7 @@ mod segment; use alloy_primitives::BlockNumber; pub use compression::Compression; pub use filters::{Filters, InclusionFilter, PerfectHashingFunction}; -pub use segment::{SegmentHeader, SnapshotSegment}; +pub use segment::{SegmentConfig, SegmentHeader, SnapshotSegment}; /// Default snapshot block count. pub const BLOCKS_PER_SNAPSHOT: u64 = 500_000; diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 1854f10921b2..6df100976c02 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -36,11 +36,14 @@ pub enum SnapshotSegment { impl SnapshotSegment { /// Returns the default configuration of the segment. - pub const fn config(&self) -> (Filters, Compression) { - let default_config = ( - Filters::WithFilters(InclusionFilter::Cuckoo, super::PerfectHashingFunction::Fmph), - Compression::Lz4, - ); + pub const fn config(&self) -> SegmentConfig { + let default_config = SegmentConfig { + filters: Filters::WithFilters( + InclusionFilter::Cuckoo, + super::PerfectHashingFunction::Fmph, + ), + compression: Compression::Lz4, + }; match self { SnapshotSegment::Headers => default_config, @@ -133,3 +136,12 @@ impl SegmentHeader { } } } + +/// Configuration used on the segment. +#[derive(Debug, Clone, Copy)] +pub struct SegmentConfig { + /// Inclusion filters used on the segment + pub filters: Filters, + /// Compression used on the segment + pub compression: Compression, +} diff --git a/crates/snapshot/src/segments/headers.rs b/crates/snapshot/src/segments/headers.rs index 874bf98da4ce..b295f0024336 100644 --- a/crates/snapshot/src/segments/headers.rs +++ b/crates/snapshot/src/segments/headers.rs @@ -5,7 +5,7 @@ use reth_db::{ }; use reth_interfaces::RethResult; use reth_primitives::{ - snapshot::{Compression, Filters}, + snapshot::{Compression, Filters, SegmentConfig}, BlockNumber, SnapshotSegment, }; use reth_provider::DatabaseProviderRO; @@ -14,21 +14,19 @@ use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Headers] part of data. #[derive(Debug)] pub struct Headers { - compression: Compression, - filters: Filters, + config: SegmentConfig, } impl Headers { /// Creates new instance of [Headers] snapshot segment. pub fn new(compression: Compression, filters: Filters) -> Self { - Self { compression, filters } + Self { config: SegmentConfig { compression, filters } } } } impl Default for Headers { fn default() -> Self { - let (filters, compression) = SnapshotSegment::Headers.config(); - Self { compression, filters } + Self { config: SnapshotSegment::Headers.config() } } } @@ -48,8 +46,7 @@ impl Segment for Headers { provider, directory, Self::segment(), - self.filters, - self.compression, + self.config, range.clone(), range_len, || { @@ -70,7 +67,7 @@ impl Segment for Headers { // Generate list of hashes for filters & PHF let mut cursor = provider.tx_ref().cursor_read::>()?; let mut hashes = None; - if self.filters.has_filters() { + if self.config.filters.has_filters() { hashes = Some( cursor .walk(Some(RawKey::from(*range.start())))? diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 98a733430360..f3ed9b0c4055 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -15,7 +15,9 @@ use reth_db::{ use reth_interfaces::RethResult; use reth_nippy_jar::NippyJar; use reth_primitives::{ - snapshot::{Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentHeader}, + snapshot::{ + Compression, Filters, InclusionFilter, PerfectHashingFunction, SegmentConfig, SegmentHeader, + }, BlockNumber, SnapshotSegment, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; @@ -57,8 +59,7 @@ pub(crate) fn prepare_jar( provider: &DatabaseProviderRO<'_, DB>, directory: PathBuf, segment: SnapshotSegment, - filters: Filters, - compression: Compression, + segment_config: SegmentConfig, block_range: RangeInclusive, total_rows: usize, prepare_compression: impl Fn() -> RethResult>, @@ -70,7 +71,7 @@ pub(crate) fn prepare_jar( SegmentHeader::new(block_range, tx_range, segment), ); - nippy_jar = match compression { + nippy_jar = match segment_config.compression { Compression::Lz4 => nippy_jar.with_lz4(), Compression::Zstd => nippy_jar.with_zstd(false, 0), Compression::ZstdWithDictionary => { @@ -83,7 +84,7 @@ pub(crate) fn prepare_jar( Compression::Uncompressed => nippy_jar, }; - if let Filters::WithFilters(inclusion_filter, phf) = filters { + if let Filters::WithFilters(inclusion_filter, phf) = segment_config.filters { nippy_jar = match inclusion_filter { InclusionFilter::Cuckoo => nippy_jar.with_cuckoo_filter(total_rows), }; diff --git a/crates/snapshot/src/segments/receipts.rs b/crates/snapshot/src/segments/receipts.rs index d9b00afc3bd3..6e328da460ce 100644 --- a/crates/snapshot/src/segments/receipts.rs +++ b/crates/snapshot/src/segments/receipts.rs @@ -2,7 +2,7 @@ use crate::segments::{prepare_jar, Segment}; use reth_db::{database::Database, snapshot::create_snapshot_T1, tables}; use reth_interfaces::RethResult; use reth_primitives::{ - snapshot::{Compression, Filters, SegmentHeader}, + snapshot::{Compression, Filters, SegmentConfig, SegmentHeader}, BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; @@ -11,21 +11,19 @@ use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Receipts] part of data. #[derive(Debug)] pub struct Receipts { - compression: Compression, - filters: Filters, -} - -impl Default for Receipts { - fn default() -> Self { - let (filters, compression) = SnapshotSegment::Receipts.config(); - Self { compression, filters } - } + config: SegmentConfig, } impl Receipts { /// Creates new instance of [Receipts] snapshot segment. pub fn new(compression: Compression, filters: Filters) -> Self { - Self { compression, filters } + Self { config: SegmentConfig { compression, filters } } + } +} + +impl Default for Receipts { + fn default() -> Self { + Self { config: SnapshotSegment::Receipts.config() } } } @@ -47,8 +45,7 @@ impl Segment for Receipts { provider, directory, Self::segment(), - self.filters, - self.compression, + self.config, block_range, tx_range_len, || { @@ -62,7 +59,7 @@ impl Segment for Receipts { // Generate list of hashes for filters & PHF let mut hashes = None; - if self.filters.has_filters() { + if self.config.filters.has_filters() { hashes = Some( provider .transaction_hashes_by_range(*tx_range.start()..(*tx_range.end() + 1))? diff --git a/crates/snapshot/src/segments/transactions.rs b/crates/snapshot/src/segments/transactions.rs index a18f73e727b4..5042e0eb2bd9 100644 --- a/crates/snapshot/src/segments/transactions.rs +++ b/crates/snapshot/src/segments/transactions.rs @@ -2,7 +2,7 @@ use crate::segments::{prepare_jar, Segment}; use reth_db::{database::Database, snapshot::create_snapshot_T1, tables}; use reth_interfaces::RethResult; use reth_primitives::{ - snapshot::{Compression, Filters, SegmentHeader}, + snapshot::{Compression, Filters, SegmentConfig, SegmentHeader}, BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; @@ -11,21 +11,19 @@ use std::{ops::RangeInclusive, path::PathBuf}; /// Snapshot segment responsible for [SnapshotSegment::Transactions] part of data. #[derive(Debug)] pub struct Transactions { - compression: Compression, - filters: Filters, + config: SegmentConfig, } impl Transactions { /// Creates new instance of [Transactions] snapshot segment. pub fn new(compression: Compression, filters: Filters) -> Self { - Self { compression, filters } + Self { config: SegmentConfig { compression, filters } } } } impl Default for Transactions { fn default() -> Self { - let (filters, compression) = SnapshotSegment::Transactions.config(); - Self { compression, filters } + Self { config: SnapshotSegment::Transactions.config() } } } @@ -47,8 +45,7 @@ impl Segment for Transactions { provider, directory, Self::segment(), - self.filters, - self.compression, + self.config, block_range, tx_range_len, || { @@ -62,7 +59,7 @@ impl Segment for Transactions { // Generate list of hashes for filters & PHF let mut hashes = None; - if self.filters.has_filters() { + if self.config.filters.has_filters() { hashes = Some( provider .transaction_hashes_by_range(*tx_range.start()..(*tx_range.end() + 1))? From 00ec5f57a2926e983727f6b9fa5207940d2c2c56 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Wed, 8 Nov 2023 16:16:04 +0000 Subject: [PATCH 17/49] add docs for directory --- crates/snapshot/src/segments/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index f3ed9b0c4055..3f5af5d69830 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -27,7 +27,8 @@ pub(crate) type Rows = [Vec>; COLUMNS]; /// A segment represents a snapshotting of some portion of the data. pub trait Segment { - /// Snapshot data using the provided range. + /// Snapshot data using the provided range. The `directory` parameter determines the snapshot + /// file's save location. fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, @@ -54,7 +55,8 @@ pub trait Segment { } } -/// Returns a [`NippyJar`] according to the desired configuration. +/// Returns a [`NippyJar`] according to the desired configuration. The `directory` parameter +/// determines the snapshot file's save location. pub(crate) fn prepare_jar( provider: &DatabaseProviderRO<'_, DB>, directory: PathBuf, From c1dd713bbb6765a2181334c333f795052bc17ef1 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Wed, 8 Nov 2023 16:27:22 +0000 Subject: [PATCH 18/49] replace PathBuf usage --- bin/reth/src/db/snapshots/headers.rs | 2 +- bin/reth/src/db/snapshots/receipts.rs | 2 +- bin/reth/src/db/snapshots/transactions.rs | 2 +- crates/snapshot/src/segments/headers.rs | 4 ++-- crates/snapshot/src/segments/mod.rs | 8 ++++---- crates/snapshot/src/segments/receipts.rs | 4 ++-- crates/snapshot/src/segments/transactions.rs | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index d97a52b92eaf..96160b7b8c60 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -35,7 +35,7 @@ impl Command { let segment = segments::Headers::new(compression, filters); - segment.snapshot::(provider, "".into(), range.clone())?; + segment.snapshot::(provider, PathBuf::default(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index 4ded466082c6..26e75b5dc369 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -36,7 +36,7 @@ impl Command { let segment = segments::Receipts::new(compression, filters); - segment.snapshot::(provider, "".into(), range.clone())?; + segment.snapshot::(provider, PathBuf::default(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index 220e3cfd7a05..c7ac400d0684 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -36,7 +36,7 @@ impl Command { let segment = segments::Transactions::new(compression, filters); - segment.snapshot::(provider, "".into(), range.clone())?; + segment.snapshot::(provider, PathBuf::default(), range.clone())?; // Default name doesn't have any configuration let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); diff --git a/crates/snapshot/src/segments/headers.rs b/crates/snapshot/src/segments/headers.rs index b295f0024336..89fc009f9fd7 100644 --- a/crates/snapshot/src/segments/headers.rs +++ b/crates/snapshot/src/segments/headers.rs @@ -9,7 +9,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::DatabaseProviderRO; -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{ops::RangeInclusive, path::Path}; /// Snapshot segment responsible for [SnapshotSegment::Headers] part of data. #[derive(Debug)] @@ -38,7 +38,7 @@ impl Segment for Headers { fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, - directory: PathBuf, + directory: impl AsRef, range: RangeInclusive, ) -> RethResult<()> { let range_len = range.clone().count(); diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 3f5af5d69830..140ea90fb354 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -21,7 +21,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{ops::RangeInclusive, path::Path}; pub(crate) type Rows = [Vec>; COLUMNS]; @@ -32,7 +32,7 @@ pub trait Segment { fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, - directory: PathBuf, + directory: impl AsRef, range: RangeInclusive, ) -> RethResult<()>; @@ -59,7 +59,7 @@ pub trait Segment { /// determines the snapshot file's save location. pub(crate) fn prepare_jar( provider: &DatabaseProviderRO<'_, DB>, - directory: PathBuf, + directory: impl AsRef, segment: SnapshotSegment, segment_config: SegmentConfig, block_range: RangeInclusive, @@ -69,7 +69,7 @@ pub(crate) fn prepare_jar( let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; let mut nippy_jar = NippyJar::new( COLUMNS, - &directory.join(segment.filename(&block_range).as_str()), + &directory.as_ref().join(segment.filename(&block_range).as_str()), SegmentHeader::new(block_range, tx_range, segment), ); diff --git a/crates/snapshot/src/segments/receipts.rs b/crates/snapshot/src/segments/receipts.rs index 6e328da460ce..75e8aaa8995d 100644 --- a/crates/snapshot/src/segments/receipts.rs +++ b/crates/snapshot/src/segments/receipts.rs @@ -6,7 +6,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{ops::RangeInclusive, path::Path}; /// Snapshot segment responsible for [SnapshotSegment::Receipts] part of data. #[derive(Debug)] @@ -35,7 +35,7 @@ impl Segment for Receipts { fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, - directory: PathBuf, + directory: impl AsRef, block_range: RangeInclusive, ) -> RethResult<()> { let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; diff --git a/crates/snapshot/src/segments/transactions.rs b/crates/snapshot/src/segments/transactions.rs index 5042e0eb2bd9..b9cccfd20d37 100644 --- a/crates/snapshot/src/segments/transactions.rs +++ b/crates/snapshot/src/segments/transactions.rs @@ -6,7 +6,7 @@ use reth_primitives::{ BlockNumber, SnapshotSegment, TxNumber, }; use reth_provider::{DatabaseProviderRO, TransactionsProviderExt}; -use std::{ops::RangeInclusive, path::PathBuf}; +use std::{ops::RangeInclusive, path::Path}; /// Snapshot segment responsible for [SnapshotSegment::Transactions] part of data. #[derive(Debug)] @@ -35,7 +35,7 @@ impl Segment for Transactions { fn snapshot( &self, provider: &DatabaseProviderRO<'_, DB>, - directory: PathBuf, + directory: impl AsRef, block_range: RangeInclusive, ) -> RethResult<()> { let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; From 704bbfd0a05ff409c44afc0639aa8382aca50424 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 9 Nov 2023 19:47:48 +0000 Subject: [PATCH 19/49] snapshot file name contains tx range as well --- bin/reth/src/cli/mod.rs | 2 +- bin/reth/src/db/snapshots/headers.rs | 19 +++++++---- bin/reth/src/db/snapshots/receipts.rs | 15 +++++---- bin/reth/src/db/snapshots/transactions.rs | 12 ++++--- crates/interfaces/src/provider.rs | 3 ++ crates/primitives/src/snapshot/mod.rs | 21 +++++++++++- crates/primitives/src/snapshot/segment.rs | 41 +++++++++++++++++------ crates/snapshot/src/segments/mod.rs | 2 +- crates/snapshot/src/snapshotter.rs | 28 ++++++---------- examples/network.rs | 2 +- 10 files changed, 95 insertions(+), 50 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index d48afb627cc5..1361b53d2d11 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -363,7 +363,7 @@ mod tests { let end = format!("reth/logs/{}", SUPPORTED_CHAINS[0]); assert!(log_dir.as_ref().ends_with(end), "{:?}", log_dir); - let mut iter = SUPPORTED_CHAINS.into_iter(); + let mut iter = SUPPORTED_CHAINS.iter(); iter.next(); for chain in iter { let mut reth = Cli::<()>::try_parse_from(["reth", "node", "--chain", chain]).unwrap(); diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 96160b7b8c60..443b7e1c1aa9 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -10,7 +10,8 @@ use reth_primitives::{ BlockHash, ChainSpec, Header, SnapshotSegment, }; use reth_provider::{ - providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory, + providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, + ProviderFactory, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; use std::{ @@ -38,9 +39,10 @@ impl Command { segment.snapshot::(provider, PathBuf::default(), range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); + let tx_range = provider.transaction_range_by_block_range(range.clone())?; + let default_name: PathBuf = SnapshotSegment::Headers.filename(&range, &tx_range).into(); let new_name: PathBuf = SnapshotSegment::Headers - .filename_with_configuration(filters, compression, &range) + .filename_with_configuration(filters, compression, &range, &tx_range) .into(); std::fs::rename(default_name, new_name)?; @@ -63,12 +65,17 @@ impl Command { Filters::WithoutFilters }; - let range = self.from..=(self.from + self.block_interval - 1); + let block_range = self.from..=(self.from + self.block_interval - 1); - let mut row_indexes = range.clone().collect::>(); + let mut row_indexes = block_range.clone().collect::>(); let mut rng = rand::thread_rng(); + + let tx_range = ProviderFactory::new(open_db_read_only(db_path, log_level)?, chain.clone()) + .provider()? + .transaction_range_by_block_range(block_range.clone())?; + let path = SnapshotSegment::Headers - .filename_with_configuration(filters, compression, &range) + .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); let jar_provider = diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index 26e75b5dc369..c7280d45a8c6 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -27,21 +27,22 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let range = self.from..=(self.from + self.block_interval - 1); + let block_range = self.from..=(self.from + self.block_interval - 1); let filters = if self.with_filters { Filters::WithFilters(inclusion_filter, phf) } else { Filters::WithoutFilters }; - let segment = segments::Receipts::new(compression, filters); - - segment.snapshot::(provider, PathBuf::default(), range.clone())?; + let segment: segments::Receipts = segments::Receipts::new(compression, filters); + segment.snapshot::(provider, PathBuf::default(), block_range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); + let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; + let default_name: PathBuf = + SnapshotSegment::Receipts.filename(&block_range, &tx_range).into(); let new_name: PathBuf = SnapshotSegment::Receipts - .filename_with_configuration(filters, compression, &range) + .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); std::fs::rename(default_name, new_name)?; @@ -75,7 +76,7 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); let path = SnapshotSegment::Receipts - .filename_with_configuration(filters, compression, &block_range) + .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index c7ac400d0684..f9245e2961d6 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -27,7 +27,7 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let range = self.from..=(self.from + self.block_interval - 1); + let block_range = self.from..=(self.from + self.block_interval - 1); let filters = if self.with_filters { Filters::WithFilters(inclusion_filter, phf) } else { @@ -36,12 +36,14 @@ impl Command { let segment = segments::Transactions::new(compression, filters); - segment.snapshot::(provider, PathBuf::default(), range.clone())?; + segment.snapshot::(provider, PathBuf::default(), block_range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); + let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; + let default_name: PathBuf = + SnapshotSegment::Transactions.filename(&block_range, &tx_range).into(); let new_name: PathBuf = SnapshotSegment::Transactions - .filename_with_configuration(filters, compression, &range) + .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); std::fs::rename(default_name, new_name)?; @@ -75,7 +77,7 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); let path = SnapshotSegment::Transactions - .filename_with_configuration(filters, compression, &block_range) + .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); let jar_provider = diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index da079fcdd1e2..645b70fc3577 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -94,6 +94,9 @@ pub enum ProviderError { /// Provider does not support this particular request. #[error("this provider does not support this request")] UnsupportedProvider, + /// Snapshot file is not found for block. + #[error("not able to find snapshot file for block {0}")] + MissingSnapshot(BlockNumber), } /// A root mismatch error at a given block height. diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index 8c595c75c1c7..ed3f1a3a41d7 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -4,11 +4,13 @@ mod compression; mod filters; mod segment; -use alloy_primitives::BlockNumber; +use alloy_primitives::{BlockNumber, TxNumber}; pub use compression::Compression; pub use filters::{Filters, InclusionFilter, PerfectHashingFunction}; pub use segment::{SegmentConfig, SegmentHeader, SnapshotSegment}; +use std::{ops::RangeInclusive, path::Path}; + /// Default snapshot block count. pub const BLOCKS_PER_SNAPSHOT: u64 = 500_000; @@ -36,3 +38,20 @@ impl HighestSnapshots { } } } + +/// Given the snapshot's location, it returns an iterator over the existing snapshots in the format +/// of a tuple composed by the segment, block range and transaction range. +pub fn iter_snapshots( + path: impl AsRef, +) -> Result< + impl Iterator, RangeInclusive)>, + std::io::Error, +> { + let entries = std::fs::read_dir(path.as_ref())?.filter_map(Result::ok); + Ok(entries.filter_map(|entry| { + if let Ok(true) = entry.metadata().map(|metadata| metadata.is_file()) { + return SnapshotSegment::parse_filename(&entry.file_name().to_string_lossy()) + } + None + })) +} diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 6df100976c02..32ca75f65aa8 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -53,10 +53,21 @@ impl SnapshotSegment { } /// Returns the default file name for the provided segment and range. - pub fn filename(&self, range: &RangeInclusive) -> String { + pub fn filename( + &self, + block_range: &RangeInclusive, + tx_range: &RangeInclusive, + ) -> String { // ATTENTION: if changing the name format, be sure to reflect those changes in // [`Self::parse_filename`.] - format!("snapshot_{}_{}_{}", self.as_ref(), range.start(), range.end(),) + format!( + "snapshot_{}_{}_{}_{}_{}", + self.as_ref(), + block_range.start(), + block_range.end(), + tx_range.start(), + tx_range.end(), + ) } /// Returns file name for the provided segment and range, alongisde filters, compression. @@ -64,9 +75,10 @@ impl SnapshotSegment { &self, filters: Filters, compression: Compression, - range: &RangeInclusive, + block_range: &RangeInclusive, + tx_range: &RangeInclusive, ) -> String { - let prefix = self.filename(range); + let prefix = self.filename(block_range, tx_range); let filters_name = match filters { Filters::WithFilters(inclusion_filter, phf) => { @@ -81,17 +93,26 @@ impl SnapshotSegment { } /// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range. - pub fn parse_filename(name: &str) -> Option<(Self, RangeInclusive)> { + pub fn parse_filename( + name: &str, + ) -> Option<(Self, RangeInclusive, RangeInclusive)> { let parts: Vec<&str> = name.split('_').collect(); - if let (Ok(segment), true) = (Self::from_str(parts[1]), parts.len() >= 4) { - let start: u64 = parts[2].parse().unwrap_or(0); - let end: u64 = parts[3].parse().unwrap_or(0); + if let (Ok(segment), true) = (Self::from_str(parts[1]), parts.len() >= 6) { + let block_start: u64 = parts[2].parse().unwrap_or(0); + let block_end: u64 = parts[3].parse().unwrap_or(0); + + if block_start <= block_end || parts[0] != "snapshot" { + return None + } + + let tx_start: u64 = parts[4].parse().unwrap_or(0); + let tx_end: u64 = parts[5].parse().unwrap_or(0); - if start <= end || parts[0] != "snapshot" { + if tx_start <= tx_end { return None } - return Some((segment, start..=end)) + return Some((segment, block_start..=block_end, tx_start..=tx_end)) } None } diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 140ea90fb354..6238c106ed2d 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -69,7 +69,7 @@ pub(crate) fn prepare_jar( let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; let mut nippy_jar = NippyJar::new( COLUMNS, - &directory.as_ref().join(segment.filename(&block_range).as_str()), + &directory.as_ref().join(segment.filename(&block_range, &tx_range).as_str()), SegmentHeader::new(block_range, tx_range, segment), ); diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 634e7ef4b45b..1779acd856e6 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -4,9 +4,10 @@ use crate::{segments, segments::Segment, SnapshotterError}; use reth_db::database::Database; use reth_interfaces::{RethError, RethResult}; use reth_primitives::{ - snapshot::HighestSnapshots, BlockNumber, ChainSpec, SnapshotSegment, TxNumber, + snapshot::{iter_snapshots, HighestSnapshots}, + BlockNumber, ChainSpec, SnapshotSegment, TxNumber, }; -use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory}; +use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory, TransactionsProviderExt}; use std::{collections::HashMap, ops::RangeInclusive, path::PathBuf, sync::Arc}; use tokio::sync::watch; use tracing::warn; @@ -153,15 +154,8 @@ impl Snapshotter { // It walks over the directory and parses the snapshot filenames extracing `SnapshotSegment` // and their inclusive range. It then takes the maximum block number for each specific // segment. - for (segment, range) in std::fs::read_dir(&self.snapshots_path) + for (segment, block_range, _) in iter_snapshots(&self.snapshots_path) .map_err(|err| RethError::Custom(err.to_string()))? - .filter_map(Result::ok) - .filter_map(|entry| { - if let Ok(true) = entry.metadata().map(|metadata| metadata.is_file()) { - return SnapshotSegment::parse_filename(&entry.file_name().to_string_lossy()) - } - None - }) { let max_segment_block = match segment { SnapshotSegment::Headers => &mut self.highest_snapshots.headers, @@ -169,8 +163,8 @@ impl Snapshotter { SnapshotSegment::Receipts => &mut self.highest_snapshots.receipts, }; - if max_segment_block.map_or(true, |block| block < *range.end()) { - *max_segment_block = Some(*range.end()); + if max_segment_block.map_or(true, |block| block < *block_range.end()) { + *max_segment_block = Some(*block_range.end()); } } @@ -220,13 +214,11 @@ impl Snapshotter { ) -> RethResult<()> { if let Some(block_range) = block_range { let temp = self.snapshots_path.join(TEMPORARY_SUBDIRECTORY); - let filename = S::segment().filename(&block_range); + let provider = self.provider_factory.provider()?; + let tx_range = provider.transaction_range_by_block_range(block_range.clone())?; + let filename = S::segment().filename(&block_range, &tx_range); - S::default().snapshot::( - &self.provider_factory.provider()?, - temp.clone(), - block_range.clone(), - )?; + S::default().snapshot::(&provider, temp.clone(), block_range)?; std::fs::rename(temp.join(&filename), self.snapshots_path.join(filename)) .map_err(|err| RethError::Custom(err.to_string()))?; diff --git a/examples/network.rs b/examples/network.rs index 8fe6a6a8ec78..18bf5cbcf981 100644 --- a/examples/network.rs +++ b/examples/network.rs @@ -7,7 +7,7 @@ //! ``` use futures::StreamExt; -use reth_network::{config::rng_secret_key, NetworkConfig, NetworkManager}; +use reth_network::{config::rng_secret_key, NetworkConfig, NetworkEvents, NetworkManager}; use reth_provider::test_utils::NoopProvider; #[tokio::main] From d21f5cd5fc9231c54c3b92638a3108f7ce765bb9 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 9 Nov 2023 20:15:22 +0000 Subject: [PATCH 20/49] add get_segment_provider from block and from tx --- bin/reth/src/db/snapshots/headers.rs | 7 +- bin/reth/src/db/snapshots/receipts.rs | 7 +- bin/reth/src/db/snapshots/transactions.rs | 7 +- crates/interfaces/src/provider.rs | 4 +- .../src/providers/snapshot/manager.rs | 169 +++++++++++++++--- .../provider/src/providers/snapshot/mod.rs | 6 +- 6 files changed, 164 insertions(+), 36 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 443b7e1c1aa9..8b31ace853cb 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -78,8 +78,11 @@ impl Command { .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); - let jar_provider = - provider.get_segment_provider(SnapshotSegment::Headers, self.from, Some(path))?; + let jar_provider = provider.get_segment_provider_from_block( + SnapshotSegment::Headers, + self.from, + Some(path), + )?; let mut cursor = jar_provider.cursor()?; for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] { diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index c7280d45a8c6..29b4e3397c5b 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -80,8 +80,11 @@ impl Command { .into(); let provider = SnapshotProvider::default(); - let jar_provider = - provider.get_segment_provider(SnapshotSegment::Receipts, self.from, Some(path))?; + let jar_provider = provider.get_segment_provider_from_block( + SnapshotSegment::Receipts, + self.from, + Some(path), + )?; let mut cursor = jar_provider.cursor()?; for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] { diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index f9245e2961d6..6275630d0aac 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -80,8 +80,11 @@ impl Command { .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); - let jar_provider = - provider.get_segment_provider(SnapshotSegment::Transactions, self.from, Some(path))?; + let jar_provider = provider.get_segment_provider_from_block( + SnapshotSegment::Transactions, + self.from, + Some(path), + )?; let mut cursor = jar_provider.cursor()?; for bench_kind in [BenchKind::Walk, BenchKind::RandomAll] { diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index 645b70fc3577..5e67b9025087 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -95,8 +95,8 @@ pub enum ProviderError { #[error("this provider does not support this request")] UnsupportedProvider, /// Snapshot file is not found for block. - #[error("not able to find snapshot file for block {0}")] - MissingSnapshot(BlockNumber), + #[error("not able to find snapshot file")] + MissingSnapshot, } /// A root mismatch error at a given block height. diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index f4c19d24f1e4..2556b59edc8c 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -1,23 +1,41 @@ use super::{LoadedJar, SnapshotJarProvider}; use crate::{BlockHashReader, BlockNumReader, HeaderProvider, TransactionsProvider}; use dashmap::DashMap; -use reth_interfaces::RethResult; +use parking_lot::RwLock; +use reth_interfaces::{provider::ProviderError, RethResult}; use reth_nippy_jar::NippyJar; use reth_primitives::{ - snapshot::{HighestSnapshots, BLOCKS_PER_SNAPSHOT}, - Address, BlockHash, BlockHashOrNumber, BlockNumber, ChainInfo, Header, SealedHeader, - SnapshotSegment, TransactionMeta, TransactionSigned, TransactionSignedNoHash, TxHash, TxNumber, - B256, U256, + snapshot::HighestSnapshots, Address, BlockHash, BlockHashOrNumber, BlockNumber, ChainInfo, + Header, SealedHeader, SnapshotSegment, TransactionMeta, TransactionSigned, + TransactionSignedNoHash, TxHash, TxNumber, B256, U256, +}; +use revm::primitives::HashMap; +use std::{ + collections::BTreeMap, + ops::{RangeBounds, RangeInclusive}, + path::PathBuf, }; -use std::{ops::RangeBounds, path::PathBuf}; use tokio::sync::watch; -/// SnapshotProvider +/// Alias type for a map that can be queried for transaction/block ranges from a block/transaction +/// segment respectively. It uses `BlockNumber` to represent the block end of a snapshot range or +/// `TxNumber` to represent the transaction end of a snapshot range. +/// +/// Can be in one of the two formats: +/// - `HashMap>>` +/// - `HashMap>>` +type SegmentRanges = HashMap>>; + +/// [`SnapshotProvider`] manages all existing [`SnapshotJarProvider`]. #[derive(Debug, Default)] pub struct SnapshotProvider { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, + /// Available snapshot ranges on disk indexed by max blocks. + snapshots_block_index: RwLock, + /// Available snapshot ranges on disk indexed by max transactions. + snapshots_tx_index: RwLock, /// Tracks the latest and highest snapshot of every segment. highest_tracker: Option>>, /// Directory where snapshots are located @@ -27,7 +45,13 @@ pub struct SnapshotProvider { impl SnapshotProvider { /// Creates a new [`SnapshotProvider`]. pub fn new(path: PathBuf) -> Self { - Self { map: Default::default(), highest_tracker: None, path } + Self { + map: Default::default(), + snapshots_block_index: Default::default(), + snapshots_tx_index: Default::default(), + highest_tracker: None, + path, + } } /// Adds a highest snapshot tracker to the provider @@ -39,30 +63,122 @@ impl SnapshotProvider { self } - /// Gets the provider of the requested segment and range. - pub fn get_segment_provider( + /// Gets the [`SnapshotJarProvider`] of the requested segment and block. + pub fn get_segment_provider_from_block( &self, segment: SnapshotSegment, block: BlockNumber, - mut path: Option, + path: Option, ) -> RethResult> { - // TODO this invalidates custom length snapshots. - let snapshot = block / BLOCKS_PER_SNAPSHOT; - let key = (snapshot, segment); + self.get_segment_provider( + segment, + || self.get_segment_ranges_from_block(segment, block), + path, + ) + } - if let Some(jar) = self.map.get(&key) { - return Ok(jar.into()) + /// Gets the [`SnapshotJarProvider`] of the requested segment and transaction. + pub fn get_segment_provider_from_transaction( + &self, + segment: SnapshotSegment, + tx: TxNumber, + path: Option, + ) -> RethResult> { + self.get_segment_provider( + segment, + || self.get_segment_ranges_from_transaction(segment, tx), + path, + ) + } + + /// Gets the [`SnapshotJarProvider`] of the requested segment and block or transaction. + pub fn get_segment_provider( + &self, + segment: SnapshotSegment, + fn_ranges: impl Fn() -> Option<(RangeInclusive, RangeInclusive)>, + path: Option, + ) -> RethResult> { + // If we have a path, then get the block range and transaction range from its name. + // Otherwise, check `self.available_snapshots` + let snapshot_ranges = match path { + Some(path) => SnapshotSegment::parse_filename( + &path.file_name().ok_or_else(|| ProviderError::MissingSnapshot)?.to_string_lossy(), + ) + .and_then(|(parsed_segment, block_range, tx_range)| { + if parsed_segment == segment { + return Some((block_range, tx_range)); + } + None + }), + None => fn_ranges(), + }; + + // Return cached `LoadedJar` or insert it for the first time, and then, return it. + match snapshot_ranges { + Some((block_range, tx_range)) => { + let key = (*block_range.end(), segment); + if let Some(jar) = self.map.get(&key) { + Ok(jar.into()) + } else { + self.map.insert( + key, + LoadedJar::new(NippyJar::load( + &self.path.join(segment.filename(&block_range, &tx_range)), + )?)?, + ); + Ok(self.map.get(&key).expect("qed").into()) + } + } + None => Err(ProviderError::MissingSnapshot.into()), } + } - if let Some(path) = &path { - self.map.insert(key, LoadedJar::new(NippyJar::load(path)?)?); - } else { - path = Some(self.path.join(segment.filename( - &((snapshot * BLOCKS_PER_SNAPSHOT)..=((snapshot + 1) * BLOCKS_PER_SNAPSHOT - 1)), - ))); + /// Gets a snapshot segment's block range and transaction range from the provider inner block + /// index. + fn get_segment_ranges_from_block( + &self, + segment: SnapshotSegment, + block: u64, + ) -> Option<(RangeInclusive, RangeInclusive)> { + let snapshots = self.snapshots_block_index.read(); + if let Some(segment_snapshots) = snapshots.get(&segment) { + // It's more probable that the request comes from a newer block height, so we iterate + // the snapshots in reverse. + let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + + while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { + let block_start = + snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); + if block_start <= block { + return Some((block_start..=*block_end, tx_range.clone())); + } + } } + None + } - self.get_segment_provider(segment, block, path) + /// Gets a snapshot segment's block range and transaction range from the provider inner + /// transaction index. + fn get_segment_ranges_from_transaction( + &self, + segment: SnapshotSegment, + tx: u64, + ) -> Option<(RangeInclusive, RangeInclusive)> { + let snapshots = self.snapshots_tx_index.read(); + if let Some(segment_snapshots) = snapshots.get(&segment) { + // It's more probable that the request comes from a newer tx height, so we iterate + // the snapshots in reverse. + let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + + while let Some((tx_end, block_range)) = snapshots_rev_iter.next() { + let tx_start = + snapshots_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0); + if tx_start <= tx { + return Some((block_range.clone(), tx_start..=*tx_end)); + } + } + } + None } /// Gets the highest snapshot if it exists for a snapshot segment. @@ -79,7 +195,8 @@ impl HeaderProvider for SnapshotProvider { } fn header_by_number(&self, num: BlockNumber) -> RethResult> { - self.get_segment_provider(SnapshotSegment::Headers, num, None)?.header_by_number(num) + self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + .header_by_number(num) } fn header_td(&self, _block_hash: &BlockHash) -> RethResult> { @@ -144,9 +261,7 @@ impl TransactionsProvider for SnapshotProvider { } fn transaction_by_id(&self, num: TxNumber) -> RethResult> { - // TODO `num` is provided after checking the index - let block_num = num; - self.get_segment_provider(SnapshotSegment::Transactions, block_num, None)? + self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)? .transaction_by_id(num) } diff --git a/crates/storage/provider/src/providers/snapshot/mod.rs b/crates/storage/provider/src/providers/snapshot/mod.rs index f7fed480f4b9..c843a8165bc2 100644 --- a/crates/storage/provider/src/providers/snapshot/mod.rs +++ b/crates/storage/provider/src/providers/snapshot/mod.rs @@ -136,7 +136,11 @@ mod test { let db_provider = factory.provider().unwrap(); let manager = SnapshotProvider::default(); let jar_provider = manager - .get_segment_provider(SnapshotSegment::Headers, 0, Some(snap_file.path().into())) + .get_segment_provider_from_block( + SnapshotSegment::Headers, + 0, + Some(snap_file.path().into()), + ) .unwrap(); assert!(!headers.is_empty()); From 2e7f810b78ed3f564670760d8fe0026f30aef44f Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 9 Nov 2023 21:21:12 +0000 Subject: [PATCH 21/49] add find_snapshot to SnapshotProvider --- .../src/providers/snapshot/manager.rs | 65 +++++++++++++++---- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 2556b59edc8c..a3cf9e1acb40 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -116,23 +116,34 @@ impl SnapshotProvider { // Return cached `LoadedJar` or insert it for the first time, and then, return it. match snapshot_ranges { Some((block_range, tx_range)) => { - let key = (*block_range.end(), segment); - if let Some(jar) = self.map.get(&key) { - Ok(jar.into()) - } else { - self.map.insert( - key, - LoadedJar::new(NippyJar::load( - &self.path.join(segment.filename(&block_range, &tx_range)), - )?)?, - ); - Ok(self.map.get(&key).expect("qed").into()) - } + self.get_or_create_jar_provider(segment, &block_range, &tx_range) } None => Err(ProviderError::MissingSnapshot.into()), } } + /// Given a segment, block range and transaction range it returns a cached + /// [`SnapshotJarProvider`]. TODO: we should check the size and pop N if there's too many. + fn get_or_create_jar_provider( + &self, + segment: SnapshotSegment, + block_range: &RangeInclusive, + tx_range: &RangeInclusive, + ) -> Result, reth_interfaces::RethError> { + let key = (*block_range.end(), segment); + if let Some(jar) = self.map.get(&key) { + Ok(jar.into()) + } else { + self.map.insert( + key, + LoadedJar::new(NippyJar::load( + &self.path.join(segment.filename(block_range, tx_range)), + )?)?, + ); + Ok(self.map.get(&key).expect("qed").into()) + } + } + /// Gets a snapshot segment's block range and transaction range from the provider inner block /// index. fn get_segment_ranges_from_block( @@ -187,6 +198,36 @@ impl SnapshotProvider { .as_ref() .and_then(|tracker| tracker.borrow().and_then(|highest| highest.highest(segment))) } + + /// Iterates through segment snapshots in reverse order, executing a function until it + /// some object. Useful for finding objects by [`TxHash`] or [`BlockHash`]. + pub fn find_snapshot( + &self, + segment: SnapshotSegment, + func: impl Fn(SnapshotJarProvider<'_>) -> RethResult>, + ) -> RethResult> { + let snapshots = self.snapshots_block_index.read(); + if let Some(segment_snapshots) = snapshots.get(&segment) { + // It's more probable that the request comes from a newer block height, so we iterate + // the snapshots in reverse. + let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + + while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { + let block_start = + snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); + + if let Some(res) = func(self.get_or_create_jar_provider( + segment, + &(block_start..=*block_end), + tx_range, + )?)? { + return Ok(Some(res)) + } + } + } + + Ok(None) + } } impl HeaderProvider for SnapshotProvider { From 82442193e618ab6f3c95ed6994244e4e978b9de5 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 9 Nov 2023 21:27:42 +0000 Subject: [PATCH 22/49] implement some more methods for SnapshotProvider --- .../src/providers/snapshot/manager.rs | 77 ++++++++++++++----- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index a3cf9e1acb40..d78f3e1ad418 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -2,6 +2,10 @@ use super::{LoadedJar, SnapshotJarProvider}; use crate::{BlockHashReader, BlockNumReader, HeaderProvider, TransactionsProvider}; use dashmap::DashMap; use parking_lot::RwLock; +use reth_db::{ + codecs::CompactU256, + snapshot::{HeaderMask, TransactionMask}, +}; use reth_interfaces::{provider::ProviderError, RethResult}; use reth_nippy_jar::NippyJar; use reth_primitives::{ @@ -199,7 +203,7 @@ impl SnapshotProvider { .and_then(|tracker| tracker.borrow().and_then(|highest| highest.highest(segment))) } - /// Iterates through segment snapshots in reverse order, executing a function until it + /// Iterates through segment snapshots in reverse order, executing a function until it returns /// some object. Useful for finding objects by [`TxHash`] or [`BlockHash`]. pub fn find_snapshot( &self, @@ -231,8 +235,18 @@ impl SnapshotProvider { } impl HeaderProvider for SnapshotProvider { - fn header(&self, _block_hash: &BlockHash) -> RethResult> { - todo!() + fn header(&self, block_hash: &BlockHash) -> RethResult> { + self.find_snapshot(SnapshotSegment::Headers, |jar_provider| { + Ok(jar_provider + .cursor()? + .get_two::>(block_hash.into())? + .and_then(|(header, hash)| { + if &hash == block_hash { + return Some(header) + } + None + })) + }) } fn header_by_number(&self, num: BlockNumber) -> RethResult> { @@ -240,12 +254,18 @@ impl HeaderProvider for SnapshotProvider { .header_by_number(num) } - fn header_td(&self, _block_hash: &BlockHash) -> RethResult> { - todo!() + fn header_td(&self, block_hash: &BlockHash) -> RethResult> { + self.find_snapshot(SnapshotSegment::Headers, |jar_provider| { + Ok(jar_provider + .cursor()? + .get_two::>(block_hash.into())? + .and_then(|(td, hash)| (&hash == block_hash).then_some(td.0))) + }) } - fn header_td_by_number(&self, _number: BlockNumber) -> RethResult> { - todo!(); + fn header_td_by_number(&self, num: BlockNumber) -> RethResult> { + self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + .header_td_by_number(num) } fn headers_range(&self, _range: impl RangeBounds) -> RethResult> { @@ -259,14 +279,15 @@ impl HeaderProvider for SnapshotProvider { todo!(); } - fn sealed_header(&self, _number: BlockNumber) -> RethResult> { - todo!(); + fn sealed_header(&self, num: BlockNumber) -> RethResult> { + self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)? + .sealed_header(num) } } impl BlockHashReader for SnapshotProvider { - fn block_hash(&self, _number: u64) -> RethResult> { - todo!() + fn block_hash(&self, num: u64) -> RethResult> { + self.get_segment_provider_from_block(SnapshotSegment::Headers, num, None)?.block_hash(num) } fn canonical_hashes_range( @@ -297,8 +318,19 @@ impl BlockNumReader for SnapshotProvider { } impl TransactionsProvider for SnapshotProvider { - fn transaction_id(&self, _tx_hash: TxHash) -> RethResult> { - todo!() + fn transaction_id(&self, tx_hash: TxHash) -> RethResult> { + self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| { + let mut cursor = jar_provider.cursor()?; + if cursor + .get_one::>((&tx_hash).into())? + .and_then(|tx| (tx.hash() == tx_hash).then_some(tx)) + .is_some() + { + Ok(Some(cursor.number())) + } else { + Ok(None) + } + }) } fn transaction_by_id(&self, num: TxNumber) -> RethResult> { @@ -308,13 +340,20 @@ impl TransactionsProvider for SnapshotProvider { fn transaction_by_id_no_hash( &self, - _id: TxNumber, + num: TxNumber, ) -> RethResult> { - todo!() + self.get_segment_provider_from_transaction(SnapshotSegment::Transactions, num, None)? + .transaction_by_id_no_hash(num) } - fn transaction_by_hash(&self, _hash: TxHash) -> RethResult> { - todo!() + fn transaction_by_hash(&self, hash: TxHash) -> RethResult> { + self.find_snapshot(SnapshotSegment::Transactions, |jar_provider| { + Ok(jar_provider + .cursor()? + .get_one::>((&hash).into())? + .map(|tx| tx.with_hash()) + .and_then(|tx| (tx.hash_ref() == &hash).then_some(tx))) + }) } fn transaction_by_hash_with_meta( @@ -353,7 +392,7 @@ impl TransactionsProvider for SnapshotProvider { todo!() } - fn transaction_sender(&self, _id: TxNumber) -> RethResult> { - todo!() + fn transaction_sender(&self, id: TxNumber) -> RethResult> { + Ok(self.transaction_by_id_no_hash(id)?.and_then(|tx| tx.recover_signer())) } } From d0699f786b6e864321f2053143fdd87f6dec996e Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:40:46 +0000 Subject: [PATCH 23/49] Update crates/snapshot/src/snapshotter.rs Co-authored-by: Alexey Shekhirin --- crates/snapshot/src/snapshotter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 3ba37d5f1316..c675a3ed33b0 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -20,7 +20,7 @@ pub type SnapshotterWithResult = (Snapshotter, SnapshotterResult); pub struct Snapshotter { /// Provider factory provider_factory: ProviderFactory, - /// Highest snapshot block number for each + /// Highest snapshotted block numbers for each segment highest_snapshots: HighestSnapshots, /// Channel sender to notify other components of the new highest snapshot values highest_snapshots_notifier: watch::Sender>, From 892b2600a5fea8aff4bbc40d5ce64423e9eb2ab6 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:40:54 +0000 Subject: [PATCH 24/49] Update crates/snapshot/src/snapshotter.rs Co-authored-by: Alexey Shekhirin --- crates/snapshot/src/snapshotter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index c675a3ed33b0..6bc722f0f688 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -22,7 +22,7 @@ pub struct Snapshotter { provider_factory: ProviderFactory, /// Highest snapshotted block numbers for each segment highest_snapshots: HighestSnapshots, - /// Channel sender to notify other components of the new highest snapshot values + /// Channel sender to notify other components of the new highest snapshots highest_snapshots_notifier: watch::Sender>, /// Channel receiver to be cloned and shared that already comes with the newest value highest_snapshots_tracker: HighestSnapshotsTracker, From 756d00b51bb2fc1cdd551078301c9e78a0af7d8a Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:41:07 +0000 Subject: [PATCH 25/49] Update crates/storage/provider/src/providers/snapshot/manager.rs Co-authored-by: Alexey Shekhirin --- crates/storage/provider/src/providers/snapshot/manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 946e8349a7cb..1b26f1db633d 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -18,7 +18,7 @@ pub struct SnapshotProvider { /// Maintains a map which allows for concurrent access to different `NippyJars`, over different /// segments and ranges. map: DashMap<(BlockNumber, SnapshotSegment), LoadedJar>, - /// Tracks the latest and highest snapshot of every segment. + /// Tracks the highest snapshot of every segment. highest_tracker: Option>>, } From 5891d0a3c9160969bd0a382932fc49986cde67fd Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:53:58 +0000 Subject: [PATCH 26/49] Update crates/primitives/src/snapshot/segment.rs Co-authored-by: Alexey Shekhirin --- crates/primitives/src/snapshot/segment.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 1854f10921b2..309d2c4ba7fe 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -52,7 +52,7 @@ impl SnapshotSegment { /// Returns the default file name for the provided segment and range. pub fn filename(&self, range: &RangeInclusive) -> String { // ATTENTION: if changing the name format, be sure to reflect those changes in - // [`Self::parse_filename`.] + // [`Self::parse_filename`]. format!("snapshot_{}_{}_{}", self.as_ref(), range.start(), range.end(),) } From d2c21e92ba5fec58adaf2b6e83ea766aab9a1c9d Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 13 Nov 2023 11:54:12 +0000 Subject: [PATCH 27/49] Update crates/snapshot/src/snapshotter.rs Co-authored-by: Alexey Shekhirin --- crates/snapshot/src/snapshotter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index c3ad10710588..c3e587343eac 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -132,7 +132,7 @@ impl Snapshotter { /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and /// notifies every tracker. fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> { - // It walks over the directory and parses the snapshot filenames extracing `SnapshotSegment` + // It walks over the directory and parses the snapshot filenames extracting `SnapshotSegment` // and their inclusive range. It then takes the maximum block number for each specific // segment. for (segment, range) in std::fs::read_dir(&self.snapshots_path) From 4574c68d4241da84dc9fcde263d5ea61b2af5bbc Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 11:58:53 +0000 Subject: [PATCH 28/49] add fn block_range to snapshot cmd --- bin/reth/src/db/snapshots/headers.rs | 4 ++-- bin/reth/src/db/snapshots/mod.rs | 7 ++++++- bin/reth/src/db/snapshots/receipts.rs | 2 +- bin/reth/src/db/snapshots/transactions.rs | 2 +- crates/snapshot/src/snapshotter.rs | 6 +++--- 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index cc3bd0a0b152..932711ff189e 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -26,7 +26,7 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let range = self.from..=(self.from + self.block_interval - 1); + let range = self.block_range(); let filters = if self.with_filters { Filters::WithFilters(inclusion_filter, phf) } else { @@ -63,7 +63,7 @@ impl Command { Filters::WithoutFilters }; - let range = self.from..=(self.from + self.block_interval - 1); + let range = self.block_range(); let mut row_indexes = range.clone().collect::>(); let mut rng = rand::thread_rng(); diff --git a/bin/reth/src/db/snapshots/mod.rs b/bin/reth/src/db/snapshots/mod.rs index efce4878393a..80f0813c539d 100644 --- a/bin/reth/src/db/snapshots/mod.rs +++ b/bin/reth/src/db/snapshots/mod.rs @@ -7,7 +7,7 @@ use reth_primitives::{ BlockNumber, ChainSpec, SnapshotSegment, }; use reth_provider::ProviderFactory; -use std::{path::Path, sync::Arc}; +use std::{ops::RangeInclusive, path::Path, sync::Arc}; mod bench; mod headers; @@ -130,4 +130,9 @@ impl Command { Ok(()) } + + /// Gives out the inclusive block range for the snapshot requested by the user. + fn block_range(&self) -> RangeInclusive { + self.from..=(self.from + self.block_interval - 1) + } } diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index 70a95f31b9b8..bf4889421a31 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -27,7 +27,7 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let range = self.from..=(self.from + self.block_interval - 1); + let range = self.block_range(); let filters = if self.with_filters { Filters::WithFilters(inclusion_filter, phf) } else { diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index a2ced80a3f6a..44094a95fcdf 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -27,7 +27,7 @@ impl Command { inclusion_filter: InclusionFilter, phf: PerfectHashingFunction, ) -> eyre::Result<()> { - let range = self.from..=(self.from + self.block_interval - 1); + let range = self.block_range(); let filters = if self.with_filters { Filters::WithFilters(inclusion_filter, phf) } else { diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index d29500f40a30..a6f85a48b9fc 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -132,9 +132,9 @@ impl Snapshotter { /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and /// notifies every tracker. fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> { - // It walks over the directory and parses the snapshot filenames extracting `SnapshotSegment` - // and their inclusive range. It then takes the maximum block number for each specific - // segment. + // It walks over the directory and parses the snapshot filenames extracting + // `SnapshotSegment` and their inclusive range. It then takes the maximum block + // number for each specific segment. for (segment, range) in std::fs::read_dir(&self.snapshots_path) .map_err(|err| RethError::Custom(err.to_string()))? .filter_map(Result::ok) From d47812a728d66fcde315aba901ceb7cac572b373 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 12:10:26 +0000 Subject: [PATCH 29/49] add read_dir to primitives fs --- crates/interfaces/src/error.rs | 6 ++++++ crates/primitives/src/fs.rs | 17 ++++++++++++++++- crates/snapshot/src/snapshotter.rs | 10 ++-------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/crates/interfaces/src/error.rs b/crates/interfaces/src/error.rs index b972124fdba4..e40a1abd5f67 100644 --- a/crates/interfaces/src/error.rs +++ b/crates/interfaces/src/error.rs @@ -39,6 +39,12 @@ impl From for RethError { } } +impl From for RethError { + fn from(err: reth_primitives::fs::FsPathError) -> Self { + RethError::Custom(err.to_string()) + } +} + // We don't want these types to be too large because they're used in a lot of places. const _SIZE_ASSERTIONS: () = { // Main error. diff --git a/crates/primitives/src/fs.rs b/crates/primitives/src/fs.rs index f31b279c5b58..cf7825217d6a 100644 --- a/crates/primitives/src/fs.rs +++ b/crates/primitives/src/fs.rs @@ -1,6 +1,7 @@ //! Wrapper for `std::fs` methods use std::{ - fs, io, + fs::{self, ReadDir}, + io, path::{Path, PathBuf}, }; @@ -30,6 +31,9 @@ pub enum FsPathError { /// Provides additional path context for [`std::fs::remove_dir`]. #[error("failed to remove dir {path:?}: {source}")] RemoveDir { source: io::Error, path: PathBuf }, + /// Provides additional path context for [`std::fs::read_dir`]. + #[error("failed to read dir {path:?}: {source}")] + ReadDir { source: io::Error, path: PathBuf }, /// Provides additional path context for [`std::fs::File::open`]. #[error("failed to open file {path:?}: {source}")] Open { source: io::Error, path: PathBuf }, @@ -77,6 +81,11 @@ impl FsPathError { FsPathError::RemoveDir { source, path: path.into() } } + /// Returns the complementary error variant for [`std::fs::read_dir`]. + pub fn read_dir(source: io::Error, path: impl Into) -> Self { + FsPathError::ReadDir { source, path: path.into() } + } + /// Returns the complementary error variant for [`std::fs::File::open`]. pub fn open(source: io::Error, path: impl Into) -> Self { FsPathError::Open { source, path: path.into() } @@ -108,3 +117,9 @@ pub fn create_dir_all(path: impl AsRef) -> Result<()> { let path = path.as_ref(); fs::create_dir_all(path).map_err(|err| FsPathError::create_dir(err, path)) } + +/// Wrapper for `std::fs::read_dir` +pub fn read_dir(path: impl AsRef) -> Result { + let path = path.as_ref(); + fs::read_dir(path).map_err(|err| FsPathError::read_dir(err, path)) +} diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index a6f85a48b9fc..c8790d336adf 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -91,12 +91,7 @@ impl Snapshotter { ) -> RethResult { // Create directory for snapshots if it doesn't exist. if !snapshots_path.exists() { - std::fs::create_dir_all(&snapshots_path).map_err(|e| { - RethError::Custom(format!( - "Could not create snapshots directory {}: {e}", - snapshots_path.display() - )) - })?; + reth_primitives::fs::create_dir_all(&snapshots_path)?; } let (highest_snapshots_notifier, highest_snapshots_tracker) = watch::channel(None); @@ -135,8 +130,7 @@ impl Snapshotter { // It walks over the directory and parses the snapshot filenames extracting // `SnapshotSegment` and their inclusive range. It then takes the maximum block // number for each specific segment. - for (segment, range) in std::fs::read_dir(&self.snapshots_path) - .map_err(|err| RethError::Custom(err.to_string()))? + for (segment, range) in reth_primitives::fs::read_dir(&self.snapshots_path)? .filter_map(Result::ok) .filter_map(|entry| { if let Ok(true) = entry.metadata().map(|metadata| metadata.is_file()) { From 1be9e291019e6e5f8ca7c7f6d56451b0aba1b06b Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 12:29:31 +0000 Subject: [PATCH 30/49] add HighestSnapshots::as_mut(segment) --- crates/primitives/src/snapshot/mod.rs | 9 +++++++++ crates/snapshot/src/snapshotter.rs | 7 +------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index 3a97749c8db0..58e6bef49078 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -38,6 +38,15 @@ impl HighestSnapshots { SnapshotSegment::Receipts => self.receipts, } } + + /// Returns a mutable reference to a snapshot segment + pub fn as_mut(&mut self, segment: SnapshotSegment) -> &mut Option { + match segment { + SnapshotSegment::Headers => &mut self.headers, + SnapshotSegment::Transactions => &mut self.transactions, + SnapshotSegment::Receipts => &mut self.receipts, + } + } } /// Given the snapshot's location, it returns an iterator over the existing snapshots in the format diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 462c1305776b..0075d662dffd 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -154,12 +154,7 @@ impl Snapshotter { // and their inclusive range. It then takes the maximum block number for each specific // segment. for (segment, block_range, _) in iter_snapshots(&self.snapshots_path)? { - let max_segment_block = match segment { - SnapshotSegment::Headers => &mut self.highest_snapshots.headers, - SnapshotSegment::Transactions => &mut self.highest_snapshots.transactions, - SnapshotSegment::Receipts => &mut self.highest_snapshots.receipts, - }; - + let max_segment_block = self.highest_snapshots.as_mut(segment); if max_segment_block.map_or(true, |block| block < *block_range.end()) { *max_segment_block = Some(*block_range.end()); } From 95e965affcc2f0ee48fc49c8dc708873d5e6dd48 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 12:40:37 +0000 Subject: [PATCH 31/49] trait Segment requires Default --- crates/snapshot/src/segments/mod.rs | 2 +- crates/snapshot/src/snapshotter.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/snapshot/src/segments/mod.rs b/crates/snapshot/src/segments/mod.rs index 140ea90fb354..37727953a59e 100644 --- a/crates/snapshot/src/segments/mod.rs +++ b/crates/snapshot/src/segments/mod.rs @@ -26,7 +26,7 @@ use std::{ops::RangeInclusive, path::Path}; pub(crate) type Rows = [Vec>; COLUMNS]; /// A segment represents a snapshotting of some portion of the data. -pub trait Segment { +pub trait Segment: Default { /// Snapshot data using the provided range. The `directory` parameter determines the snapshot /// file's save location. fn snapshot( diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 38067978e25e..2869da5d970b 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -212,7 +212,7 @@ impl Snapshotter { /// this block range and segment. /// /// If it succeeds, then we move the snapshot file from the temporary directory to its main one. - fn run_segment( + fn run_segment( &self, block_range: Option>, ) -> RethResult<()> { From 12aaa26dea13eca46fcc4ff5d5b08e5452150d26 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 12:59:26 +0000 Subject: [PATCH 32/49] add tests for snapshot segment filenames --- crates/primitives/src/snapshot/segment.rs | 90 ++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index e468b733ee63..a417f78f5070 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -101,14 +101,14 @@ impl SnapshotSegment { let block_start: u64 = parts[2].parse().unwrap_or(0); let block_end: u64 = parts[3].parse().unwrap_or(0); - if block_start <= block_end || parts[0] != "snapshot" { + if block_start >= block_end || parts[0] != "snapshot" { return None } let tx_start: u64 = parts[4].parse().unwrap_or(0); let tx_end: u64 = parts[5].parse().unwrap_or(0); - if tx_start <= tx_end { + if tx_start >= tx_end { return None } @@ -166,3 +166,89 @@ pub struct SegmentConfig { /// Compression used on the segment pub compression: Compression, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_filename() { + let test_vectors = [ + (SnapshotSegment::Headers, 2..=30, 0..=1, "snapshot_headers_2_30_0_1", None), + ( + SnapshotSegment::Receipts, + 30..=300, + 110..=1000, + "snapshot_receipts_30_300_110_1000", + None, + ), + ( + SnapshotSegment::Transactions, + 1_123_233..=11_223_233, + 1_123_233..=2_123_233, + "snapshot_transactions_1123233_11223233_1123233_2123233", + None, + ), + ( + SnapshotSegment::Headers, + 2..=30, + 0..=1, + "snapshot_headers_2_30_0_1_cuckoo-fmph_lz4", + Some(( + Compression::Lz4, + Filters::WithFilters( + InclusionFilter::Cuckoo, + crate::snapshot::PerfectHashingFunction::Fmph, + ), + )), + ), + ( + SnapshotSegment::Headers, + 2..=30, + 0..=1, + "snapshot_headers_2_30_0_1_cuckoo-fmph_zstd", + Some(( + Compression::Zstd, + Filters::WithFilters( + InclusionFilter::Cuckoo, + crate::snapshot::PerfectHashingFunction::Fmph, + ), + )), + ), + ( + SnapshotSegment::Headers, + 2..=30, + 0..=1, + "snapshot_headers_2_30_0_1_cuckoo-fmph_zstd-dict", + Some(( + Compression::ZstdWithDictionary, + Filters::WithFilters( + InclusionFilter::Cuckoo, + crate::snapshot::PerfectHashingFunction::Fmph, + ), + )), + ), + ]; + + for (segment, block_range, tx_range, filename, configuration) in test_vectors { + if let Some((compression, filters)) = configuration { + assert_eq!( + segment.filename_with_configuration( + filters, + compression, + &block_range, + &tx_range + ), + filename + ); + } else { + assert_eq!(segment.filename(&block_range, &tx_range), filename); + } + + assert_eq!( + SnapshotSegment::parse_filename(filename), + Some((segment, block_range, tx_range)) + ); + } + } +} From 8dded9ed542eeb331dcedca7d8aa8151468efb83 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:12:36 +0000 Subject: [PATCH 33/49] redo parse_filename --- crates/primitives/src/snapshot/segment.rs | 29 ++++++++++------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index a417f78f5070..d17bdb308753 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -96,25 +96,20 @@ impl SnapshotSegment { pub fn parse_filename( name: &str, ) -> Option<(Self, RangeInclusive, RangeInclusive)> { - let parts: Vec<&str> = name.split('_').collect(); - if let (Ok(segment), true) = (Self::from_str(parts[1]), parts.len() >= 6) { - let block_start: u64 = parts[2].parse().unwrap_or(0); - let block_end: u64 = parts[3].parse().unwrap_or(0); - - if block_start >= block_end || parts[0] != "snapshot" { - return None - } + let mut parts = name.split('_'); + if parts.next() != Some("snapshot") { + return None; + } - let tx_start: u64 = parts[4].parse().unwrap_or(0); - let tx_end: u64 = parts[5].parse().unwrap_or(0); + let segment = Self::from_str(parts.next()?).ok()?; + let (block_start, block_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); + let (tx_start, tx_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); - if tx_start >= tx_end { - return None - } - - return Some((segment, block_start..=block_end, tx_start..=tx_end)) + if block_start >= block_end || tx_start >= tx_end { + return None; } - None + + Some((segment, block_start..=block_end, tx_start..=tx_end)) } } @@ -250,5 +245,7 @@ mod tests { Some((segment, block_range, tx_range)) ); } + + assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_1_1"), None); } } From 4d0f75297a81aa5e49d484bf055897359c275f9a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:15:16 +0000 Subject: [PATCH 34/49] add a couple more tests cases --- crates/primitives/src/snapshot/segment.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index d17bdb308753..86616405e74b 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -105,7 +105,7 @@ impl SnapshotSegment { let (block_start, block_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); let (tx_start, tx_end) = (parts.next()?.parse().ok()?, parts.next()?.parse().ok()?); - if block_start >= block_end || tx_start >= tx_end { + if block_start >= block_end || tx_start > tx_end { return None; } @@ -246,6 +246,7 @@ mod tests { ); } - assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_1_1"), None); + assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_3_2"), None); + assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_1"), None); } } From dd272d1b38202d837670477426bd2b5b77ab6a2f Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:17:39 +0000 Subject: [PATCH 35/49] option ? operator --- .../src/providers/snapshot/manager.rs | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index aa09e056d330..c3d676ca7d5d 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -156,17 +156,17 @@ impl SnapshotProvider { block: u64, ) -> Option<(RangeInclusive, RangeInclusive)> { let snapshots = self.snapshots_block_index.read(); - if let Some(segment_snapshots) = snapshots.get(&segment) { - // It's more probable that the request comes from a newer block height, so we iterate - // the snapshots in reverse. - let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + let segment_snapshots = snapshots.get(&segment)?; - while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { - let block_start = - snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); - if block_start <= block { - return Some((block_start..=*block_end, tx_range.clone())); - } + // It's more probable that the request comes from a newer block height, so we iterate + // the snapshots in reverse. + let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + + while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { + let block_start = + snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); + if block_start <= block { + return Some((block_start..=*block_end, tx_range.clone())); } } None @@ -180,17 +180,16 @@ impl SnapshotProvider { tx: u64, ) -> Option<(RangeInclusive, RangeInclusive)> { let snapshots = self.snapshots_tx_index.read(); - if let Some(segment_snapshots) = snapshots.get(&segment) { - // It's more probable that the request comes from a newer tx height, so we iterate - // the snapshots in reverse. - let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + let segment_snapshots = snapshots.get(&segment)?; - while let Some((tx_end, block_range)) = snapshots_rev_iter.next() { - let tx_start = - snapshots_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0); - if tx_start <= tx { - return Some((block_range.clone(), tx_start..=*tx_end)); - } + // It's more probable that the request comes from a newer tx height, so we iterate + // the snapshots in reverse. + let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); + + while let Some((tx_end, block_range)) = snapshots_rev_iter.next() { + let tx_start = snapshots_rev_iter.peek().map(|(tx_end, _)| *tx_end + 1).unwrap_or(0); + if tx_start <= tx { + return Some((block_range.clone(), tx_start..=*tx_end)); } } None From 5c8406cadbdff287a8a5d286b1b35c8c9503245a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:22:05 +0000 Subject: [PATCH 36/49] clippy --- crates/snapshot/src/snapshotter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index cd22e151c387..44077d853d55 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -5,7 +5,7 @@ use reth_db::database::Database; use reth_interfaces::{RethError, RethResult}; use reth_primitives::{ snapshot::{iter_snapshots, HighestSnapshots}, - BlockNumber, ChainSpec, SnapshotSegment, TxNumber, + BlockNumber, ChainSpec, TxNumber, }; use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory, TransactionsProviderExt}; use std::{collections::HashMap, ops::RangeInclusive, path::PathBuf, sync::Arc}; From 23b525feafbb12fac8e856b6f091047089da8cf1 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:29:35 +0000 Subject: [PATCH 37/49] add docs to parse_filename --- crates/primitives/src/snapshot/segment.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 86616405e74b..a7b6beee9e8c 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -92,7 +92,19 @@ impl SnapshotSegment { format!("{prefix}_{}_{}", filters_name, compression.as_ref()) } - /// Takes a filename and parses the [`SnapshotSegment`] and its inclusive range. + /// Parses a filename into a `SnapshotSegment` and its corresponding block and transaction ranges. + /// + /// The filename is expected to follow the format: "snapshot_{segment}_{block_start}_{block_end}_{tx_start}_{tx_end}". + /// This function checks for the correct prefix ("snapshot"), and then parses the segment and the inclusive + /// ranges for blocks and transactions. It ensures that the start of each range is less than the end. + /// + /// # Returns + /// - `Some((segment, block_range, tx_range))` if parsing is successful and all conditions are met. + /// - `None` if any condition fails, such as an incorrect prefix, parsing error, or invalid range. + /// + /// # Note + /// This function is tightly coupled with the naming convention defined in [`Self::filename`]. + /// Any changes in the filename format in `filename` should be reflected here. pub fn parse_filename( name: &str, ) -> Option<(Self, RangeInclusive, RangeInclusive)> { From 5886cdfbe3220a75e3550245ce8cf6cb6bc03218 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 13:46:10 +0000 Subject: [PATCH 38/49] use OsStr instead on parse_filename --- crates/primitives/src/snapshot/mod.rs | 2 +- crates/primitives/src/snapshot/segment.rs | 33 +++++++++++-------- .../src/providers/snapshot/manager.rs | 2 +- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/primitives/src/snapshot/mod.rs b/crates/primitives/src/snapshot/mod.rs index 58e6bef49078..b8fe3a1e85cf 100644 --- a/crates/primitives/src/snapshot/mod.rs +++ b/crates/primitives/src/snapshot/mod.rs @@ -60,7 +60,7 @@ pub fn iter_snapshots( let entries = crate::fs::read_dir(path.as_ref())?.filter_map(Result::ok); Ok(entries.filter_map(|entry| { if entry.metadata().map_or(false, |metadata| metadata.is_file()) { - return SnapshotSegment::parse_filename(&entry.file_name().to_string_lossy()) + return SnapshotSegment::parse_filename(&entry.file_name()) } None })) diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index a7b6beee9e8c..4b6cadbdd0da 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -3,7 +3,7 @@ use crate::{ BlockNumber, TxNumber, }; use serde::{Deserialize, Serialize}; -use std::{ops::RangeInclusive, str::FromStr}; +use std::{ffi::OsStr, ops::RangeInclusive, str::FromStr}; use strum::{AsRefStr, EnumString}; #[derive( @@ -92,23 +92,28 @@ impl SnapshotSegment { format!("{prefix}_{}_{}", filters_name, compression.as_ref()) } - /// Parses a filename into a `SnapshotSegment` and its corresponding block and transaction ranges. + /// Parses a filename into a `SnapshotSegment` and its corresponding block and transaction + /// ranges. + /// + /// The filename is expected to follow the format: + /// "snapshot_{segment}_{block_start}_{block_end}_{tx_start}_{tx_end}". This function checks + /// for the correct prefix ("snapshot"), and then parses the segment and the inclusive + /// ranges for blocks and transactions. It ensures that the start of each range is less than the + /// end. /// - /// The filename is expected to follow the format: "snapshot_{segment}_{block_start}_{block_end}_{tx_start}_{tx_end}". - /// This function checks for the correct prefix ("snapshot"), and then parses the segment and the inclusive - /// ranges for blocks and transactions. It ensures that the start of each range is less than the end. - /// /// # Returns - /// - `Some((segment, block_range, tx_range))` if parsing is successful and all conditions are met. - /// - `None` if any condition fails, such as an incorrect prefix, parsing error, or invalid range. - /// + /// - `Some((segment, block_range, tx_range))` if parsing is successful and all conditions are + /// met. + /// - `None` if any condition fails, such as an incorrect prefix, parsing error, or invalid + /// range. + /// /// # Note /// This function is tightly coupled with the naming convention defined in [`Self::filename`]. /// Any changes in the filename format in `filename` should be reflected here. pub fn parse_filename( - name: &str, + name: &OsStr, ) -> Option<(Self, RangeInclusive, RangeInclusive)> { - let mut parts = name.split('_'); + let mut parts = name.to_str()?.split('_'); if parts.next() != Some("snapshot") { return None; } @@ -253,12 +258,12 @@ mod tests { } assert_eq!( - SnapshotSegment::parse_filename(filename), + SnapshotSegment::parse_filename(OsStr::new(filename)), Some((segment, block_range, tx_range)) ); } - assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_3_2"), None); - assert_eq!(SnapshotSegment::parse_filename("snapshot_headers_2_30_1"), None); + assert_eq!(SnapshotSegment::parse_filename(OsStr::new("snapshot_headers_2_30_3_2")), None); + assert_eq!(SnapshotSegment::parse_filename(OsStr::new("snapshot_headers_2_30_1")), None); } } diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index c3d676ca7d5d..c5286e6602fa 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -106,7 +106,7 @@ impl SnapshotProvider { // Otherwise, check `self.available_snapshots` let snapshot_ranges = match path { Some(path) => SnapshotSegment::parse_filename( - &path.file_name().ok_or_else(|| ProviderError::MissingSnapshot)?.to_string_lossy(), + path.file_name().ok_or_else(|| ProviderError::MissingSnapshot)?, ) .and_then(|(parsed_segment, block_range, tx_range)| { if parsed_segment == segment { From 8932ab3a59c61a3dc20bcdf96ec3c6a0df147ece Mon Sep 17 00:00:00 2001 From: joshieDo Date: Mon, 13 Nov 2023 14:16:04 +0000 Subject: [PATCH 39/49] swap some pathbufs for path --- bin/reth/src/db/snapshots/headers.rs | 4 ++-- bin/reth/src/db/snapshots/receipts.rs | 4 ++-- bin/reth/src/db/snapshots/transactions.rs | 4 ++-- crates/snapshot/src/snapshotter.rs | 11 ++++++++--- .../provider/src/providers/snapshot/manager.rs | 12 ++++++------ .../storage/provider/src/providers/snapshot/mod.rs | 13 +++++-------- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index ac13b9d391d8..5a0b9745d6d2 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -74,14 +74,14 @@ impl Command { .provider()? .transaction_range_by_block_range(block_range.clone())?; - let path = SnapshotSegment::Headers + let path: PathBuf = SnapshotSegment::Headers .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider_from_block( SnapshotSegment::Headers, self.from, - Some(path), + Some(&path), )?; let mut cursor = jar_provider.cursor()?; diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index af94074ecf64..76344c448462 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -75,7 +75,7 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Receipts + let path: PathBuf = SnapshotSegment::Receipts .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); @@ -83,7 +83,7 @@ impl Command { let jar_provider = provider.get_segment_provider_from_block( SnapshotSegment::Receipts, self.from, - Some(path), + Some(&path), )?; let mut cursor = jar_provider.cursor()?; diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index b4b7d31b4b2a..50795533f618 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -76,14 +76,14 @@ impl Command { let mut row_indexes = tx_range.clone().collect::>(); - let path = SnapshotSegment::Transactions + let path: PathBuf = SnapshotSegment::Transactions .filename_with_configuration(filters, compression, &block_range, &tx_range) .into(); let provider = SnapshotProvider::default(); let jar_provider = provider.get_segment_provider_from_block( SnapshotSegment::Transactions, self.from, - Some(path), + Some(&path), )?; let mut cursor = jar_provider.cursor()?; diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 44077d853d55..5d311d372bcb 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -8,7 +8,12 @@ use reth_primitives::{ BlockNumber, ChainSpec, TxNumber, }; use reth_provider::{BlockReader, DatabaseProviderRO, ProviderFactory, TransactionsProviderExt}; -use std::{collections::HashMap, ops::RangeInclusive, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + ops::RangeInclusive, + path::{Path, PathBuf}, + sync::Arc, +}; use tokio::sync::watch; use tracing::warn; @@ -90,7 +95,7 @@ impl Snapshotter { /// Creates a new [Snapshotter]. pub fn new( db: DB, - snapshots_path: PathBuf, + snapshots_path: impl AsRef, chain_spec: Arc, block_interval: u64, ) -> RethResult { @@ -98,7 +103,7 @@ impl Snapshotter { let mut snapshotter = Self { provider_factory: ProviderFactory::new(db, chain_spec), - snapshots_path, + snapshots_path: snapshots_path.as_ref().into(), // TODO(alexey): fill from on-disk snapshot data highest_snapshots: HighestSnapshots::default(), highest_snapshots_notifier, diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index c5286e6602fa..39408748b353 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -17,7 +17,7 @@ use revm::primitives::HashMap; use std::{ collections::BTreeMap, ops::{RangeBounds, RangeInclusive}, - path::PathBuf, + path::{Path, PathBuf}, }; use tokio::sync::watch; @@ -48,13 +48,13 @@ pub struct SnapshotProvider { impl SnapshotProvider { /// Creates a new [`SnapshotProvider`]. - pub fn new(path: PathBuf) -> Self { + pub fn new(path: impl AsRef) -> Self { Self { map: Default::default(), snapshots_block_index: Default::default(), snapshots_tx_index: Default::default(), highest_tracker: None, - path, + path: path.as_ref().to_path_buf(), } } @@ -72,7 +72,7 @@ impl SnapshotProvider { &self, segment: SnapshotSegment, block: BlockNumber, - path: Option, + path: Option<&Path>, ) -> RethResult> { self.get_segment_provider( segment, @@ -86,7 +86,7 @@ impl SnapshotProvider { &self, segment: SnapshotSegment, tx: TxNumber, - path: Option, + path: Option<&Path>, ) -> RethResult> { self.get_segment_provider( segment, @@ -100,7 +100,7 @@ impl SnapshotProvider { &self, segment: SnapshotSegment, fn_ranges: impl Fn() -> Option<(RangeInclusive, RangeInclusive)>, - path: Option, + path: Option<&Path>, ) -> RethResult> { // If we have a path, then get the block range and transaction range from its name. // Otherwise, check `self.available_snapshots` diff --git a/crates/storage/provider/src/providers/snapshot/mod.rs b/crates/storage/provider/src/providers/snapshot/mod.rs index c843a8165bc2..cde7dbfc1ac8 100644 --- a/crates/storage/provider/src/providers/snapshot/mod.rs +++ b/crates/storage/provider/src/providers/snapshot/mod.rs @@ -66,7 +66,8 @@ mod test { // Data sources let db = create_test_rw_db(); let factory = ProviderFactory::new(&db, MAINNET.clone()); - let snap_file = tempfile::NamedTempFile::new().unwrap(); + let snap_path = tempfile::tempdir().unwrap(); + let snap_file = snap_path.path().join(SnapshotSegment::Headers.filename(&range, &range)); // Setup data let mut headers = random_header_range( @@ -96,7 +97,7 @@ mod test { let with_compression = true; let with_filter = true; - let mut nippy_jar = NippyJar::new(3, snap_file.path(), segment_header); + let mut nippy_jar = NippyJar::new(3, snap_file.as_path(), segment_header); if with_compression { nippy_jar = nippy_jar.with_zstd(false, 0); @@ -134,13 +135,9 @@ mod test { // Use providers to query Header data and compare if it matches { let db_provider = factory.provider().unwrap(); - let manager = SnapshotProvider::default(); + let manager = SnapshotProvider::new(snap_path.path()); let jar_provider = manager - .get_segment_provider_from_block( - SnapshotSegment::Headers, - 0, - Some(snap_file.path().into()), - ) + .get_segment_provider_from_block(SnapshotSegment::Headers, 0, Some(&snap_file)) .unwrap(); assert!(!headers.is_empty()); From 7054744956f05904cd1c11cb01c649c7d24f2a9c Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 15:01:20 +0000 Subject: [PATCH 40/49] add reth_primitives::fs::rename --- bin/reth/src/db/snapshots/headers.rs | 10 ++++------ bin/reth/src/db/snapshots/receipts.rs | 10 ++++------ bin/reth/src/db/snapshots/transactions.rs | 10 ++++------ crates/primitives/src/fs.rs | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 932711ff189e..ee89995c2c30 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -38,12 +38,10 @@ impl Command { segment.snapshot::(provider, range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Headers.filename(&range).into(); - let new_name: PathBuf = SnapshotSegment::Headers - .filename_with_configuration(filters, compression, &range) - .into(); - - std::fs::rename(default_name, new_name)?; + reth_primitives::fs::rename( + SnapshotSegment::Headers.filename(&range), + SnapshotSegment::Headers.filename_with_configuration(filters, compression, &range), + )?; Ok(()) } diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index bf4889421a31..9c3b94ca4580 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -39,12 +39,10 @@ impl Command { segment.snapshot::(provider, range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Receipts.filename(&range).into(); - let new_name: PathBuf = SnapshotSegment::Receipts - .filename_with_configuration(filters, compression, &range) - .into(); - - std::fs::rename(default_name, new_name)?; + reth_primitives::fs::rename( + SnapshotSegment::Receipts.filename(&range), + SnapshotSegment::Receipts.filename_with_configuration(filters, compression, &range), + )?; Ok(()) } diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index 44094a95fcdf..b5bd726fd9e4 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -39,12 +39,10 @@ impl Command { segment.snapshot::(provider, range.clone())?; // Default name doesn't have any configuration - let default_name: PathBuf = SnapshotSegment::Transactions.filename(&range).into(); - let new_name: PathBuf = SnapshotSegment::Transactions - .filename_with_configuration(filters, compression, &range) - .into(); - - std::fs::rename(default_name, new_name)?; + reth_primitives::fs::rename( + SnapshotSegment::Transactions.filename(&range), + SnapshotSegment::Transactions.filename_with_configuration(filters, compression, &range), + )?; Ok(()) } diff --git a/crates/primitives/src/fs.rs b/crates/primitives/src/fs.rs index cf7825217d6a..3388d477003d 100644 --- a/crates/primitives/src/fs.rs +++ b/crates/primitives/src/fs.rs @@ -34,6 +34,9 @@ pub enum FsPathError { /// Provides additional path context for [`std::fs::read_dir`]. #[error("failed to read dir {path:?}: {source}")] ReadDir { source: io::Error, path: PathBuf }, + /// Provides additional context for [`std::fs::rename`]. + #[error("failed to rename from {from:?} to {to:?}: {source}")] + Rename { source: io::Error, from: PathBuf, to: PathBuf }, /// Provides additional path context for [`std::fs::File::open`]. #[error("failed to open file {path:?}: {source}")] Open { source: io::Error, path: PathBuf }, @@ -90,6 +93,11 @@ impl FsPathError { pub fn open(source: io::Error, path: impl Into) -> Self { FsPathError::Open { source, path: path.into() } } + + /// Returns the complementary error variant for [`std::fs::rename`]. + pub fn rename(source: io::Error, from: impl Into, to: impl Into) -> Self { + FsPathError::Rename { source, from: from.into(), to: to.into() } + } } type Result = std::result::Result; @@ -123,3 +131,10 @@ pub fn read_dir(path: impl AsRef) -> Result { let path = path.as_ref(); fs::read_dir(path).map_err(|err| FsPathError::read_dir(err, path)) } + +/// Wrapper for `std::fs::rename` +pub fn rename(from: impl AsRef, to: impl AsRef) -> Result<()> { + let from = from.as_ref(); + let to = to.as_ref(); + fs::rename(from, to).map_err(|err| FsPathError::rename(err, from, to)) +} From a865b5935012810643a35a4b540c9435c069a863 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 15:04:12 +0000 Subject: [PATCH 41/49] use reth fs rename --- crates/snapshot/src/snapshotter.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 2869da5d970b..cd583b5e14eb 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -226,8 +226,7 @@ impl Snapshotter { block_range.clone(), )?; - std::fs::rename(temp.join(&filename), self.snapshots_path.join(filename)) - .map_err(|err| RethError::Custom(err.to_string()))?; + reth_primitives::fs::rename(temp.join(&filename), self.snapshots_path.join(filename))?; } Ok(()) } From 77094e5060333385e75aee14aaca708dc56c669f Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 15:09:11 +0000 Subject: [PATCH 42/49] clippy --- bin/reth/src/cli/mod.rs | 2 +- bin/reth/src/db/snapshots/headers.rs | 2 +- bin/reth/src/db/snapshots/receipts.rs | 2 +- bin/reth/src/db/snapshots/transactions.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index d48afb627cc5..1361b53d2d11 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -363,7 +363,7 @@ mod tests { let end = format!("reth/logs/{}", SUPPORTED_CHAINS[0]); assert!(log_dir.as_ref().ends_with(end), "{:?}", log_dir); - let mut iter = SUPPORTED_CHAINS.into_iter(); + let mut iter = SUPPORTED_CHAINS.iter(); iter.next(); for chain in iter { let mut reth = Cli::<()>::try_parse_from(["reth", "node", "--chain", chain]).unwrap(); diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index ee89995c2c30..8b737e833b22 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -14,7 +14,7 @@ use reth_provider::{ }; use reth_snapshot::{segments, segments::Segment}; use std::{ - path::{Path, PathBuf}, + path::{Path}, sync::Arc, }; diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index 9c3b94ca4580..e555c0beb1d1 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -15,7 +15,7 @@ use reth_provider::{ }; use reth_snapshot::{segments, segments::Segment}; use std::{ - path::{Path, PathBuf}, + path::{Path}, sync::Arc, }; diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index b5bd726fd9e4..54b6da65df0c 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -15,7 +15,7 @@ use reth_provider::{ }; use reth_snapshot::{segments, segments::Segment}; use std::{ - path::{Path, PathBuf}, + path::{Path}, sync::Arc, }; From 4f440b3b2348824d0af3ac8263d6642272f7ae85 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 15:11:48 +0000 Subject: [PATCH 43/49] remove done TODO reminder --- crates/snapshot/src/snapshotter.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 5d311d372bcb..8fb66104f985 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -104,7 +104,6 @@ impl Snapshotter { let mut snapshotter = Self { provider_factory: ProviderFactory::new(db, chain_spec), snapshots_path: snapshots_path.as_ref().into(), - // TODO(alexey): fill from on-disk snapshot data highest_snapshots: HighestSnapshots::default(), highest_snapshots_notifier, highest_snapshots_tracker, From 622159b58cb1e1172fa35c6821ca85f37af7cdcf Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:43:53 +0000 Subject: [PATCH 44/49] Update crates/primitives/src/fs.rs Co-authored-by: Matthias Seitz --- crates/primitives/src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives/src/fs.rs b/crates/primitives/src/fs.rs index 3388d477003d..8e4e50acd6e8 100644 --- a/crates/primitives/src/fs.rs +++ b/crates/primitives/src/fs.rs @@ -35,7 +35,7 @@ pub enum FsPathError { #[error("failed to read dir {path:?}: {source}")] ReadDir { source: io::Error, path: PathBuf }, /// Provides additional context for [`std::fs::rename`]. - #[error("failed to rename from {from:?} to {to:?}: {source}")] + #[error("failed to rename {from:?} to {to:?}: {source}")] Rename { source: io::Error, from: PathBuf, to: PathBuf }, /// Provides additional path context for [`std::fs::File::open`]. #[error("failed to open file {path:?}: {source}")] From 975b47d6393c0c78d8ad78d46d8b7ae8d0a13459 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 16:11:10 +0000 Subject: [PATCH 45/49] fmt --- bin/reth/src/db/snapshots/headers.rs | 5 +---- bin/reth/src/db/snapshots/receipts.rs | 5 +---- bin/reth/src/db/snapshots/transactions.rs | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 8b737e833b22..b09b99ebc648 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -13,10 +13,7 @@ use reth_provider::{ providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory, }; use reth_snapshot::{segments, segments::Segment}; -use std::{ - path::{Path}, - sync::Arc, -}; +use std::{path::Path, sync::Arc}; impl Command { pub(crate) fn generate_headers_snapshot( diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index e555c0beb1d1..b0475eeff1b5 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -14,10 +14,7 @@ use reth_provider::{ ReceiptProvider, TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{ - path::{Path}, - sync::Arc, -}; +use std::{path::Path, sync::Arc}; impl Command { pub(crate) fn generate_receipts_snapshot( diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index 54b6da65df0c..9d3530d40286 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -14,10 +14,7 @@ use reth_provider::{ TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{ - path::{Path}, - sync::Arc, -}; +use std::{path::Path, sync::Arc}; impl Command { pub(crate) fn generate_transactions_snapshot( From 986abfd67065ea2cb9f6a9891496db6cb783ab9a Mon Sep 17 00:00:00 2001 From: joshieDo Date: Tue, 14 Nov 2023 21:28:59 +0000 Subject: [PATCH 46/49] add missing imports --- bin/reth/src/db/snapshots/headers.rs | 5 ++++- bin/reth/src/db/snapshots/receipts.rs | 5 ++++- bin/reth/src/db/snapshots/transactions.rs | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/db/snapshots/headers.rs b/bin/reth/src/db/snapshots/headers.rs index 5a944aae1fd2..8b9ea080aac2 100644 --- a/bin/reth/src/db/snapshots/headers.rs +++ b/bin/reth/src/db/snapshots/headers.rs @@ -13,7 +13,10 @@ use reth_provider::{ providers::SnapshotProvider, DatabaseProviderRO, HeaderProvider, ProviderError, ProviderFactory, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_headers_snapshot( diff --git a/bin/reth/src/db/snapshots/receipts.rs b/bin/reth/src/db/snapshots/receipts.rs index fa92c04026fb..84e47acf4ba8 100644 --- a/bin/reth/src/db/snapshots/receipts.rs +++ b/bin/reth/src/db/snapshots/receipts.rs @@ -14,7 +14,10 @@ use reth_provider::{ ReceiptProvider, TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_receipts_snapshot( diff --git a/bin/reth/src/db/snapshots/transactions.rs b/bin/reth/src/db/snapshots/transactions.rs index 8cc4c396d953..0b7d8b0163db 100644 --- a/bin/reth/src/db/snapshots/transactions.rs +++ b/bin/reth/src/db/snapshots/transactions.rs @@ -14,7 +14,10 @@ use reth_provider::{ TransactionsProvider, TransactionsProviderExt, }; use reth_snapshot::{segments, segments::Segment}; -use std::{path::Path, sync::Arc}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; impl Command { pub(crate) fn generate_transactions_snapshot( From 6b2de08c44d3d33ef0ce32331c6b73505b395147 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Wed, 15 Nov 2023 14:45:16 +0000 Subject: [PATCH 47/49] typo --- crates/snapshot/src/snapshotter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 205600017bee..23d6851f2dec 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -154,7 +154,7 @@ impl Snapshotter { /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and /// notifies every tracker. fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> { - // It walks over the directory and parses the snapshot filenames extracing `SnapshotSegment` + // It walks over the directory and parses the snapshot filenames extracting `SnapshotSegment` // and their inclusive range. It then takes the maximum block number for each specific // segment. for (segment, block_range, _) in iter_snapshots(&self.snapshots_path)? { From 08850f0cb2fd908e0180206f79fb2bc5fb742326 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Wed, 15 Nov 2023 14:49:55 +0000 Subject: [PATCH 48/49] add doc to unwrap or 0 --- crates/snapshot/src/snapshotter.rs | 6 +++--- crates/storage/provider/src/providers/snapshot/manager.rs | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 23d6851f2dec..d9c1f6aeb003 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -154,9 +154,9 @@ impl Snapshotter { /// Looks into the snapshot directory to find the highest snapshotted block of each segment, and /// notifies every tracker. fn update_highest_snapshots_tracker(&mut self) -> RethResult<()> { - // It walks over the directory and parses the snapshot filenames extracting `SnapshotSegment` - // and their inclusive range. It then takes the maximum block number for each specific - // segment. + // It walks over the directory and parses the snapshot filenames extracting + // `SnapshotSegment` and their inclusive range. It then takes the maximum block + // number for each specific segment. for (segment, block_range, _) in iter_snapshots(&self.snapshots_path)? { let max_segment_block = self.highest_snapshots.as_mut(segment); if max_segment_block.map_or(true, |block| block < *block_range.end()) { diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 39408748b353..0f8472a0e265 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -163,6 +163,8 @@ impl SnapshotProvider { let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { + // `unwrap_or(0) is safe here as it sets block_start to 0 if the iterator is empty, + // indicating the lowest height snapshot has been reached. let block_start = snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); if block_start <= block { @@ -216,6 +218,8 @@ impl SnapshotProvider { let mut snapshots_rev_iter = segment_snapshots.iter().rev().peekable(); while let Some((block_end, tx_range)) = snapshots_rev_iter.next() { + // `unwrap_or(0) is safe here as it sets block_start to 0 if the iterator + // is empty, indicating the lowest height snapshot has been reached. let block_start = snapshots_rev_iter.peek().map(|(block_end, _)| *block_end + 1).unwrap_or(0); From 8881812ef7c066753809afe7561993ccb6442486 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Wed, 15 Nov 2023 15:18:19 +0000 Subject: [PATCH 49/49] add different missing snapshot variants --- crates/interfaces/src/provider.rs | 16 ++++++-- crates/primitives/src/snapshot/segment.rs | 2 + .../src/providers/snapshot/manager.rs | 37 ++++++++++--------- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index 5e67b9025087..a5a72c942ad6 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -1,6 +1,8 @@ use reth_primitives::{ - Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, TxHashOrNumber, TxNumber, B256, + Address, BlockHash, BlockHashOrNumber, BlockNumber, GotExpected, SnapshotSegment, + TxHashOrNumber, TxNumber, B256, }; +use std::path::PathBuf; use thiserror::Error; /// Bundled errors variants thrown by various providers. @@ -94,9 +96,15 @@ pub enum ProviderError { /// Provider does not support this particular request. #[error("this provider does not support this request")] UnsupportedProvider, - /// Snapshot file is not found for block. - #[error("not able to find snapshot file")] - MissingSnapshot, + /// Snapshot file is not found at specified path. + #[error("not able to find {0} snapshot file at {1}")] + MissingSnapshotPath(SnapshotSegment, PathBuf), + /// Snapshot file is not found for requested block. + #[error("not able to find {0} snapshot file for block number {1}")] + MissingSnapshotBlock(SnapshotSegment, BlockNumber), + /// Snapshot file is not found for requested transaction. + #[error("not able to find {0} snapshot file for transaction id {1}")] + MissingSnapshotTx(SnapshotSegment, TxNumber), } /// A root mismatch error at a given block height. diff --git a/crates/primitives/src/snapshot/segment.rs b/crates/primitives/src/snapshot/segment.rs index 4b6cadbdd0da..d8357fc169cb 100644 --- a/crates/primitives/src/snapshot/segment.rs +++ b/crates/primitives/src/snapshot/segment.rs @@ -2,6 +2,7 @@ use crate::{ snapshot::{Compression, Filters, InclusionFilter}, BlockNumber, TxNumber, }; +use derive_more::Display; use serde::{Deserialize, Serialize}; use std::{ffi::OsStr, ops::RangeInclusive, str::FromStr}; use strum::{AsRefStr, EnumString}; @@ -19,6 +20,7 @@ use strum::{AsRefStr, EnumString}; Serialize, EnumString, AsRefStr, + Display, )] #[cfg_attr(feature = "clap", derive(clap::ValueEnum))] /// Segment of the data that can be snapshotted. diff --git a/crates/storage/provider/src/providers/snapshot/manager.rs b/crates/storage/provider/src/providers/snapshot/manager.rs index 0f8472a0e265..7cdbdae316b1 100644 --- a/crates/storage/provider/src/providers/snapshot/manager.rs +++ b/crates/storage/provider/src/providers/snapshot/manager.rs @@ -78,7 +78,8 @@ impl SnapshotProvider { segment, || self.get_segment_ranges_from_block(segment, block), path, - ) + )? + .ok_or_else(|| ProviderError::MissingSnapshotBlock(segment, block).into()) } /// Gets the [`SnapshotJarProvider`] of the requested segment and transaction. @@ -92,7 +93,8 @@ impl SnapshotProvider { segment, || self.get_segment_ranges_from_transaction(segment, tx), path, - ) + )? + .ok_or_else(|| ProviderError::MissingSnapshotTx(segment, tx).into()) } /// Gets the [`SnapshotJarProvider`] of the requested segment and block or transaction. @@ -101,29 +103,30 @@ impl SnapshotProvider { segment: SnapshotSegment, fn_ranges: impl Fn() -> Option<(RangeInclusive, RangeInclusive)>, path: Option<&Path>, - ) -> RethResult> { + ) -> RethResult>> { // If we have a path, then get the block range and transaction range from its name. // Otherwise, check `self.available_snapshots` let snapshot_ranges = match path { - Some(path) => SnapshotSegment::parse_filename( - path.file_name().ok_or_else(|| ProviderError::MissingSnapshot)?, - ) - .and_then(|(parsed_segment, block_range, tx_range)| { - if parsed_segment == segment { - return Some((block_range, tx_range)); - } - None - }), + Some(path) => { + SnapshotSegment::parse_filename(path.file_name().ok_or_else(|| { + ProviderError::MissingSnapshotPath(segment, path.to_path_buf()) + })?) + .and_then(|(parsed_segment, block_range, tx_range)| { + if parsed_segment == segment { + return Some((block_range, tx_range)); + } + None + }) + } None => fn_ranges(), }; // Return cached `LoadedJar` or insert it for the first time, and then, return it. - match snapshot_ranges { - Some((block_range, tx_range)) => { - self.get_or_create_jar_provider(segment, &block_range, &tx_range) - } - None => Err(ProviderError::MissingSnapshot.into()), + if let Some((block_range, tx_range)) = snapshot_ranges { + return Ok(Some(self.get_or_create_jar_provider(segment, &block_range, &tx_range)?)); } + + Ok(None) } /// Given a segment, block range and transaction range it returns a cached