From 57a96194f0efe35989e2422d2148dc5af07ab2e6 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 24 Apr 2024 04:23:50 +0300 Subject: [PATCH 01/14] WIP: refactor commit_changes_with_height_update --- crates/fuel-core/src/database.rs | 149 +++++++++--------- .../fuel-core/src/service/genesis/importer.rs | 3 + .../src/service/genesis/importer/off_chain.rs | 36 +++++ crates/storage/src/transactional.rs | 3 +- 4 files changed, 119 insertions(+), 72 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 707fd00222b..60511d93037 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -300,41 +300,92 @@ impl AtomicView for Database { impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - commit_changes_with_height_update(self, changes, |iter| { - iter.iter_all::(Some(IterDirection::Reverse)) + let ignore_heights = false; + let new_height = if ignore_heights { + None + } else { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some(IterDirection::Reverse)) .map(|result| result.map(|(height, _)| height)) - .try_collect() - }) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(|h| u32::from(*h).into()).collect(), + } + .into()); + } + new_heights.iter().next().copied() + }; + commit_changes_with_height_update(self, changes, new_height) } } impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - commit_changes_with_height_update(self, changes, |iter| { - iter.iter_all::(Some(IterDirection::Reverse)) + let ignore_heights = false; + let new_height = if ignore_heights { + None + } else { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some(IterDirection::Reverse)) .map(|result| result.map(|(_, height)| height)) - .try_collect() - }) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(|h| u32::from(*h).into()).collect(), + } + .into()); + } + new_heights.iter().next().copied() + }; + commit_changes_with_height_update(self, changes, new_height) } } #[cfg(feature = "relayer")] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - commit_changes_with_height_update(self, changes, |iter| { - iter.iter_all::(Some( - IterDirection::Reverse, - )) - .map(|result| result.map(|(height, _)| height)) - .try_collect() - }) + let ignore_heights = false; + let new_height = if ignore_heights { + None + } else { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some( + IterDirection::Reverse, + )) + .map(|result| result.map(|(height, _)| height)) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(|h| u64::from(*h)).collect(), + } + .into()); + } + new_heights.iter().next().copied() + }; + commit_changes_with_height_update(self, changes, new_height) } } #[cfg(not(feature = "relayer"))] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - commit_changes_with_height_update(self, changes, |_| Ok(vec![])) + commit_changes_with_height_update(self, changes, vec![]) } } @@ -367,10 +418,8 @@ impl DatabaseHeight for DaBlockHeight { fn commit_changes_with_height_update( database: &mut Database, - changes: Changes, - heights_lookup: impl Fn( - &ChangesIterator, - ) -> StorageResult>, + mut changes: Changes, + new_height: Option, ) -> StorageResult<()> where Description: DatabaseDescription, @@ -378,32 +427,10 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { - // Gets the all new heights from the `changes` - let iterator = ChangesIterator::::new(&changes); - let new_heights = heights_lookup(&iterator)?; - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), - } - .into()); - } - - let new_height = new_heights.into_iter().last(); - let prev_height = *database.height.lock(); - - match (prev_height, new_height) { - (None, None) => { - // We are inside the regenesis process if the old and new heights are not set. - // In this case, we continue to commit until we discover a new height. - // This height will be the start of the database. - } - (Some(prev_height), Some(new_height)) => { + if let Some(new_height) = dbg!(new_height) { + // If height is already set, check that the new height is valid + if let Some(prev_height) = dbg!(*database.height.lock()) { // Each new commit should be linked to the previous commit to create a monotonically growing database. - let next_expected_height = prev_height .advance_height() .ok_or(DatabaseError::FailedToAdvanceHeight)?; @@ -418,24 +445,8 @@ where .into()); } } - (None, Some(_)) => { - // The new height is finally found; starting at this point, - // all next commits should be linked(the height should increase each time by one). - } - (Some(prev_height), None) => { - // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. - // The commit always should contain all data for the corresponding height. - return Err(DatabaseError::NewHeightIsNotSet { - prev_height: prev_height.as_u64(), - } - .into()); - } - }; - let updated_changes = if let Some(new_height) = new_height { - // We want to update the metadata table to include a new height. - // For that, we are building a new storage transaction around `changes`. - // Modifying this transaction will include all required updates into the `changes`. + // We want to also update the metadata table to include a new height. let mut transaction = StorageTransaction::transaction( &database, ConflictPolicy::Overwrite, @@ -450,20 +461,16 @@ where height: new_height, }, )?; - - transaction.into_changes() - } else { - changes + changes = transaction.into_changes(); }; let mut guard = database.height.lock(); - database - .data - .as_ref() - .commit_changes(new_height, updated_changes)?; + database.data.as_ref().commit_changes(new_height, changes)?; // Update the block height - *guard = new_height; + if new_height.is_some() { + *guard = new_height; + } Ok(()) } diff --git a/crates/fuel-core/src/service/genesis/importer.rs b/crates/fuel-core/src/service/genesis/importer.rs index 72867f16d80..a6cf88f1928 100644 --- a/crates/fuel-core/src/service/genesis/importer.rs +++ b/crates/fuel-core/src/service/genesis/importer.rs @@ -18,6 +18,7 @@ use crate::{ }, fuel_core_graphql_api::storage::messages::SpentMessages, graphql_api::storage::{ + blocks::FuelBlockIdsToHeights, coins::OwnedCoins, contracts::ContractsInfo, messages::OwnedMessageIds, @@ -133,6 +134,8 @@ impl SnapshotImporter { self.spawn_worker_off_chain::()?; self.spawn_worker_off_chain::()?; self.spawn_worker_off_chain::()?; + self.spawn_worker_off_chain::()?; + self.spawn_worker_off_chain::()?; self.task_manager.wait().await?; diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index f73d7b95b23..8f7accdd244 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -291,3 +291,39 @@ impl ImportTable for Handler { Ok(()) } } + +impl ImportTable for Handler { + type TableInSnapshot = FuelBlocks; + type TableBeingWritten = FuelBlockIdsToHeights; + type DbDesc = OffChain; + + fn process( + &mut self, + group: Vec>, + tx: &mut StorageTransaction<&mut Database>, + ) -> anyhow::Result<()> { + for entry in group { + tx.storage_as_mut::() + .insert(&entry.value.id(), &entry.key)?; + } + Ok(()) + } +} + +impl ImportTable for Handler { + type TableInSnapshot = OldFuelBlocks; + type TableBeingWritten = FuelBlockIdsToHeights; + type DbDesc = OffChain; + + fn process( + &mut self, + group: Vec>, + tx: &mut StorageTransaction<&mut Database>, + ) -> anyhow::Result<()> { + for entry in group { + tx.storage_as_mut::() + .insert(&entry.value.id(), &entry.key)?; + } + Ok(()) + } +} diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 4351fde6d3b..1b6808a54cd 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -118,6 +118,7 @@ pub trait Modifiable { } /// The type describing the list of changes to the storage. +/// ColumnId -> (Key -> Operation) pub type Changes = HashMap, WriteOperation>>; impl From> for Changes { @@ -188,7 +189,7 @@ where Storage: Modifiable, { /// Commits the changes into the storage. - pub fn commit(mut self) -> StorageResult { + pub fn commit(mut self, _x: bool) -> StorageResult { let changes = core::mem::take(&mut self.inner.changes); self.inner.storage.commit_changes(changes)?; Ok(self.inner.storage) From 2a503bd0ced1cd25612a141ed5915fdd0ead7258 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Wed, 24 Apr 2024 06:43:13 +0300 Subject: [PATCH 02/14] Make it actually work --- crates/fuel-core/src/database.rs | 207 ++++++++++++------ crates/fuel-core/src/database/storage.rs | 16 +- crates/fuel-core/src/executor.rs | 6 +- .../service/genesis/importer/import_task.rs | 8 +- .../src/service/genesis/importer/off_chain.rs | 9 +- crates/services/executor/src/executor.rs | 21 +- .../upgradable-executor/src/executor.rs | 19 +- crates/storage/src/structured_storage.rs | 9 +- crates/storage/src/test_helpers.rs | 3 +- crates/storage/src/transactional.rs | 79 ++++++- 10 files changed, 266 insertions(+), 111 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 60511d93037..eb93c6f5402 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -41,6 +41,7 @@ use fuel_core_storage::{ Changes, ConflictPolicy, Modifiable, + ModifyHeightPolicy, StorageTransaction, }, Error as StorageError, @@ -299,54 +300,78 @@ impl AtomicView for Database { } impl Modifiable for Database { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - let ignore_heights = false; - let new_height = if ignore_heights { - None - } else { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some(IterDirection::Reverse)) - .map(|result| result.map(|(height, _)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(|h| u32::from(*h).into()).collect(), + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + let new_height = match dbg!(height_policy) { + ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, + ModifyHeightPolicy::Update => { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(height, _)| height)) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights + .iter() + .map(|h| u32::from(*h).into()) + .collect(), + } + .into()); } - .into()); + new_heights + .iter() + .next() + .copied() + .map(UpdateHeight::SetIncremented) + .unwrap_or(UpdateHeight::NoChange) } - new_heights.iter().next().copied() }; commit_changes_with_height_update(self, changes, new_height) } } impl Modifiable for Database { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - let ignore_heights = false; - let new_height = if ignore_heights { - None - } else { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some(IterDirection::Reverse)) - .map(|result| result.map(|(_, height)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(|h| u32::from(*h).into()).collect(), + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + let new_height = match dbg!(height_policy) { + ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, + ModifyHeightPolicy::Update => { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(_, height)| height)) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights + .iter() + .map(|h| u32::from(*h).into()) + .collect(), + } + .into()); } - .into()); + new_heights + .iter() + .next() + .copied() + .map(UpdateHeight::SetIncremented) + .unwrap_or(UpdateHeight::NoChange) } - new_heights.iter().next().copied() }; commit_changes_with_height_update(self, changes, new_height) } @@ -354,29 +379,38 @@ impl Modifiable for Database { #[cfg(feature = "relayer")] impl Modifiable for Database { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - let ignore_heights = false; - let new_height = if ignore_heights { - None - } else { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some( - IterDirection::Reverse, - )) - .map(|result| result.map(|(height, _)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(|h| u64::from(*h)).collect(), + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + let new_height = match dbg!(height_policy) { + ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, + ModifyHeightPolicy::Update => { + let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) + .iter_all::(Some( + IterDirection::Reverse, + )) + .map(|result| result.map(|(height, _)| height)) + .try_collect()?; + new_heights.dedup(); + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(|h| u64::from(*h)).collect(), + } + .into()); } - .into()); + new_heights + .iter() + .next() + .copied() + .map(UpdateHeight::SetIncremented) + .unwrap_or(UpdateHeight::NoChange) } - new_heights.iter().next().copied() }; commit_changes_with_height_update(self, changes, new_height) } @@ -416,10 +450,20 @@ impl DatabaseHeight for DaBlockHeight { } } +enum UpdateHeight { + /// Set new height, that must be one more than the previous one. + SetIncremented(Height), + /// This commit doesn't contain any height. + /// This causes an error if the previous height is set. + NoChange, + /// Do not perform any height update. + Ignore, +} + fn commit_changes_with_height_update( database: &mut Database, mut changes: Changes, - new_height: Option, + update_height: UpdateHeight, ) -> StorageResult<()> where Description: DatabaseDescription, @@ -427,25 +471,42 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { - if let Some(new_height) = dbg!(new_height) { - // If height is already set, check that the new height is valid - if let Some(prev_height) = dbg!(*database.height.lock()) { - // Each new commit should be linked to the previous commit to create a monotonically growing database. - let next_expected_height = prev_height - .advance_height() - .ok_or(DatabaseError::FailedToAdvanceHeight)?; - - // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 - // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. - if next_expected_height > new_height { - return Err(DatabaseError::HeightsAreNotLinked { + let prev_height = *database.height.lock(); + let new_height = match update_height { + UpdateHeight::SetIncremented(new_height) => { + // If height is already set, check that the new height is valid + if let Some(prev_height) = prev_height { + // Each new commit should be linked to the previous commit to create a monotonically growing database. + let next_expected_height = prev_height + .advance_height() + .ok_or(DatabaseError::FailedToAdvanceHeight)?; + + // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 + // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. + if next_expected_height > new_height { + return Err(DatabaseError::HeightsAreNotLinked { + prev_height: prev_height.as_u64(), + new_height: new_height.as_u64(), + } + .into()); + } + } + Some(new_height) + } + UpdateHeight::NoChange => { + if let Some(prev_height) = prev_height { + return Err(DatabaseError::NewHeightIsNotSet { prev_height: prev_height.as_u64(), - new_height: new_height.as_u64(), } .into()); + } else { + None } } + UpdateHeight::Ignore => None, + }; + if let Some(new_height) = new_height { // We want to also update the metadata table to include a new height. let mut transaction = StorageTransaction::transaction( &database, diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index acabb92b5fc..44936645121 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -76,7 +76,7 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().insert(key, value)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(prev) } @@ -87,7 +87,7 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().remove(key)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(prev) } } @@ -145,7 +145,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::write(&mut transaction, key, buf)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(prev) } @@ -160,7 +160,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::replace(&mut transaction, key, buf)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(prev) } @@ -171,7 +171,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::take(&mut transaction, key)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(prev) } } @@ -197,7 +197,7 @@ where Default::default(), ); StorageBatchMutate::init_storage(&mut transaction, set)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(()) } @@ -213,7 +213,7 @@ where Default::default(), ); StorageBatchMutate::insert_batch(&mut transaction, set)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(()) } @@ -228,7 +228,7 @@ where Default::default(), ); StorageBatchMutate::remove_batch(&mut transaction, set)?; - self.commit_changes(transaction.into_changes())?; + self.commit_changes(transaction.into_changes(), Default::default())?; Ok(()) } } diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 1d6c48f017a..510b3b03792 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -480,7 +480,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes) + .commit_changes(changes, Default::default()) .unwrap(); assert_eq!(skipped_transactions.len(), 1); @@ -563,7 +563,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes) + .commit_changes(changes, Default::default()) .unwrap(); assert_eq!(skipped_transactions.len(), 0); @@ -786,7 +786,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes) + .commit_changes(changes, Default::default()) .unwrap(); let receipts = tx_status[0].result.receipts(); diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index b80d72d5845..122f7eee66e 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -4,6 +4,7 @@ use fuel_core_storage::{ structured_storage::TableWithBlueprint, transactional::{ Modifiable, + ModifyHeightPolicy, StorageTransaction, WriteTransaction, }, @@ -46,6 +47,8 @@ pub trait ImportTable { type TableBeingWritten: TableWithBlueprint; type DbDesc: DatabaseDescription; + const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Update; + fn process( &mut self, group: Vec>, @@ -119,7 +122,10 @@ where }) .try_for_each(|(index, group)| { let group = group?; - let mut tx = db.write_transaction(); + let mut tx = db + .write_transaction() + .with_modify_height_policy(Logic::MODIFY_HEIGHT_POLICY); + self.handler.process(group, &mut tx)?; let progress_name = diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 8f7accdd244..0af1a87eb55 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -34,7 +34,10 @@ use fuel_core_storage::{ SealedBlockConsensus, Transactions, }, - transactional::StorageTransaction, + transactional::{ + ModifyHeightPolicy, + StorageTransaction, + }, StorageAsMut, }; use fuel_core_types::services::executor::Event; @@ -297,6 +300,8 @@ impl ImportTable for Handler { type TableBeingWritten = FuelBlockIdsToHeights; type DbDesc = OffChain; + const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Ignore; + fn process( &mut self, group: Vec>, @@ -315,6 +320,8 @@ impl ImportTable for Handler { type TableBeingWritten = FuelBlockIdsToHeights; type DbDesc = OffChain; + const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Ignore; + fn process( &mut self, group: Vec>, diff --git a/crates/services/executor/src/executor.rs b/crates/services/executor/src/executor.rs index 56e292537c8..d701399b184 100644 --- a/crates/services/executor/src/executor.rs +++ b/crates/services/executor/src/executor.rs @@ -650,9 +650,12 @@ where } let changes_from_thread = thread_block_transaction.into_changes(); - block_with_relayer_data_transaction.commit_changes(changes_from_thread)?; - self.block_st_transaction - .commit_changes(block_with_relayer_data_transaction.into_changes())?; + block_with_relayer_data_transaction + .commit_changes(changes_from_thread, Default::default())?; + self.block_st_transaction.commit_changes( + block_with_relayer_data_transaction.into_changes(), + Default::default(), + )?; if execution_kind != ExecutionKind::DryRun && !data.found_mint { return Err(ExecutorError::MintMissing) @@ -788,9 +791,12 @@ where } let changes_from_thread = thread_block_transaction.into_changes(); - block_with_relayer_data_transaction.commit_changes(changes_from_thread)?; - self.block_st_transaction - .commit_changes(block_with_relayer_data_transaction.into_changes())?; + block_with_relayer_data_transaction + .commit_changes(changes_from_thread, Default::default())?; + self.block_st_transaction.commit_changes( + block_with_relayer_data_transaction.into_changes(), + Default::default(), + )?; if !data.found_mint { return Err(ExecutorError::MintMissing) @@ -1328,8 +1334,9 @@ where // only commit state changes if execution was a success if !reverted { self.log_backtrace(&vm, &receipts); + let height_policy = sub_block_db_commit.height_policy(); let changes = sub_block_db_commit.into_changes(); - tx_st_transaction.commit_changes(changes)?; + tx_st_transaction.commit_changes(changes, height_policy)?; } // update block commitment diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index fa1e48d230c..79172d3d9e7 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -201,7 +201,8 @@ where ) -> fuel_core_types::services::executor::Result { let (result, changes) = self.execute_without_commit(block)?.into(); - self.storage_view_provider.commit_changes(changes)?; + self.storage_view_provider + .commit_changes(changes, Default::default())?; Ok(result) } @@ -213,7 +214,8 @@ where ) -> fuel_core_types::services::executor::Result { let (result, changes) = self.validate_without_commit(block)?.into(); - self.storage_view_provider.commit_changes(changes)?; + self.storage_view_provider + .commit_changes(changes, Default::default())?; Ok(result) } } @@ -586,7 +588,10 @@ mod test { kv_store::Value, structured_storage::test::InMemoryStorage, tables::ConsensusParametersVersions, - transactional::WriteTransaction, + transactional::{ + ModifyHeightPolicy, + WriteTransaction, + }, Result as StorageResult, StorageAsMut, }; @@ -641,8 +646,12 @@ mod test { } impl Modifiable for Storage { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.0.commit_changes(changes) + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + self.0.commit_changes(changes, height_policy) } } diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index 11fb9fa986a..ad6a83da91c 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -30,6 +30,7 @@ use crate::{ transactional::{ Changes, Modifiable, + ModifyHeightPolicy, }, Error as StorageError, Mappable, @@ -201,8 +202,12 @@ impl Modifiable for StructuredStorage where S: Modifiable, { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.inner.commit_changes(changes) + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + self.inner.commit_changes(changes, height_policy) } } diff --git a/crates/storage/src/test_helpers.rs b/crates/storage/src/test_helpers.rs index 9e11db66f08..1d2f5bc9eae 100644 --- a/crates/storage/src/test_helpers.rs +++ b/crates/storage/src/test_helpers.rs @@ -4,6 +4,7 @@ use crate::{ transactional::{ Changes, Modifiable, + ModifyHeightPolicy, }, Error as StorageError, Mappable, @@ -83,7 +84,7 @@ mockall::mock! { } impl Modifiable for Basic { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; + fn commit_changes(&mut self, changes: Changes, _ignored: ModifyHeightPolicy) -> StorageResult<()>; } } diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 1b6808a54cd..878b2801c2e 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -58,6 +58,7 @@ pub type StorageTransaction = StructuredStorage>; pub struct InMemoryTransaction { pub(crate) changes: Changes, pub(crate) policy: ConflictPolicy, + pub(crate) height_policy: ModifyHeightPolicy, pub(crate) storage: S, } @@ -67,15 +68,27 @@ impl StorageTransaction { StructuredStorage::new(InMemoryTransaction { changes, policy, + height_policy: ModifyHeightPolicy::Update, storage, }) } - /// Creates a new instance of the structured storage with a `ConflictPolicy`. + /// Returns a new version with updated `ConflictPolicy`. pub fn with_policy(self, policy: ConflictPolicy) -> Self { StructuredStorage::new(InMemoryTransaction { changes: self.inner.changes, policy, + height_policy: self.inner.height_policy, + storage: self.inner.storage, + }) + } + + /// Returns a new version with updated `ModifyHeightPolicy`. + pub fn with_modify_height_policy(self, height_policy: ModifyHeightPolicy) -> Self { + StructuredStorage::new(InMemoryTransaction { + changes: self.inner.changes, + policy: self.inner.policy, + height_policy, storage: self.inner.storage, }) } @@ -85,6 +98,7 @@ impl StorageTransaction { StructuredStorage::new(InMemoryTransaction { changes, policy: self.inner.policy, + height_policy: self.inner.height_policy, storage: self.inner.storage, }) } @@ -94,6 +108,11 @@ impl StorageTransaction { self.inner.changes } + /// Returns the height update policy + pub fn height_policy(&self) -> ModifyHeightPolicy { + self.inner.height_policy + } + /// Resets the changes to the storage. pub fn reset_changes(&mut self) { self.inner.changes = Default::default(); @@ -114,7 +133,26 @@ pub enum ConflictPolicy { #[impl_tools::autoimpl(for &mut T, Box)] pub trait Modifiable { /// Commits the changes into the storage. - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()>; +} + +/// Whether to update the height information when committing. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModifyHeightPolicy { + /// Update height information when committing. + /// Errors if attempting to write to multiple heights in a single commit. + /// Errors if the commit doesn't update the height. + /// Note that targets which do not track height will ignore this policy, + /// unless they can pass it to the underlying storage which does so. + #[default] + Update, + /// Do not update height information when committing. + /// Allows multiple heights to be written in a single commit. + Ignore, } /// The type describing the list of changes to the storage. @@ -189,15 +227,26 @@ where Storage: Modifiable, { /// Commits the changes into the storage. - pub fn commit(mut self, _x: bool) -> StorageResult { + pub fn commit(mut self) -> StorageResult { let changes = core::mem::take(&mut self.inner.changes); - self.inner.storage.commit_changes(changes)?; + self.inner + .storage + .commit_changes(changes, self.inner.height_policy)?; Ok(self.inner.storage) } } impl Modifiable for InMemoryTransaction { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + /// Panics height policy mismatches the height policy fo the transaction. + fn commit_changes( + &mut self, + changes: Changes, + height_policy: ModifyHeightPolicy, + ) -> StorageResult<()> { + assert!( + self.height_policy == height_policy, + "Height policy mismatch" + ); for (column, value) in changes.into_iter() { let btree = self.changes.entry(column).or_default(); for (k, v) in value { @@ -392,7 +441,11 @@ mod test { }; impl Modifiable for InMemoryStorage { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + fn commit_changes( + &mut self, + changes: Changes, + _: ModifyHeightPolicy, + ) -> StorageResult<()> { for (column, value) in changes.into_iter() { for (key, value) in value { match value { @@ -446,8 +499,12 @@ mod test { let changes1 = sub_transaction1.into(); let changes2 = sub_transaction2.into(); - transactions.commit_changes(changes1).unwrap(); - transactions.commit_changes(changes2).unwrap(); + transactions + .commit_changes(changes1, Default::default()) + .unwrap(); + transactions + .commit_changes(changes2, Default::default()) + .unwrap(); } #[test] @@ -470,9 +527,11 @@ mod test { let changes1 = sub_transaction1.into(); let changes2 = sub_transaction2.into(); - transactions.commit_changes(changes1).unwrap(); transactions - .commit_changes(changes2) + .commit_changes(changes1, Default::default()) + .unwrap(); + transactions + .commit_changes(changes2, Default::default()) .expect_err("Should fails because of the modification for the same key"); } From 46d6c8a3ad55006d84296047b1a3c9535adcc44e Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 01:49:42 +0300 Subject: [PATCH 03/14] Revert changes to commit --- crates/fuel-core/src/database.rs | 234 +++++++----------- crates/fuel-core/src/database/storage.rs | 16 +- crates/fuel-core/src/executor.rs | 6 +- .../service/genesis/importer/import_task.rs | 8 +- .../src/service/genesis/importer/off_chain.rs | 9 +- crates/services/executor/src/executor.rs | 21 +- .../upgradable-executor/src/executor.rs | 19 +- crates/storage/src/structured_storage.rs | 9 +- crates/storage/src/test_helpers.rs | 3 +- crates/storage/src/transactional.rs | 78 +----- 10 files changed, 120 insertions(+), 283 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index eb93c6f5402..707fd00222b 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -41,7 +41,6 @@ use fuel_core_storage::{ Changes, ConflictPolicy, Modifiable, - ModifyHeightPolicy, StorageTransaction, }, Error as StorageError, @@ -300,126 +299,42 @@ impl AtomicView for Database { } impl Modifiable for Database { - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - let new_height = match dbg!(height_policy) { - ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, - ModifyHeightPolicy::Update => { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some(IterDirection::Reverse)) - .map(|result| result.map(|(height, _)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights - .iter() - .map(|h| u32::from(*h).into()) - .collect(), - } - .into()); - } - new_heights - .iter() - .next() - .copied() - .map(UpdateHeight::SetIncremented) - .unwrap_or(UpdateHeight::NoChange) - } - }; - commit_changes_with_height_update(self, changes, new_height) + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(height, _)| height)) + .try_collect() + }) } } impl Modifiable for Database { - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - let new_height = match dbg!(height_policy) { - ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, - ModifyHeightPolicy::Update => { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some(IterDirection::Reverse)) - .map(|result| result.map(|(_, height)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights - .iter() - .map(|h| u32::from(*h).into()) - .collect(), - } - .into()); - } - new_heights - .iter() - .next() - .copied() - .map(UpdateHeight::SetIncremented) - .unwrap_or(UpdateHeight::NoChange) - } - }; - commit_changes_with_height_update(self, changes, new_height) + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some(IterDirection::Reverse)) + .map(|result| result.map(|(_, height)| height)) + .try_collect() + }) } } #[cfg(feature = "relayer")] impl Modifiable for Database { - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - let new_height = match dbg!(height_policy) { - ModifyHeightPolicy::Ignore => UpdateHeight::Ignore, - ModifyHeightPolicy::Update => { - let mut new_heights: Vec<_> = ChangesIterator::::new(&changes) - .iter_all::(Some( - IterDirection::Reverse, - )) - .map(|result| result.map(|(height, _)| height)) - .try_collect()?; - new_heights.dedup(); - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(|h| u64::from(*h)).collect(), - } - .into()); - } - new_heights - .iter() - .next() - .copied() - .map(UpdateHeight::SetIncremented) - .unwrap_or(UpdateHeight::NoChange) - } - }; - commit_changes_with_height_update(self, changes, new_height) + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + commit_changes_with_height_update(self, changes, |iter| { + iter.iter_all::(Some( + IterDirection::Reverse, + )) + .map(|result| result.map(|(height, _)| height)) + .try_collect() + }) } } #[cfg(not(feature = "relayer"))] impl Modifiable for Database { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - commit_changes_with_height_update(self, changes, vec![]) + commit_changes_with_height_update(self, changes, |_| Ok(vec![])) } } @@ -450,20 +365,12 @@ impl DatabaseHeight for DaBlockHeight { } } -enum UpdateHeight { - /// Set new height, that must be one more than the previous one. - SetIncremented(Height), - /// This commit doesn't contain any height. - /// This causes an error if the previous height is set. - NoChange, - /// Do not perform any height update. - Ignore, -} - fn commit_changes_with_height_update( database: &mut Database, - mut changes: Changes, - update_height: UpdateHeight, + changes: Changes, + heights_lookup: impl Fn( + &ChangesIterator, + ) -> StorageResult>, ) -> StorageResult<()> where Description: DatabaseDescription, @@ -471,43 +378,64 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { + // Gets the all new heights from the `changes` + let iterator = ChangesIterator::::new(&changes); + let new_heights = heights_lookup(&iterator)?; + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), + } + .into()); + } + + let new_height = new_heights.into_iter().last(); let prev_height = *database.height.lock(); - let new_height = match update_height { - UpdateHeight::SetIncremented(new_height) => { - // If height is already set, check that the new height is valid - if let Some(prev_height) = prev_height { - // Each new commit should be linked to the previous commit to create a monotonically growing database. - let next_expected_height = prev_height - .advance_height() - .ok_or(DatabaseError::FailedToAdvanceHeight)?; - - // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 - // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. - if next_expected_height > new_height { - return Err(DatabaseError::HeightsAreNotLinked { - prev_height: prev_height.as_u64(), - new_height: new_height.as_u64(), - } - .into()); - } - } - Some(new_height) + + match (prev_height, new_height) { + (None, None) => { + // We are inside the regenesis process if the old and new heights are not set. + // In this case, we continue to commit until we discover a new height. + // This height will be the start of the database. } - UpdateHeight::NoChange => { - if let Some(prev_height) = prev_height { - return Err(DatabaseError::NewHeightIsNotSet { + (Some(prev_height), Some(new_height)) => { + // Each new commit should be linked to the previous commit to create a monotonically growing database. + + let next_expected_height = prev_height + .advance_height() + .ok_or(DatabaseError::FailedToAdvanceHeight)?; + + // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 + // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. + if next_expected_height > new_height { + return Err(DatabaseError::HeightsAreNotLinked { prev_height: prev_height.as_u64(), + new_height: new_height.as_u64(), } .into()); - } else { - None } } - UpdateHeight::Ignore => None, + (None, Some(_)) => { + // The new height is finally found; starting at this point, + // all next commits should be linked(the height should increase each time by one). + } + (Some(prev_height), None) => { + // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. + // The commit always should contain all data for the corresponding height. + return Err(DatabaseError::NewHeightIsNotSet { + prev_height: prev_height.as_u64(), + } + .into()); + } }; - if let Some(new_height) = new_height { - // We want to also update the metadata table to include a new height. + let updated_changes = if let Some(new_height) = new_height { + // We want to update the metadata table to include a new height. + // For that, we are building a new storage transaction around `changes`. + // Modifying this transaction will include all required updates into the `changes`. let mut transaction = StorageTransaction::transaction( &database, ConflictPolicy::Overwrite, @@ -522,16 +450,20 @@ where height: new_height, }, )?; - changes = transaction.into_changes(); + + transaction.into_changes() + } else { + changes }; let mut guard = database.height.lock(); - database.data.as_ref().commit_changes(new_height, changes)?; + database + .data + .as_ref() + .commit_changes(new_height, updated_changes)?; // Update the block height - if new_height.is_some() { - *guard = new_height; - } + *guard = new_height; Ok(()) } diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index 44936645121..acabb92b5fc 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -76,7 +76,7 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().insert(key, value)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } @@ -87,7 +87,7 @@ where Default::default(), ); let prev = transaction.storage_as_mut::().remove(key)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } } @@ -145,7 +145,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::write(&mut transaction, key, buf)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } @@ -160,7 +160,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::replace(&mut transaction, key, buf)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } @@ -171,7 +171,7 @@ where Default::default(), ); let prev = <_ as StorageWrite>::take(&mut transaction, key)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(prev) } } @@ -197,7 +197,7 @@ where Default::default(), ); StorageBatchMutate::init_storage(&mut transaction, set)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(()) } @@ -213,7 +213,7 @@ where Default::default(), ); StorageBatchMutate::insert_batch(&mut transaction, set)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(()) } @@ -228,7 +228,7 @@ where Default::default(), ); StorageBatchMutate::remove_batch(&mut transaction, set)?; - self.commit_changes(transaction.into_changes(), Default::default())?; + self.commit_changes(transaction.into_changes())?; Ok(()) } } diff --git a/crates/fuel-core/src/executor.rs b/crates/fuel-core/src/executor.rs index 510b3b03792..1d6c48f017a 100644 --- a/crates/fuel-core/src/executor.rs +++ b/crates/fuel-core/src/executor.rs @@ -480,7 +480,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes, Default::default()) + .commit_changes(changes) .unwrap(); assert_eq!(skipped_transactions.len(), 1); @@ -563,7 +563,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes, Default::default()) + .commit_changes(changes) .unwrap(); assert_eq!(skipped_transactions.len(), 0); @@ -786,7 +786,7 @@ mod tests { .into(); producer .storage_view_provider - .commit_changes(changes, Default::default()) + .commit_changes(changes) .unwrap(); let receipts = tx_status[0].result.receipts(); diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index 122f7eee66e..b80d72d5845 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -4,7 +4,6 @@ use fuel_core_storage::{ structured_storage::TableWithBlueprint, transactional::{ Modifiable, - ModifyHeightPolicy, StorageTransaction, WriteTransaction, }, @@ -47,8 +46,6 @@ pub trait ImportTable { type TableBeingWritten: TableWithBlueprint; type DbDesc: DatabaseDescription; - const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Update; - fn process( &mut self, group: Vec>, @@ -122,10 +119,7 @@ where }) .try_for_each(|(index, group)| { let group = group?; - let mut tx = db - .write_transaction() - .with_modify_height_policy(Logic::MODIFY_HEIGHT_POLICY); - + let mut tx = db.write_transaction(); self.handler.process(group, &mut tx)?; let progress_name = diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 0af1a87eb55..8f7accdd244 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -34,10 +34,7 @@ use fuel_core_storage::{ SealedBlockConsensus, Transactions, }, - transactional::{ - ModifyHeightPolicy, - StorageTransaction, - }, + transactional::StorageTransaction, StorageAsMut, }; use fuel_core_types::services::executor::Event; @@ -300,8 +297,6 @@ impl ImportTable for Handler { type TableBeingWritten = FuelBlockIdsToHeights; type DbDesc = OffChain; - const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Ignore; - fn process( &mut self, group: Vec>, @@ -320,8 +315,6 @@ impl ImportTable for Handler { type TableBeingWritten = FuelBlockIdsToHeights; type DbDesc = OffChain; - const MODIFY_HEIGHT_POLICY: ModifyHeightPolicy = ModifyHeightPolicy::Ignore; - fn process( &mut self, group: Vec>, diff --git a/crates/services/executor/src/executor.rs b/crates/services/executor/src/executor.rs index d701399b184..56e292537c8 100644 --- a/crates/services/executor/src/executor.rs +++ b/crates/services/executor/src/executor.rs @@ -650,12 +650,9 @@ where } let changes_from_thread = thread_block_transaction.into_changes(); - block_with_relayer_data_transaction - .commit_changes(changes_from_thread, Default::default())?; - self.block_st_transaction.commit_changes( - block_with_relayer_data_transaction.into_changes(), - Default::default(), - )?; + block_with_relayer_data_transaction.commit_changes(changes_from_thread)?; + self.block_st_transaction + .commit_changes(block_with_relayer_data_transaction.into_changes())?; if execution_kind != ExecutionKind::DryRun && !data.found_mint { return Err(ExecutorError::MintMissing) @@ -791,12 +788,9 @@ where } let changes_from_thread = thread_block_transaction.into_changes(); - block_with_relayer_data_transaction - .commit_changes(changes_from_thread, Default::default())?; - self.block_st_transaction.commit_changes( - block_with_relayer_data_transaction.into_changes(), - Default::default(), - )?; + block_with_relayer_data_transaction.commit_changes(changes_from_thread)?; + self.block_st_transaction + .commit_changes(block_with_relayer_data_transaction.into_changes())?; if !data.found_mint { return Err(ExecutorError::MintMissing) @@ -1334,9 +1328,8 @@ where // only commit state changes if execution was a success if !reverted { self.log_backtrace(&vm, &receipts); - let height_policy = sub_block_db_commit.height_policy(); let changes = sub_block_db_commit.into_changes(); - tx_st_transaction.commit_changes(changes, height_policy)?; + tx_st_transaction.commit_changes(changes)?; } // update block commitment diff --git a/crates/services/upgradable-executor/src/executor.rs b/crates/services/upgradable-executor/src/executor.rs index 79172d3d9e7..fa1e48d230c 100644 --- a/crates/services/upgradable-executor/src/executor.rs +++ b/crates/services/upgradable-executor/src/executor.rs @@ -201,8 +201,7 @@ where ) -> fuel_core_types::services::executor::Result { let (result, changes) = self.execute_without_commit(block)?.into(); - self.storage_view_provider - .commit_changes(changes, Default::default())?; + self.storage_view_provider.commit_changes(changes)?; Ok(result) } @@ -214,8 +213,7 @@ where ) -> fuel_core_types::services::executor::Result { let (result, changes) = self.validate_without_commit(block)?.into(); - self.storage_view_provider - .commit_changes(changes, Default::default())?; + self.storage_view_provider.commit_changes(changes)?; Ok(result) } } @@ -588,10 +586,7 @@ mod test { kv_store::Value, structured_storage::test::InMemoryStorage, tables::ConsensusParametersVersions, - transactional::{ - ModifyHeightPolicy, - WriteTransaction, - }, + transactional::WriteTransaction, Result as StorageResult, StorageAsMut, }; @@ -646,12 +641,8 @@ mod test { } impl Modifiable for Storage { - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - self.0.commit_changes(changes, height_policy) + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.0.commit_changes(changes) } } diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index ad6a83da91c..11fb9fa986a 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -30,7 +30,6 @@ use crate::{ transactional::{ Changes, Modifiable, - ModifyHeightPolicy, }, Error as StorageError, Mappable, @@ -202,12 +201,8 @@ impl Modifiable for StructuredStorage where S: Modifiable, { - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - self.inner.commit_changes(changes, height_policy) + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.inner.commit_changes(changes) } } diff --git a/crates/storage/src/test_helpers.rs b/crates/storage/src/test_helpers.rs index 1d2f5bc9eae..9e11db66f08 100644 --- a/crates/storage/src/test_helpers.rs +++ b/crates/storage/src/test_helpers.rs @@ -4,7 +4,6 @@ use crate::{ transactional::{ Changes, Modifiable, - ModifyHeightPolicy, }, Error as StorageError, Mappable, @@ -84,7 +83,7 @@ mockall::mock! { } impl Modifiable for Basic { - fn commit_changes(&mut self, changes: Changes, _ignored: ModifyHeightPolicy) -> StorageResult<()>; + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; } } diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 878b2801c2e..4351fde6d3b 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -58,7 +58,6 @@ pub type StorageTransaction = StructuredStorage>; pub struct InMemoryTransaction { pub(crate) changes: Changes, pub(crate) policy: ConflictPolicy, - pub(crate) height_policy: ModifyHeightPolicy, pub(crate) storage: S, } @@ -68,27 +67,15 @@ impl StorageTransaction { StructuredStorage::new(InMemoryTransaction { changes, policy, - height_policy: ModifyHeightPolicy::Update, storage, }) } - /// Returns a new version with updated `ConflictPolicy`. + /// Creates a new instance of the structured storage with a `ConflictPolicy`. pub fn with_policy(self, policy: ConflictPolicy) -> Self { StructuredStorage::new(InMemoryTransaction { changes: self.inner.changes, policy, - height_policy: self.inner.height_policy, - storage: self.inner.storage, - }) - } - - /// Returns a new version with updated `ModifyHeightPolicy`. - pub fn with_modify_height_policy(self, height_policy: ModifyHeightPolicy) -> Self { - StructuredStorage::new(InMemoryTransaction { - changes: self.inner.changes, - policy: self.inner.policy, - height_policy, storage: self.inner.storage, }) } @@ -98,7 +85,6 @@ impl StorageTransaction { StructuredStorage::new(InMemoryTransaction { changes, policy: self.inner.policy, - height_policy: self.inner.height_policy, storage: self.inner.storage, }) } @@ -108,11 +94,6 @@ impl StorageTransaction { self.inner.changes } - /// Returns the height update policy - pub fn height_policy(&self) -> ModifyHeightPolicy { - self.inner.height_policy - } - /// Resets the changes to the storage. pub fn reset_changes(&mut self) { self.inner.changes = Default::default(); @@ -133,30 +114,10 @@ pub enum ConflictPolicy { #[impl_tools::autoimpl(for &mut T, Box)] pub trait Modifiable { /// Commits the changes into the storage. - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()>; -} - -/// Whether to update the height information when committing. -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] -pub enum ModifyHeightPolicy { - /// Update height information when committing. - /// Errors if attempting to write to multiple heights in a single commit. - /// Errors if the commit doesn't update the height. - /// Note that targets which do not track height will ignore this policy, - /// unless they can pass it to the underlying storage which does so. - #[default] - Update, - /// Do not update height information when committing. - /// Allows multiple heights to be written in a single commit. - Ignore, + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()>; } /// The type describing the list of changes to the storage. -/// ColumnId -> (Key -> Operation) pub type Changes = HashMap, WriteOperation>>; impl From> for Changes { @@ -229,24 +190,13 @@ where /// Commits the changes into the storage. pub fn commit(mut self) -> StorageResult { let changes = core::mem::take(&mut self.inner.changes); - self.inner - .storage - .commit_changes(changes, self.inner.height_policy)?; + self.inner.storage.commit_changes(changes)?; Ok(self.inner.storage) } } impl Modifiable for InMemoryTransaction { - /// Panics height policy mismatches the height policy fo the transaction. - fn commit_changes( - &mut self, - changes: Changes, - height_policy: ModifyHeightPolicy, - ) -> StorageResult<()> { - assert!( - self.height_policy == height_policy, - "Height policy mismatch" - ); + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { for (column, value) in changes.into_iter() { let btree = self.changes.entry(column).or_default(); for (k, v) in value { @@ -441,11 +391,7 @@ mod test { }; impl Modifiable for InMemoryStorage { - fn commit_changes( - &mut self, - changes: Changes, - _: ModifyHeightPolicy, - ) -> StorageResult<()> { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { for (column, value) in changes.into_iter() { for (key, value) in value { match value { @@ -499,12 +445,8 @@ mod test { let changes1 = sub_transaction1.into(); let changes2 = sub_transaction2.into(); - transactions - .commit_changes(changes1, Default::default()) - .unwrap(); - transactions - .commit_changes(changes2, Default::default()) - .unwrap(); + transactions.commit_changes(changes1).unwrap(); + transactions.commit_changes(changes2).unwrap(); } #[test] @@ -527,11 +469,9 @@ mod test { let changes1 = sub_transaction1.into(); let changes2 = sub_transaction2.into(); + transactions.commit_changes(changes1).unwrap(); transactions - .commit_changes(changes1, Default::default()) - .unwrap(); - transactions - .commit_changes(changes2, Default::default()) + .commit_changes(changes2) .expect_err("Should fails because of the modification for the same key"); } From ba36f0158fa6312d8040d53294ada39bf76f47c6 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 04:26:25 +0300 Subject: [PATCH 04/14] Metadata using RwLock --- crates/fuel-core/src/database.rs | 143 ++++++++++-------- .../src/database/database_description.rs | 13 +- crates/fuel-core/src/database/metadata.rs | 37 +++-- .../fuel-core/src/service/genesis/exporter.rs | 2 +- .../service/genesis/importer/import_task.rs | 6 +- crates/fuel-core/src/state.rs | 8 +- .../src/state/in_memory/memory_store.rs | 10 +- crates/fuel-core/src/state/rocks_db.rs | 16 +- crates/services/src/lib.rs | 8 +- crates/services/src/service.rs | 50 +----- crates/services/src/sync.rs | 80 ++++++++++ 11 files changed, 220 insertions(+), 153 deletions(-) create mode 100644 crates/services/src/sync.rs diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 707fd00222b..817975677de 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -18,7 +18,7 @@ use crate::{ }, }; use fuel_core_chain_config::TableEntry; -use fuel_core_services::SharedMutex; +use fuel_core_services::SharedRwLock; use fuel_core_storage::{ self, blueprint::BlueprintInspect, @@ -85,12 +85,19 @@ pub mod state; pub mod storage; pub mod transactions; +/// Cached values from Metadata table, used to speed up commits. +#[derive(Clone, Debug, Default)] +struct CachedMetadata { + height: SharedRwLock>, + genesis_active: SharedRwLock, +} + #[derive(Clone, Debug)] pub struct Database where Description: DatabaseDescription, { - height: SharedMutex>, + cached_metadata: CachedMetadata, data: DataSource, } @@ -128,15 +135,22 @@ where Self: StorageInspect, Error = StorageError>, { pub fn new(data_source: DataSource) -> Self { - let mut database = Self { - height: SharedMutex::new(None), + let database = Self { + cached_metadata: CachedMetadata { + height: SharedRwLock::new(None), + genesis_active: SharedRwLock::new(false), + }, data: data_source, }; let height = database .latest_height() .expect("Failed to get latest height during creation of the database"); + let genesis_active = database + .genesis_active() + .expect("Failed to get genesis active state during creation of the database"); - database.height = SharedMutex::new(height); + *database.cached_metadata.height.write() = height; + *database.cached_metadata.genesis_active.write() = genesis_active; database } @@ -157,8 +171,8 @@ where pub fn in_memory() -> Self { let data = Arc::>::new(MemoryStore::default()); Self { - height: SharedMutex::new(None), data, + ..Default::default() } } @@ -167,8 +181,8 @@ where let data = Arc::>::new(RocksDb::default_open_temp(None).unwrap()); Self { - height: SharedMutex::new(None), data, + ..Default::default() } } } @@ -247,7 +261,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.cached_metadata.height.read() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -267,7 +281,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.cached_metadata.height.read() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -286,7 +300,7 @@ impl AtomicView for Database { type Height = DaBlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.cached_metadata.height.read() } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -378,58 +392,64 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { - // Gets the all new heights from the `changes` - let iterator = ChangesIterator::::new(&changes); - let new_heights = heights_lookup(&iterator)?; - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), - } - .into()); - } - - let new_height = new_heights.into_iter().last(); - let prev_height = *database.height.lock(); - - match (prev_height, new_height) { - (None, None) => { - // We are inside the regenesis process if the old and new heights are not set. - // In this case, we continue to commit until we discover a new height. - // This height will be the start of the database. + // DB consistency checks are only enforced when genesis flag is not set. + let new_height = if *database.cached_metadata.genesis_active.read() { + None + } else { + // Gets the all new heights from the `changes` + let iterator = ChangesIterator::::new(&changes); + let new_heights = heights_lookup(&iterator)?; + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), + } + .into()); } - (Some(prev_height), Some(new_height)) => { - // Each new commit should be linked to the previous commit to create a monotonically growing database. - let next_expected_height = prev_height - .advance_height() - .ok_or(DatabaseError::FailedToAdvanceHeight)?; + let new_height = new_heights.into_iter().last(); + let prev_height = *database.cached_metadata.height.read(); - // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 - // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. - if next_expected_height > new_height { - return Err(DatabaseError::HeightsAreNotLinked { + match (prev_height, new_height) { + (None, None) => { + // We are inside the regenesis process if the old and new heights are not set. + // In this case, we continue to commit until we discover a new height. + // This height will be the start of the database. + } + (Some(prev_height), Some(new_height)) => { + // Each new commit should be linked to the previous commit to create a monotonically growing database. + + let next_expected_height = prev_height + .advance_height() + .ok_or(DatabaseError::FailedToAdvanceHeight)?; + + // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 + // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. + if next_expected_height > new_height { + return Err(DatabaseError::HeightsAreNotLinked { + prev_height: prev_height.as_u64(), + new_height: new_height.as_u64(), + } + .into()); + } + } + (None, Some(_)) => { + // The new height is finally found; starting at this point, + // all next commits should be linked(the height should increase each time by one). + } + (Some(prev_height), None) => { + // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. + // The commit always should contain all data for the corresponding height. + return Err(DatabaseError::NewHeightIsNotSet { prev_height: prev_height.as_u64(), - new_height: new_height.as_u64(), } .into()); } - } - (None, Some(_)) => { - // The new height is finally found; starting at this point, - // all next commits should be linked(the height should increase each time by one). - } - (Some(prev_height), None) => { - // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. - // The commit always should contain all data for the corresponding height. - return Err(DatabaseError::NewHeightIsNotSet { - prev_height: prev_height.as_u64(), - } - .into()); - } + }; + new_height }; let updated_changes = if let Some(new_height) = new_height { @@ -448,6 +468,7 @@ where &DatabaseMetadata::V1 { version: Description::version(), height: new_height, + genesis_active: false, }, )?; @@ -456,14 +477,14 @@ where changes }; - let mut guard = database.height.lock(); - database - .data - .as_ref() - .commit_changes(new_height, updated_changes)?; + // Atomically commit the changes to the database, and to the mutex-protected field. + let mut guard = database.cached_metadata.height.write(); + database.data.as_ref().commit_changes(updated_changes)?; // Update the block height - *guard = new_height; + if let Some(new_height) = new_height { + *guard = Some(new_height); + } Ok(()) } diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index f62c6a11695..e7ba588db32 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -28,7 +28,11 @@ pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { /// The metadata of the database contains information about the version and its height. #[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { - V1 { version: u32, height: Height }, + V1 { + version: u32, + height: Height, + genesis_active: bool, + }, } impl DatabaseMetadata { @@ -45,4 +49,11 @@ impl DatabaseMetadata { Self::V1 { height, .. } => height, } } + + /// Returns if the genesis is currently in progress. + pub fn genesis_active(&self) -> bool { + match self { + Self::V1 { genesis_active, .. } => *genesis_active, + } + } } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index d9f8672b073..288b63cd33d 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + use crate::database::{ database_description::{ DatabaseDescription, @@ -47,29 +49,34 @@ where Description: DatabaseDescription, Self: StorageInspect, Error = StorageError>, { + /// Reads the metadata from the database. + fn read_metadata( + &self, + ) -> StorageResult>>> { + self.storage::>().get(&()) + } + /// Ensures the version is correct. pub fn check_version(&self) -> StorageResult<()> { - let Some(metadata) = self.storage::>().get(&())? - else { - return Ok(()); - }; - - if metadata.version() != Description::version() { - return Err(DatabaseError::InvalidDatabaseVersion { - found: metadata.version(), - expected: Description::version(), + if let Some(metadata) = self.read_metadata()? { + if metadata.version() != Description::version() { + return Err(DatabaseError::InvalidDatabaseVersion { + found: metadata.version(), + expected: Description::version(), + } + .into()); } - .into()) } - Ok(()) } + /// Reads latest height from the underlying database, bypassing cache pub fn latest_height(&self) -> StorageResult> { - let metadata = self.storage::>().get(&())?; - - let metadata = metadata.map(|metadata| *metadata.height()); + Ok(self.read_metadata()?.map(|m| *m.height())) + } - Ok(metadata) + /// Reads latest genesis active state from the underlying database, bypassing cache + pub fn genesis_active(&self) -> StorageResult { + Ok(self.read_metadata()?.map_or(false, |m| m.genesis_active())) } } diff --git a/crates/fuel-core/src/service/genesis/exporter.rs b/crates/fuel-core/src/service/genesis/exporter.rs index badbb01a471..e38d511d4c5 100644 --- a/crates/fuel-core/src/service/genesis/exporter.rs +++ b/crates/fuel-core/src/service/genesis/exporter.rs @@ -167,7 +167,7 @@ where TableEntry: serde::Serialize, StateConfigBuilder: AddTable, DbDesc: DatabaseDescription, - DbDesc::Height: Send, + DbDesc::Height: Send + Sync, { let mut writer = self.create_writer()?; let group_size = self.group_size; diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index b80d72d5845..f6073fe2d19 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -567,11 +567,7 @@ mod tests { } impl TransactableStorage for BrokenTransactions { - fn commit_changes( - &self, - _: Option, - _: Changes, - ) -> StorageResult<()> { + fn commit_changes(&self, _: Changes) -> StorageResult<()> { Err(anyhow::anyhow!("I refuse to work!").into()) } } diff --git a/crates/fuel-core/src/state.rs b/crates/fuel-core/src/state.rs index a707ba30b27..aecb67aa43b 100644 --- a/crates/fuel-core/src/state.rs +++ b/crates/fuel-core/src/state.rs @@ -33,11 +33,7 @@ where pub trait TransactableStorage: IterableStore + Debug + Send + Sync { /// Commits the changes into the storage. - fn commit_changes( - &self, - height: Option, - changes: Changes, - ) -> StorageResult<()>; + fn commit_changes(&self, changes: Changes) -> StorageResult<()>; } // It is used only to allow conversion of the `StorageTransaction` into the `DataSource`. @@ -47,7 +43,7 @@ impl TransactableStorage where S: IterableStore + Debug + Send + Sync, { - fn commit_changes(&self, _: Option, _: Changes) -> StorageResult<()> { + fn commit_changes(&self, _: Changes) -> StorageResult<()> { unimplemented!() } } diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index 28f1fca30ce..9dc171f9d1d 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -114,11 +114,7 @@ impl TransactableStorage for MemoryStore, - changes: Changes, - ) -> StorageResult<()> { + fn commit_changes(&self, changes: Changes) -> StorageResult<()> { for (column, btree) in changes.into_iter() { let mut lock = self.inner[column as usize] .lock() @@ -162,7 +158,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(Default::default(), changes)?; + self.commit_changes(changes)?; Ok(len) } @@ -170,7 +166,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(Default::default(), changes)?; + self.commit_changes(changes)?; Ok(()) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index a51732c7171..b5ac56ec962 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -525,11 +525,7 @@ impl TransactableStorage for RocksDb, - changes: Changes, - ) -> StorageResult<()> { + fn commit_changes(&self, changes: Changes) -> StorageResult<()> { let mut batch = WriteBatch::default(); for (column, ops) in changes { @@ -596,7 +592,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(Default::default(), changes)?; + self.commit_changes(changes)?; Ok(len) } @@ -605,7 +601,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(Default::default(), changes)?; + self.commit_changes(changes)?; Ok(()) } } @@ -680,8 +676,7 @@ mod tests { )]), )]; - db.commit_changes(Default::default(), HashMap::from_iter(ops)) - .unwrap(); + db.commit_changes(HashMap::from_iter(ops)).unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap().unwrap(), value) } @@ -697,8 +692,7 @@ mod tests { Column::Metadata.id(), BTreeMap::from_iter(vec![(key.clone(), WriteOperation::Remove)]), )]; - db.commit_changes(Default::default(), HashMap::from_iter(ops)) - .unwrap(); + db.commit_changes(HashMap::from_iter(ops)).unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap(), None); } diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 12333c5b50d..29e326c6ecd 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -7,6 +7,7 @@ mod service; mod state; +mod sync; /// Re-exports for streaming utilities pub mod stream { @@ -45,10 +46,13 @@ pub use service::{ RunnableTask, Service, ServiceRunner, - Shared, - SharedMutex, }; pub use state::{ State, StateWatcher, }; +pub use sync::{ + Shared, + SharedMutex, + SharedRwLock, +}; diff --git a/crates/services/src/service.rs b/crates/services/src/service.rs index d2ba048cad5..108c5cdbbe5 100644 --- a/crates/services/src/service.rs +++ b/crates/services/src/service.rs @@ -1,9 +1,11 @@ -use crate::state::{ - State, - StateWatcher, +use crate::{ + state::{ + State, + StateWatcher, + }, + Shared, }; use anyhow::anyhow; -use core::ops::Deref; use fuel_core_metrics::{ future_tracker::FutureTracker, services::{ @@ -16,27 +18,6 @@ use std::any::Any; use tokio::sync::watch; use tracing::Instrument; -/// Alias for `Arc` -pub type Shared = std::sync::Arc; - -/// A mutex that can safely be in async contexts and avoids deadlocks. -#[derive(Default, Debug)] -pub struct SharedMutex(Shared>); - -impl Clone for SharedMutex { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl Deref for SharedMutex { - type Target = Shared>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - /// Used if services have no asynchronously shared data #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EmptyShared; @@ -440,25 +421,6 @@ where got_panic } -impl SharedMutex { - /// Creates a new `SharedMutex` with the given value. - pub fn new(t: T) -> Self { - Self(Shared::new(parking_lot::Mutex::new(t))) - } - - /// Apply a function to the inner value and return a value. - pub fn apply(&self, f: impl FnOnce(&mut T) -> R) -> R { - let mut t = self.0.lock(); - f(&mut t) - } -} - -impl From for SharedMutex { - fn from(t: T) -> Self { - Self::new(t) - } -} - fn panic_to_string(e: Box) -> String { match e.downcast::() { Ok(v) => *v, diff --git a/crates/services/src/sync.rs b/crates/services/src/sync.rs new file mode 100644 index 00000000000..bf849dec71d --- /dev/null +++ b/crates/services/src/sync.rs @@ -0,0 +1,80 @@ +//! Wrappers for synchronization constainers. + +use std::ops::Deref; + +/// Alias for `Arc` +pub type Shared = std::sync::Arc; + +/// A mutex that can safely be in async contexts and avoids deadlocks. +#[derive(Default, Debug)] +pub struct SharedMutex(Shared>); + +impl Clone for SharedMutex { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Deref for SharedMutex { + type Target = Shared>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl SharedMutex { + /// Creates a new `SharedMutex` with the given value. + pub fn new(t: T) -> Self { + Self(Shared::new(parking_lot::Mutex::new(t))) + } + + /// Apply a function to the inner value and return a value. + pub fn apply(&self, f: impl FnOnce(&mut T) -> R) -> R { + let mut t = self.0.lock(); + f(&mut t) + } +} + +impl From for SharedMutex { + fn from(t: T) -> Self { + Self::new(t) + } +} + +/// An RwLock that can safely be in async contexts and avoids deadlocks. +#[derive(Default, Debug)] +pub struct SharedRwLock(Shared>); + +impl Clone for SharedRwLock { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Deref for SharedRwLock { + type Target = Shared>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl SharedRwLock { + /// Creates a new `SharedRwLock` with the given value. + pub fn new(t: T) -> Self { + Self(Shared::new(parking_lot::RwLock::new(t))) + } + + /// Apply a function to the inner value and return a value. + pub fn apply(&self, f: impl FnOnce(&mut T) -> R) -> R { + let mut t = self.0.write(); + f(&mut t) + } +} + +impl From for SharedRwLock { + fn from(t: T) -> Self { + Self::new(t) + } +} From 562bca3e6a512f3b89b0e1dda0b5fc1e5115cfea Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 05:16:31 +0300 Subject: [PATCH 05/14] Add genesis flag to db handler --- crates/client/src/lib.rs | 2 +- crates/fuel-core/src/combined_database.rs | 6 +++ crates/fuel-core/src/database.rs | 49 +++++++++---------- .../src/database/database_description.rs | 13 +---- crates/fuel-core/src/database/metadata.rs | 37 ++++++-------- crates/fuel-core/src/service/genesis.rs | 4 ++ crates/types/src/lib.rs | 2 +- 7 files changed, 50 insertions(+), 63 deletions(-) diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 728cd0be4dd..d11117374a1 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -#![deny(warnings)] +// #![deny(warnings)] pub mod client; pub mod schema; diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 9679045f0da..927add3b600 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -124,6 +124,12 @@ impl CombinedDatabase { Ok(()) } + pub fn set_genesis_active(&self, active: bool) { + self.on_chain.set_genesis_active(active); + self.off_chain.set_genesis_active(active); + self.relayer.set_genesis_active(active); + } + pub fn on_chain(&self) -> &Database { &self.on_chain } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 817975677de..ec27d2c0f3f 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -85,19 +85,15 @@ pub mod state; pub mod storage; pub mod transactions; -/// Cached values from Metadata table, used to speed up commits. -#[derive(Clone, Debug, Default)] -struct CachedMetadata { - height: SharedRwLock>, - genesis_active: SharedRwLock, -} - #[derive(Clone, Debug)] pub struct Database where Description: DatabaseDescription, { - cached_metadata: CachedMetadata, + /// Cached value from Metadata table, used to speed up lookups. + height: SharedRwLock>, + /// If set, some consistency checks are not performed. + genesis_active: SharedRwLock, data: DataSource, } @@ -136,21 +132,14 @@ where { pub fn new(data_source: DataSource) -> Self { let database = Self { - cached_metadata: CachedMetadata { - height: SharedRwLock::new(None), - genesis_active: SharedRwLock::new(false), - }, + height: SharedRwLock::new(None), + genesis_active: SharedRwLock::new(false), data: data_source, }; let height = database .latest_height() .expect("Failed to get latest height during creation of the database"); - let genesis_active = database - .genesis_active() - .expect("Failed to get genesis active state during creation of the database"); - - *database.cached_metadata.height.write() = height; - *database.cached_metadata.genesis_active.write() = genesis_active; + *database.height.write() = height; database } @@ -162,6 +151,11 @@ where Ok(Database::new(Arc::new(db))) } + + /// Set the genesis flag that controls some consistency checks. + pub fn set_genesis_active(&self, active: bool) { + *self.genesis_active.write() = active; + } } impl Database @@ -172,7 +166,8 @@ where let data = Arc::>::new(MemoryStore::default()); Self { data, - ..Default::default() + height: SharedRwLock::new(None), + genesis_active: SharedRwLock::new(false), } } @@ -182,7 +177,8 @@ where Arc::>::new(RocksDb::default_open_temp(None).unwrap()); Self { data, - ..Default::default() + height: SharedRwLock::new(None), + genesis_active: SharedRwLock::new(false), } } } @@ -261,7 +257,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.cached_metadata.height.read() + *self.height.read() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -281,7 +277,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.cached_metadata.height.read() + *self.height.read() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -300,7 +296,7 @@ impl AtomicView for Database { type Height = DaBlockHeight; fn latest_height(&self) -> Option { - *self.cached_metadata.height.read() + *self.height.read() } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -393,7 +389,7 @@ where StorageMutate, Error = StorageError>, { // DB consistency checks are only enforced when genesis flag is not set. - let new_height = if *database.cached_metadata.genesis_active.read() { + let new_height = if *database.genesis_active.read() { None } else { // Gets the all new heights from the `changes` @@ -411,7 +407,7 @@ where } let new_height = new_heights.into_iter().last(); - let prev_height = *database.cached_metadata.height.read(); + let prev_height = *database.height.read(); match (prev_height, new_height) { (None, None) => { @@ -468,7 +464,6 @@ where &DatabaseMetadata::V1 { version: Description::version(), height: new_height, - genesis_active: false, }, )?; @@ -478,7 +473,7 @@ where }; // Atomically commit the changes to the database, and to the mutex-protected field. - let mut guard = database.cached_metadata.height.write(); + let mut guard = database.height.write(); database.data.as_ref().commit_changes(updated_changes)?; // Update the block height diff --git a/crates/fuel-core/src/database/database_description.rs b/crates/fuel-core/src/database/database_description.rs index e7ba588db32..f62c6a11695 100644 --- a/crates/fuel-core/src/database/database_description.rs +++ b/crates/fuel-core/src/database/database_description.rs @@ -28,11 +28,7 @@ pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { /// The metadata of the database contains information about the version and its height. #[derive(Copy, Clone, Debug, serde::Serialize, serde::Deserialize)] pub enum DatabaseMetadata { - V1 { - version: u32, - height: Height, - genesis_active: bool, - }, + V1 { version: u32, height: Height }, } impl DatabaseMetadata { @@ -49,11 +45,4 @@ impl DatabaseMetadata { Self::V1 { height, .. } => height, } } - - /// Returns if the genesis is currently in progress. - pub fn genesis_active(&self) -> bool { - match self { - Self::V1 { genesis_active, .. } => *genesis_active, - } - } } diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index 288b63cd33d..d9f8672b073 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use crate::database::{ database_description::{ DatabaseDescription, @@ -49,34 +47,29 @@ where Description: DatabaseDescription, Self: StorageInspect, Error = StorageError>, { - /// Reads the metadata from the database. - fn read_metadata( - &self, - ) -> StorageResult>>> { - self.storage::>().get(&()) - } - /// Ensures the version is correct. pub fn check_version(&self) -> StorageResult<()> { - if let Some(metadata) = self.read_metadata()? { - if metadata.version() != Description::version() { - return Err(DatabaseError::InvalidDatabaseVersion { - found: metadata.version(), - expected: Description::version(), - } - .into()); + let Some(metadata) = self.storage::>().get(&())? + else { + return Ok(()); + }; + + if metadata.version() != Description::version() { + return Err(DatabaseError::InvalidDatabaseVersion { + found: metadata.version(), + expected: Description::version(), } + .into()) } + Ok(()) } - /// Reads latest height from the underlying database, bypassing cache pub fn latest_height(&self) -> StorageResult> { - Ok(self.read_metadata()?.map(|m| *m.height())) - } + let metadata = self.storage::>().get(&())?; + + let metadata = metadata.map(|metadata| *metadata.height()); - /// Reads latest genesis active state from the underlying database, bypassing cache - pub fn genesis_active(&self) -> StorageResult { - Ok(self.read_metadata()?.map_or(false, |m| m.genesis_active())) + Ok(metadata) } } diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index 0bd08cd910f..80e3feb0282 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -70,6 +70,8 @@ pub async fn execute_genesis_block( let genesis_block = create_genesis_block(config); tracing::info!("Genesis block created: {:?}", genesis_block.header()); + db.set_genesis_active(true); + SnapshotImporter::import( db.clone(), genesis_block.clone(), @@ -78,6 +80,8 @@ pub async fn execute_genesis_block( ) .await?; + db.set_genesis_active(false); + let genesis_progress_on_chain: Vec = db .on_chain() .iter_all::>(None) diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 1cefcb66edc..2d3f01f8d02 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -6,7 +6,7 @@ #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] #![deny(missing_docs)] -#![deny(warnings)] +// #![deny(warnings)] #[doc(no_inline)] pub use fuel_vm_private::{ From 336378a79cf5b2f008288f737fd0d76db8a28f7d Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 05:20:42 +0300 Subject: [PATCH 06/14] More docs --- crates/fuel-core/src/combined_database.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 927add3b600..a0094309a0b 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -124,6 +124,7 @@ impl CombinedDatabase { Ok(()) } + /// Set the genesis flag that controls some consistency checks. pub fn set_genesis_active(&self, active: bool) { self.on_chain.set_genesis_active(active); self.off_chain.set_genesis_active(active); From 9c0fd526e8716876cc2ed8dacf50a28c2ad8626d Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 05:22:16 +0300 Subject: [PATCH 07/14] Add changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37660541ac..0f4db6650f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [#1856](https://github.com/FuelLabs/fuel-core/pull/1856): Replaced instances of `Union` with `Enum` for GraphQL definitions of `ConsensusParametersVersion` and related types. This is needed because `Union` does not support multiple `Version`s inside discriminants or empty variants. +### Added + +- [#1860](https://github.com/FuelLabs/fuel-core/pull/1860): Regenesis now preserves `FuelBlockIdsToHeights` off-chain table. + ### Changed - [#1837](https://github.com/FuelLabs/fuel-core/pull/1837): Refactor the executor and separate validation from the other use cases From d50083ecfa41b70887efd8393fe09c67ea55fdce Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 05:30:52 +0300 Subject: [PATCH 08/14] Cleanup --- crates/types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 2d3f01f8d02..1cefcb66edc 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -6,7 +6,7 @@ #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] #![deny(missing_docs)] -// #![deny(warnings)] +#![deny(warnings)] #[doc(no_inline)] pub use fuel_vm_private::{ From 8c861e02fd02f75d8de423fc6b992e15453c1a0b Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Thu, 25 Apr 2024 05:37:13 +0300 Subject: [PATCH 09/14] Simplify locking --- crates/client/src/lib.rs | 2 +- crates/fuel-core/src/database.rs | 34 ++++++++++++++--------------- crates/services/src/lib.rs | 1 - crates/services/src/sync.rs | 37 -------------------------------- 4 files changed, 18 insertions(+), 56 deletions(-) diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index d11117374a1..728cd0be4dd 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -1,7 +1,7 @@ #![deny(clippy::arithmetic_side_effects)] #![deny(clippy::cast_possible_truncation)] #![deny(unused_crate_dependencies)] -// #![deny(warnings)] +#![deny(warnings)] pub mod client; pub mod schema; diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index ec27d2c0f3f..ce5397597ba 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -18,7 +18,7 @@ use crate::{ }, }; use fuel_core_chain_config::TableEntry; -use fuel_core_services::SharedRwLock; +use fuel_core_services::SharedMutex; use fuel_core_storage::{ self, blueprint::BlueprintInspect, @@ -91,9 +91,9 @@ where Description: DatabaseDescription, { /// Cached value from Metadata table, used to speed up lookups. - height: SharedRwLock>, + height: SharedMutex>, /// If set, some consistency checks are not performed. - genesis_active: SharedRwLock, + genesis_active: SharedMutex, data: DataSource, } @@ -132,14 +132,14 @@ where { pub fn new(data_source: DataSource) -> Self { let database = Self { - height: SharedRwLock::new(None), - genesis_active: SharedRwLock::new(false), + height: SharedMutex::new(None), + genesis_active: SharedMutex::new(false), data: data_source, }; let height = database .latest_height() .expect("Failed to get latest height during creation of the database"); - *database.height.write() = height; + *database.height.lock() = height; database } @@ -154,7 +154,7 @@ where /// Set the genesis flag that controls some consistency checks. pub fn set_genesis_active(&self, active: bool) { - *self.genesis_active.write() = active; + *self.genesis_active.lock() = active; } } @@ -166,8 +166,8 @@ where let data = Arc::>::new(MemoryStore::default()); Self { data, - height: SharedRwLock::new(None), - genesis_active: SharedRwLock::new(false), + height: SharedMutex::new(None), + genesis_active: SharedMutex::new(false), } } @@ -177,8 +177,8 @@ where Arc::>::new(RocksDb::default_open_temp(None).unwrap()); Self { data, - height: SharedRwLock::new(None), - genesis_active: SharedRwLock::new(false), + height: SharedMutex::new(None), + genesis_active: SharedMutex::new(false), } } } @@ -257,7 +257,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.read() + *self.height.lock() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -277,7 +277,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.read() + *self.height.lock() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -296,7 +296,7 @@ impl AtomicView for Database { type Height = DaBlockHeight; fn latest_height(&self) -> Option { - *self.height.read() + *self.height.lock() } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -389,7 +389,7 @@ where StorageMutate, Error = StorageError>, { // DB consistency checks are only enforced when genesis flag is not set. - let new_height = if *database.genesis_active.read() { + let new_height = if *database.genesis_active.lock() { None } else { // Gets the all new heights from the `changes` @@ -407,7 +407,7 @@ where } let new_height = new_heights.into_iter().last(); - let prev_height = *database.height.read(); + let prev_height = *database.height.lock(); match (prev_height, new_height) { (None, None) => { @@ -473,7 +473,7 @@ where }; // Atomically commit the changes to the database, and to the mutex-protected field. - let mut guard = database.height.write(); + let mut guard = database.height.lock(); database.data.as_ref().commit_changes(updated_changes)?; // Update the block height diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 29e326c6ecd..be22f30180e 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -54,5 +54,4 @@ pub use state::{ pub use sync::{ Shared, SharedMutex, - SharedRwLock, }; diff --git a/crates/services/src/sync.rs b/crates/services/src/sync.rs index bf849dec71d..056cfcc653b 100644 --- a/crates/services/src/sync.rs +++ b/crates/services/src/sync.rs @@ -41,40 +41,3 @@ impl From for SharedMutex { Self::new(t) } } - -/// An RwLock that can safely be in async contexts and avoids deadlocks. -#[derive(Default, Debug)] -pub struct SharedRwLock(Shared>); - -impl Clone for SharedRwLock { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl Deref for SharedRwLock { - type Target = Shared>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl SharedRwLock { - /// Creates a new `SharedRwLock` with the given value. - pub fn new(t: T) -> Self { - Self(Shared::new(parking_lot::RwLock::new(t))) - } - - /// Apply a function to the inner value and return a value. - pub fn apply(&self, f: impl FnOnce(&mut T) -> R) -> R { - let mut t = self.0.write(); - f(&mut t) - } -} - -impl From for SharedRwLock { - fn from(t: T) -> Self { - Self::new(t) - } -} From df13a4afe4fc8a75ef76261e868f2274aa54070c Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 29 Apr 2024 06:11:36 +0300 Subject: [PATCH 10/14] Use UncheckedDatabase type for genesis imports --- crates/fuel-core/src/combined_database.rs | 7 - crates/fuel-core/src/database.rs | 249 +++++++++++++----- crates/fuel-core/src/database/storage.rs | 28 ++ crates/fuel-core/src/service/genesis.rs | 4 - .../fuel-core/src/service/genesis/importer.rs | 4 +- .../service/genesis/importer/import_task.rs | 46 ++-- .../src/service/genesis/importer/off_chain.rs | 34 +-- .../src/service/genesis/importer/on_chain.rs | 24 +- 8 files changed, 266 insertions(+), 130 deletions(-) diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index a0094309a0b..9679045f0da 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -124,13 +124,6 @@ impl CombinedDatabase { Ok(()) } - /// Set the genesis flag that controls some consistency checks. - pub fn set_genesis_active(&self, active: bool) { - self.on_chain.set_genesis_active(active); - self.off_chain.set_genesis_active(active); - self.relayer.set_genesis_active(active); - } - pub fn on_chain(&self) -> &Database { &self.on_chain } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index ce5397597ba..4494f8c82fa 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -85,6 +85,26 @@ pub mod state; pub mod storage; pub mod transactions; +#[derive(Clone, Debug)] +pub struct UncheckedDatabase +where + Description: DatabaseDescription, +{ + data: DataSource, +} + +impl UncheckedDatabase +where + Description: DatabaseDescription, +{ + pub fn into_checked(self) -> Database { + Database { + height: SharedMutex::new(None), + data: self.data, + } + } +} + #[derive(Clone, Debug)] pub struct Database where @@ -92,8 +112,6 @@ where { /// Cached value from Metadata table, used to speed up lookups. height: SharedMutex>, - /// If set, some consistency checks are not performed. - genesis_active: SharedMutex, data: DataSource, } @@ -133,7 +151,6 @@ where pub fn new(data_source: DataSource) -> Self { let database = Self { height: SharedMutex::new(None), - genesis_active: SharedMutex::new(false), data: data_source, }; let height = database @@ -152,34 +169,42 @@ where Ok(Database::new(Arc::new(db))) } - /// Set the genesis flag that controls some consistency checks. - pub fn set_genesis_active(&self, active: bool) { - *self.genesis_active.lock() = active; + /// Converts to an unchecked database. + /// Panics if the height is already set. + pub fn into_unchecked(self) -> UncheckedDatabase { + assert!(!self.height.lock().is_some()); + UncheckedDatabase { data: self.data } } } -impl Database +impl UncheckedDatabase where Description: DatabaseDescription, { pub fn in_memory() -> Self { let data = Arc::>::new(MemoryStore::default()); - Self { - data, - height: SharedMutex::new(None), - genesis_active: SharedMutex::new(false), - } + Self { data } } #[cfg(feature = "rocksdb")] pub fn rocksdb_temp() -> Self { let data = Arc::>::new(RocksDb::default_open_temp(None).unwrap()); - Self { - data, - height: SharedMutex::new(None), - genesis_active: SharedMutex::new(false), - } + Self { data } + } +} + +impl Database +where + Description: DatabaseDescription, +{ + pub fn in_memory() -> Self { + UncheckedDatabase::in_memory().into_checked() + } + + #[cfg(feature = "rocksdb")] + pub fn rocksdb_temp() -> Self { + UncheckedDatabase::in_memory().into_checked() } } @@ -232,6 +257,74 @@ where } } +impl KeyValueInspect for UncheckedDatabase +where + Description: DatabaseDescription, +{ + type Column = Description::Column; + + fn exists(&self, key: &[u8], column: Self::Column) -> StorageResult { + self.data.as_ref().exists(key, column) + } + + fn size_of_value( + &self, + key: &[u8], + column: Self::Column, + ) -> StorageResult> { + self.data.as_ref().size_of_value(key, column) + } + + fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { + self.data.as_ref().get(key, column) + } + + fn read( + &self, + key: &[u8], + column: Self::Column, + buf: &mut [u8], + ) -> StorageResult> { + self.data.as_ref().read(key, column, buf) + } +} + +impl IterableStore for UncheckedDatabase +where + Description: DatabaseDescription, +{ + fn iter_store( + &self, + column: Self::Column, + prefix: Option<&[u8]>, + start: Option<&[u8]>, + direction: IterDirection, + ) -> BoxedIter { + self.data + .as_ref() + .iter_store(column, prefix, start, direction) + } +} + +/// Construct an ephemeral database +/// uses rocksdb when rocksdb features are enabled +/// uses in-memory when rocksdb features are disabled +impl Default for UncheckedDatabase +where + Description: DatabaseDescription, +{ + fn default() -> Self { + #[cfg(not(feature = "rocksdb"))] + { + Self::in_memory() + } + #[cfg(feature = "rocksdb")] + { + Self::rocksdb_temp() + } + } +} + /// Construct an ephemeral database /// uses rocksdb when rocksdb features are enabled /// uses in-memory when rocksdb features are disabled @@ -348,6 +441,32 @@ impl Modifiable for Database { } } +impl Modifiable for UncheckedDatabase { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.data.as_ref().commit_changes(changes) + } +} + +impl Modifiable for UncheckedDatabase { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.data.as_ref().commit_changes(changes) + } +} + +#[cfg(feature = "relayer")] +impl Modifiable for UncheckedDatabase { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.data.as_ref().commit_changes(changes) + } +} + +#[cfg(not(feature = "relayer"))] +impl Modifiable for UncheckedDatabase { + fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { + self.data.as_ref().commit_changes(changes) + } +} + trait DatabaseHeight: Sized { fn as_u64(&self) -> u64; @@ -388,64 +507,58 @@ where for<'a> StorageTransaction<&'a &'a mut Database>: StorageMutate, Error = StorageError>, { - // DB consistency checks are only enforced when genesis flag is not set. - let new_height = if *database.genesis_active.lock() { - None - } else { - // Gets the all new heights from the `changes` - let iterator = ChangesIterator::::new(&changes); - let new_heights = heights_lookup(&iterator)?; - - // Changes for each block should be committed separately. - // If we have more than one height, it means we are mixing commits - // for several heights in one batch - return error in this case. - if new_heights.len() > 1 { - return Err(DatabaseError::MultipleHeightsInCommit { - heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), - } - .into()); + // Gets the all new heights from the `changes` + let iterator = ChangesIterator::::new(&changes); + let new_heights = heights_lookup(&iterator)?; + + // Changes for each block should be committed separately. + // If we have more than one height, it means we are mixing commits + // for several heights in one batch - return error in this case. + if new_heights.len() > 1 { + return Err(DatabaseError::MultipleHeightsInCommit { + heights: new_heights.iter().map(DatabaseHeight::as_u64).collect(), } + .into()); + } - let new_height = new_heights.into_iter().last(); - let prev_height = *database.height.lock(); + let new_height = new_heights.into_iter().last(); + let prev_height = *database.height.lock(); - match (prev_height, new_height) { - (None, None) => { - // We are inside the regenesis process if the old and new heights are not set. - // In this case, we continue to commit until we discover a new height. - // This height will be the start of the database. - } - (Some(prev_height), Some(new_height)) => { - // Each new commit should be linked to the previous commit to create a monotonically growing database. - - let next_expected_height = prev_height - .advance_height() - .ok_or(DatabaseError::FailedToAdvanceHeight)?; - - // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 - // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. - if next_expected_height > new_height { - return Err(DatabaseError::HeightsAreNotLinked { - prev_height: prev_height.as_u64(), - new_height: new_height.as_u64(), - } - .into()); - } - } - (None, Some(_)) => { - // The new height is finally found; starting at this point, - // all next commits should be linked(the height should increase each time by one). - } - (Some(prev_height), None) => { - // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. - // The commit always should contain all data for the corresponding height. - return Err(DatabaseError::NewHeightIsNotSet { + match (prev_height, new_height) { + (None, None) => { + // We are inside the regenesis process if the old and new heights are not set. + // In this case, we continue to commit until we discover a new height. + // This height will be the start of the database. + } + (Some(prev_height), Some(new_height)) => { + // Each new commit should be linked to the previous commit to create a monotonically growing database. + + let next_expected_height = prev_height + .advance_height() + .ok_or(DatabaseError::FailedToAdvanceHeight)?; + + // TODO: After https://github.com/FuelLabs/fuel-core/issues/451 + // we can replace `next_expected_height > new_height` with `next_expected_height != new_height`. + if next_expected_height > new_height { + return Err(DatabaseError::HeightsAreNotLinked { prev_height: prev_height.as_u64(), + new_height: new_height.as_u64(), } .into()); } - }; - new_height + } + (None, Some(_)) => { + // The new height is finally found; starting at this point, + // all next commits should be linked(the height should increase each time by one). + } + (Some(prev_height), None) => { + // In production, we shouldn't have cases where we call `commit_chagnes` with intermediate changes. + // The commit always should contain all data for the corresponding height. + return Err(DatabaseError::NewHeightIsNotSet { + prev_height: prev_height.as_u64(), + } + .into()); + } }; let updated_changes = if let Some(new_height) = new_height { diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index acabb92b5fc..a3e8b4807b2 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -30,6 +30,34 @@ use fuel_core_storage::{ StorageWrite, }; +use super::UncheckedDatabase; + +impl StorageInspect for UncheckedDatabase +where + Description: DatabaseDescription, + M: Mappable, + for<'a> StructuredStorage<&'a Self>: StorageInspect, +{ + type Error = StorageError; + + fn get(&self, key: &M::Key) -> StorageResult>> { + let storage = StructuredStorage::new(self); + let value = storage.storage::().get(key)?; + + if let Some(cow) = value { + Ok(Some(Cow::Owned(cow.into_owned()))) + } else { + Ok(None) + } + } + + fn contains_key(&self, key: &M::Key) -> StorageResult { + StructuredStorage::new(self) + .storage::() + .contains_key(key) + } +} + impl StorageInspect for Database where Description: DatabaseDescription, diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index a72add0e66c..e4ec83e8422 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -72,8 +72,6 @@ pub async fn execute_genesis_block( let genesis_block = create_genesis_block(config); tracing::info!("Genesis block created: {:?}", genesis_block.header()); - db.set_genesis_active(true); - SnapshotImporter::import( db.clone(), genesis_block.clone(), @@ -82,8 +80,6 @@ pub async fn execute_genesis_block( ) .await?; - db.set_genesis_active(false); - let genesis_progress_on_chain: Vec = db .on_chain() .iter_all::>(None) diff --git a/crates/fuel-core/src/service/genesis/importer.rs b/crates/fuel-core/src/service/genesis/importer.rs index 3a7617ccc10..7d613c97751 100644 --- a/crates/fuel-core/src/service/genesis/importer.rs +++ b/crates/fuel-core/src/service/genesis/importer.rs @@ -155,7 +155,7 @@ impl SnapshotImporter { let block_height = *self.genesis_block.header().height(); let da_block_height = self.genesis_block.header().da_height; - let db = self.db.on_chain().clone(); + let db = self.db.on_chain().clone().into_unchecked(); let migration_name = migration_name::(); let progress_reporter = self @@ -203,7 +203,7 @@ impl SnapshotImporter { let block_height = *self.genesis_block.header().height(); let da_block_height = self.genesis_block.header().da_height; - let db = self.db.off_chain().clone(); + let db = self.db.off_chain().clone().into_unchecked(); let migration_name = migration_name::(); let progress_reporter = self diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index 260c3b06d50..bdfdad8703d 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -19,7 +19,7 @@ use crate::{ GenesisMetadata, GenesisProgressMutate, }, - Database, + UncheckedDatabase, }, service::genesis::{ progress::ProgressReporter, @@ -36,7 +36,7 @@ where handler: Handler, skip: usize, groups: Groups, - db: Database, + db: UncheckedDatabase, reporter: ProgressReporter, } @@ -48,7 +48,7 @@ pub trait ImportTable { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()>; } @@ -56,12 +56,12 @@ impl ImportTask where DbDesc: DatabaseDescription, Logic: ImportTable, - Database: StorageInspect>, + UncheckedDatabase: StorageInspect>, { pub fn new( handler: Logic, groups: GroupGenerator, - db: Database, + db: UncheckedDatabase, reporter: ProgressReporter, ) -> Self { let progress_name = @@ -95,9 +95,9 @@ where Value = usize, OwnedValue = usize, >, - Database: + UncheckedDatabase: StorageInspect> + WriteTransaction + Modifiable, - for<'a> StorageTransaction<&'a mut Database>: + for<'a> StorageTransaction<&'a mut UncheckedDatabase>: StorageMutate, Error = fuel_core_storage::Error>, { pub fn run(mut self, cancel_token: CancellationToken) -> anyhow::Result<()> { @@ -137,7 +137,10 @@ where #[cfg(test)] mod tests { use crate::{ - database::genesis_progress::GenesisProgressInspect, + database::{ + genesis_progress::GenesisProgressInspect, + UncheckedDatabase, + }, service::genesis::{ importer::{ import_task::ImportTask, @@ -227,7 +230,7 @@ mod tests { where L: FnMut( TableEntry, - &mut StorageTransaction<&mut Database>, + &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()>, { type TableInSnapshot = Coins; @@ -236,7 +239,7 @@ mod tests { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group .into_iter() @@ -287,7 +290,7 @@ mod tests { Ok(()) }), data.as_ok_groups(), - Database::default(), + UncheckedDatabase::default(), ProgressReporter::default(), ); @@ -317,7 +320,7 @@ mod tests { Ok(()) }), data.as_ok_groups(), - db.on_chain().clone(), + db.on_chain().clone().into_unchecked(), ProgressReporter::default(), ); @@ -332,7 +335,7 @@ mod tests { fn changes_to_db_by_handler_are_behind_a_transaction() { // given let groups = TestData::new(1); - let outer_db = Database::default(); + let outer_db = UncheckedDatabase::default(); let utxo_id = UtxoId::new(Default::default(), 0); let runner = ImportTask::new( @@ -369,7 +372,10 @@ mod tests { .unwrap()); } - fn insert_a_coin(tx: &mut StorageTransaction<&mut Database>, utxo_id: &UtxoId) { + fn insert_a_coin( + tx: &mut StorageTransaction<&mut UncheckedDatabase>, + utxo_id: &UtxoId, + ) { let coin: CompressedCoin = CompressedCoinV1::default().into(); tx.storage_as_mut::().insert(utxo_id, &coin).unwrap(); @@ -379,7 +385,7 @@ mod tests { fn tx_reverted_if_handler_fails() { // given let groups = TestData::new(1); - let db = Database::default(); + let db = UncheckedDatabase::default(); let utxo_id = UtxoId::new(Default::default(), 0); let runner = ImportTask::new( @@ -406,7 +412,7 @@ mod tests { let runner = ImportTask::new( TestHandler::new(|_, _| bail!("Some error")), groups.as_ok_groups(), - Database::default(), + Default::default(), ProgressReporter::default(), ); @@ -424,7 +430,7 @@ mod tests { let runner = ImportTask::new( TestHandler::new(|_, _| Ok(())), groups, - Database::default(), + Default::default(), ProgressReporter::default(), ); @@ -439,7 +445,7 @@ mod tests { fn succesfully_processed_batch_updates_the_genesis_progress() { // given let data = TestData::new(2); - let db = Database::default(); + let db = UncheckedDatabase::default(); let runner = ImportTask::new( TestHandler::new(|_, _| Ok(())), data.as_ok_groups(), @@ -475,7 +481,7 @@ mod tests { Ok(()) }), rx, - Database::default(), + Default::default(), ProgressReporter::default(), ) }; @@ -562,7 +568,7 @@ mod tests { let runner = ImportTask::new( TestHandler::new(|_, _| Ok(())), groups.as_ok_groups(), - Database::new(Arc::new(BrokenTransactions::new())), + Database::new(Arc::new(BrokenTransactions::new())).into_unchecked(), ProgressReporter::default(), ); diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index 8f7accdd244..c3b8fcb82cf 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use crate::{ database::{ database_description::off_chain::OffChain, - Database, + UncheckedDatabase, }, fuel_core_graphql_api::storage::messages::SpentMessages, graphql_api::{ @@ -52,7 +52,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for tx_status in group { tx.storage::() @@ -70,7 +70,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage::() @@ -88,7 +88,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage::() @@ -106,7 +106,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let events = group .into_iter() @@ -124,7 +124,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) @@ -142,7 +142,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let transactions = group.iter().map(|TableEntry { value, .. }| value); worker_service::process_transactions(transactions, tx)?; @@ -158,7 +158,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let transactions = group.iter().map(|TableEntry { value, .. }| value); worker_service::process_transactions(transactions, tx)?; @@ -174,7 +174,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -192,7 +192,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -210,7 +210,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -228,7 +228,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -246,7 +246,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let transactions = group .iter() @@ -264,7 +264,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { let transactions = group .iter() @@ -282,7 +282,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() @@ -300,7 +300,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() @@ -318,7 +318,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() diff --git a/crates/fuel-core/src/service/genesis/importer/on_chain.rs b/crates/fuel-core/src/service/genesis/importer/on_chain.rs index eac8ee71699..14eba85ec66 100644 --- a/crates/fuel-core/src/service/genesis/importer/on_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/on_chain.rs @@ -6,7 +6,7 @@ use crate::database::{ balances::BalancesInitializer, database_description::on_chain::OnChain, state::StateInitializer, - Database, + UncheckedDatabase, }; use anyhow::anyhow; use fuel_core_chain_config::TableEntry; @@ -41,7 +41,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|coin| { init_coin(tx, &coin, self.block_height)?; @@ -58,7 +58,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group .into_iter() @@ -74,7 +74,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|transaction| { tx.storage_as_mut::() @@ -93,7 +93,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|contract| { init_contract_raw_code(tx, &contract)?; @@ -110,7 +110,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|contract| { init_contract_latest_utxo(tx, &contract, self.block_height)?; @@ -127,7 +127,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { tx.update_contract_states(group)?; Ok(()) @@ -142,7 +142,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut Database>, + tx: &mut StorageTransaction<&mut UncheckedDatabase>, ) -> anyhow::Result<()> { tx.update_contract_balances(group)?; Ok(()) @@ -150,7 +150,7 @@ impl ImportTable for Handler { } fn init_coin( - transaction: &mut StorageTransaction<&mut Database>, + transaction: &mut StorageTransaction<&mut UncheckedDatabase>, coin: &TableEntry, height: BlockHeight, ) -> anyhow::Result<()> { @@ -185,7 +185,7 @@ fn init_coin( } fn init_contract_latest_utxo( - transaction: &mut StorageTransaction<&mut Database>, + transaction: &mut StorageTransaction<&mut UncheckedDatabase>, entry: &TableEntry, height: BlockHeight, ) -> anyhow::Result<()> { @@ -209,7 +209,7 @@ fn init_contract_latest_utxo( } fn init_contract_raw_code( - transaction: &mut StorageTransaction<&mut Database>, + transaction: &mut StorageTransaction<&mut UncheckedDatabase>, entry: &TableEntry, ) -> anyhow::Result<()> { let contract = entry.value.as_ref(); @@ -228,7 +228,7 @@ fn init_contract_raw_code( } fn init_da_message( - transaction: &mut StorageTransaction<&mut Database>, + transaction: &mut StorageTransaction<&mut UncheckedDatabase>, msg: TableEntry, da_height: DaBlockHeight, ) -> anyhow::Result<()> { From 451f8be5df981875679a0930ce803be367ba9ded Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 29 Apr 2024 06:19:28 +0300 Subject: [PATCH 11/14] Restore TransactableStorage::commit height parameter --- crates/fuel-core/src/database.rs | 13 ++++++++----- .../src/service/genesis/importer/import_task.rs | 6 +++++- crates/fuel-core/src/state.rs | 8 ++++++-- .../fuel-core/src/state/in_memory/memory_store.rs | 10 +++++++--- crates/fuel-core/src/state/rocks_db.rs | 14 +++++++++----- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 4494f8c82fa..012b94ceb32 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -443,27 +443,27 @@ impl Modifiable for Database { impl Modifiable for UncheckedDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(changes) + self.data.as_ref().commit_changes(None, changes) } } impl Modifiable for UncheckedDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(changes) + self.data.as_ref().commit_changes(None, changes) } } #[cfg(feature = "relayer")] impl Modifiable for UncheckedDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(changes) + self.data.as_ref().commit_changes(None, changes) } } #[cfg(not(feature = "relayer"))] impl Modifiable for UncheckedDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(changes) + self.data.as_ref().commit_changes(None, changes) } } @@ -587,7 +587,10 @@ where // Atomically commit the changes to the database, and to the mutex-protected field. let mut guard = database.height.lock(); - database.data.as_ref().commit_changes(updated_changes)?; + database + .data + .as_ref() + .commit_changes(new_height, updated_changes)?; // Update the block height if let Some(new_height) = new_height { diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index bdfdad8703d..1e6e12b5683 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -556,7 +556,11 @@ mod tests { } impl TransactableStorage for BrokenTransactions { - fn commit_changes(&self, _: Changes) -> StorageResult<()> { + fn commit_changes( + &self, + _: Option, + _: Changes, + ) -> StorageResult<()> { Err(anyhow::anyhow!("I refuse to work!").into()) } } diff --git a/crates/fuel-core/src/state.rs b/crates/fuel-core/src/state.rs index aecb67aa43b..a707ba30b27 100644 --- a/crates/fuel-core/src/state.rs +++ b/crates/fuel-core/src/state.rs @@ -33,7 +33,11 @@ where pub trait TransactableStorage: IterableStore + Debug + Send + Sync { /// Commits the changes into the storage. - fn commit_changes(&self, changes: Changes) -> StorageResult<()>; + fn commit_changes( + &self, + height: Option, + changes: Changes, + ) -> StorageResult<()>; } // It is used only to allow conversion of the `StorageTransaction` into the `DataSource`. @@ -43,7 +47,7 @@ impl TransactableStorage where S: IterableStore + Debug + Send + Sync, { - fn commit_changes(&self, _: Changes) -> StorageResult<()> { + fn commit_changes(&self, _: Option, _: Changes) -> StorageResult<()> { unimplemented!() } } diff --git a/crates/fuel-core/src/state/in_memory/memory_store.rs b/crates/fuel-core/src/state/in_memory/memory_store.rs index 9dc171f9d1d..bd8b0564f38 100644 --- a/crates/fuel-core/src/state/in_memory/memory_store.rs +++ b/crates/fuel-core/src/state/in_memory/memory_store.rs @@ -114,7 +114,11 @@ impl TransactableStorage for MemoryStore StorageResult<()> { + fn commit_changes( + &self, + _: Option, + changes: Changes, + ) -> StorageResult<()> { for (column, btree) in changes.into_iter() { let mut lock = self.inner[column as usize] .lock() @@ -158,7 +162,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(None, changes)?; Ok(len) } @@ -166,7 +170,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(None, changes)?; Ok(()) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index b5ac56ec962..d44b5e484f1 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -525,7 +525,11 @@ impl TransactableStorage for RocksDb StorageResult<()> { + fn commit_changes( + &self, + _: Option, + changes: Changes, + ) -> StorageResult<()> { let mut batch = WriteBatch::default(); for (column, ops) in changes { @@ -592,7 +596,7 @@ mod tests { let mut transaction = self.read_transaction(); let len = transaction.write(key, column, buf)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(None, changes)?; Ok(len) } @@ -601,7 +605,7 @@ mod tests { let mut transaction = self.read_transaction(); transaction.delete(key, column)?; let changes = transaction.into_changes(); - self.commit_changes(changes)?; + self.commit_changes(None, changes)?; Ok(()) } } @@ -676,7 +680,7 @@ mod tests { )]), )]; - db.commit_changes(HashMap::from_iter(ops)).unwrap(); + db.commit_changes(None, HashMap::from_iter(ops)).unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap().unwrap(), value) } @@ -692,7 +696,7 @@ mod tests { Column::Metadata.id(), BTreeMap::from_iter(vec![(key.clone(), WriteOperation::Remove)]), )]; - db.commit_changes(HashMap::from_iter(ops)).unwrap(); + db.commit_changes(None, HashMap::from_iter(ops)).unwrap(); assert_eq!(db.get(&key, Column::Metadata).unwrap(), None); } From 28de830962ff8b6501fc742aeae75e1e46c70ebf Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 29 Apr 2024 06:44:59 +0300 Subject: [PATCH 12/14] Make Database use UncheckedDatabase internally --- crates/fuel-core/src/database.rs | 33 +++++++++++--------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 012b94ceb32..50e02a77754 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -98,9 +98,9 @@ where Description: DatabaseDescription, { pub fn into_checked(self) -> Database { - Database { + Database:: { height: SharedMutex::new(None), - data: self.data, + inner: self, } } } @@ -112,7 +112,7 @@ where { /// Cached value from Metadata table, used to speed up lookups. height: SharedMutex>, - data: DataSource, + inner: UncheckedDatabase, } impl Database { @@ -149,16 +149,7 @@ where Self: StorageInspect, Error = StorageError>, { pub fn new(data_source: DataSource) -> Self { - let database = Self { - height: SharedMutex::new(None), - data: data_source, - }; - let height = database - .latest_height() - .expect("Failed to get latest height during creation of the database"); - *database.height.lock() = height; - - database + (UncheckedDatabase { data: data_source }).into_checked() } #[cfg(feature = "rocksdb")] @@ -173,7 +164,7 @@ where /// Panics if the height is already set. pub fn into_unchecked(self) -> UncheckedDatabase { assert!(!self.height.lock().is_some()); - UncheckedDatabase { data: self.data } + self.inner } } @@ -215,7 +206,7 @@ where type Column = Description::Column; fn exists(&self, key: &[u8], column: Self::Column) -> StorageResult { - self.data.as_ref().exists(key, column) + KeyValueInspect::exists(&self.inner, key, column) } fn size_of_value( @@ -223,11 +214,11 @@ where key: &[u8], column: Self::Column, ) -> StorageResult> { - self.data.as_ref().size_of_value(key, column) + KeyValueInspect::size_of_value(&self.inner, key, column) } fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { - self.data.as_ref().get(key, column) + KeyValueInspect::get(&self.inner, key, column) } fn read( @@ -236,7 +227,7 @@ where column: Self::Column, buf: &mut [u8], ) -> StorageResult> { - self.data.as_ref().read(key, column, buf) + KeyValueInspect::read(&self.inner, key, column, buf) } } @@ -251,9 +242,7 @@ where start: Option<&[u8]>, direction: IterDirection, ) -> BoxedIter { - self.data - .as_ref() - .iter_store(column, prefix, start, direction) + IterableStore::iter_store(&self.inner, column, prefix, start, direction) } } @@ -588,8 +577,8 @@ where // Atomically commit the changes to the database, and to the mutex-protected field. let mut guard = database.height.lock(); database + .inner .data - .as_ref() .commit_changes(new_height, updated_changes)?; // Update the block height From 8ba1fc2687757e0813635ef8f33d221db1f45498 Mon Sep 17 00:00:00 2001 From: Hannes Karppila Date: Mon, 29 Apr 2024 06:56:24 +0300 Subject: [PATCH 13/14] Restore height fetch on db creation --- crates/fuel-core/src/database.rs | 12 ++++++++++-- crates/fuel-core/src/database/metadata.rs | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 50e02a77754..430002fce68 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -146,10 +146,18 @@ where impl Database where Description: DatabaseDescription, - Self: StorageInspect, Error = StorageError>, + UncheckedDatabase: + StorageInspect, Error = StorageError>, { pub fn new(data_source: DataSource) -> Self { - (UncheckedDatabase { data: data_source }).into_checked() + let inner = UncheckedDatabase { data: data_source }; + let height = inner + .latest_height() + .expect("Failed to get latest height during creation of the database"); + Self { + height: SharedMutex::new(height), + inner, + } } #[cfg(feature = "rocksdb")] diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index d9f8672b073..c7ea2fea87e 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -5,6 +5,7 @@ use crate::database::{ }, Database, Error as DatabaseError, + UncheckedDatabase, }; use fuel_core_storage::{ blueprint::plain::Plain, @@ -42,7 +43,7 @@ where } } -impl Database +impl UncheckedDatabase where Description: DatabaseDescription, Self: StorageInspect, Error = StorageError>, @@ -73,3 +74,19 @@ where Ok(metadata) } } + +impl Database +where + Description: DatabaseDescription, + UncheckedDatabase: + StorageInspect, Error = StorageError>, +{ + /// Ensures the version is correct. + pub fn check_version(&self) -> StorageResult<()> { + self.inner.check_version() + } + + pub fn latest_height(&self) -> StorageResult> { + self.inner.latest_height() + } +} From 97d681c31e35874b2b24b368f1c23142e7db2cba Mon Sep 17 00:00:00 2001 From: xgreenx Date: Mon, 29 Apr 2024 10:24:22 +0200 Subject: [PATCH 14/14] Used stages to reuse most of the functionality --- benches/benches/block_target_gas.rs | 1 + benches/benches/vm.rs | 3 +- benches/benches/vm_set/blockchain.rs | 7 +- benches/src/lib.rs | 12 +- crates/fuel-core/src/combined_database.rs | 33 +++ crates/fuel-core/src/database.rs | 198 +++++++----------- .../src/database/genesis_progress.rs | 4 +- crates/fuel-core/src/database/metadata.rs | 19 +- crates/fuel-core/src/database/storage.rs | 43 +--- crates/fuel-core/src/service/genesis.rs | 1 + .../fuel-core/src/service/genesis/importer.rs | 12 +- .../service/genesis/importer/import_task.rs | 40 ++-- .../src/service/genesis/importer/off_chain.rs | 34 +-- .../src/service/genesis/importer/on_chain.rs | 24 +-- crates/services/src/sync.rs | 4 +- 15 files changed, 184 insertions(+), 251 deletions(-) diff --git a/benches/benches/block_target_gas.rs b/benches/benches/block_target_gas.rs index aac9ef2a128..341848f4c01 100644 --- a/benches/benches/block_target_gas.rs +++ b/benches/benches/block_target_gas.rs @@ -20,6 +20,7 @@ use fuel_core::{ database::{ balances::BalancesInitializer, state::StateInitializer, + Database, }, service::{ config::Trigger, diff --git a/benches/benches/vm.rs b/benches/benches/vm.rs index 16e7cf2461e..799d222a0eb 100644 --- a/benches/benches/vm.rs +++ b/benches/benches/vm.rs @@ -15,6 +15,7 @@ use std::sync::Arc; use crate::vm_initialization::vm_initialization; use contract::*; +use fuel_core::database::GenesisDatabase; use fuel_core_benches::*; use fuel_core_storage::transactional::IntoTransaction; use fuel_core_types::fuel_asm::Instruction; @@ -47,7 +48,7 @@ where let relayer_database_tx = block_database_tx.into_transaction(); let thread_database_tx = relayer_database_tx.into_transaction(); let tx_database_tx = thread_database_tx.into_transaction(); - let database = Database::new(Arc::new(tx_database_tx)); + let database = GenesisDatabase::new(Arc::new(tx_database_tx)); *vm.as_mut().database_mut() = database.into_transaction(); let mut total = core::time::Duration::ZERO; diff --git a/benches/benches/vm_set/blockchain.rs b/benches/benches/vm_set/blockchain.rs index ada8d1116f2..59581834499 100644 --- a/benches/benches/vm_set/blockchain.rs +++ b/benches/benches/vm_set/blockchain.rs @@ -16,6 +16,7 @@ use fuel_core::{ balances::BalancesInitializer, database_description::on_chain::OnChain, state::StateInitializer, + GenesisDatabase, }, service::Config, state::rocks_db::{ @@ -63,7 +64,7 @@ use rand::{ }; pub struct BenchDb { - db: Database, + db: GenesisDatabase, /// Used for RAII cleanup. Contents of this directory are deleted on drop. _tmp_dir: ShallowTempDir, } @@ -79,7 +80,7 @@ impl BenchDb { let state_size = crate::utils::get_state_size(); - let mut database = Database::new(db); + let mut database = GenesisDatabase::new(db); database.init_contract_state( contract_id, (0..state_size).map(|_| { @@ -125,7 +126,7 @@ impl BenchDb { } /// Creates a `VmDatabase` instance. - fn to_vm_database(&self) -> VmStorage> { + fn to_vm_database(&self) -> VmStorage> { let consensus = ConsensusHeader { prev_root: Default::default(), height: 1.into(), diff --git a/benches/src/lib.rs b/benches/src/lib.rs index 2807ddf0fc5..af672011f08 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -1,7 +1,6 @@ pub mod default_gas_costs; pub mod import; -pub use fuel_core::database::Database; pub use fuel_core_storage::vm_storage::VmStorage; use fuel_core_types::{ fuel_asm::{ @@ -31,13 +30,14 @@ use fuel_core_types::{ }, }; +use fuel_core::database::GenesisDatabase; use fuel_core_storage::transactional::StorageTransaction; pub use rand::Rng; use std::iter; const LARGE_GAS_LIMIT: u64 = u64::MAX - 1001; -fn new_db() -> VmStorage> { +fn new_db() -> VmStorage> { // when rocksdb is enabled, this creates a new db instance with a temporary path VmStorage::default() } @@ -90,7 +90,7 @@ pub struct VmBench { pub inputs: Vec, pub outputs: Vec, pub witnesses: Vec, - pub db: Option>>, + pub db: Option>>, pub instruction: Instruction, pub prepare_call: Option, pub dummy_contract: Option, @@ -101,7 +101,7 @@ pub struct VmBench { #[derive(Debug, Clone)] pub struct VmBenchPrepared { - pub vm: Interpreter>, Script>, + pub vm: Interpreter>, Script>, pub instruction: Instruction, pub diff: diff::Diff, } @@ -149,7 +149,7 @@ impl VmBench { pub fn contract_using_db( rng: &mut R, - mut db: VmStorage>, + mut db: VmStorage>, instruction: Instruction, ) -> anyhow::Result where @@ -208,7 +208,7 @@ impl VmBench { .with_prepare_call(prepare_call)) } - pub fn with_db(mut self, db: VmStorage>) -> Self { + pub fn with_db(mut self, db: VmStorage>) -> Self { self.db.replace(db); self } diff --git a/crates/fuel-core/src/combined_database.rs b/crates/fuel-core/src/combined_database.rs index 9679045f0da..6023c580a47 100644 --- a/crates/fuel-core/src/combined_database.rs +++ b/crates/fuel-core/src/combined_database.rs @@ -6,6 +6,7 @@ use crate::{ relayer::Relayer, }, Database, + GenesisDatabase, Result as DatabaseResult, }, service::DbType, @@ -178,4 +179,36 @@ impl CombinedDatabase { Ok(state_config) } + + /// Converts the combined database into a genesis combined database. + pub fn into_genesis(self) -> CombinedGenesisDatabase { + CombinedGenesisDatabase { + on_chain: self.on_chain.into_genesis(), + off_chain: self.off_chain.into_genesis(), + relayer: self.relayer.into_genesis(), + } + } +} + +/// A genesis database that combines the on-chain, off-chain and relayer +/// genesis databases into one entity. +#[derive(Default, Clone)] +pub struct CombinedGenesisDatabase { + on_chain: GenesisDatabase, + off_chain: GenesisDatabase, + relayer: GenesisDatabase, +} + +impl CombinedGenesisDatabase { + pub fn on_chain(&self) -> &GenesisDatabase { + &self.on_chain + } + + pub fn off_chain(&self) -> &GenesisDatabase { + &self.off_chain + } + + pub fn relayer(&self) -> &GenesisDatabase { + &self.relayer + } } diff --git a/crates/fuel-core/src/database.rs b/crates/fuel-core/src/database.rs index 430002fce68..de14aac4f2e 100644 --- a/crates/fuel-core/src/database.rs +++ b/crates/fuel-core/src/database.rs @@ -85,34 +85,38 @@ pub mod state; pub mod storage; pub mod transactions; -#[derive(Clone, Debug)] -pub struct UncheckedDatabase +#[derive(Default, Debug, Copy, Clone)] +pub struct GenesisStage; + +#[derive(Debug, Clone)] +pub struct RegularStage where Description: DatabaseDescription, { - data: DataSource, + /// Cached value from Metadata table, used to speed up lookups. + height: SharedMutex>, } -impl UncheckedDatabase +impl Default for RegularStage where Description: DatabaseDescription, { - pub fn into_checked(self) -> Database { - Database:: { + fn default() -> Self { + Self { height: SharedMutex::new(None), - inner: self, } } } +pub type GenesisDatabase = Database; + #[derive(Clone, Debug)] -pub struct Database +pub struct Database> where Description: DatabaseDescription, { - /// Cached value from Metadata table, used to speed up lookups. - height: SharedMutex>, - inner: UncheckedDatabase, + data: DataSource, + stage: Stage, } impl Database { @@ -143,21 +147,38 @@ where } } +impl GenesisDatabase +where + Description: DatabaseDescription, +{ + pub fn new(data_source: DataSource) -> Self { + Self { + stage: GenesisStage, + data: data_source, + } + } +} + impl Database where Description: DatabaseDescription, - UncheckedDatabase: + Database: StorageInspect, Error = StorageError>, { pub fn new(data_source: DataSource) -> Self { - let inner = UncheckedDatabase { data: data_source }; - let height = inner + let mut database = Self { + stage: RegularStage { + height: SharedMutex::new(None), + }, + data: data_source, + }; + let height = database .latest_height() .expect("Failed to get latest height during creation of the database"); - Self { - height: SharedMutex::new(height), - inner, - } + + database.stage.height = SharedMutex::new(height); + + database } #[cfg(feature = "rocksdb")] @@ -165,96 +186,46 @@ where use anyhow::Context; let db = RocksDb::::default_open(path, capacity.into()).map_err(Into::::into).with_context(|| format!("Failed to open rocksdb, you may need to wipe a pre-existing incompatible db e.g. `rm -rf {path:?}`"))?; - Ok(Database::new(Arc::new(db))) + Ok(Self::new(Arc::new(db))) } /// Converts to an unchecked database. /// Panics if the height is already set. - pub fn into_unchecked(self) -> UncheckedDatabase { - assert!(!self.height.lock().is_some()); - self.inner + pub fn into_genesis(self) -> GenesisDatabase { + assert!( + !self.stage.height.lock().is_some(), + "Height is already set for `{}`", + Description::name() + ); + GenesisDatabase::new(self.data) } } -impl UncheckedDatabase +impl Database where Description: DatabaseDescription, + Stage: Default, { pub fn in_memory() -> Self { let data = Arc::>::new(MemoryStore::default()); - Self { data } + Self { + data, + stage: Stage::default(), + } } #[cfg(feature = "rocksdb")] pub fn rocksdb_temp() -> Self { let data = Arc::>::new(RocksDb::default_open_temp(None).unwrap()); - Self { data } - } -} - -impl Database -where - Description: DatabaseDescription, -{ - pub fn in_memory() -> Self { - UncheckedDatabase::in_memory().into_checked() - } - - #[cfg(feature = "rocksdb")] - pub fn rocksdb_temp() -> Self { - UncheckedDatabase::in_memory().into_checked() - } -} - -impl KeyValueInspect for Database -where - Description: DatabaseDescription, -{ - type Column = Description::Column; - - fn exists(&self, key: &[u8], column: Self::Column) -> StorageResult { - KeyValueInspect::exists(&self.inner, key, column) - } - - fn size_of_value( - &self, - key: &[u8], - column: Self::Column, - ) -> StorageResult> { - KeyValueInspect::size_of_value(&self.inner, key, column) - } - - fn get(&self, key: &[u8], column: Self::Column) -> StorageResult> { - KeyValueInspect::get(&self.inner, key, column) - } - - fn read( - &self, - key: &[u8], - column: Self::Column, - buf: &mut [u8], - ) -> StorageResult> { - KeyValueInspect::read(&self.inner, key, column, buf) - } -} - -impl IterableStore for Database -where - Description: DatabaseDescription, -{ - fn iter_store( - &self, - column: Self::Column, - prefix: Option<&[u8]>, - start: Option<&[u8]>, - direction: IterDirection, - ) -> BoxedIter { - IterableStore::iter_store(&self.inner, column, prefix, start, direction) + Self { + data, + stage: Stage::default(), + } } } -impl KeyValueInspect for UncheckedDatabase +impl KeyValueInspect for Database where Description: DatabaseDescription, { @@ -286,7 +257,7 @@ where } } -impl IterableStore for UncheckedDatabase +impl IterableStore for Database where Description: DatabaseDescription, { @@ -306,28 +277,10 @@ where /// Construct an ephemeral database /// uses rocksdb when rocksdb features are enabled /// uses in-memory when rocksdb features are disabled -impl Default for UncheckedDatabase -where - Description: DatabaseDescription, -{ - fn default() -> Self { - #[cfg(not(feature = "rocksdb"))] - { - Self::in_memory() - } - #[cfg(feature = "rocksdb")] - { - Self::rocksdb_temp() - } - } -} - -/// Construct an ephemeral database -/// uses rocksdb when rocksdb features are enabled -/// uses in-memory when rocksdb features are disabled -impl Default for Database +impl Default for Database where Description: DatabaseDescription, + Stage: Default, { fn default() -> Self { #[cfg(not(feature = "rocksdb"))] @@ -347,7 +300,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.stage.height.lock() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -367,7 +320,7 @@ impl AtomicView for Database { type Height = BlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.stage.height.lock() } fn view_at(&self, _: &BlockHeight) -> StorageResult { @@ -386,7 +339,7 @@ impl AtomicView for Database { type Height = DaBlockHeight; fn latest_height(&self) -> Option { - *self.height.lock() + *self.stage.height.lock() } fn view_at(&self, _: &Self::Height) -> StorageResult { @@ -438,27 +391,19 @@ impl Modifiable for Database { } } -impl Modifiable for UncheckedDatabase { +impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } } -impl Modifiable for UncheckedDatabase { +impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } } -#[cfg(feature = "relayer")] -impl Modifiable for UncheckedDatabase { - fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { - self.data.as_ref().commit_changes(None, changes) - } -} - -#[cfg(not(feature = "relayer"))] -impl Modifiable for UncheckedDatabase { +impl Modifiable for GenesisDatabase { fn commit_changes(&mut self, changes: Changes) -> StorageResult<()> { self.data.as_ref().commit_changes(None, changes) } @@ -519,7 +464,7 @@ where } let new_height = new_heights.into_iter().last(); - let prev_height = *database.height.lock(); + let prev_height = *database.stage.height.lock(); match (prev_height, new_height) { (None, None) => { @@ -583,11 +528,8 @@ where }; // Atomically commit the changes to the database, and to the mutex-protected field. - let mut guard = database.height.lock(); - database - .inner - .data - .commit_changes(new_height, updated_changes)?; + let mut guard = database.stage.height.lock(); + database.data.commit_changes(new_height, updated_changes)?; // Update the block height if let Some(new_height) = new_height { diff --git a/crates/fuel-core/src/database/genesis_progress.rs b/crates/fuel-core/src/database/genesis_progress.rs index 81823d571ba..9e847f38dba 100644 --- a/crates/fuel-core/src/database/genesis_progress.rs +++ b/crates/fuel-core/src/database/genesis_progress.rs @@ -6,7 +6,7 @@ use super::{ on_chain::OnChain, DatabaseDescription, }, - Database, + GenesisDatabase, }; use fuel_core_chain_config::GenesisCommitment; use fuel_core_executor::refs::ContractRef; @@ -102,7 +102,7 @@ where } } -impl Database { +impl GenesisDatabase { pub fn genesis_coins_root(&self) -> Result { let coins = self.iter_all::(None); diff --git a/crates/fuel-core/src/database/metadata.rs b/crates/fuel-core/src/database/metadata.rs index c7ea2fea87e..83f78d0992f 100644 --- a/crates/fuel-core/src/database/metadata.rs +++ b/crates/fuel-core/src/database/metadata.rs @@ -5,7 +5,6 @@ use crate::database::{ }, Database, Error as DatabaseError, - UncheckedDatabase, }; use fuel_core_storage::{ blueprint::plain::Plain, @@ -43,7 +42,7 @@ where } } -impl UncheckedDatabase +impl Database where Description: DatabaseDescription, Self: StorageInspect, Error = StorageError>, @@ -74,19 +73,3 @@ where Ok(metadata) } } - -impl Database -where - Description: DatabaseDescription, - UncheckedDatabase: - StorageInspect, Error = StorageError>, -{ - /// Ensures the version is correct. - pub fn check_version(&self) -> StorageResult<()> { - self.inner.check_version() - } - - pub fn latest_height(&self) -> StorageResult> { - self.inner.latest_height() - } -} diff --git a/crates/fuel-core/src/database/storage.rs b/crates/fuel-core/src/database/storage.rs index a3e8b4807b2..cf6b55823e9 100644 --- a/crates/fuel-core/src/database/storage.rs +++ b/crates/fuel-core/src/database/storage.rs @@ -30,35 +30,7 @@ use fuel_core_storage::{ StorageWrite, }; -use super::UncheckedDatabase; - -impl StorageInspect for UncheckedDatabase -where - Description: DatabaseDescription, - M: Mappable, - for<'a> StructuredStorage<&'a Self>: StorageInspect, -{ - type Error = StorageError; - - fn get(&self, key: &M::Key) -> StorageResult>> { - let storage = StructuredStorage::new(self); - let value = storage.storage::().get(key)?; - - if let Some(cow) = value { - Ok(Some(Cow::Owned(cow.into_owned()))) - } else { - Ok(None) - } - } - - fn contains_key(&self, key: &M::Key) -> StorageResult { - StructuredStorage::new(self) - .storage::() - .contains_key(key) - } -} - -impl StorageInspect for Database +impl StorageInspect for Database where Description: DatabaseDescription, M: Mappable, @@ -85,7 +57,7 @@ where } #[cfg(feature = "test-helpers")] -impl StorageMutate for Database +impl StorageMutate for Database where Description: DatabaseDescription, M: Mappable, @@ -120,7 +92,7 @@ where } } -impl StorageSize for Database +impl StorageSize for Database where Description: DatabaseDescription, M: Mappable, @@ -131,7 +103,8 @@ where } } -impl MerkleRootStorage for Database +impl MerkleRootStorage + for Database where Description: DatabaseDescription, M: Mappable, @@ -142,7 +115,7 @@ where } } -impl StorageRead for Database +impl StorageRead for Database where Description: DatabaseDescription, M: Mappable, @@ -158,7 +131,7 @@ where } #[cfg(feature = "test-helpers")] -impl StorageWrite for Database +impl StorageWrite for Database where Description: DatabaseDescription, M: Mappable, @@ -205,7 +178,7 @@ where } #[cfg(feature = "test-helpers")] -impl StorageBatchMutate for Database +impl StorageBatchMutate for Database where Description: DatabaseDescription, M: Mappable, diff --git a/crates/fuel-core/src/service/genesis.rs b/crates/fuel-core/src/service/genesis.rs index e4ec83e8422..c0f94383706 100644 --- a/crates/fuel-core/src/service/genesis.rs +++ b/crates/fuel-core/src/service/genesis.rs @@ -71,6 +71,7 @@ pub async fn execute_genesis_block( ) -> anyhow::Result> { let genesis_block = create_genesis_block(config); tracing::info!("Genesis block created: {:?}", genesis_block.header()); + let db = db.clone().into_genesis(); SnapshotImporter::import( db.clone(), diff --git a/crates/fuel-core/src/service/genesis/importer.rs b/crates/fuel-core/src/service/genesis/importer.rs index 7d613c97751..178f3b360f8 100644 --- a/crates/fuel-core/src/service/genesis/importer.rs +++ b/crates/fuel-core/src/service/genesis/importer.rs @@ -3,7 +3,7 @@ use super::{ task_manager::TaskManager, }; use crate::{ - combined_database::CombinedDatabase, + combined_database::CombinedGenesisDatabase, database::database_description::{ off_chain::OffChain, on_chain::OnChain, @@ -68,7 +68,7 @@ mod on_chain; const GROUPS_NUMBER_FOR_PARALLELIZATION: usize = 10; pub struct SnapshotImporter { - db: CombinedDatabase, + db: CombinedGenesisDatabase, task_manager: TaskManager<()>, genesis_block: Block, snapshot_reader: SnapshotReader, @@ -77,7 +77,7 @@ pub struct SnapshotImporter { impl SnapshotImporter { fn new( - db: CombinedDatabase, + db: CombinedGenesisDatabase, genesis_block: Block, snapshot_reader: SnapshotReader, watcher: StateWatcher, @@ -94,7 +94,7 @@ impl SnapshotImporter { } pub async fn import( - db: CombinedDatabase, + db: CombinedGenesisDatabase, genesis_block: Block, snapshot_reader: SnapshotReader, watcher: StateWatcher, @@ -155,7 +155,7 @@ impl SnapshotImporter { let block_height = *self.genesis_block.header().height(); let da_block_height = self.genesis_block.header().da_height; - let db = self.db.on_chain().clone().into_unchecked(); + let db = self.db.on_chain().clone(); let migration_name = migration_name::(); let progress_reporter = self @@ -203,7 +203,7 @@ impl SnapshotImporter { let block_height = *self.genesis_block.header().height(); let da_block_height = self.genesis_block.header().da_height; - let db = self.db.off_chain().clone().into_unchecked(); + let db = self.db.off_chain().clone(); let migration_name = migration_name::(); let progress_reporter = self diff --git a/crates/fuel-core/src/service/genesis/importer/import_task.rs b/crates/fuel-core/src/service/genesis/importer/import_task.rs index 1e6e12b5683..4325d6165e0 100644 --- a/crates/fuel-core/src/service/genesis/importer/import_task.rs +++ b/crates/fuel-core/src/service/genesis/importer/import_task.rs @@ -19,7 +19,7 @@ use crate::{ GenesisMetadata, GenesisProgressMutate, }, - UncheckedDatabase, + GenesisDatabase, }, service::genesis::{ progress::ProgressReporter, @@ -36,7 +36,7 @@ where handler: Handler, skip: usize, groups: Groups, - db: UncheckedDatabase, + db: GenesisDatabase, reporter: ProgressReporter, } @@ -48,7 +48,7 @@ pub trait ImportTable { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()>; } @@ -56,12 +56,12 @@ impl ImportTask where DbDesc: DatabaseDescription, Logic: ImportTable, - UncheckedDatabase: StorageInspect>, + GenesisDatabase: StorageInspect>, { pub fn new( handler: Logic, groups: GroupGenerator, - db: UncheckedDatabase, + db: GenesisDatabase, reporter: ProgressReporter, ) -> Self { let progress_name = @@ -95,9 +95,9 @@ where Value = usize, OwnedValue = usize, >, - UncheckedDatabase: + GenesisDatabase: StorageInspect> + WriteTransaction + Modifiable, - for<'a> StorageTransaction<&'a mut UncheckedDatabase>: + for<'a> StorageTransaction<&'a mut GenesisDatabase>: StorageMutate, Error = fuel_core_storage::Error>, { pub fn run(mut self, cancel_token: CancellationToken) -> anyhow::Result<()> { @@ -139,7 +139,7 @@ mod tests { use crate::{ database::{ genesis_progress::GenesisProgressInspect, - UncheckedDatabase, + GenesisDatabase, }, service::genesis::{ importer::{ @@ -199,11 +199,9 @@ mod tests { }; use crate::{ - combined_database::CombinedDatabase, database::{ database_description::on_chain::OnChain, genesis_progress::GenesisProgressMutate, - Database, }, state::{ in_memory::memory_store::MemoryStore, @@ -230,7 +228,7 @@ mod tests { where L: FnMut( TableEntry, - &mut StorageTransaction<&mut UncheckedDatabase>, + &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()>, { type TableInSnapshot = Coins; @@ -239,7 +237,7 @@ mod tests { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group .into_iter() @@ -290,7 +288,7 @@ mod tests { Ok(()) }), data.as_ok_groups(), - UncheckedDatabase::default(), + GenesisDatabase::default(), ProgressReporter::default(), ); @@ -307,9 +305,9 @@ mod tests { let data = TestData::new(2); let mut called_with = vec![]; - let mut db = CombinedDatabase::default(); + let mut db = GenesisDatabase::::default(); GenesisProgressMutate::::update_genesis_progress( - db.on_chain_mut(), + &mut db, &migration_name::(), 0, ) @@ -320,7 +318,7 @@ mod tests { Ok(()) }), data.as_ok_groups(), - db.on_chain().clone().into_unchecked(), + db, ProgressReporter::default(), ); @@ -335,7 +333,7 @@ mod tests { fn changes_to_db_by_handler_are_behind_a_transaction() { // given let groups = TestData::new(1); - let outer_db = UncheckedDatabase::default(); + let outer_db = GenesisDatabase::default(); let utxo_id = UtxoId::new(Default::default(), 0); let runner = ImportTask::new( @@ -373,7 +371,7 @@ mod tests { } fn insert_a_coin( - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, utxo_id: &UtxoId, ) { let coin: CompressedCoin = CompressedCoinV1::default().into(); @@ -385,7 +383,7 @@ mod tests { fn tx_reverted_if_handler_fails() { // given let groups = TestData::new(1); - let db = UncheckedDatabase::default(); + let db = GenesisDatabase::default(); let utxo_id = UtxoId::new(Default::default(), 0); let runner = ImportTask::new( @@ -445,7 +443,7 @@ mod tests { fn succesfully_processed_batch_updates_the_genesis_progress() { // given let data = TestData::new(2); - let db = UncheckedDatabase::default(); + let db = GenesisDatabase::default(); let runner = ImportTask::new( TestHandler::new(|_, _| Ok(())), data.as_ok_groups(), @@ -572,7 +570,7 @@ mod tests { let runner = ImportTask::new( TestHandler::new(|_, _| Ok(())), groups.as_ok_groups(), - Database::new(Arc::new(BrokenTransactions::new())).into_unchecked(), + GenesisDatabase::new(Arc::new(BrokenTransactions::new())), ProgressReporter::default(), ); diff --git a/crates/fuel-core/src/service/genesis/importer/off_chain.rs b/crates/fuel-core/src/service/genesis/importer/off_chain.rs index c3b8fcb82cf..58cce0dd211 100644 --- a/crates/fuel-core/src/service/genesis/importer/off_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/off_chain.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use crate::{ database::{ database_description::off_chain::OffChain, - UncheckedDatabase, + GenesisDatabase, }, fuel_core_graphql_api::storage::messages::SpentMessages, graphql_api::{ @@ -52,7 +52,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for tx_status in group { tx.storage::() @@ -70,7 +70,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage::() @@ -88,7 +88,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage::() @@ -106,7 +106,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let events = group .into_iter() @@ -124,7 +124,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let events = group.into_iter().map(|TableEntry { value, key }| { Cow::Owned(Event::CoinCreated(value.uncompress(key))) @@ -142,7 +142,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let transactions = group.iter().map(|TableEntry { value, .. }| value); worker_service::process_transactions(transactions, tx)?; @@ -158,7 +158,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let transactions = group.iter().map(|TableEntry { value, .. }| value); worker_service::process_transactions(transactions, tx)?; @@ -174,7 +174,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -192,7 +192,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -210,7 +210,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -228,7 +228,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let blocks = group .iter() @@ -246,7 +246,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let transactions = group .iter() @@ -264,7 +264,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { let transactions = group .iter() @@ -282,7 +282,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() @@ -300,7 +300,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() @@ -318,7 +318,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { for entry in group { tx.storage_as_mut::() diff --git a/crates/fuel-core/src/service/genesis/importer/on_chain.rs b/crates/fuel-core/src/service/genesis/importer/on_chain.rs index 14eba85ec66..e75fd097995 100644 --- a/crates/fuel-core/src/service/genesis/importer/on_chain.rs +++ b/crates/fuel-core/src/service/genesis/importer/on_chain.rs @@ -6,7 +6,7 @@ use crate::database::{ balances::BalancesInitializer, database_description::on_chain::OnChain, state::StateInitializer, - UncheckedDatabase, + GenesisDatabase, }; use anyhow::anyhow; use fuel_core_chain_config::TableEntry; @@ -41,7 +41,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|coin| { init_coin(tx, &coin, self.block_height)?; @@ -58,7 +58,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group .into_iter() @@ -74,7 +74,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|transaction| { tx.storage_as_mut::() @@ -93,7 +93,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|contract| { init_contract_raw_code(tx, &contract)?; @@ -110,7 +110,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { group.into_iter().try_for_each(|contract| { init_contract_latest_utxo(tx, &contract, self.block_height)?; @@ -127,7 +127,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { tx.update_contract_states(group)?; Ok(()) @@ -142,7 +142,7 @@ impl ImportTable for Handler { fn process( &mut self, group: Vec>, - tx: &mut StorageTransaction<&mut UncheckedDatabase>, + tx: &mut StorageTransaction<&mut GenesisDatabase>, ) -> anyhow::Result<()> { tx.update_contract_balances(group)?; Ok(()) @@ -150,7 +150,7 @@ impl ImportTable for Handler { } fn init_coin( - transaction: &mut StorageTransaction<&mut UncheckedDatabase>, + transaction: &mut StorageTransaction<&mut GenesisDatabase>, coin: &TableEntry, height: BlockHeight, ) -> anyhow::Result<()> { @@ -185,7 +185,7 @@ fn init_coin( } fn init_contract_latest_utxo( - transaction: &mut StorageTransaction<&mut UncheckedDatabase>, + transaction: &mut StorageTransaction<&mut GenesisDatabase>, entry: &TableEntry, height: BlockHeight, ) -> anyhow::Result<()> { @@ -209,7 +209,7 @@ fn init_contract_latest_utxo( } fn init_contract_raw_code( - transaction: &mut StorageTransaction<&mut UncheckedDatabase>, + transaction: &mut StorageTransaction<&mut GenesisDatabase>, entry: &TableEntry, ) -> anyhow::Result<()> { let contract = entry.value.as_ref(); @@ -228,7 +228,7 @@ fn init_contract_raw_code( } fn init_da_message( - transaction: &mut StorageTransaction<&mut UncheckedDatabase>, + transaction: &mut StorageTransaction<&mut GenesisDatabase>, msg: TableEntry, da_height: DaBlockHeight, ) -> anyhow::Result<()> { diff --git a/crates/services/src/sync.rs b/crates/services/src/sync.rs index 056cfcc653b..7d869b67203 100644 --- a/crates/services/src/sync.rs +++ b/crates/services/src/sync.rs @@ -1,6 +1,6 @@ -//! Wrappers for synchronization constainers. +//! Wrappers for synchronization containers. -use std::ops::Deref; +use core::ops::Deref; /// Alias for `Arc` pub type Shared = std::sync::Arc;