-
Notifications
You must be signed in to change notification settings - Fork 2.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use a separate database for each data domain (#1629)
Closes #1568 The change splits the `Database` into 3 databases: - `Database<OnChain>` - Stores only data required for normal work of the blockchain. - `Database<OffChain>` - Stores only data used by the off-chain services like GraphQL. - `Database<Relayer>` - Stores relayer-related data like events(messages or transactions) from L1. The `Database<Description>` type has a generic `Description` that implements the `DatabaseDescription` trait: ```rust /// The description of the database that makes it unique. pub trait DatabaseDescription: 'static + Clone + Debug + Send + Sync { /// The type of the column used by the database. type Column: StorageColumn + strum::EnumCount + enum_iterator::Sequence; /// The type of the height of the database used to track commits. type Height: Copy; /// Returns the expected version of the database. fn version() -> u32; /// Returns the name of the database. fn name() -> &'static str; /// Returns the column used to store the metadata. fn metadata_column() -> Self::Column; /// Returns the prefix for the column. fn prefix(column: &Self::Column) -> Option<usize>; } ``` Each database has its folder, defined by the `DatabaseDescription::name`, where actual data is stored. <img width="353" alt="image" src="https://github.com/FuelLabs/fuel-core/assets/18346821/8b642384-0dd4-4668-a415-0748be3e88f0"> Each database has its own `Column` type that describes all columns, avoiding overlaps with other tables. The change updates a little bit `StrucutredStorage` implementation and `TableWithBlueprint` to be more flexible and use the `Column` defined by the table, instead of hardcoded `fuel_core_storage::column::Column`. Other small changes: - Unified the logic of storing the database's metadata. It will be useful for #1589 to implement a unified `commit_chagnes` function. - The `latest_height` function now uses the height from the metadata table. - Removed relayers-related tables and columns from the `fuel-core-storage` crate. - Removed part of GraphQL tables and columns from the `fuel-core-storage`. The last part will be removed during #1583. - Moved `tx_count` metrics from `BlockImporter` to GraphQL off-chain worker. Any statistic that requires a persistent state in the database may be done outside of the blockchain. - Remove `chain_name` from the database. The `ConsensusParameters` already contains this information. - Removed the `checkpoint` function from the `RocksDB` since it is not used. Later it will be added again back but with another implementation during #1589. - Removed `Column::ForeignColumn`, since each database has its own `Column` type. Removed the macro rules added to handle `ForeignColumn`.
- Loading branch information
Showing
74 changed files
with
1,745 additions
and
1,086 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| use crate::database::{ | ||
| database_description::{ | ||
| off_chain::OffChain, | ||
| on_chain::OnChain, | ||
| relayer::Relayer, | ||
| }, | ||
| Database, | ||
| Result as DatabaseResult, | ||
| }; | ||
| use fuel_core_storage::Result as StorageResult; | ||
| use fuel_core_types::{ | ||
| blockchain::primitives::DaBlockHeight, | ||
| fuel_types::BlockHeight, | ||
| }; | ||
|
|
||
| /// A database that combines the on-chain, off-chain and relayer databases into one entity. | ||
| #[derive(Default, Clone)] | ||
| pub struct CombinedDatabase { | ||
| on_chain: Database<OnChain>, | ||
| off_chain: Database<OffChain>, | ||
| relayer: Database<Relayer>, | ||
| } | ||
|
|
||
| impl CombinedDatabase { | ||
| pub fn new( | ||
| on_chain: Database<OnChain>, | ||
| off_chain: Database<OffChain>, | ||
| relayer: Database<Relayer>, | ||
| ) -> Self { | ||
| Self { | ||
| on_chain, | ||
| off_chain, | ||
| relayer, | ||
| } | ||
| } | ||
|
|
||
| #[cfg(feature = "rocksdb")] | ||
| pub fn open(path: &std::path::Path, capacity: usize) -> DatabaseResult<Self> { | ||
| // TODO: Use different cache sizes for different databases | ||
| let on_chain = Database::open(path, capacity)?; | ||
| let off_chain = Database::open(path, capacity)?; | ||
| let relayer = Database::open(path, capacity)?; | ||
| Ok(Self { | ||
| on_chain, | ||
| off_chain, | ||
| relayer, | ||
| }) | ||
| } | ||
|
|
||
| pub fn in_memory() -> Self { | ||
| Self::new( | ||
| Database::in_memory(), | ||
| Database::in_memory(), | ||
| Database::in_memory(), | ||
| ) | ||
| } | ||
|
|
||
| pub fn init( | ||
| &mut self, | ||
| block_height: &BlockHeight, | ||
| da_block_height: &DaBlockHeight, | ||
| ) -> StorageResult<()> { | ||
| self.on_chain.init(block_height)?; | ||
| self.off_chain.init(block_height)?; | ||
| self.relayer.init(da_block_height)?; | ||
| Ok(()) | ||
| } | ||
|
|
||
| pub fn on_chain(&self) -> &Database<OnChain> { | ||
| &self.on_chain | ||
| } | ||
|
|
||
| #[cfg(any(feature = "test-helpers", test))] | ||
| pub fn on_chain_mut(&mut self) -> &mut Database<OnChain> { | ||
| &mut self.on_chain | ||
| } | ||
|
|
||
| pub fn off_chain(&self) -> &Database<OffChain> { | ||
| &self.off_chain | ||
| } | ||
|
|
||
| pub fn relayer(&self) -> &Database<Relayer> { | ||
| &self.relayer | ||
| } | ||
|
|
||
| pub fn flush(self) -> DatabaseResult<()> { | ||
| self.on_chain.flush()?; | ||
| self.off_chain.flush()?; | ||
| self.relayer.flush()?; | ||
| Ok(()) | ||
| } | ||
| } |
Oops, something went wrong.