feat: Add new chunk count column for tracking #342
Conversation
… index and lenght of columns
client/file-manager/src/rocksdb.rs
Outdated
| @@ -20,23 +20,50 @@ use crate::{ | |||
| LOG_TARGET, | |||
| }; | |||
| use codec::{Decode, Encode}; | |||
| use strum::{EnumCount, VariantArray}; | |||
|
|
|||
| #[derive(Debug, Clone, Copy, EnumCount, VariantArray)] | |||
|
|
||
| #[derive(Debug, Clone, Copy, EnumCount, VariantArray)] | ||
| pub enum Column { | ||
| Metadata, |
There was a problem hiding this comment.
We should have docs for the Metadata member as well, and if possible for all the columns
| @@ -58,7 +85,8 @@ fn open_or_creating_rocksdb(db_path: String) -> io::Result<kvdb_rocksdb::Databas | |||
| Ok(db) | |||
| } | |||
|
|
|||
| /// Storage backend for RocksDB. | |||
| /// Storage backend implementation for RocksDB. | |||
There was a problem hiding this comment.
Love the improved docs on all functions!
| @@ -46,7 +73,7 @@ fn open_or_creating_rocksdb(db_path: String) -> io::Result<kvdb_rocksdb::Databas | |||
| path.push(db_path.as_str()); | |||
| path.push("storagehub/file_storage/"); | |||
|
|
|||
| let db_config = kvdb_rocksdb::DatabaseConfig::with_columns(8); | |||
| let db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUMBER_OF_COLUMNS); | |||
There was a problem hiding this comment.
Nice improvement.
Indeed in the past I had errors when forgetting to increase the columns number hardcoded value. It would avoid it in the future.
| @@ -15,10 +15,14 @@ pub enum FileStorageWriteError { | |||
| FailedToGetFileChunk, | |||
| /// File metadata fingerprint does not match the stored file fingerprint. | |||
| FingerprintAndStoredFileMismatch, | |||
| /// Failed to construct file trie. | |||
| FailedToContructFileTrie, | |||
There was a problem hiding this comment.
| FailedToContructFileTrie, | |
| FailedToConstructFileTrie, |
| /// Exclude* columns stores keys of 32 bytes representing the `file_key` with empty values. | ||
| /// | ||
| /// These columns are used primarily to mark file keys as being excluded from certain operations. | ||
| ExcludeFile, | ||
| ExcludeUser, | ||
| ExcludeBucket, | ||
| ExcludeFingerprint, |
There was a problem hiding this comment.
I'd copy and paste the doc comment for the other items, or make one for each. Otherwise this doc comment will only be linked to ExcludeFile. So for example, if you hover over ExcludeBucket, you won't see any doc comment.
There was a problem hiding this comment.
I'll add this in another PR to avoid restarting CI for such a small change
Adds new
CHUNK_COUNT_COLUMNto track number of chunks for a given file key. This avoids having to iterate through the entire file trie to count all the chunks and conclude if we have uploaded all the chunks yet for a file.These changes also include a refactor of our column definitions to have consistent index retrievals and column length constant. In addition, I added some documentation to these columns to have a clear definition for their purpose in the context of RocksDB file storage.
Resolves issue #20