From 6e18fadb25b786dbaa9dce31a7bdace677cd6365 Mon Sep 17 00:00:00 2001 From: Marco Granelli Date: Thu, 23 Feb 2023 19:16:05 +0100 Subject: [PATCH 1/3] Updates RocksDB dump --- apps/src/lib/cli.rs | 8 ++++++++ apps/src/lib/node/ledger/mod.rs | 3 ++- apps/src/lib/node/ledger/storage/rocksdb.rs | 21 +++++++++++++++------ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/apps/src/lib/cli.rs b/apps/src/lib/cli.rs index 9c23cadb46..d8f074fb1d 100644 --- a/apps/src/lib/cli.rs +++ b/apps/src/lib/cli.rs @@ -1631,6 +1631,7 @@ pub mod args { arg_default_from_ctx("gas-token", DefaultFn(|| "NAM".into())); const GENESIS_PATH: Arg = arg("genesis-path"); const GENESIS_VALIDATOR: ArgOpt = arg("genesis-validator").opt(); + const HISTORIC: ArgFlag = flag("historic"); const LEDGER_ADDRESS_ABOUT: &str = "Address of a ledger node as \"{scheme}://{host}:{port}\". If the \ scheme is not supplied, it is assumed to be TCP."; @@ -1769,6 +1770,7 @@ pub mod args { // TODO: allow to specify height // pub block_height: Option, pub out_file_path: PathBuf, + pub historic: bool, } impl Args for LedgerDumpDb { @@ -1777,9 +1779,12 @@ pub mod args { let out_file_path = OUT_FILE_PATH_OPT .parse(matches) .unwrap_or_else(|| PathBuf::from("db_dump".to_string())); + let historic = HISTORIC.parse(matches); + Self { // block_height, out_file_path, + historic, } } @@ -1793,6 +1798,9 @@ pub mod args { Defaults to \"db_dump.{block_height}.toml\" in the \ current working directory.", )) + .arg(HISTORIC.def().about( + "If provided, dump also the diff of the last height", + )) } } diff --git a/apps/src/lib/node/ledger/mod.rs b/apps/src/lib/node/ledger/mod.rs index 7bf950cc8d..c499a21d2a 100644 --- a/apps/src/lib/node/ledger/mod.rs +++ b/apps/src/lib/node/ledger/mod.rs @@ -206,6 +206,7 @@ pub fn dump_db( args::LedgerDumpDb { // block_height, out_file_path, + historic, }: args::LedgerDumpDb, ) { use namada::ledger::storage::DB; @@ -214,7 +215,7 @@ pub fn dump_db( let db_path = config.shell.db_dir(&chain_id); let db = storage::PersistentDB::open(db_path, None); - db.dump_last_block(out_file_path); + db.dump_last_block(out_file_path, historic); } /// Runs and monitors a few concurrent tasks. diff --git a/apps/src/lib/node/ledger/storage/rocksdb.rs b/apps/src/lib/node/ledger/storage/rocksdb.rs index 82d3faee4f..65f0f2c16b 100644 --- a/apps/src/lib/node/ledger/storage/rocksdb.rs +++ b/apps/src/lib/node/ledger/storage/rocksdb.rs @@ -242,10 +242,14 @@ impl RocksDB { } /// Dump last known block - pub fn dump_last_block(&self, out_file_path: std::path::PathBuf) { + pub fn dump_last_block( + &self, + out_file_path: std::path::PathBuf, + historic: bool, + ) { use std::io::Write; - // Fine the last block height + // Find the last block height let height: BlockHeight = types::decode( self.0 .get("height") @@ -295,10 +299,15 @@ impl RocksDB { } }; - // Dump accounts subspace and block height data - dump_it("subspace".to_string()); - let block_prefix = format!("{}/", height.raw()); - dump_it(block_prefix); + let prefix = if historic { + // Dump subspace and diffs from last block height + height.raw() + } else { + // Dump only accounts subspace + "subspace".to_string() + }; + + dump_it(prefix); println!("Done writing to {}", full_path.to_string_lossy()); } From 1933207fb51ca9f39510abd7c4811cf89e074f5e Mon Sep 17 00:00:00 2001 From: Marco Granelli Date: Wed, 1 Mar 2023 12:21:14 +0100 Subject: [PATCH 2/3] Improves iterator in `dump-db` --- apps/src/lib/node/ledger/storage/rocksdb.rs | 58 +++++++++++---------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/apps/src/lib/node/ledger/storage/rocksdb.rs b/apps/src/lib/node/ledger/storage/rocksdb.rs index 65f0f2c16b..87eca68af8 100644 --- a/apps/src/lib/node/ledger/storage/rocksdb.rs +++ b/apps/src/lib/node/ledger/storage/rocksdb.rs @@ -1,7 +1,6 @@ //! The persistent storage in RocksDB. //! //! The current storage tree is: -//! - `chain_id` //! - `height`: the last committed block height //! - `tx_queue`: txs to be decrypted in the next block //! - `pred`: predecessor values of the top-level keys of the same name @@ -15,6 +14,7 @@ //! - `next_epoch_min_start_time` //! - `subspace`: accounts sub-spaces //! - `{address}/{dyn}`: any byte data associated with accounts +//! - `results`: block results //! - `h`: for each block at height `h`: //! - `tree`: merkle tree //! - `root`: root hash @@ -277,37 +277,39 @@ impl RocksDB { println!("Will write to {} ...", full_path.to_string_lossy()); let mut dump_it = |prefix: String| { - for next in self.0.iterator(IteratorMode::From( - prefix.as_bytes(), - Direction::Forward, - )) { - match next { - Err(e) => { - eprintln!( - "Something failed in a \"{prefix}\" iterator: {e}" - ) - } - Ok((raw_key, raw_val)) => { - let key = std::str::from_utf8(&raw_key) - .expect("All keys should be valid UTF-8 strings"); - let val = HEXLOWER.encode(&raw_val); - let bytes = format!("\"{key}\" = \"{val}\"\n"); - file.write_all(bytes.as_bytes()) - .expect("Unable to write to output file"); - } - }; + let mut read_opts = ReadOptions::default(); + read_opts.set_total_order_seek(true); + + let mut upper_prefix = prefix.clone().into_bytes(); + if let Some(last) = upper_prefix.pop() { + upper_prefix.push(last + 1); } - }; + read_opts.set_iterate_upper_bound(upper_prefix); - let prefix = if historic { - // Dump subspace and diffs from last block height - height.raw() - } else { - // Dump only accounts subspace - "subspace".to_string() + let iter = self.0.iterator_opt( + IteratorMode::From(prefix.as_bytes(), Direction::Forward), + read_opts, + ); + + for (key, raw_val, _gas) in PersistentPrefixIterator( + PrefixIterator::new(iter, String::default()), + // Empty string to prevent prefix stripping, the prefix is + // already in the enclosed iterator + ) { + let val = HEXLOWER.encode(&raw_val); + let bytes = format!("\"{key}\" = \"{val}\"\n"); + file.write_all(bytes.as_bytes()) + .expect("Unable to write to output file"); + } }; - dump_it(prefix); + if historic { + // Dump the keys prepended with the selected block height (includes + // subspace diff keys) + dump_it(height.raw()); + } + + dump_it("subspace".to_string()); println!("Done writing to {}", full_path.to_string_lossy()); } From d001d5b42b86dad6c76ba9c27888561f801a25f4 Mon Sep 17 00:00:00 2001 From: Marco Granelli Date: Tue, 7 Mar 2023 18:37:04 +0100 Subject: [PATCH 3/3] changelog: add #1184 --- .changelog/unreleased/bug-fixes/1184-rocksdb-dump.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/unreleased/bug-fixes/1184-rocksdb-dump.md diff --git a/.changelog/unreleased/bug-fixes/1184-rocksdb-dump.md b/.changelog/unreleased/bug-fixes/1184-rocksdb-dump.md new file mode 100644 index 0000000000..19ad1dd0d0 --- /dev/null +++ b/.changelog/unreleased/bug-fixes/1184-rocksdb-dump.md @@ -0,0 +1,3 @@ +- Fixed dump-db node utility which was not iterating on db keys correctly + leading to duplicates in the dump. Added an historic flag to also dump the + diff keys. ([#1184](https://github.com/anoma/namada/pull/1184)) \ No newline at end of file