From e82be11a03981951a87841a816b9d0bef92d3f9b Mon Sep 17 00:00:00 2001 From: Gijs van Dam Date: Tue, 23 May 2023 18:57:05 +0200 Subject: [PATCH 1/3] `dump-db` custom height --- apps/src/lib/cli.rs | 36 ++++++------- apps/src/lib/node/ledger/mod.rs | 4 +- apps/src/lib/node/ledger/storage/rocksdb.rs | 56 ++++++++++++++++++--- 3 files changed, 68 insertions(+), 28 deletions(-) diff --git a/apps/src/lib/cli.rs b/apps/src/lib/cli.rs index 1c0bdba287d..c95c75bd589 100644 --- a/apps/src/lib/cli.rs +++ b/apps/src/lib/cli.rs @@ -2341,7 +2341,7 @@ pub mod args { }), ); pub const BLOCK_HEIGHT: Arg = arg("block-height"); - // pub const BLOCK_HEIGHT_OPT: ArgOpt = arg_opt("height"); + pub const BLOCK_HEIGHT_OPT: ArgOpt = arg_opt("height"); pub const BROADCAST_ONLY: ArgFlag = flag("broadcast-only"); pub const CHAIN_ID: Arg = arg("chain-id"); pub const CHAIN_ID_OPT: ArgOpt = CHAIN_ID.opt(); @@ -2608,41 +2608,41 @@ pub mod args { #[derive(Clone, Debug)] pub struct LedgerDumpDb { // TODO: allow to specify height - // pub block_height: Option, + pub block_height: Option, pub out_file_path: PathBuf, pub historic: bool, } impl Args for LedgerDumpDb { fn parse(matches: &ArgMatches) -> Self { - // let block_height = BLOCK_HEIGHT_OPT.parse(matches); + let block_height = BLOCK_HEIGHT_OPT.parse(matches); let out_file_path = OUT_FILE_PATH_OPT .parse(matches) .unwrap_or_else(|| PathBuf::from("db_dump".to_string())); let historic = HISTORIC.parse(matches); Self { - // block_height, + block_height, out_file_path, historic, } } fn def(app: App) -> App { - app - // .arg(BLOCK_HEIGHT_OPT.def().help( - // "The block height to dump. Defaults to latest committed - // block.", )) - .arg(OUT_FILE_PATH_OPT.def().help( - "Path for the output file (omitting file extension). \ - Defaults to \"db_dump.{block_height}.toml\" in the \ - current working directory.", - )) - .arg( - HISTORIC.def().help( - "If provided, dump also the diff of the last height", - ), - ) + app.arg(BLOCK_HEIGHT_OPT.def().help( + "The block height to dump. Defaults to latest committed + block.", + )) + .arg(OUT_FILE_PATH_OPT.def().help( + "Path for the output file (omitting file extension). Defaults \ + to \"db_dump.{block_height}.toml\" in the current working \ + directory.", + )) + .arg( + HISTORIC + .def() + .help("If provided, dump also the diff of the last height"), + ) } } diff --git a/apps/src/lib/node/ledger/mod.rs b/apps/src/lib/node/ledger/mod.rs index 37b447b4daa..f01b7cc909c 100644 --- a/apps/src/lib/node/ledger/mod.rs +++ b/apps/src/lib/node/ledger/mod.rs @@ -215,7 +215,7 @@ pub fn reset(config: config::Ledger) -> Result<(), shell::Error> { pub fn dump_db( config: config::Ledger, args::LedgerDumpDb { - // block_height, + block_height, out_file_path, historic, }: args::LedgerDumpDb, @@ -226,7 +226,7 @@ pub fn dump_db( let db_path = config.shell.db_dir(&chain_id); let db = storage::PersistentDB::open(db_path, None); - db.dump_last_block(out_file_path, historic); + db.dump_block(out_file_path, historic, &mut block_height.clone()); } /// Roll Namada state back to the previous height diff --git a/apps/src/lib/node/ledger/storage/rocksdb.rs b/apps/src/lib/node/ledger/storage/rocksdb.rs index f679ca92324..c51b2206710 100644 --- a/apps/src/lib/node/ledger/storage/rocksdb.rs +++ b/apps/src/lib/node/ledger/storage/rocksdb.rs @@ -38,6 +38,7 @@ use std::path::Path; use std::str::FromStr; use std::sync::Mutex; +use ark_serialize::Write; use borsh::{BorshDeserialize, BorshSerialize}; use data_encoding::HEXLOWER; use namada::core::types::ethereum_structs; @@ -252,16 +253,18 @@ impl RocksDB { } /// Dump last known block - pub fn dump_last_block( + pub fn dump_block( &self, out_file_path: std::path::PathBuf, historic: bool, + height: &mut Option, ) { // Find the last block height let state_cf = self .get_column_family(STATE_CF) .expect("State column family should exist"); - let height: BlockHeight = types::decode( + + let last_height: BlockHeight = types::decode( self.0 .get_cf(state_cf, "height") .expect("Unable to read DB") @@ -269,6 +272,8 @@ impl RocksDB { ) .expect("Unable to decode block height"); + let height = height.get_or_insert(last_height); + let full_path = out_file_path .with_file_name(format!( "{}_{height}", @@ -306,10 +311,47 @@ impl RocksDB { } // subspace - let cf = self - .get_column_family(SUBSPACE_CF) - .expect("Subspace column family should exist"); - self.dump_it(cf, None, &mut file); + if *height != last_height { + // Restoring subspace at specified height + + let restored_subspace = self + .iter_prefix(&Key::default()) + .par_bridge() + .fold( + || "".to_string(), + |mut cur, (key, _value, _gas)| match self + .read_subspace_val_with_height( + &Key::from(key.to_db_key()), + *height, + last_height, + ) + .expect("Unable to find subspace key") + { + Some(value) => { + let val = HEXLOWER.encode(&value); + let new_line = format!("\"{key}\" = \"{val}\"\n"); + cur.push_str(new_line.as_str()); + cur + } + None => cur, + }, + ) + .reduce( + || "".to_string(), + |mut a: String, b: String| { + a.push_str(&b); + a + }, + ); + file.write_all(restored_subspace.as_bytes()) + .expect("Unable to write to output file"); + } else { + // Just dump the current subspace + let cf = self + .get_column_family(SUBSPACE_CF) + .expect("Subspace column family should exist"); + self.dump_it(cf, None, &mut file); + } println!("Done writing to {}", full_path.to_string_lossy()); } @@ -321,8 +363,6 @@ impl RocksDB { prefix: Option, file: &mut File, ) { - use std::io::Write; - let read_opts = make_iter_read_opts(prefix.clone()); let iter = if let Some(prefix) = prefix { self.0.iterator_cf_opt( From 95c758da3ac15e82621e4a599b5d8d52d4262dd4 Mon Sep 17 00:00:00 2001 From: Marco Granelli Date: Fri, 11 Aug 2023 16:54:29 +0200 Subject: [PATCH 2/3] Buffer write for db-dump. Changes arg to ownership --- apps/src/lib/node/ledger/mod.rs | 2 +- apps/src/lib/node/ledger/storage/rocksdb.rs | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apps/src/lib/node/ledger/mod.rs b/apps/src/lib/node/ledger/mod.rs index f01b7cc909c..200d7b8ac71 100644 --- a/apps/src/lib/node/ledger/mod.rs +++ b/apps/src/lib/node/ledger/mod.rs @@ -226,7 +226,7 @@ pub fn dump_db( let db_path = config.shell.db_dir(&chain_id); let db = storage::PersistentDB::open(db_path, None); - db.dump_block(out_file_path, historic, &mut block_height.clone()); + db.dump_block(out_file_path, historic, block_height); } /// Roll Namada state back to the previous height diff --git a/apps/src/lib/node/ledger/storage/rocksdb.rs b/apps/src/lib/node/ledger/storage/rocksdb.rs index c51b2206710..61eb2c32e52 100644 --- a/apps/src/lib/node/ledger/storage/rocksdb.rs +++ b/apps/src/lib/node/ledger/storage/rocksdb.rs @@ -34,6 +34,7 @@ //! - `header`: block's header use std::fs::File; +use std::io::BufWriter; use std::path::Path; use std::str::FromStr; use std::sync::Mutex; @@ -257,7 +258,7 @@ impl RocksDB { &self, out_file_path: std::path::PathBuf, historic: bool, - height: &mut Option, + height: Option, ) { // Find the last block height let state_cf = self @@ -272,7 +273,7 @@ impl RocksDB { ) .expect("Unable to decode block height"); - let height = height.get_or_insert(last_height); + let height = height.unwrap_or(last_height); let full_path = out_file_path .with_file_name(format!( @@ -311,18 +312,17 @@ impl RocksDB { } // subspace - if *height != last_height { + if height != last_height { // Restoring subspace at specified height - let restored_subspace = self - .iter_prefix(&Key::default()) + .iter_prefix(None) .par_bridge() .fold( || "".to_string(), |mut cur, (key, _value, _gas)| match self .read_subspace_val_with_height( &Key::from(key.to_db_key()), - *height, + height, last_height, ) .expect("Unable to find subspace key") @@ -374,6 +374,7 @@ impl RocksDB { self.0.iterator_cf_opt(cf, read_opts, IteratorMode::Start) }; + let mut buf = BufWriter::new(file); for (key, raw_val, _gas) in PersistentPrefixIterator( PrefixIterator::new(iter, String::default()), // Empty string to prevent prefix stripping, the prefix is @@ -381,9 +382,10 @@ impl RocksDB { ) { let val = HEXLOWER.encode(&raw_val); let bytes = format!("\"{key}\" = \"{val}\"\n"); - file.write_all(bytes.as_bytes()) - .expect("Unable to write to output file"); + buf.write_all(bytes.as_bytes()) + .expect("Unable to write to buffer"); } + buf.flush().expect("Unable to write to output file"); } /// Rollback to previous block. Given the inner working of tendermint From c9950211326e35fb293e09b908b116a1c03650cf Mon Sep 17 00:00:00 2001 From: Marco Granelli Date: Fri, 11 Aug 2023 17:12:50 +0200 Subject: [PATCH 3/3] changelog: #1468 --- .../unreleased/improvements/1468-custom-height-db-dump.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .changelog/unreleased/improvements/1468-custom-height-db-dump.md diff --git a/.changelog/unreleased/improvements/1468-custom-height-db-dump.md b/.changelog/unreleased/improvements/1468-custom-height-db-dump.md new file mode 100644 index 00000000000..36328a6aa7e --- /dev/null +++ b/.changelog/unreleased/improvements/1468-custom-height-db-dump.md @@ -0,0 +1,2 @@ +- Adds the possibility to dump the state of the db at a custom height. + ([\#1468](https://github.com/anoma/namada/pull/1468)) \ No newline at end of file