Merge remote-tracking branch 'namada/grarco/rocksdb-dump' (#1184) int…

…o main

* namada/grarco/rocksdb-dump:
  changelog: add #1184
  Improves iterator in `dump-db`
  Updates RocksDB dump

.changelog/unreleased/bug-fixes/1184-rocksdb-dump.md

@@ -0,0 +1,3 @@
+- Fixed dump-db node utility which was not iterating on db keys correctly
+  leading to duplicates in the dump. Added an historic flag to also dump the
+  diff keys. ([#1184](https://github.com/anoma/namada/pull/1184))

apps/src/lib/cli.rs

@@ -1630,6 +1630,7 @@ pub mod args {
         arg_default_from_ctx("gas-token", DefaultFn(|| "NAM".into()));
     const GENESIS_PATH: Arg<PathBuf> = arg("genesis-path");
     const GENESIS_VALIDATOR: ArgOpt<String> = arg("genesis-validator").opt();
+    const HISTORIC: ArgFlag = flag("historic");
     const LEDGER_ADDRESS_ABOUT: &str =
         "Address of a ledger node as \"{scheme}://{host}:{port}\". If the \
          scheme is not supplied, it is assumed to be TCP.";
@@ -1773,6 +1774,7 @@ pub mod args {
         // TODO: allow to specify height
         // pub block_height: Option<BlockHeight>,
         pub out_file_path: PathBuf,
+        pub historic: bool,
     }
 
     impl Args for LedgerDumpDb {
@@ -1781,9 +1783,12 @@ pub mod args {
             let out_file_path = OUT_FILE_PATH_OPT
                 .parse(matches)
                 .unwrap_or_else(|| PathBuf::from("db_dump".to_string()));
+            let historic = HISTORIC.parse(matches);
+
             Self {
                 // block_height,
                 out_file_path,
+                historic,
             }
         }
 
@@ -1797,6 +1802,9 @@ pub mod args {
                      Defaults to \"db_dump.{block_height}.toml\" in the \
                      current working directory.",
                 ))
+                .arg(HISTORIC.def().about(
+                    "If provided, dump also the diff of the last height",
+                ))
         }
     }
 

apps/src/lib/node/ledger/mod.rs

@@ -211,6 +211,7 @@ pub fn dump_db(
     args::LedgerDumpDb {
         // block_height,
         out_file_path,
+        historic,
     }: args::LedgerDumpDb,
 ) {
     use namada::ledger::storage::DB;
@@ -219,7 +220,7 @@ pub fn dump_db(
     let db_path = config.shell.db_dir(&chain_id);
 
     let db = storage::PersistentDB::open(db_path, None);
-    db.dump_last_block(out_file_path);
+    db.dump_last_block(out_file_path, historic);
 }
 
 /// Runs and monitors a few concurrent tasks.

apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -1,7 +1,6 @@
 //! The persistent storage in RocksDB.
 //!
 //! The current storage tree is:
-//! - `chain_id`
 //! - `height`: the last committed block height
 //! - `tx_queue`: txs to be decrypted in the next block
 //! - `pred`: predecessor values of the top-level keys of the same name
@@ -15,6 +14,7 @@
 //!   - `next_epoch_min_start_time`
 //! - `subspace`: accounts sub-spaces
 //!   - `{address}/{dyn}`: any byte data associated with accounts
+//! - `results`: block results
 //! - `h`: for each block at height `h`:
 //!   - `tree`: merkle tree
 //!     - `root`: root hash
@@ -243,10 +243,14 @@ impl RocksDB {
     }
 
     /// Dump last known block
-    pub fn dump_last_block(&self, out_file_path: std::path::PathBuf) {
+    pub fn dump_last_block(
+        &self,
+        out_file_path: std::path::PathBuf,
+        historic: bool,
+    ) {
         use std::io::Write;
 
-        // Fine the last block height
+        // Find the last block height
         let height: BlockHeight = types::decode(
             self.0
                 .get("height")
@@ -274,32 +278,39 @@ impl RocksDB {
         println!("Will write to {} ...", full_path.to_string_lossy());
 
         let mut dump_it = |prefix: String| {
-            for next in self.0.iterator(IteratorMode::From(
-                prefix.as_bytes(),
-                Direction::Forward,
-            )) {
-                match next {
-                    Err(e) => {
-                        eprintln!(
-                            "Something failed in a \"{prefix}\" iterator: {e}"
-                        )
-                    }
-                    Ok((raw_key, raw_val)) => {
-                        let key = std::str::from_utf8(&raw_key)
-                            .expect("All keys should be valid UTF-8 strings");
-                        let val = HEXLOWER.encode(&raw_val);
-                        let bytes = format!("\"{key}\" = \"{val}\"\n");
-                        file.write_all(bytes.as_bytes())
-                            .expect("Unable to write to output file");
-                    }
-                };
+            let mut read_opts = ReadOptions::default();
+            read_opts.set_total_order_seek(true);
+
+            let mut upper_prefix = prefix.clone().into_bytes();
+            if let Some(last) = upper_prefix.pop() {
+                upper_prefix.push(last + 1);
+            }
+            read_opts.set_iterate_upper_bound(upper_prefix);
+
+            let iter = self.0.iterator_opt(
+                IteratorMode::From(prefix.as_bytes(), Direction::Forward),
+                read_opts,
+            );
+
+            for (key, raw_val, _gas) in PersistentPrefixIterator(
+                PrefixIterator::new(iter, String::default()),
+                // Empty string to prevent prefix stripping, the prefix is
+                // already in the enclosed iterator
+            ) {
+                let val = HEXLOWER.encode(&raw_val);
+                let bytes = format!("\"{key}\" = \"{val}\"\n");
+                file.write_all(bytes.as_bytes())
+                    .expect("Unable to write to output file");
             }
         };
 
-        // Dump accounts subspace and block height data
+        if historic {
+            // Dump the keys prepended with the selected block height (includes
+            // subspace diff keys)
+            dump_it(height.raw());
+        }
+
         dump_it("subspace".to_string());
-        let block_prefix = format!("{}/", height.raw());
-        dump_it(block_prefix);
 
         println!("Done writing to {}", full_path.to_string_lossy());
     }