Merge remote-tracking branch 'namada/tomas/sorted-prefix-iter' (#458)

- updated `shared/src/types/key/mod.rs` to use `data_encoding::HEXLOWER` instead of `hex`

* namada/tomas/sorted-prefix-iter:
  [ci skip] wasm checksums update
  changelog: add #458
  tests: extend prefix iter tests for reverse order
  add support for rev_iter_prefix in storage and VP and tx envs
  test/vm_host_env: check prefix iter order in tx and VP
  shared/storage/key: add support for int/uint keys that maintain order
  deps: replace hex with data-encoding

.changelog/unreleased/improvements/409-sorted-prefix-iter.md

@@ -0,0 +1,3 @@
+- Fix order of prefix iterator to be sorted by storage
+  keys and add support for a reverse order prefix iterator.
+  ([#409](https://github.com/anoma/namada/issues/409))

apps/Cargo.toml

@@ -62,6 +62,7 @@ clap = {git = "https://github.com/clap-rs/clap/", tag = "v3.0.0-beta.2", default
 color-eyre = "0.5.10"
 config = "0.11.0"
 curl = "0.4.43"
+data-encoding = "2.3.2"
 derivative = "2.2.0"
 directories = "4.0.1"
 ed25519-consensus = "1.2.0"
@@ -71,7 +72,6 @@ eyre = "0.6.5"
 flate2 = "1.0.22"
 file-lock = "2.0.2"
 futures = "0.3"
-hex = "0.4.3"
 itertools = "0.10.1"
 jsonpath_lib = "0.3.0"
 libc = "0.2.97"

apps/src/lib/client/rpc.rs

@@ -11,6 +11,7 @@ use async_std::fs::{self};
 use async_std::path::PathBuf;
 use async_std::prelude::*;
 use borsh::BorshDeserialize;
+use data_encoding::HEXLOWER;
 use itertools::Itertools;
 use namada::ledger::governance::storage as gov_storage;
 use namada::ledger::governance::utils::Votes;
@@ -81,7 +82,7 @@ pub async fn query_raw_bytes(_ctx: Context, args: args::QueryRawBytes) {
         .unwrap();
     match response.code {
         Code::Ok => {
-            println!("{}", hex::encode(&response.value));
+            println!("{}", HEXLOWER.encode(&response.value));
         }
         Code::Err(err) => {
             eprintln!(

apps/src/lib/config/genesis.rs

@@ -26,7 +26,7 @@ pub mod genesis_config {
     use std::path::Path;
     use std::str::FromStr;
 
-    use hex;
+    use data_encoding::HEXLOWER;
     use namada::ledger::governance::parameters::GovParams;
     use namada::ledger::parameters::{EpochDuration, Parameters};
     use namada::ledger::pos::types::BasisPoints;
@@ -50,12 +50,12 @@ pub mod genesis_config {
 
     impl HexString {
         pub fn to_bytes(&self) -> Result<Vec<u8>, HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             Ok(bytes)
         }
 
         pub fn to_sha256_bytes(&self) -> Result<[u8; 32], HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             let slice = bytes.as_slice();
             let array: [u8; 32] = slice.try_into()?;
             Ok(array)
@@ -76,15 +76,15 @@ pub mod genesis_config {
     #[derive(Error, Debug)]
     pub enum HexKeyError {
         #[error("Invalid hex string: {0:?}")]
-        InvalidHexString(hex::FromHexError),
+        InvalidHexString(data_encoding::DecodeError),
         #[error("Invalid sha256 checksum: {0}")]
         InvalidSha256(TryFromSliceError),
         #[error("Invalid public key: {0}")]
         InvalidPublicKey(ParsePublicKeyError),
     }
 
-    impl From<hex::FromHexError> for HexKeyError {
-        fn from(err: hex::FromHexError) -> Self {
+    impl From<data_encoding::DecodeError> for HexKeyError {
+        fn from(err: data_encoding::DecodeError) -> Self {
             Self::InvalidHexString(err)
         }
     }

apps/src/lib/node/gossip/p2p/identity.rs

@@ -21,6 +21,7 @@ pub struct Identity {
 // TODO this is needed because libp2p does not export ed255519 serde
 // feature maybe a MR for libp2p to export theses functions ?
 mod keypair_serde {
+    use data_encoding::HEXLOWER;
     use libp2p::identity::ed25519::Keypair;
     use serde::de::Error;
     use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -33,15 +34,16 @@ mod keypair_serde {
         S: Serializer,
     {
         let bytes = value.encode();
-        let string = hex::encode(&bytes[..]);
+        let string = HEXLOWER.encode(&bytes[..]);
         string.serialize(serializer)
     }
     pub fn deserialize<'d, D>(deserializer: D) -> Result<Keypair, D::Error>
     where
         D: Deserializer<'d>,
     {
         let string = String::deserialize(deserializer)?;
-        let mut bytes = hex::decode(&string).map_err(Error::custom)?;
+        let mut bytes =
+            HEXLOWER.decode(string.as_ref()).map_err(Error::custom)?;
         Keypair::decode(bytes.as_mut()).map_err(Error::custom)
     }
 }

apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -806,24 +806,36 @@ impl<'iter> DBIter<'iter> for RocksDB {
         &'iter self,
         prefix: &Key,
     ) -> PersistentPrefixIterator<'iter> {
-        let db_prefix = "subspace/".to_owned();
-        let prefix = format!("{}{}", db_prefix, prefix);
+        iter_prefix(self, prefix, Direction::Forward)
+    }
 
-        let mut read_opts = ReadOptions::default();
-        // don't use the prefix bloom filter
-        read_opts.set_total_order_seek(true);
-        let mut upper_prefix = prefix.clone().into_bytes();
-        if let Some(last) = upper_prefix.pop() {
-            upper_prefix.push(last + 1);
-        }
-        read_opts.set_iterate_upper_bound(upper_prefix);
+    fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter {
+        iter_prefix(self, prefix, Direction::Reverse)
+    }
+}
 
-        let iter = self.0.iterator_opt(
-            IteratorMode::From(prefix.as_bytes(), Direction::Forward),
-            read_opts,
-        );
-        PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
+fn iter_prefix<'iter>(
+    db: &'iter RocksDB,
+    prefix: &Key,
+    direction: Direction,
+) -> PersistentPrefixIterator<'iter> {
+    let db_prefix = "subspace/".to_owned();
+    let prefix = format!("{}{}", db_prefix, prefix);
+
+    let mut read_opts = ReadOptions::default();
+    // don't use the prefix bloom filter
+    read_opts.set_total_order_seek(true);
+    let mut upper_prefix = prefix.clone().into_bytes();
+    if let Some(last) = upper_prefix.pop() {
+        upper_prefix.push(last + 1);
     }
+    read_opts.set_iterate_upper_bound(upper_prefix);
+
+    let iter = db.0.iterator_opt(
+        IteratorMode::From(prefix.as_bytes(), direction),
+        read_opts,
+    );
+    PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
 }
 
 #[derive(Debug)]

apps/src/lib/wallet/keys.rs

@@ -5,6 +5,7 @@ use std::rc::Rc;
 use std::str::FromStr;
 
 use borsh::{BorshDeserialize, BorshSerialize};
+use data_encoding::HEXLOWER;
 use namada::types::key::*;
 use orion::{aead, kdf};
 use serde::{Deserialize, Serialize};
@@ -108,15 +109,15 @@ pub struct EncryptedKeypair(Vec<u8>);
 
 impl Display for EncryptedKeypair {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", hex::encode(&self.0))
+        write!(f, "{}", HEXLOWER.encode(self.0.as_ref()))
     }
 }
 
 impl FromStr for EncryptedKeypair {
-    type Err = hex::FromHexError;
+    type Err = data_encoding::DecodeError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        hex::decode(s).map(Self)
+        HEXLOWER.decode(s.as_ref()).map(Self)
     }
 }
 

apps/src/lib/wasm_loader/mod.rs

@@ -4,8 +4,8 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
+use data_encoding::HEXLOWER;
 use futures::future::join_all;
-use hex;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use thiserror::Error;
@@ -144,7 +144,7 @@ pub async fn pre_fetch_wasm(wasm_directory: impl AsRef<Path>) {
                 Ok(bytes) => {
                     let mut hasher = Sha256::new();
                     hasher.update(bytes);
-                    let result = hex::encode(hasher.finalize());
+                    let result = HEXLOWER.encode(&hasher.finalize());
                     let derived_name = format!(
                         "{}.{}.wasm",
                         &name.split('.').collect::<Vec<&str>>()[0],

shared/Cargo.toml

@@ -52,11 +52,11 @@ borsh = "0.9.0"
 chrono = "0.4.19"
 # Using unreleased commit on top of version 0.5.0 that adds Sync to the CLruCache
 clru = {git = "https://github.com/marmeladema/clru-rs.git", rev = "71ca566"}
+data-encoding = "2.3.2"
 derivative = "2.2.0"
 ed25519-consensus = "1.2.0"
 ferveo = {optional = true, git = "https://github.com/anoma/ferveo"}
 ferveo-common = {git = "https://github.com/anoma/ferveo"}
-hex = "0.4.3"
 tpke = {package = "group-threshold-cryptography", optional = true, git = "https://github.com/anoma/ferveo"}
 # TODO using the same version of tendermint-rs as we do here.
 ibc = {git = "https://github.com/heliaxdev/ibc-rs", rev = "30b3495ac56c6c37c99bc69ef9f2e84c3309c6cc", default-features = false}

shared/src/ledger/native_vp.rs

@@ -188,6 +188,13 @@ where
         .into_storage_result()
     }
 
+    fn rev_iter_prefix(
+        &self,
+        prefix: &crate::types::storage::Key,
+    ) -> storage_api::Result<Self::PrefixIter> {
+        self.ctx.rev_iter_prefix(prefix).into_storage_result()
+    }
+
     fn iter_next(
         &self,
         iter: &mut Self::PrefixIter,
@@ -258,6 +265,13 @@ where
         .into_storage_result()
     }
 
+    fn rev_iter_prefix(
+        &self,
+        prefix: &crate::types::storage::Key,
+    ) -> storage_api::Result<Self::PrefixIter> {
+        self.ctx.rev_iter_prefix(prefix).into_storage_result()
+    }
+
     fn iter_next(
         &self,
         iter: &mut Self::PrefixIter,
@@ -377,6 +391,18 @@ where
         .into_storage_result()
     }
 
+    fn rev_iter_prefix(
+        &self,
+        prefix: &Key,
+    ) -> Result<Self::PrefixIter, storage_api::Error> {
+        vp_env::rev_iter_prefix(
+            &mut *self.gas_meter.borrow_mut(),
+            self.storage,
+            prefix,
+        )
+        .into_storage_result()
+    }
+
     fn eval(
         &self,
         vp_code: Vec<u8>,

shared/src/ledger/storage/mockdb.rs

@@ -431,7 +431,28 @@ impl<'iter> DBIter<'iter> for MockDB {
         let db_prefix = "subspace/".to_owned();
         let prefix = format!("{}{}", db_prefix, prefix);
         let iter = self.0.borrow().clone().into_iter();
-        MockPrefixIterator::new(MockIterator { prefix, iter }, db_prefix)
+        MockPrefixIterator::new(
+            MockIterator {
+                prefix,
+                iter,
+                reverse_order: false,
+            },
+            db_prefix,
+        )
+    }
+
+    fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter {
+        let db_prefix = "subspace/".to_owned();
+        let prefix = format!("{}{}", db_prefix, prefix);
+        let iter = self.0.borrow().clone().into_iter();
+        MockPrefixIterator::new(
+            MockIterator {
+                prefix,
+                iter,
+                reverse_order: true,
+            },
+            db_prefix,
+        )
     }
 }
 
@@ -441,6 +462,8 @@ pub struct MockIterator {
     prefix: String,
     /// The concrete iterator
     pub iter: btree_map::IntoIter<String, Vec<u8>>,
+    /// Is the iterator in reverse order?
+    reverse_order: bool,
 }
 
 /// A prefix iterator for the [`MockDB`].
@@ -450,12 +473,23 @@ impl Iterator for MockIterator {
     type Item = KVBytes;
 
     fn next(&mut self) -> Option<Self::Item> {
-        for (key, val) in &mut self.iter {
-            if key.starts_with(&self.prefix) {
-                return Some((
-                    Box::from(key.as_bytes()),
-                    Box::from(val.as_slice()),
-                ));
+        if self.reverse_order {
+            for (key, val) in (&mut self.iter).rev() {
+                if key.starts_with(&self.prefix) {
+                    return Some((
+                        Box::from(key.as_bytes()),
+                        Box::from(val.as_slice()),
+                    ));
+                }
+            }
+        } else {
+            for (key, val) in &mut self.iter {
+                if key.starts_with(&self.prefix) {
+                    return Some((
+                        Box::from(key.as_bytes()),
+                        Box::from(val.as_slice()),
+                    ));
+                }
             }
         }
         None