diff --git a/.changelog/unreleased/improvements/409-sorted-prefix-iter.md b/.changelog/unreleased/improvements/409-sorted-prefix-iter.md new file mode 100644 index 0000000000..2f95505960 --- /dev/null +++ b/.changelog/unreleased/improvements/409-sorted-prefix-iter.md @@ -0,0 +1,3 @@ +- Fix order of prefix iterator to be sorted by storage + keys and add support for a reverse order prefix iterator. + ([#409](https://github.com/anoma/namada/issues/409)) \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index e39aa6e45d..cd8e0eee61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3843,12 +3843,12 @@ dependencies = [ "byte-unit", "chrono", "clru", + "data-encoding", "derivative", "ed25519-consensus", "ferveo", "ferveo-common", "group-threshold-cryptography", - "hex", "ibc", "ibc-proto", "ics23", @@ -3905,6 +3905,7 @@ dependencies = [ "color-eyre", "config", "curl", + "data-encoding", "derivative", "directories", "ed25519-consensus", @@ -3915,7 +3916,6 @@ dependencies = [ "flate2", "futures 0.3.21", "git2", - "hex", "itertools 0.10.3", "jsonpath_lib", "libc", @@ -4008,13 +4008,13 @@ dependencies = [ "chrono", "color-eyre", "concat-idents", + "data-encoding", "derivative", "escargot", "expectrl", "eyre", "file-serve", "fs_extra", - "hex", "itertools 0.10.3", "libp2p", "namada", diff --git a/apps/Cargo.toml b/apps/Cargo.toml index 7a4dffac76..3fcd3af977 100644 --- a/apps/Cargo.toml +++ b/apps/Cargo.toml @@ -62,6 +62,7 @@ clap = {git = "https://github.com/clap-rs/clap/", tag = "v3.0.0-beta.2", default color-eyre = "0.5.10" config = "0.11.0" curl = "0.4.43" +data-encoding = "2.3.2" derivative = "2.2.0" directories = "4.0.1" ed25519-consensus = "1.2.0" @@ -71,7 +72,6 @@ eyre = "0.6.5" flate2 = "1.0.22" file-lock = "2.0.2" futures = "0.3" -hex = "0.4.3" itertools = "0.10.1" jsonpath_lib = "0.3.0" libc = "0.2.97" diff --git a/apps/src/lib/client/rpc.rs b/apps/src/lib/client/rpc.rs index 665a14fe2c..a89b302b36 100644 --- a/apps/src/lib/client/rpc.rs +++ b/apps/src/lib/client/rpc.rs @@ -11,6 +11,7 @@ use async_std::fs::{self}; use async_std::path::PathBuf; use async_std::prelude::*; use borsh::BorshDeserialize; +use data_encoding::HEXLOWER; use itertools::Itertools; use namada::ledger::governance::storage as gov_storage; use namada::ledger::governance::utils::Votes; @@ -81,7 +82,7 @@ pub async fn query_raw_bytes(_ctx: Context, args: args::QueryRawBytes) { .unwrap(); match response.code { Code::Ok => { - println!("{}", hex::encode(&response.value)); + println!("{}", HEXLOWER.encode(&response.value)); } Code::Err(err) => { eprintln!( diff --git a/apps/src/lib/config/genesis.rs b/apps/src/lib/config/genesis.rs index 5bd3dc803f..9c1fff3dcc 100644 --- a/apps/src/lib/config/genesis.rs +++ b/apps/src/lib/config/genesis.rs @@ -26,7 +26,7 @@ pub mod genesis_config { use std::path::Path; use std::str::FromStr; - use hex; + use data_encoding::HEXLOWER; use namada::ledger::governance::parameters::GovParams; use namada::ledger::parameters::{EpochDuration, Parameters}; use namada::ledger::pos::types::BasisPoints; @@ -50,12 +50,12 @@ pub mod genesis_config { impl HexString { pub fn to_bytes(&self) -> Result, HexKeyError> { - let bytes = hex::decode(&self.0)?; + let bytes = HEXLOWER.decode(self.0.as_ref())?; Ok(bytes) } pub fn to_sha256_bytes(&self) -> Result<[u8; 32], HexKeyError> { - let bytes = hex::decode(&self.0)?; + let bytes = HEXLOWER.decode(self.0.as_ref())?; let slice = bytes.as_slice(); let array: [u8; 32] = slice.try_into()?; Ok(array) @@ -76,15 +76,15 @@ pub mod genesis_config { #[derive(Error, Debug)] pub enum HexKeyError { #[error("Invalid hex string: {0:?}")] - InvalidHexString(hex::FromHexError), + InvalidHexString(data_encoding::DecodeError), #[error("Invalid sha256 checksum: {0}")] InvalidSha256(TryFromSliceError), #[error("Invalid public key: {0}")] InvalidPublicKey(ParsePublicKeyError), } - impl From for HexKeyError { - fn from(err: hex::FromHexError) -> Self { + impl From for HexKeyError { + fn from(err: data_encoding::DecodeError) -> Self { Self::InvalidHexString(err) } } diff --git a/apps/src/lib/node/gossip/p2p/identity.rs b/apps/src/lib/node/gossip/p2p/identity.rs index 42442054c8..3227060014 100644 --- a/apps/src/lib/node/gossip/p2p/identity.rs +++ b/apps/src/lib/node/gossip/p2p/identity.rs @@ -21,6 +21,7 @@ pub struct Identity { // TODO this is needed because libp2p does not export ed255519 serde // feature maybe a MR for libp2p to export theses functions ? mod keypair_serde { + use data_encoding::HEXLOWER; use libp2p::identity::ed25519::Keypair; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -33,7 +34,7 @@ mod keypair_serde { S: Serializer, { let bytes = value.encode(); - let string = hex::encode(&bytes[..]); + let string = HEXLOWER.encode(&bytes[..]); string.serialize(serializer) } pub fn deserialize<'d, D>(deserializer: D) -> Result @@ -41,7 +42,8 @@ mod keypair_serde { D: Deserializer<'d>, { let string = String::deserialize(deserializer)?; - let mut bytes = hex::decode(&string).map_err(Error::custom)?; + let mut bytes = + HEXLOWER.decode(string.as_ref()).map_err(Error::custom)?; Keypair::decode(bytes.as_mut()).map_err(Error::custom) } } diff --git a/apps/src/lib/node/ledger/storage/rocksdb.rs b/apps/src/lib/node/ledger/storage/rocksdb.rs index 71ed305ea5..44f55fa8e3 100644 --- a/apps/src/lib/node/ledger/storage/rocksdb.rs +++ b/apps/src/lib/node/ledger/storage/rocksdb.rs @@ -806,24 +806,36 @@ impl<'iter> DBIter<'iter> for RocksDB { &'iter self, prefix: &Key, ) -> PersistentPrefixIterator<'iter> { - let db_prefix = "subspace/".to_owned(); - let prefix = format!("{}{}", db_prefix, prefix); + iter_prefix(self, prefix, Direction::Forward) + } - let mut read_opts = ReadOptions::default(); - // don't use the prefix bloom filter - read_opts.set_total_order_seek(true); - let mut upper_prefix = prefix.clone().into_bytes(); - if let Some(last) = upper_prefix.pop() { - upper_prefix.push(last + 1); - } - read_opts.set_iterate_upper_bound(upper_prefix); + fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter { + iter_prefix(self, prefix, Direction::Reverse) + } +} - let iter = self.0.iterator_opt( - IteratorMode::From(prefix.as_bytes(), Direction::Forward), - read_opts, - ); - PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix)) +fn iter_prefix<'iter>( + db: &'iter RocksDB, + prefix: &Key, + direction: Direction, +) -> PersistentPrefixIterator<'iter> { + let db_prefix = "subspace/".to_owned(); + let prefix = format!("{}{}", db_prefix, prefix); + + let mut read_opts = ReadOptions::default(); + // don't use the prefix bloom filter + read_opts.set_total_order_seek(true); + let mut upper_prefix = prefix.clone().into_bytes(); + if let Some(last) = upper_prefix.pop() { + upper_prefix.push(last + 1); } + read_opts.set_iterate_upper_bound(upper_prefix); + + let iter = db.0.iterator_opt( + IteratorMode::From(prefix.as_bytes(), direction), + read_opts, + ); + PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix)) } #[derive(Debug)] diff --git a/apps/src/lib/wallet/keys.rs b/apps/src/lib/wallet/keys.rs index 1c521e7515..e922c7df5a 100644 --- a/apps/src/lib/wallet/keys.rs +++ b/apps/src/lib/wallet/keys.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use std::str::FromStr; use borsh::{BorshDeserialize, BorshSerialize}; +use data_encoding::HEXLOWER; use namada::types::key::*; use orion::{aead, kdf}; use serde::{Deserialize, Serialize}; @@ -108,15 +109,15 @@ pub struct EncryptedKeypair(Vec); impl Display for EncryptedKeypair { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.0)) + write!(f, "{}", HEXLOWER.encode(self.0.as_ref())) } } impl FromStr for EncryptedKeypair { - type Err = hex::FromHexError; + type Err = data_encoding::DecodeError; fn from_str(s: &str) -> Result { - hex::decode(s).map(Self) + HEXLOWER.decode(s.as_ref()).map(Self) } } diff --git a/apps/src/lib/wasm_loader/mod.rs b/apps/src/lib/wasm_loader/mod.rs index b6cb424457..e41efdbf77 100644 --- a/apps/src/lib/wasm_loader/mod.rs +++ b/apps/src/lib/wasm_loader/mod.rs @@ -4,8 +4,8 @@ use std::collections::HashMap; use std::fs; use std::path::Path; +use data_encoding::HEXLOWER; use futures::future::join_all; -use hex; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use thiserror::Error; @@ -144,7 +144,7 @@ pub async fn pre_fetch_wasm(wasm_directory: impl AsRef) { Ok(bytes) => { let mut hasher = Sha256::new(); hasher.update(bytes); - let result = hex::encode(hasher.finalize()); + let result = HEXLOWER.encode(&hasher.finalize()); let derived_name = format!( "{}.{}.wasm", &name.split('.').collect::>()[0], diff --git a/shared/Cargo.toml b/shared/Cargo.toml index 241fd034da..7bff772557 100644 --- a/shared/Cargo.toml +++ b/shared/Cargo.toml @@ -52,11 +52,11 @@ borsh = "0.9.0" chrono = "0.4.19" # Using unreleased commit on top of version 0.5.0 that adds Sync to the CLruCache clru = {git = "https://github.com/marmeladema/clru-rs.git", rev = "71ca566"} +data-encoding = "2.3.2" derivative = "2.2.0" ed25519-consensus = "1.2.0" ferveo = {optional = true, git = "https://github.com/anoma/ferveo"} ferveo-common = {git = "https://github.com/anoma/ferveo"} -hex = "0.4.3" tpke = {package = "group-threshold-cryptography", optional = true, git = "https://github.com/anoma/ferveo"} # TODO using the same version of tendermint-rs as we do here. ibc = {git = "https://github.com/heliaxdev/ibc-rs", rev = "30b3495ac56c6c37c99bc69ef9f2e84c3309c6cc", default-features = false} diff --git a/shared/src/ledger/native_vp.rs b/shared/src/ledger/native_vp.rs index b98794d12c..2b7d41e795 100644 --- a/shared/src/ledger/native_vp.rs +++ b/shared/src/ledger/native_vp.rs @@ -188,6 +188,13 @@ where .into_storage_result() } + fn rev_iter_prefix( + &self, + prefix: &crate::types::storage::Key, + ) -> storage_api::Result { + self.ctx.rev_iter_prefix(prefix).into_storage_result() + } + fn iter_next( &self, iter: &mut Self::PrefixIter, @@ -258,6 +265,13 @@ where .into_storage_result() } + fn rev_iter_prefix( + &self, + prefix: &crate::types::storage::Key, + ) -> storage_api::Result { + self.ctx.rev_iter_prefix(prefix).into_storage_result() + } + fn iter_next( &self, iter: &mut Self::PrefixIter, @@ -377,6 +391,18 @@ where .into_storage_result() } + fn rev_iter_prefix( + &self, + prefix: &Key, + ) -> Result { + vp_env::rev_iter_prefix( + &mut *self.gas_meter.borrow_mut(), + self.storage, + prefix, + ) + .into_storage_result() + } + fn eval( &self, vp_code: Vec, diff --git a/shared/src/ledger/storage/mockdb.rs b/shared/src/ledger/storage/mockdb.rs index 3cc7e8863c..a9e3e8057b 100644 --- a/shared/src/ledger/storage/mockdb.rs +++ b/shared/src/ledger/storage/mockdb.rs @@ -431,7 +431,28 @@ impl<'iter> DBIter<'iter> for MockDB { let db_prefix = "subspace/".to_owned(); let prefix = format!("{}{}", db_prefix, prefix); let iter = self.0.borrow().clone().into_iter(); - MockPrefixIterator::new(MockIterator { prefix, iter }, db_prefix) + MockPrefixIterator::new( + MockIterator { + prefix, + iter, + reverse_order: false, + }, + db_prefix, + ) + } + + fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter { + let db_prefix = "subspace/".to_owned(); + let prefix = format!("{}{}", db_prefix, prefix); + let iter = self.0.borrow().clone().into_iter(); + MockPrefixIterator::new( + MockIterator { + prefix, + iter, + reverse_order: true, + }, + db_prefix, + ) } } @@ -441,6 +462,8 @@ pub struct MockIterator { prefix: String, /// The concrete iterator pub iter: btree_map::IntoIter>, + /// Is the iterator in reverse order? + reverse_order: bool, } /// A prefix iterator for the [`MockDB`]. @@ -450,12 +473,23 @@ impl Iterator for MockIterator { type Item = KVBytes; fn next(&mut self) -> Option { - for (key, val) in &mut self.iter { - if key.starts_with(&self.prefix) { - return Some(( - Box::from(key.as_bytes()), - Box::from(val.as_slice()), - )); + if self.reverse_order { + for (key, val) in (&mut self.iter).rev() { + if key.starts_with(&self.prefix) { + return Some(( + Box::from(key.as_bytes()), + Box::from(val.as_slice()), + )); + } + } + } else { + for (key, val) in &mut self.iter { + if key.starts_with(&self.prefix) { + return Some(( + Box::from(key.as_bytes()), + Box::from(val.as_slice()), + )); + } } } None diff --git a/shared/src/ledger/storage/mod.rs b/shared/src/ledger/storage/mod.rs index 235e2164e3..1c4e4efd94 100644 --- a/shared/src/ledger/storage/mod.rs +++ b/shared/src/ledger/storage/mod.rs @@ -243,8 +243,13 @@ pub trait DBIter<'iter> { /// The concrete type of the iterator type PrefixIter: Debug + Iterator, u64)>; - /// Read account subspace key value pairs with the given prefix from the DB + /// Read account subspace key value pairs with the given prefix from the DB, + /// ordered by the storage keys. fn iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter; + + /// Read account subspace key value pairs with the given prefix from the DB, + /// reverse ordered by the storage keys. + fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter; } /// Atomic batch write. @@ -411,7 +416,7 @@ where } } - /// Returns a prefix iterator and the gas cost + /// Returns a prefix iterator, ordered by storage keys, and the gas cost pub fn iter_prefix( &self, prefix: &Key, @@ -419,6 +424,15 @@ where (self.db.iter_prefix(prefix), prefix.len() as _) } + /// Returns a prefix iterator, reverse ordered by storage keys, and the gas + /// cost + pub fn rev_iter_prefix( + &self, + prefix: &Key, + ) -> (>::PrefixIter, u64) { + (self.db.rev_iter_prefix(prefix), prefix.len() as _) + } + /// Write a value to the specified subspace and returns the gas cost and the /// size difference pub fn write( @@ -718,6 +732,13 @@ where Ok(self.db.iter_prefix(prefix)) } + fn rev_iter_prefix( + &'iter self, + prefix: &crate::types::storage::Key, + ) -> std::result::Result { + Ok(self.db.rev_iter_prefix(prefix)) + } + fn iter_next( &self, iter: &mut Self::PrefixIter, diff --git a/shared/src/ledger/storage_api/mod.rs b/shared/src/ledger/storage_api/mod.rs index 23035c4c53..06ec8361f5 100644 --- a/shared/src/ledger/storage_api/mod.rs +++ b/shared/src/ledger/storage_api/mod.rs @@ -54,14 +54,8 @@ pub trait StorageRead<'iter> { /// Storage `has_key` in. It will try to read from the storage. fn has_key(&self, key: &storage::Key) -> Result; - /// Storage prefix iterator for. It will try to read from the storage. - fn iter_next( - &self, - iter: &mut Self::PrefixIter, - ) -> Result)>>; - - /// Storage prefix iterator. It will try to get an iterator from the - /// storage. + /// Storage prefix iterator ordered by the storage keys. It will try to get + /// an iterator from the storage. /// /// For a more user-friendly iterator API, use [`fn@iter_prefix`] or /// [`fn@iter_prefix_bytes`] instead. @@ -70,6 +64,22 @@ pub trait StorageRead<'iter> { prefix: &storage::Key, ) -> Result; + /// Storage prefix iterator in reverse order of the storage keys. It will + /// try to get an iterator from the storage. + /// + /// For a more user-friendly iterator API, use [`fn@rev_iter_prefix`] or + /// [`fn@rev_iter_prefix_bytes`] instead. + fn rev_iter_prefix( + &'iter self, + prefix: &storage::Key, + ) -> Result; + + /// Storage prefix iterator. It will try to read from the storage. + fn iter_next( + &self, + iter: &mut Self::PrefixIter, + ) -> Result)>>; + /// Getting the chain ID. fn get_chain_id(&self) -> Result; @@ -109,7 +119,7 @@ pub trait StorageWrite { fn delete(&mut self, key: &storage::Key) -> Result<()>; } -/// Iterate items matching the given prefix. +/// Iterate items matching the given prefix, ordered by the storage keys. pub fn iter_prefix_bytes<'a>( storage: &'a impl StorageRead<'a>, prefix: &crate::types::storage::Key, @@ -137,7 +147,8 @@ pub fn iter_prefix_bytes<'a>( Ok(iter) } -/// Iterate Borsh encoded items matching the given prefix. +/// Iterate Borsh encoded items matching the given prefix, ordered by the +/// storage keys. pub fn iter_prefix<'a, T>( storage: &'a impl StorageRead<'a>, prefix: &crate::types::storage::Key, @@ -174,3 +185,71 @@ where }); Ok(iter) } + +/// Iterate items matching the given prefix, reverse ordered by the storage +/// keys. +pub fn rev_iter_prefix_bytes<'a>( + storage: &'a impl StorageRead<'a>, + prefix: &crate::types::storage::Key, +) -> Result)>> + 'a> { + let iter = storage.rev_iter_prefix(prefix)?; + let iter = itertools::unfold(iter, |iter| { + match storage.iter_next(iter) { + Ok(Some((key, val))) => { + let key = match storage::Key::parse(key).into_storage_result() { + Ok(key) => key, + Err(err) => { + // Propagate key encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + Some(Ok((key, val))) + } + Ok(None) => None, + Err(err) => { + // Propagate `iter_next` errors into Iterator's Item + Some(Err(err)) + } + } + }); + Ok(iter) +} + +/// Iterate Borsh encoded items matching the given prefix, reverse ordered by +/// the storage keys. +pub fn rev_iter_prefix<'a, T>( + storage: &'a impl StorageRead<'a>, + prefix: &crate::types::storage::Key, +) -> Result> + 'a> +where + T: BorshDeserialize, +{ + let iter = storage.rev_iter_prefix(prefix)?; + let iter = itertools::unfold(iter, |iter| { + match storage.iter_next(iter) { + Ok(Some((key, val))) => { + let key = match storage::Key::parse(key).into_storage_result() { + Ok(key) => key, + Err(err) => { + // Propagate key encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + let val = match T::try_from_slice(&val).into_storage_result() { + Ok(val) => val, + Err(err) => { + // Propagate val encoding errors into Iterator's Item + return Some(Err(err)); + } + }; + Some(Ok((key, val))) + } + Ok(None) => None, + Err(err) => { + // Propagate `iter_next` errors into Iterator's Item + Some(Err(err)) + } + } + }); + Ok(iter) +} diff --git a/shared/src/ledger/vp_env.rs b/shared/src/ledger/vp_env.rs index 8398f37cda..3f1b22ddc6 100644 --- a/shared/src/ledger/vp_env.rs +++ b/shared/src/ledger/vp_env.rs @@ -65,13 +65,20 @@ pub trait VpEnv<'view> { /// current transaction is being applied. fn get_block_epoch(&'view self) -> Result; - /// Storage prefix iterator. It will try to get an iterator from the - /// storage. + /// Storage prefix iterator, ordered by storage keys. It will try to get an + /// iterator from the storage. fn iter_prefix( &'view self, prefix: &Key, ) -> Result; + /// Storage prefix iterator, reverse ordered by storage keys. It will try to + /// get an iterator from the storage. + fn rev_iter_prefix( + &self, + prefix: &Key, + ) -> Result; + /// Evaluate a validity predicate with given data. The address, changed /// storage keys and verifiers will have the same values as the input to /// caller's validity predicate. @@ -427,7 +434,8 @@ where Ok(epoch) } -/// Storage prefix iterator. It will try to get an iterator from the storage. +/// Storage prefix iterator, ordered by storage keys. It will try to get an +/// iterator from the storage. pub fn iter_prefix<'a, DB, H>( gas_meter: &mut VpGasMeter, storage: &'a Storage, @@ -442,6 +450,22 @@ where Ok(iter) } +/// Storage prefix iterator, reverse ordered by storage keys. It will try to get +/// an iterator from the storage. +pub fn rev_iter_prefix<'a, DB, H>( + gas_meter: &mut VpGasMeter, + storage: &'a Storage, + prefix: &Key, +) -> EnvResult<>::PrefixIter> +where + DB: storage::DB + for<'iter> storage::DBIter<'iter>, + H: StorageHasher, +{ + let (iter, gas) = storage.rev_iter_prefix(prefix); + add_gas(gas_meter, gas)?; + Ok(iter) +} + /// Storage prefix iterator for prior state (before tx execution). It will try /// to read from the storage. pub fn iter_pre_next( diff --git a/shared/src/proto/mod.rs b/shared/src/proto/mod.rs index aa971f0b96..1ee5da63c8 100644 --- a/shared/src/proto/mod.rs +++ b/shared/src/proto/mod.rs @@ -9,6 +9,7 @@ pub use types::{ #[cfg(test)] mod tests { + use data_encoding::HEXLOWER; use generated::types::Tx; use prost::Message; @@ -23,8 +24,8 @@ mod tests { }; let mut tx_bytes = vec![]; tx.encode(&mut tx_bytes).unwrap(); - let tx_hex = hex::encode(tx_bytes); - let tx_from_hex = hex::decode(tx_hex).unwrap(); + let tx_hex = HEXLOWER.encode(&tx_bytes); + let tx_from_hex = HEXLOWER.decode(tx_hex.as_ref()).unwrap(); let tx_from_bytes = Tx::decode(&tx_from_hex[..]).unwrap(); assert_eq!(tx, tx_from_bytes); } diff --git a/shared/src/proto/types.rs b/shared/src/proto/types.rs index bc1f072e45..a2359d3cf2 100644 --- a/shared/src/proto/types.rs +++ b/shared/src/proto/types.rs @@ -4,6 +4,7 @@ use std::fmt::Display; use std::hash::{Hash, Hasher}; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::HEXLOWER; use prost::Message; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -374,7 +375,7 @@ impl>> From for IntentId { impl Display for IntentId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.0)) + write!(f, "{}", HEXLOWER.encode(&self.0)) } } diff --git a/shared/src/types/key/common.rs b/shared/src/types/key/common.rs index d3401258d1..552c5314ae 100644 --- a/shared/src/types/key/common.rs +++ b/shared/src/types/key/common.rs @@ -4,6 +4,7 @@ use std::fmt::Display; use std::str::FromStr; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::HEXLOWER; use namada_proof_of_stake::types::PublicKeyTmRawHash; #[cfg(feature = "rand")] use rand::{CryptoRng, RngCore}; @@ -59,7 +60,7 @@ impl super::PublicKey for PublicKey { impl Display for PublicKey { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.try_to_vec().unwrap())) + write!(f, "{}", HEXLOWER.encode(&self.try_to_vec().unwrap())) } } @@ -67,7 +68,9 @@ impl FromStr for PublicKey { type Err = ParsePublicKeyError; fn from_str(str: &str) -> Result { - let vec = hex::decode(str).map_err(ParsePublicKeyError::InvalidHex)?; + let vec = HEXLOWER + .decode(str.as_ref()) + .map_err(ParsePublicKeyError::InvalidHex)?; Self::try_from_slice(vec.as_slice()) .map_err(ParsePublicKeyError::InvalidEncoding) } @@ -154,7 +157,7 @@ impl RefTo for SecretKey { impl Display for SecretKey { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.try_to_vec().unwrap())) + write!(f, "{}", HEXLOWER.encode(&self.try_to_vec().unwrap())) } } @@ -162,7 +165,9 @@ impl FromStr for SecretKey { type Err = ParseSecretKeyError; fn from_str(str: &str) -> Result { - let vec = hex::decode(str).map_err(ParseSecretKeyError::InvalidHex)?; + let vec = HEXLOWER + .decode(str.as_ref()) + .map_err(ParseSecretKeyError::InvalidHex)?; Self::try_from_slice(vec.as_slice()) .map_err(ParseSecretKeyError::InvalidEncoding) } diff --git a/shared/src/types/key/dkg_session_keys.rs b/shared/src/types/key/dkg_session_keys.rs index 26f6fffa00..f2cafb639c 100644 --- a/shared/src/types/key/dkg_session_keys.rs +++ b/shared/src/types/key/dkg_session_keys.rs @@ -7,6 +7,7 @@ use std::str::FromStr; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::HEXLOWER; use serde::{Deserialize, Serialize}; use crate::types::address::Address; @@ -142,7 +143,7 @@ impl Display for DkgPublicKey { let vec = self .try_to_vec() .expect("Encoding public key shouldn't fail"); - write!(f, "{}", hex::encode(&vec)) + write!(f, "{}", HEXLOWER.encode(&vec)) } } @@ -150,7 +151,9 @@ impl FromStr for DkgPublicKey { type Err = ParsePublicKeyError; fn from_str(s: &str) -> Result { - let vec = hex::decode(s).map_err(ParsePublicKeyError::InvalidHex)?; + let vec = HEXLOWER + .decode(s.as_ref()) + .map_err(ParsePublicKeyError::InvalidHex)?; BorshDeserialize::try_from_slice(&vec) .map_err(ParsePublicKeyError::InvalidEncoding) } diff --git a/shared/src/types/key/ed25519.rs b/shared/src/types/key/ed25519.rs index 12e5093bd2..90ab540f19 100644 --- a/shared/src/types/key/ed25519.rs +++ b/shared/src/types/key/ed25519.rs @@ -6,6 +6,7 @@ use std::io::Write; use std::str::FromStr; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::HEXLOWER; #[cfg(feature = "rand")] use rand::{CryptoRng, RngCore}; use serde::{Deserialize, Serialize}; @@ -107,7 +108,7 @@ impl Ord for PublicKey { impl Display for PublicKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.0.to_bytes())) + write!(f, "{}", HEXLOWER.encode(&self.0.to_bytes())) } } @@ -115,7 +116,9 @@ impl FromStr for PublicKey { type Err = ParsePublicKeyError; fn from_str(s: &str) -> Result { - let vec = hex::decode(s).map_err(ParsePublicKeyError::InvalidHex)?; + let vec = HEXLOWER + .decode(s.as_ref()) + .map_err(ParsePublicKeyError::InvalidHex)?; BorshDeserialize::try_from_slice(&vec) .map_err(ParsePublicKeyError::InvalidEncoding) } @@ -204,7 +207,7 @@ impl BorshSchema for SecretKey { impl Display for SecretKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", hex::encode(&self.0.to_bytes())) + write!(f, "{}", HEXLOWER.encode(&self.0.to_bytes())) } } @@ -212,7 +215,9 @@ impl FromStr for SecretKey { type Err = ParseSecretKeyError; fn from_str(s: &str) -> Result { - let vec = hex::decode(s).map_err(ParseSecretKeyError::InvalidHex)?; + let vec = HEXLOWER + .decode(s.as_ref()) + .map_err(ParseSecretKeyError::InvalidHex)?; BorshDeserialize::try_from_slice(&vec) .map_err(ParseSecretKeyError::InvalidEncoding) } diff --git a/shared/src/types/key/mod.rs b/shared/src/types/key/mod.rs index 9b560a4c84..b6ffa5da2f 100644 --- a/shared/src/types/key/mod.rs +++ b/shared/src/types/key/mod.rs @@ -7,6 +7,7 @@ use std::hash::Hash; use std::str::FromStr; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::HEXUPPER; #[cfg(feature = "rand")] use rand::{CryptoRng, RngCore}; use serde::{Deserialize, Serialize}; @@ -79,7 +80,7 @@ pub enum VerifySigError { #[derive(Error, Debug)] pub enum ParsePublicKeyError { #[error("Invalid public key hex: {0}")] - InvalidHex(hex::FromHexError), + InvalidHex(data_encoding::DecodeError), #[error("Invalid public key encoding: {0}")] InvalidEncoding(std::io::Error), #[error("Parsed public key does not belong to desired scheme")] @@ -90,7 +91,7 @@ pub enum ParsePublicKeyError { #[derive(Error, Debug)] pub enum ParseSignatureError { #[error("Invalid signature hex: {0}")] - InvalidHex(hex::FromHexError), + InvalidHex(data_encoding::DecodeError), #[error("Invalid signature encoding: {0}")] InvalidEncoding(std::io::Error), #[error("Parsed signature does not belong to desired scheme")] @@ -101,7 +102,7 @@ pub enum ParseSignatureError { #[derive(Error, Debug)] pub enum ParseSecretKeyError { #[error("Invalid secret key hex: {0}")] - InvalidHex(hex::FromHexError), + InvalidHex(data_encoding::DecodeError), #[error("Invalid secret key encoding: {0}")] InvalidEncoding(std::io::Error), #[error("Parsed secret key does not belong to desired scheme")] @@ -345,7 +346,7 @@ pub fn tm_consensus_key_raw_hash(pk: &common::PublicKey) -> String { /// Convert Tendermint validator's raw hash bytes to Anoma raw hash string pub fn tm_raw_hash_to_string(raw_hash: impl AsRef<[u8]>) -> String { - hex::encode_upper(raw_hash) + HEXUPPER.encode(raw_hash.as_ref()) } /// Helpers for testing with keys. diff --git a/shared/src/types/storage.rs b/shared/src/types/storage.rs index f60bb66ccf..f77fb2567c 100644 --- a/shared/src/types/storage.rs +++ b/shared/src/types/storage.rs @@ -6,6 +6,7 @@ use std::ops::{Add, Div, Mul, Rem, Sub}; use std::str::FromStr; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; +use data_encoding::BASE32HEX_NOPAD; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -27,6 +28,8 @@ pub enum Error { ParseAddressFromKey, #[error("Reserved prefix or string is specified: {0}")] InvalidKeySeg(String), + #[error("Error parsing key segment {0}")] + ParseKeySeg(String), } /// Result for functions that may fail @@ -193,6 +196,7 @@ impl Header { BorshDeserialize, BorshSchema, Debug, + Default, Eq, PartialEq, Ord, @@ -446,14 +450,12 @@ impl KeySeg for String { impl KeySeg for BlockHeight { fn parse(string: String) -> Result { - let h = string.parse::().map_err(|e| Error::Temporary { - error: format!("Unexpected height value {}, {}", string, e), - })?; + let h: u64 = KeySeg::parse(string)?; Ok(BlockHeight(h)) } fn raw(&self) -> String { - format!("{}", self.0) + self.0.raw() } fn to_db_key(&self) -> DbKeySeg { @@ -481,6 +483,67 @@ impl KeySeg for Address { } } +/// Implement [`KeySeg`] for a type via base32hex of its BE bytes (using +/// `to_le_bytes()` and `from_le_bytes` methods) that maintains sort order of +/// the original data. +// TODO this could be a bit more efficient without the string conversion (atm +// with base32hex), if we can use bytes for storage key directly (which we can +// with rockDB, but atm, we're calling `to_string()` using the custom `Display` +// impl from here) +macro_rules! impl_int_key_seg { + ($unsigned:ty, $signed:ty, $len:literal) => { + impl KeySeg for $unsigned { + fn parse(string: String) -> Result { + let bytes = + BASE32HEX_NOPAD.decode(string.as_ref()).map_err(|err| { + Error::ParseKeySeg(format!( + "Failed parsing {} with {}", + string, err + )) + })?; + let mut fixed_bytes = [0; $len]; + fixed_bytes.copy_from_slice(&bytes); + Ok(<$unsigned>::from_be_bytes(fixed_bytes)) + } + + fn raw(&self) -> String { + BASE32HEX_NOPAD.encode(&self.to_be_bytes()) + } + + fn to_db_key(&self) -> DbKeySeg { + DbKeySeg::StringSeg(self.raw()) + } + } + + impl KeySeg for $signed { + fn parse(string: String) -> Result { + // get signed int from a unsigned int complemented with a min + // value + let complemented = <$unsigned>::parse(string)?; + let signed = (complemented as $signed) ^ <$signed>::MIN; + Ok(signed) + } + + fn raw(&self) -> String { + // signed int is converted to unsigned int that preserves the + // order by complementing it with a min value + let complemented = (*self ^ <$signed>::MIN) as $unsigned; + complemented.raw() + } + + fn to_db_key(&self) -> DbKeySeg { + DbKeySeg::StringSeg(self.raw()) + } + } + }; +} + +impl_int_key_seg!(u8, i8, 1); +impl_int_key_seg!(u16, i16, 2); +impl_int_key_seg!(u32, i32, 4); +impl_int_key_seg!(u64, i64, 8); +impl_int_key_seg!(u128, i128, 16); + /// Epoch identifier. Epochs are identified by consecutive numbers. #[derive( Clone, diff --git a/shared/src/vm/host_env.rs b/shared/src/vm/host_env.rs index fc7fd33e0c..533a05d94e 100644 --- a/shared/src/vm/host_env.rs +++ b/shared/src/vm/host_env.rs @@ -665,7 +665,7 @@ where /// Storage prefix iterator function exposed to the wasm VM Tx environment. /// It will try to get an iterator from the storage and return the corresponding -/// ID of the iterator. +/// ID of the iterator, ordered by storage keys. pub fn tx_iter_prefix( env: &TxVmEnv, prefix_ptr: u64, @@ -695,6 +695,38 @@ where Ok(iterators.insert(iter).id()) } +/// Storage prefix iterator function exposed to the wasm VM Tx environment. +/// It will try to get an iterator from the storage and return the corresponding +/// ID of the iterator, reverse ordered by storage keys. +pub fn tx_rev_iter_prefix( + env: &TxVmEnv, + prefix_ptr: u64, + prefix_len: u64, +) -> TxResult +where + MEM: VmMemory, + DB: storage::DB + for<'iter> storage::DBIter<'iter>, + H: StorageHasher, + CA: WasmCacheAccess, +{ + let (prefix, gas) = env + .memory + .read_string(prefix_ptr, prefix_len as _) + .map_err(|e| TxRuntimeError::MemoryError(Box::new(e)))?; + tx_add_gas(env, gas)?; + + tracing::debug!("tx_rev_iter_prefix {}, prefix {}", prefix, prefix_ptr); + + let prefix = + Key::parse(prefix).map_err(TxRuntimeError::StorageDataError)?; + + let storage = unsafe { env.ctx.storage.get() }; + let iterators = unsafe { env.ctx.iterators.get() }; + let (iter, gas) = storage.rev_iter_prefix(&prefix); + tx_add_gas(env, gas)?; + Ok(iterators.insert(iter).id()) +} + /// Storage prefix iterator next function exposed to the wasm VM Tx environment. /// It will try to read from the write log first and if no entry found then from /// the storage. @@ -1195,7 +1227,7 @@ where /// Storage prefix iterator function exposed to the wasm VM VP environment. /// It will try to get an iterator from the storage and return the corresponding -/// ID of the iterator. +/// ID of the iterator, ordered by storage keys. pub fn vp_iter_prefix( env: &VpVmEnv, prefix_ptr: u64, @@ -1225,6 +1257,38 @@ where Ok(iterators.insert(iter).id()) } +/// Storage prefix iterator function exposed to the wasm VM VP environment. +/// It will try to get an iterator from the storage and return the corresponding +/// ID of the iterator, reverse ordered by storage keys. +pub fn vp_rev_iter_prefix( + env: &VpVmEnv, + prefix_ptr: u64, + prefix_len: u64, +) -> vp_env::EnvResult +where + MEM: VmMemory, + DB: storage::DB + for<'iter> storage::DBIter<'iter>, + H: StorageHasher, + EVAL: VpEvaluator, + CA: WasmCacheAccess, +{ + let (prefix, gas) = env + .memory + .read_string(prefix_ptr, prefix_len as _) + .map_err(|e| vp_env::RuntimeError::MemoryError(Box::new(e)))?; + let gas_meter = unsafe { env.ctx.gas_meter.get() }; + vp_env::add_gas(gas_meter, gas)?; + + let prefix = + Key::parse(prefix).map_err(vp_env::RuntimeError::StorageDataError)?; + tracing::debug!("vp_rev_iter_prefix {}", prefix); + + let storage = unsafe { env.ctx.storage.get() }; + let iter = vp_env::rev_iter_prefix(gas_meter, storage, &prefix)?; + let iterators = unsafe { env.ctx.iterators.get() }; + Ok(iterators.insert(iter).id()) +} + /// Storage prefix iterator for prior state (before tx execution) function /// exposed to the wasm VM VP environment. It will try to read from the storage. /// diff --git a/shared/src/vm/wasm/host_env.rs b/shared/src/vm/wasm/host_env.rs index 1a50c5533d..06d45fa4f9 100644 --- a/shared/src/vm/wasm/host_env.rs +++ b/shared/src/vm/wasm/host_env.rs @@ -67,6 +67,7 @@ where "anoma_tx_write_temp" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_write_temp), "anoma_tx_delete" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_delete), "anoma_tx_iter_prefix" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_iter_prefix), + "anoma_tx_rev_iter_prefix" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_rev_iter_prefix), "anoma_tx_iter_next" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_iter_next), "anoma_tx_insert_verifier" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_insert_verifier), "anoma_tx_update_validity_predicate" => Function::new_native_with_env(wasm_store, env.clone(), host_env::tx_update_validity_predicate), @@ -107,6 +108,7 @@ where "anoma_vp_has_key_pre" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_has_key_pre), "anoma_vp_has_key_post" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_has_key_post), "anoma_vp_iter_prefix" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_iter_prefix), + "anoma_vp_rev_iter_prefix" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_rev_iter_prefix), "anoma_vp_iter_pre_next" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_iter_pre_next), "anoma_vp_iter_post_next" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_iter_post_next), "anoma_vp_get_chain_id" => Function::new_native_with_env(wasm_store, env.clone(), host_env::vp_get_chain_id), diff --git a/tests/Cargo.toml b/tests/Cargo.toml index cc437b3686..6872e912b7 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -31,13 +31,13 @@ namada_apps = {path = "../apps", default-features = false, features = ["testing" assert_cmd = "1.0.7" borsh = "0.9.1" color-eyre = "0.5.11" +data-encoding = "2.3.2" # NOTE: enable "print" feature to see output from builds ran by e2e tests escargot = {version = "0.5.7"} # , features = ["print"]} expectrl = {version = "=0.5.2"} eyre = "0.6.5" file-serve = "0.2.0" fs_extra = "1.2.0" -hex = "0.4.3" itertools = "0.10.0" libp2p = "0.38.0" pretty_assertions = "0.7.2" diff --git a/tests/src/e2e/ledger_tests.rs b/tests/src/e2e/ledger_tests.rs index efb763eff8..729f3ad2b9 100644 --- a/tests/src/e2e/ledger_tests.rs +++ b/tests/src/e2e/ledger_tests.rs @@ -17,6 +17,7 @@ use std::time::{Duration, Instant}; use borsh::BorshSerialize; use color_eyre::eyre::Result; +use data_encoding::HEXLOWER; use namada::types::token; use namada_apps::config::genesis::genesis_config::{ GenesisConfig, ParametersConfig, PosParamsConfig, @@ -386,7 +387,7 @@ fn ledger_txs_and_queries() -> Result<()> { &validator_one_rpc, ], // expect hex encoded of borsh encoded bytes - hex::encode(christel_balance.try_to_vec().unwrap()), + HEXLOWER.encode(&christel_balance.try_to_vec().unwrap()), ), ]; for (query_args, expected) in &query_args_and_expected_response { diff --git a/tests/src/vm_host_env/mod.rs b/tests/src/vm_host_env/mod.rs index 98f95e7fae..e585ab7924 100644 --- a/tests/src/vm_host_env/mod.rs +++ b/tests/src/vm_host_env/mod.rs @@ -157,11 +157,14 @@ mod tests { should yield an empty iterator." ); - // Write some values directly into the storage first - let prefix = Key::parse("prefix").unwrap(); + let prefix = storage::Key::parse("prefix").unwrap(); + // We'll write sub-key in some random order to check prefix iter's order + let sub_keys = [2_i32, 1, i32::MAX, -1, 260, -2, i32::MIN, 5, 0]; + + // Write the values directly into the storage first tx_host_env::with(|env| { - for i in 0..10_i32 { - let key = prefix.join(&Key::parse(i.to_string()).unwrap()); + for i in sub_keys.iter() { + let key = prefix.push(i).unwrap(); let value = i.try_to_vec().unwrap(); env.storage.write(&key, value).unwrap(); } @@ -171,11 +174,27 @@ mod tests { // Then try to iterate over their prefix let iter = namada_tx_prelude::iter_prefix(tx::ctx(), &prefix) .unwrap() - .map(|item| item.unwrap()); - let expected = (0..10).map(|i| { - (storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), i) - }); - itertools::assert_equal(iter.sorted(), expected.sorted()); + .map(Result::unwrap); + + // The order has to be sorted by sub-key value + let expected = sub_keys + .iter() + .sorted() + .map(|i| (prefix.push(i).unwrap(), *i)); + itertools::assert_equal(iter, expected); + + // Try to iterate over their prefix in reverse + let iter = namada_tx_prelude::rev_iter_prefix(tx::ctx(), &prefix) + .unwrap() + .map(Result::unwrap); + + // The order has to be reverse sorted by sub-key value + let expected = sub_keys + .iter() + .sorted() + .rev() + .map(|i| (prefix.push(i).unwrap(), *i)); + itertools::assert_equal(iter, expected); } #[test] @@ -251,7 +270,7 @@ mod tests { // We can add some data to the environment let key_raw = "key"; - let key = Key::parse(key_raw).unwrap(); + let key = storage::Key::parse(key_raw).unwrap(); let value = "test".to_string(); let value_raw = value.try_to_vec().unwrap(); vp_host_env::with(|env| { @@ -269,7 +288,7 @@ mod tests { let mut tx_env = TestTxEnv::default(); let addr = address::testing::established_address_1(); - let addr_key = Key::from(addr.to_db_key()); + let addr_key = storage::Key::from(addr.to_db_key()); // Write some value to storage let existing_key = @@ -354,12 +373,15 @@ mod tests { let mut tx_env = TestTxEnv::default(); let addr = address::testing::established_address_1(); - let addr_key = Key::from(addr.to_db_key()); + let addr_key = storage::Key::from(addr.to_db_key()); - // Write some value to storage let prefix = addr_key.join(&Key::parse("prefix").unwrap()); - for i in 0..10_i32 { - let key = prefix.join(&Key::parse(i.to_string()).unwrap()); + // We'll write sub-key in some random order to check prefix iter's order + let sub_keys = [2_i32, 1, i32::MAX, -1, 260, -2, i32::MIN, 5, 0]; + + // Write some values to storage + for i in sub_keys.iter() { + let key = prefix.push(i).unwrap(); let value = i.try_to_vec().unwrap(); tx_env.storage.write(&key, value).unwrap(); } @@ -367,8 +389,8 @@ mod tests { // In a transaction, write override the existing key's value and add // another key-value - let existing_key = prefix.join(&Key::parse(5.to_string()).unwrap()); - let new_key = prefix.join(&Key::parse(11.to_string()).unwrap()); + let existing_key = prefix.push(&5).unwrap(); + let new_key = prefix.push(&11).unwrap(); // Initialize the VP environment via a transaction vp_host_env::init_from_tx(addr, tx_env, |_addr| { @@ -383,23 +405,38 @@ mod tests { let iter_pre = namada_vp_prelude::iter_prefix(&ctx_pre, &prefix) .unwrap() .map(|item| item.unwrap()); - let expected_pre = (0..10).map(|i| { - (storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), i) - }); - itertools::assert_equal(iter_pre.sorted(), expected_pre.sorted()); + + // The order in pre has to be sorted by sub-key value + let expected_pre = sub_keys + .iter() + .sorted() + .map(|i| (prefix.push(i).unwrap(), *i)); + itertools::assert_equal(iter_pre, expected_pre); let ctx_post = vp::CTX.post(); let iter_post = namada_vp_prelude::iter_prefix(&ctx_post, &prefix) .unwrap() .map(|item| item.unwrap()); - let expected_post = (0..10).map(|i| { - let val = if i == 5 { 100 } else { i }; - ( - storage::Key::parse(format!("{}/{}", prefix, i)).unwrap(), - val, - ) + + // The order in post also has to be sorted + let expected_post = sub_keys.iter().sorted().map(|i| { + let val = if *i == 5 { 100 } else { *i }; + (prefix.push(i).unwrap(), val) }); - itertools::assert_equal(iter_post.sorted(), expected_post.sorted()); + itertools::assert_equal(iter_post, expected_post); + + // Try to iterate over their prefix in reverse + let iter_pre = namada_vp_prelude::rev_iter_prefix(&ctx_pre, &prefix) + .unwrap() + .map(|item| item.unwrap()); + + // The order in has to be reverse sorted by sub-key value + let expected_pre = sub_keys + .iter() + .sorted() + .rev() + .map(|i| (prefix.push(i).unwrap(), *i)); + itertools::assert_equal(iter_pre, expected_pre); } #[test] diff --git a/tests/src/vm_host_env/tx.rs b/tests/src/vm_host_env/tx.rs index de0eeb4d98..a27a78b09b 100644 --- a/tests/src/vm_host_env/tx.rs +++ b/tests/src/vm_host_env/tx.rs @@ -335,6 +335,7 @@ mod native_tx_host_env { )); native_host_fn!(tx_delete(key_ptr: u64, key_len: u64)); native_host_fn!(tx_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64); + native_host_fn!(tx_rev_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64); native_host_fn!(tx_iter_next(iter_id: u64) -> i64); native_host_fn!(tx_insert_verifier(addr_ptr: u64, addr_len: u64)); native_host_fn!(tx_update_validity_predicate( diff --git a/tests/src/vm_host_env/vp.rs b/tests/src/vm_host_env/vp.rs index d849a11487..88aa63d530 100644 --- a/tests/src/vm_host_env/vp.rs +++ b/tests/src/vm_host_env/vp.rs @@ -326,6 +326,7 @@ mod native_vp_host_env { native_host_fn!(vp_has_key_pre(key_ptr: u64, key_len: u64) -> i64); native_host_fn!(vp_has_key_post(key_ptr: u64, key_len: u64) -> i64); native_host_fn!(vp_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64); + native_host_fn!(vp_rev_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64); native_host_fn!(vp_iter_pre_next(iter_id: u64) -> i64); native_host_fn!(vp_iter_post_next(iter_id: u64) -> i64); native_host_fn!(vp_get_chain_id(result_ptr: u64)); diff --git a/tx_prelude/src/lib.rs b/tx_prelude/src/lib.rs index 49b3eee5d3..d099162523 100644 --- a/tx_prelude/src/lib.rs +++ b/tx_prelude/src/lib.rs @@ -21,8 +21,9 @@ pub use namada::ledger::governance::storage as gov_storage; pub use namada::ledger::parameters::storage as parameters_storage; pub use namada::ledger::storage::types::encode; pub use namada::ledger::storage_api::{ - self, iter_prefix, iter_prefix_bytes, Error, OptionExt, ResultExt, - StorageRead, StorageWrite, + self, iter_prefix, iter_prefix_bytes, rev_iter_prefix, + rev_iter_prefix_bytes, Error, OptionExt, ResultExt, StorageRead, + StorageWrite, }; pub use namada::ledger::treasury::storage as treasury_storage; pub use namada::ledger::tx_env::TxEnv; @@ -167,6 +168,17 @@ impl StorageRead<'_> for Ctx { Ok(KeyValIterator(iter_id, PhantomData)) } + fn rev_iter_prefix( + &self, + prefix: &storage::Key, + ) -> storage_api::Result { + let prefix = prefix.to_string(); + let iter_id = unsafe { + anoma_tx_rev_iter_prefix(prefix.as_ptr() as _, prefix.len() as _) + }; + Ok(KeyValIterator(iter_id, PhantomData)) + } + fn iter_next( &self, iter: &mut Self::PrefixIter, diff --git a/vm_env/src/lib.rs b/vm_env/src/lib.rs index 53c594dbab..1421dbde48 100644 --- a/vm_env/src/lib.rs +++ b/vm_env/src/lib.rs @@ -48,9 +48,17 @@ pub mod tx { // Delete the given key and its value pub fn anoma_tx_delete(key_ptr: u64, key_len: u64); - // Get an ID of a data iterator with key prefix + // Get an ID of a data iterator with key prefix, ordered by storage + // keys. pub fn anoma_tx_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64; + // Get an ID of a data iterator with key prefix, reverse ordered by + // storage keys. + pub fn anoma_tx_rev_iter_prefix( + prefix_ptr: u64, + prefix_len: u64, + ) -> u64; + // Returns the size of the value (can be 0), or -1 if there's no next // value. If a value is found, it will be placed in the read // cache, because we cannot allocate a buffer for it before we know @@ -133,9 +141,17 @@ pub mod vp { // Returns 1 if the key is present in posterior state, -1 otherwise. pub fn anoma_vp_has_key_post(key_ptr: u64, key_len: u64) -> i64; - // Get an ID of a data iterator with key prefix + // Get an ID of a data iterator with key prefix, ordered by storage + // keys. pub fn anoma_vp_iter_prefix(prefix_ptr: u64, prefix_len: u64) -> u64; + // Get an ID of a data iterator with key prefix, reverse ordered by + // storage keys. + pub fn anoma_vp_rev_iter_prefix( + prefix_ptr: u64, + prefix_len: u64, + ) -> u64; + // Read variable-length prior state when we don't know the size // up-front, returns the size of the value (can be 0), or -1 if // the key is not present. If a value is found, it will be placed in the diff --git a/vp_prelude/src/lib.rs b/vp_prelude/src/lib.rs index ad61fa1254..ed553e3075 100644 --- a/vp_prelude/src/lib.rs +++ b/vp_prelude/src/lib.rs @@ -21,8 +21,8 @@ use std::marker::PhantomData; pub use borsh::{BorshDeserialize, BorshSerialize}; pub use namada::ledger::governance::storage as gov_storage; pub use namada::ledger::storage_api::{ - self, iter_prefix, iter_prefix_bytes, Error, OptionExt, ResultExt, - StorageRead, + self, iter_prefix, iter_prefix_bytes, rev_iter_prefix, + rev_iter_prefix_bytes, Error, OptionExt, ResultExt, StorageRead, }; pub use namada::ledger::vp_env::VpEnv; pub use namada::ledger::{parameters, pos as proof_of_stake}; @@ -216,6 +216,14 @@ impl<'view> VpEnv<'view> for Ctx { iter_prefix_impl(prefix) } + fn rev_iter_prefix( + &self, + prefix: &storage::Key, + ) -> Result { + // Both `CtxPreStorageRead` and `CtxPostStorageRead` have the same impl + rev_iter_prefix_impl(prefix) + } + fn eval( &self, vp_code: Vec, @@ -298,6 +306,13 @@ impl StorageRead<'_> for CtxPreStorageRead<'_> { iter_prefix_impl(prefix) } + fn rev_iter_prefix( + &self, + prefix: &storage::Key, + ) -> storage_api::Result { + rev_iter_prefix_impl(prefix) + } + fn get_chain_id(&self) -> Result { get_chain_id() } @@ -352,6 +367,13 @@ impl StorageRead<'_> for CtxPostStorageRead<'_> { iter_prefix_impl(prefix) } + fn rev_iter_prefix( + &self, + prefix: &storage::Key, + ) -> storage_api::Result { + rev_iter_prefix_impl(prefix) + } + fn get_chain_id(&self) -> Result { get_chain_id() } @@ -379,7 +401,17 @@ fn iter_prefix_impl( Ok(KeyValIterator(iter_id, PhantomData)) } -fn get_chain_id() -> Result { +fn rev_iter_prefix_impl( + prefix: &storage::Key, +) -> Result)>, storage_api::Error> { + let prefix = prefix.to_string(); + let iter_id = unsafe { + anoma_vp_rev_iter_prefix(prefix.as_ptr() as _, prefix.len() as _) + }; + Ok(KeyValIterator(iter_id, PhantomData)) +} + +fn get_chain_id() -> Result { let result = Vec::with_capacity(CHAIN_ID_LENGTH); unsafe { anoma_vp_get_chain_id(result.as_ptr() as _); diff --git a/wasm/checksums.json b/wasm/checksums.json index 21ec7b7134..eec2ef8b36 100644 --- a/wasm/checksums.json +++ b/wasm/checksums.json @@ -1,19 +1,19 @@ { - "tx_bond.wasm": "tx_bond.84c92cb9902f25c17a3f710264a343f66528bf579f7b4664d528fe48ed13162c.wasm", - "tx_from_intent.wasm": "tx_from_intent.d7cb7e69955c1408733efbeceafa5c3613ac0436f7b31975600231f8cd094251.wasm", - "tx_ibc.wasm": "tx_ibc.8b53361ef12127c9bde2aecc8a87028e8d54a778428bf7084483b1258db42db9.wasm", - "tx_init_account.wasm": "tx_init_account.1fb31906257df2acf6b431f284369b1fa2f67ea7f2ba57927a55961077308a4e.wasm", - "tx_init_nft.wasm": "tx_init_nft.9ef7ffb8276873b9cba92e7c955f61cd5653695a73eabbb3acf523bdc0b26736.wasm", - "tx_init_proposal.wasm": "tx_init_proposal.38158cd766ad7ef89c1638f19d741cb0dff37d2fd0cb052522084a4a78b10ef5.wasm", - "tx_init_validator.wasm": "tx_init_validator.2e35bbc6729ec3874c0ec8bb9ccd43f9a6ee864bc277e7acbe9848c233849fc9.wasm", - "tx_mint_nft.wasm": "tx_mint_nft.38a3a1e768206d9ceb6b96b36e3d66903d9ff331669c393176f91fe605d245b3.wasm", - "tx_transfer.wasm": "tx_transfer.b1a1a7c26cabebd98e195cb700e2fbd81b633f2531fc4510adedf7b0a2863afc.wasm", - "tx_unbond.wasm": "tx_unbond.c0e3ff3a6946e18915237e7de94152f84cf021bd9d9071e2579f60f8bb3e7a4d.wasm", - "tx_update_vp.wasm": "tx_update_vp.d113030ac6ba66518d7672e2d708f3d24ba86635d1517c9c672c997f0c86a39b.wasm", - "tx_vote_proposal.wasm": "tx_vote_proposal.b1b33949f79049f48138bc432011e8dab6d1802d53a16cccb1634507d14b6498.wasm", - "tx_withdraw.wasm": "tx_withdraw.1164da83a420936432c6f70321743c338cf5324dfdc98b6037bc1ec50a1b43f1.wasm", - "vp_nft.wasm": "vp_nft.30196b3deeb4bda27036bcfacd826d50fc4e76eb88a767f545c5f6341792dc72.wasm", - "vp_testnet_faucet.wasm": "vp_testnet_faucet.cfb87581a363e5ccf3107a7b0a5117827af80a7548ad02a8be14d2ee14cd0ca4.wasm", - "vp_token.wasm": "vp_token.cd0271f60a73fd41151f0a34ad8de6086fe21df3062871ee68d5fd093fe663bf.wasm", - "vp_user.wasm": "vp_user.0131d76f247459f54b4cd957a9e6526937dcded1b9067d085d0d195dd4fc0730.wasm" + "tx_bond.wasm": "tx_bond.5fb96735e9d9a7c1bdc99a08add19a8ace5c6a2f8a39bd7a6cf74c4e21701ceb.wasm", + "tx_from_intent.wasm": "tx_from_intent.01289125cb64a2871d9257c53c52c28a4f1090dbfb1fa94d669b56fc0570323b.wasm", + "tx_ibc.wasm": "tx_ibc.cbc76cecb2d8877f60af8b2f399468336c485529fda42b85430c4b605ca39b43.wasm", + "tx_init_account.wasm": "tx_init_account.7db780376570c7b1865a522306e9fa5a2ff29c8e398d66b908c26bc04256eed6.wasm", + "tx_init_nft.wasm": "tx_init_nft.efdc03ebd9bea49e36ce61f93f799fe31ca76514b7f9055394618eddcd4ef8a3.wasm", + "tx_init_proposal.wasm": "tx_init_proposal.6c7ce66e5cfffd9c095db60c11c7eb7d4a3aebf686147f1ec55f3f585b7ed2b0.wasm", + "tx_init_validator.wasm": "tx_init_validator.163892a01022ab743a51ed6219c4c8775cc3a1da06ecbf9ec60a98f7abdec902.wasm", + "tx_mint_nft.wasm": "tx_mint_nft.218bf4a411361126a0ac431024277333cf68b23531a54c116eaee44d60ba1bc4.wasm", + "tx_transfer.wasm": "tx_transfer.766318b9313e5d61c615732f0474e760715998eeedb2bdbadaed3dbe4e4714bc.wasm", + "tx_unbond.wasm": "tx_unbond.38b029940a5115821ec3dc14801ae4d668ee42fddd764a93a3126faa93f2c634.wasm", + "tx_update_vp.wasm": "tx_update_vp.ab172304f3bd23f209a9005249762bd3618e9337e24dd42153d84c1f68d4a3a3.wasm", + "tx_vote_proposal.wasm": "tx_vote_proposal.2c418e22709e9edbbd72469a2b1dbdf23ee71e2461ade7d82153bb8fb2610341.wasm", + "tx_withdraw.wasm": "tx_withdraw.a7c35b925c4c2c71e05fe167933260f1bb01157eb56c52c770f4c96bd90a6714.wasm", + "vp_nft.wasm": "vp_nft.8071fb9ee39605599adbef85d9c9a3cbf9164800047b4ebba2d63aeaed358f48.wasm", + "vp_testnet_faucet.wasm": "vp_testnet_faucet.4d39a6ed4ec355b12658a5475b0c60aa0e62206cfbf70b4d931a0996d01f6085.wasm", + "vp_token.wasm": "vp_token.cb344e7b5c1bd30c00cc3bf077ef86f87f4571be495644ad91acce3ec168a810.wasm", + "vp_user.wasm": "vp_user.6f6f4d091b3e4849de10f5ba77946148b4a38d6c0b73e08a63db7e58c93e8d00.wasm" } \ No newline at end of file diff --git a/wasm/tx_template/Cargo.lock b/wasm/tx_template/Cargo.lock index 5f78ab664e..754cd87ded 100644 --- a/wasm/tx_template/Cargo.lock +++ b/wasm/tx_template/Cargo.lock @@ -602,6 +602,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derivative" version = "2.2.0" @@ -1359,10 +1365,10 @@ dependencies = [ "borsh", "chrono", "clru", + "data-encoding", "derivative", "ed25519-consensus", "ferveo-common", - "hex", "ibc", "ibc-proto", "ics23", diff --git a/wasm/vp_template/Cargo.lock b/wasm/vp_template/Cargo.lock index e2129190bb..a20f51bfdb 100644 --- a/wasm/vp_template/Cargo.lock +++ b/wasm/vp_template/Cargo.lock @@ -602,6 +602,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derivative" version = "2.2.0" @@ -1359,10 +1365,10 @@ dependencies = [ "borsh", "chrono", "clru", + "data-encoding", "derivative", "ed25519-consensus", "ferveo-common", - "hex", "ibc", "ibc-proto", "ics23", diff --git a/wasm/wasm_source/Cargo.lock b/wasm/wasm_source/Cargo.lock index 6d84006f5b..9cfe2d4105 100644 --- a/wasm/wasm_source/Cargo.lock +++ b/wasm/wasm_source/Cargo.lock @@ -602,6 +602,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derivative" version = "2.2.0" @@ -1359,10 +1365,10 @@ dependencies = [ "borsh", "chrono", "clru", + "data-encoding", "derivative", "ed25519-consensus", "ferveo-common", - "hex", "ibc", "ibc-proto", "ics23", diff --git a/wasm_for_tests/wasm_source/Cargo.lock b/wasm_for_tests/wasm_source/Cargo.lock index 49032e47d1..fb01de3457 100644 --- a/wasm_for_tests/wasm_source/Cargo.lock +++ b/wasm_for_tests/wasm_source/Cargo.lock @@ -603,6 +603,12 @@ dependencies = [ "syn", ] +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "derivative" version = "2.2.0" @@ -1370,10 +1376,10 @@ dependencies = [ "borsh", "chrono", "clru", + "data-encoding", "derivative", "ed25519-consensus", "ferveo-common", - "hex", "ibc", "ibc-proto", "ics23",