Merge branch 'yuji/e2e-reward-shielded-ibc-token' (#2601)

* origin/yuji/e2e-reward-shielded-ibc-token:
  revert LazyMap
  making token map lazy
  WIP making token map lazy
  add changelog
  reduce epochs
  add shielded-sync
  wait more epoch
  wait long
  add retries
  change the epoch query interval
  fix integration test
  propose before ibc transfer
  leave assets in ConversionState and store the hash
  merklize root of masp convert_anchor
  check nam
  move token/asset maps from conversion_state back to storage
  add e2e test

.changelog/unreleased/features/2601-update-masp-token-map.md

@@ -0,0 +1,2 @@
+- Enable to update ConversionState token map by proposal wasm tx
+  ([\#2601](https://github.com/anoma/namada/issues/2601))

.github/workflows/scripts/e2e.json

@@ -3,6 +3,7 @@
     "e2e::ibc_tests::run_ledger_ibc": 155,
     "e2e::ibc_tests::run_ledger_ibc_with_hermes": 130,
     "e2e::ibc_tests::pgf_over_ibc_with_hermes": 240,
+    "e2e::ibc_tests::proposal_ibc_token_inflation": 600,
     "e2e::eth_bridge_tests::test_add_to_bridge_pool": 10,
     "e2e::ledger_tests::double_signing_gets_slashed": 12,
     "e2e::ledger_tests::invalid_transactions": 13,
@@ -36,4 +37,4 @@
     "e2e::wallet_tests::wallet_encrypted_key_cmds": 1,
     "e2e::wallet_tests::wallet_encrypted_key_cmds_env_var": 1,
     "e2e::wallet_tests::wallet_unencrypted_key_cmds": 1
-}
+}

crates/apps/src/lib/node/ledger/shell/init_chain.rs

@@ -1,5 +1,5 @@
 //! Implementation of chain initialization for the Shell
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::ops::ControlFlow;
 
 use masp_primitives::merkle_tree::CommitmentTree;
@@ -12,6 +12,7 @@ use namada::ledger::parameters::Parameters;
 use namada::ledger::{ibc, pos};
 use namada::proof_of_stake::BecomeValidator;
 use namada::state::StorageWrite;
+use namada::token::storage_key::masp_token_map_key;
 use namada::token::{credit_tokens, write_denom};
 use namada::vm::validate_untrusted_wasm;
 use namada_sdk::eth_bridge::EthBridgeStatus;
@@ -418,6 +419,7 @@ where
 
     /// Init genesis token accounts
     fn init_token_accounts(&mut self, genesis: &genesis::chain::Finalized) {
+        let mut token_map = BTreeMap::new();
         for (alias, token) in &genesis.tokens.token {
             tracing::debug!("Initializing token {alias}");
 
@@ -438,13 +440,12 @@ where
                 // add token addresses to the masp reward conversions lookup
                 // table.
                 let alias = alias.to_string();
-                self.state
-                    .in_mem_mut()
-                    .conversion_state
-                    .tokens
-                    .insert(alias, address.clone());
+                token_map.insert(alias, address.clone());
             }
         }
+        self.state
+            .write(&masp_token_map_key(), token_map)
+            .expect("Couldn't init token accounts");
     }
 
     /// Init genesis token balances
@@ -937,7 +938,6 @@ impl<T> Policy<T> {
 
 #[cfg(all(test, not(feature = "integration")))]
 mod test {
-    use std::collections::BTreeMap;
     use std::str::FromStr;
 
     use namada::core::string_encoding::StringEncoded;

crates/apps/src/lib/node/ledger/shell/mod.rs

@@ -357,8 +357,11 @@ where
 /// Merkle tree storage key filter. Return `false` for keys that shouldn't be
 /// merklized.
 pub fn is_merklized_storage_key(key: &namada_sdk::storage::Key) -> bool {
-    !token::storage_key::is_masp_key(key)
-        && !namada::ibc::storage::is_ibc_counter_key(key)
+    !(token::storage_key::is_masp_key(key)
+        && *key != token::storage_key::masp_convert_anchor_key()
+        && *key != token::storage_key::masp_token_map_key()
+        && *key != token::storage_key::masp_assets_hash_key()
+        || namada::ibc::storage::is_ibc_counter_key(key))
 }
 
 /// Channels for communicating with an Ethereum oracle.

crates/apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -1784,9 +1784,7 @@ mod imp {
 
 #[cfg(test)]
 mod test {
-    use namada::core::address::{
-        gen_established_address, EstablishedAddressGen,
-    };
+    use namada::core::address::EstablishedAddressGen;
     use namada::core::storage::{BlockHash, Epochs};
     use namada::state::{MerkleTree, Sha256Hasher};
     use tempfile::tempdir;
@@ -1997,10 +1995,7 @@ mod test {
         let height_0 = BlockHeight(100);
         let mut pred_epochs = Epochs::default();
         pred_epochs.new_epoch(height_0);
-        let mut conversion_state_0 = ConversionState::default();
-        conversion_state_0
-            .tokens
-            .insert("dummy1".to_string(), gen_established_address("test"));
+        let conversion_state_0 = ConversionState::default();
         let to_delete_val = vec![1_u8, 1, 0, 0];
         let to_overwrite_val = vec![1_u8, 1, 1, 0];
         db.batch_write_subspace_val(
@@ -2035,10 +2030,7 @@ mod test {
         let mut batch = RocksDB::batch();
         let height_1 = BlockHeight(101);
         pred_epochs.new_epoch(height_1);
-        let mut conversion_state_1 = ConversionState::default();
-        conversion_state_1
-            .tokens
-            .insert("dummy2".to_string(), gen_established_address("test"));
+        let conversion_state_1 = ConversionState::default();
         let add_val = vec![1_u8, 0, 0, 0];
         let overwrite_val = vec![1_u8, 1, 1, 1];
         db.batch_write_subspace_val(

crates/core/src/masp.rs

@@ -1,5 +1,6 @@
 //! MASP types
 
+use std::collections::BTreeMap;
 use std::fmt::Display;
 use std::str::FromStr;
 
@@ -89,6 +90,9 @@ pub fn encode_asset_type(
     .encode()
 }
 
+/// MASP token map
+pub type TokenMap = BTreeMap<String, Address>;
+
 // enough capacity to store the payment address
 // plus the pinned/unpinned discriminant
 const PAYMENT_ADDRESS_SIZE: usize = 43 + 1;

crates/sdk/src/queries/shell.rs

@@ -12,13 +12,15 @@ use namada_core::address::Address;
 use namada_core::dec::Dec;
 use namada_core::hash::Hash;
 use namada_core::hints;
+use namada_core::masp::TokenMap;
 use namada_core::storage::{
     self, BlockHeight, BlockResults, Epoch, KeySeg, PrefixValue,
 };
 use namada_core::token::{Denomination, MaspDigitPos};
 use namada_core::uint::Uint;
 use namada_state::{DBIter, LastBlock, StateRead, StorageHasher, DB};
 use namada_storage::{ResultExt, StorageRead};
+use namada_token::storage_key::masp_token_map_key;
 #[cfg(any(test, feature = "async-client"))]
 use namada_tx::data::TxResult;
 
@@ -228,9 +230,11 @@ where
     D: 'static + DB + for<'iter> DBIter<'iter> + Sync,
     H: 'static + StorageHasher + Sync,
 {
-    let tokens = ctx.state.in_mem().conversion_state.tokens.clone();
+    let token_map_key = masp_token_map_key();
+    let token_map: TokenMap =
+        ctx.state.read(&token_map_key)?.unwrap_or_default();
     let mut data = Vec::<MaspTokenRewardData>::new();
-    for (name, token) in tokens {
+    for (name, token) in token_map {
         let max_reward_rate = ctx
             .state
             .read::<Dec>(&namada_token::storage_key::masp_max_reward_rate_key(

crates/shielded_token/src/conversion.rs

@@ -1,7 +1,11 @@
 //! MASP rewards conversions
 
 use namada_core::address::{Address, MASP};
+#[cfg(any(feature = "multicore", test))]
+use namada_core::borsh::BorshSerializeExt;
 use namada_core::dec::Dec;
+#[cfg(any(feature = "multicore", test))]
+use namada_core::hash::Hash;
 use namada_core::uint::Uint;
 use namada_parameters as parameters;
 use namada_storage::{StorageRead, StorageWrite};
@@ -11,6 +15,8 @@ use namada_trans_token::inflation::{
 use namada_trans_token::storage_key::{balance_key, minted_balance_key};
 use namada_trans_token::{read_denom, Amount, DenominatedAmount, Denomination};
 
+#[cfg(any(feature = "multicore", test))]
+use crate::storage_key::{masp_assets_hash_key, masp_token_map_key};
 use crate::storage_key::{
     masp_kd_gain_key, masp_kp_gain_key, masp_last_inflation_key,
     masp_last_locked_amount_key, masp_locked_amount_target_key,
@@ -220,12 +226,10 @@ where
     // The derived conversions will be placed in MASP address space
     let masp_addr = MASP;
 
-    let mut masp_reward_keys: Vec<_> = storage
-        .conversion_state()
-        .tokens
-        .values()
-        .cloned()
-        .collect();
+    let token_map_key = masp_token_map_key();
+    let token_map: namada_core::masp::TokenMap =
+        storage.read(&token_map_key)?.unwrap_or_default();
+    let mut masp_reward_keys: Vec<_> = token_map.values().cloned().collect();
     let mut masp_reward_denoms = BTreeMap::new();
     // Put the native rewards first because other inflation computations depend
     // on it
@@ -517,6 +521,10 @@ where
             );
         }
     }
+    // store only the assets hash because the size is quite large
+    let assets_hash =
+        Hash::sha256(storage.conversion_state().assets.serialize_to_vec());
+    storage.write(&masp_assets_hash_key(), assets_hash)?;
 
     Ok(())
 }
@@ -615,9 +623,11 @@ mod tests {
                 .unwrap();
 
                 // Insert tokens into MASP conversion state
-                s.conversion_state_mut()
-                    .tokens
-                    .insert(alias.to_string(), token_addr.clone());
+                let token_map_key = masp_token_map_key();
+                let mut token_map: namada_core::masp::TokenMap =
+                    s.read(&token_map_key).unwrap().unwrap_or_default();
+                token_map.insert(alias.to_string(), token_addr.clone());
+                s.write(&token_map_key, token_map).unwrap();
             }
         }
 

crates/shielded_token/src/storage_key.rs

@@ -17,6 +17,10 @@ pub const MASP_NOTE_COMMITMENT_TREE_KEY: &str = "commitment_tree";
 pub const MASP_NOTE_COMMITMENT_ANCHOR_PREFIX: &str = "note_commitment_anchor";
 /// Key segment prefix for the convert anchor
 pub const MASP_CONVERT_ANCHOR_KEY: &str = "convert_anchor";
+/// The key for the token map
+pub const MASP_TOKEN_MAP_KEY: &str = "tokens";
+/// The key for the asset map
+pub const MASP_ASSETS_HASH_KEY: &str = "assets_hash";
 /// Last calculated inflation value handed out
 pub const MASP_LAST_INFLATION_KEY: &str = "last_inflation";
 /// The last locked ratio
@@ -146,3 +150,17 @@ pub fn masp_convert_anchor_key() -> storage::Key {
         .push(&MASP_CONVERT_ANCHOR_KEY.to_owned())
         .expect("Cannot obtain a storage key")
 }
+
+/// Get the key for the masp token map
+pub fn masp_token_map_key() -> storage::Key {
+    storage::Key::from(address::MASP.to_db_key())
+        .push(&MASP_TOKEN_MAP_KEY.to_owned())
+        .expect("Cannot obtain a storage key")
+}
+
+/// Get the key for the masp assets' hash
+pub fn masp_assets_hash_key() -> storage::Key {
+    storage::Key::from(address::MASP.to_db_key())
+        .push(&MASP_ASSETS_HASH_KEY.to_owned())
+        .expect("Cannot obtain a storage key")
+}

crates/storage/src/conversion_state.rs

@@ -18,8 +18,6 @@ pub struct ConversionState {
     pub normed_inflation: Option<u128>,
     /// The tree currently containing all the conversions
     pub tree: FrozenCommitmentTree<sapling::Node>,
-    /// A map from token alias to actual address.
-    pub tokens: BTreeMap<String, Address>,
     /// Map assets to their latest conversion and position in Merkle tree
     #[allow(clippy::type_complexity)]
     pub assets: BTreeMap<

crates/test_utils/src/lib.rs

@@ -28,6 +28,7 @@ pub enum TestWasms {
     VpMemoryLimit,
     VpReadStorageKey,
     TxProposalMaspRewards,
+    TxProposalIbcTokenInflation,
 }
 
 impl TestWasms {
@@ -46,6 +47,9 @@ impl TestWasms {
             TestWasms::VpMemoryLimit => "vp_memory_limit.wasm",
             TestWasms::VpReadStorageKey => "vp_read_storage_key.wasm",
             TestWasms::TxProposalMaspRewards => "tx_proposal_masp_reward.wasm",
+            TestWasms::TxProposalIbcTokenInflation => {
+                "tx_proposal_ibc_token_inflation.wasm"
+            }
         };
         let cwd =
             env::current_dir().expect("Couldn't get current working directory");

crates/tests/src/e2e/helpers.rs

@@ -319,7 +319,7 @@ pub fn get_epoch(test: &Test, ledger_address: &str) -> Result<Epoch> {
         test,
         Bin::Client,
         &["epoch", "--node", ledger_address],
-        Some(10)
+        Some(20)
     )?;
     let (unread, matched) = find.exp_regex("Last committed epoch: .*")?;
     let epoch_str = strip_trailing_newline(&matched)