diff --git a/Cargo.lock b/Cargo.lock index 845d23f358..98ff5f5fed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6982,6 +6982,7 @@ dependencies = [ "account", "cumulus-pallet-parachain-system", "cumulus-pallet-xcmp-queue", + "cumulus-primitives-storage-weight-reclaim", "fp-ethereum", "fp-evm", "frame-benchmarking", @@ -7038,6 +7039,7 @@ dependencies = [ "sp-consensus-slots", "sp-core", "sp-genesis-builder", + "sp-io", "sp-runtime", "sp-std", "staging-xcm", @@ -9821,10 +9823,12 @@ dependencies = [ name = "pallet-moonbeam-lazy-migrations" version = "0.1.0" dependencies = [ + "cumulus-primitives-storage-weight-reclaim", "frame-benchmarking", "frame-support", "frame-system", "log", + "pallet-assets", "pallet-balances", "pallet-evm", "pallet-scheduler", diff --git a/Cargo.toml b/Cargo.toml index 78a320899a..16eec9526c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -295,7 +295,6 @@ cumulus-primitives-parachain-inherent = { git = "https://github.com/moonbeam-fou cumulus-primitives-timestamp = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } cumulus-primitives-utility = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } cumulus-test-relay-sproof-builder = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } -cumulus-primitives-storage-weight-reclaim = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } parachain-info = { package = "staging-parachain-info", git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } parachains-common = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false } @@ -312,6 +311,7 @@ cumulus-relay-chain-inprocess-interface = { git = "https://github.com/moonbeam-f cumulus-relay-chain-interface = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" } cumulus-relay-chain-minimal-node = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" } cumulus-relay-chain-rpc-interface = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0" } +cumulus-primitives-storage-weight-reclaim = { git = "https://github.com/moonbeam-foundation/polkadot-sdk", branch = "moonbeam-polkadot-v1.11.0", default-features = false} # Polkadot / XCM (wasm) orml-traits = { git = "https://github.com/moonbeam-foundation/open-runtime-module-library", branch = "moonbeam-polkadot-v1.11.0", default-features = false } diff --git a/pallets/moonbeam-lazy-migrations/Cargo.toml b/pallets/moonbeam-lazy-migrations/Cargo.toml index cefc23f304..cf03633392 100644 --- a/pallets/moonbeam-lazy-migrations/Cargo.toml +++ b/pallets/moonbeam-lazy-migrations/Cargo.toml @@ -1,37 +1,41 @@ [package] -authors = {workspace = true} +authors = { workspace = true } description = "A pallet for performing migrations from extrinsics" edition = "2021" name = "pallet-moonbeam-lazy-migrations" version = "0.1.0" [dependencies] -log = {workspace = true} +log = { workspace = true } # Substrate -frame-support = {workspace = true} -frame-system = {workspace = true} -pallet-scheduler = {workspace = true} -pallet-balances = {workspace = true} -parity-scale-codec = {workspace = true} -scale-info = {workspace = true, features = ["derive"]} -sp-core = {workspace = true} -sp-io = {workspace = true} -sp-runtime = {workspace = true} -sp-std = {workspace = true} +frame-support = { workspace = true } +frame-system = { workspace = true } +pallet-scheduler = { workspace = true } +pallet-assets = { workspace = true } +pallet-balances = { workspace = true } +parity-scale-codec = { workspace = true } +scale-info = { workspace = true, features = ["derive"] } +sp-core = { workspace = true } +sp-io = { workspace = true } +sp-runtime = { workspace = true } +sp-std = { workspace = true } # Frontier -pallet-evm = {workspace = true, features = ["forbid-evm-reentrancy"]} +pallet-evm = { workspace = true, features = ["forbid-evm-reentrancy"] } + +# Runtime Interfaces +cumulus-primitives-storage-weight-reclaim = { workspace = true, default-features = false } # Benchmarks -frame-benchmarking = {workspace = true, optional = true} +frame-benchmarking = { workspace = true, optional = true } [dev-dependencies] -frame-benchmarking = {workspace = true, features = ["std"]} -pallet-balances = {workspace = true, features = ["std", "insecure_zero_ed"]} -pallet-timestamp = {workspace = true, features = ["std"]} -rlp = {workspace = true, features = ["std"]} -sp-io = {workspace = true, features = ["std"]} +frame-benchmarking = { workspace = true, features = ["std"] } +pallet-balances = { workspace = true, features = ["std", "insecure_zero_ed"] } +pallet-timestamp = { workspace = true, features = ["std"] } +rlp = { workspace = true, features = ["std"] } +sp-io = { workspace = true, features = ["std"] } [features] default = ["std"] @@ -47,6 +51,8 @@ std = [ "sp-std/std", "pallet-evm/std", "pallet-timestamp/std", + "pallet-assets/std", + "cumulus-primitives-storage-weight-reclaim/std", "rlp/std", ] -try-runtime = ["frame-support/try-runtime"] \ No newline at end of file +try-runtime = ["frame-support/try-runtime"] diff --git a/pallets/moonbeam-lazy-migrations/src/lib.rs b/pallets/moonbeam-lazy-migrations/src/lib.rs index 2114608c97..58875eefee 100644 --- a/pallets/moonbeam-lazy-migrations/src/lib.rs +++ b/pallets/moonbeam-lazy-migrations/src/lib.rs @@ -38,6 +38,7 @@ const MAX_CONTRACT_CODE_SIZE: u64 = 25 * 1024; #[pallet] pub mod pallet { use super::*; + use cumulus_primitives_storage_weight_reclaim::get_proof_size; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use sp_core::H160; @@ -53,6 +54,26 @@ pub mod pallet { /// The total number of suicided contracts that were removed pub(crate) type SuicidedContractsRemoved = StorageValue<_, u32, ValueQuery>; + #[pallet::storage] + pub(crate) type StateMigrationStatusValue = + StorageValue<_, (StateMigrationStatus, u64), ValueQuery>; + + pub(crate) type StorageKey = BoundedVec>; + + #[derive(Clone, Encode, Decode, scale_info::TypeInfo, PartialEq, Eq, MaxEncodedLen, Debug)] + pub enum StateMigrationStatus { + NotStarted, + Started(StorageKey), + Error(BoundedVec>), + Complete, + } + + impl Default for StateMigrationStatus { + fn default() -> Self { + return StateMigrationStatus::NotStarted; + } + } + /// Configuration trait of this pallet. #[pallet::config] pub trait Config: frame_system::Config + pallet_evm::Config + pallet_balances::Config { @@ -71,6 +92,233 @@ pub mod pallet { ContractMetadataAlreadySet, /// Contract not exist ContractNotExist, + /// The key lengths exceeds the maximum allowed + KeyTooLong, + } + + pub(crate) const MAX_ITEM_PROOF_SIZE: u64 = 30 * 1024; // 30 KB + pub(crate) const PROOF_SIZE_BUFFER: u64 = 100 * 1024; // 100 KB + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_idle(_n: BlockNumberFor, remaining_weight: Weight) -> Weight { + let proof_size_before: u64 = get_proof_size().unwrap_or(0); + let res = Pallet::::handle_migration(remaining_weight); + let proof_size_after: u64 = get_proof_size().unwrap_or(0); + let proof_size_diff = proof_size_after.saturating_sub(proof_size_before); + + Weight::from_parts(0, proof_size_diff) + .saturating_add(T::DbWeight::get().reads_writes(res.reads, res.writes)) + } + } + + #[derive(Default, Clone, PartialEq, Eq, Encode, Decode, Debug)] + pub(crate) struct ReadWriteOps { + pub reads: u64, + pub writes: u64, + } + + impl ReadWriteOps { + pub fn new() -> Self { + Self { + reads: 0, + writes: 0, + } + } + + pub fn add_one_read(&mut self) { + self.reads += 1; + } + + pub fn add_one_write(&mut self) { + self.writes += 1; + } + + pub fn add_reads(&mut self, reads: u64) { + self.reads += reads; + } + + pub fn add_writes(&mut self, writes: u64) { + self.writes += writes; + } + } + + #[derive(Clone)] + struct StateMigrationResult { + last_key: Option, + error: Option<&'static str>, + migrated: u64, + reads: u64, + writes: u64, + } + + enum NextKeyResult { + NextKey(StorageKey), + NoMoreKeys, + Error(&'static str), + } + + impl Pallet { + /// Handle the migration of the storage keys, returns the number of read and write operations + pub(crate) fn handle_migration(remaining_weight: Weight) -> ReadWriteOps { + let mut read_write_ops = ReadWriteOps::new(); + + // maximum number of items that can be migrated in one block + let migration_limit = remaining_weight + .proof_size() + .saturating_sub(PROOF_SIZE_BUFFER) + .saturating_div(MAX_ITEM_PROOF_SIZE); + + if migration_limit == 0 { + return read_write_ops; + } + + let (status, mut migrated_keys) = StateMigrationStatusValue::::get(); + read_write_ops.add_one_read(); + + let next_key = match &status { + StateMigrationStatus::NotStarted => Default::default(), + StateMigrationStatus::Started(storage_key) => { + let (reads, next_key_result) = Pallet::::get_next_key(storage_key); + read_write_ops.add_reads(reads); + match next_key_result { + NextKeyResult::NextKey(next_key) => next_key, + NextKeyResult::NoMoreKeys => { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Complete, + migrated_keys, + )); + read_write_ops.add_one_write(); + return read_write_ops; + } + NextKeyResult::Error(e) => { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Error( + e.as_bytes().to_vec().try_into().unwrap_or_default(), + ), + migrated_keys, + )); + read_write_ops.add_one_write(); + return read_write_ops; + } + } + } + StateMigrationStatus::Complete | StateMigrationStatus::Error(_) => { + return read_write_ops; + } + }; + + let res = Pallet::::migrate_keys(next_key, migration_limit); + migrated_keys += res.migrated; + read_write_ops.add_reads(res.reads); + read_write_ops.add_writes(res.writes); + + match (res.last_key, res.error) { + (None, None) => { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Complete, + migrated_keys, + )); + read_write_ops.add_one_write(); + } + // maybe we should store the previous key in the storage as well + (_, Some(e)) => { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Error( + e.as_bytes().to_vec().try_into().unwrap_or_default(), + ), + migrated_keys, + )); + read_write_ops.add_one_write(); + } + (Some(key), None) => { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Started(key), + migrated_keys, + )); + read_write_ops.add_one_write(); + } + } + + read_write_ops + } + + /// Tries to get the next key in the storage, returns None if there are no more keys to migrate. + /// Returns an error if the key is too long. + fn get_next_key(key: &StorageKey) -> (u64, NextKeyResult) { + if let Some(next) = sp_io::storage::next_key(key) { + let next: Result = next.try_into(); + match next { + Ok(next_key) => { + if next_key.as_slice() == sp_core::storage::well_known_keys::CODE { + let (reads, next_key_res) = Pallet::::get_next_key(&next_key); + return (1 + reads, next_key_res); + } + (1, NextKeyResult::NextKey(next_key)) + } + Err(_) => (1, NextKeyResult::Error("Key too long")), + } + } else { + (1, NextKeyResult::NoMoreKeys) + } + } + + /// Migrate maximum of `limit` keys starting from `start`, returns the next key to migrate + /// Returns None if there are no more keys to migrate. + /// Returns an error if an error occurred during migration. + fn migrate_keys(start: StorageKey, limit: u64) -> StateMigrationResult { + let mut key = start; + let mut migrated = 0; + let mut next_key_reads = 0; + let mut writes = 0; + + while migrated < limit { + let data = sp_io::storage::get(&key); + if let Some(data) = data { + sp_io::storage::set(&key, &data); + writes += 1; + } + + migrated += 1; + + if migrated < limit { + let (reads, next_key_res) = Pallet::::get_next_key(&key); + next_key_reads += reads; + + match next_key_res { + NextKeyResult::NextKey(next_key) => { + key = next_key; + } + NextKeyResult::NoMoreKeys => { + return StateMigrationResult { + last_key: None, + error: None, + migrated, + reads: migrated + next_key_reads, + writes, + }; + } + NextKeyResult::Error(e) => { + return StateMigrationResult { + last_key: Some(key), + error: Some(e), + migrated, + reads: migrated + next_key_reads, + writes, + }; + } + }; + } + } + + StateMigrationResult { + last_key: Some(key), + error: None, + migrated, + reads: migrated + next_key_reads, + writes, + } + } } #[pallet::call] diff --git a/pallets/moonbeam-lazy-migrations/src/mock.rs b/pallets/moonbeam-lazy-migrations/src/mock.rs index 9836b1af65..8d9b074408 100644 --- a/pallets/moonbeam-lazy-migrations/src/mock.rs +++ b/pallets/moonbeam-lazy-migrations/src/mock.rs @@ -21,7 +21,7 @@ use crate as pallet_moonbeam_lazy_migrations; use frame_support::{ construct_runtime, ord_parameter_types, parameter_types, traits::{EqualPrivilegeOnly, Everything, SortedMembers}, - weights::{constants::RocksDbWeight, Weight}, + weights::{RuntimeDbWeight, Weight}, }; use frame_system::EnsureRoot; use pallet_evm::{AddressMapping, EnsureAddressTruncated}; @@ -54,9 +54,17 @@ parameter_types! { pub const AvailableBlockRatio: Perbill = Perbill::one(); pub const SS58Prefix: u8 = 42; } + +parameter_types! { + pub const MockDbWeight: RuntimeDbWeight = RuntimeDbWeight { + read: 1_000_000, + write: 1, + }; +} + impl frame_system::Config for Test { type BaseCallFilter = Everything; - type DbWeight = RocksDbWeight; + type DbWeight = MockDbWeight; type RuntimeOrigin = RuntimeOrigin; type RuntimeTask = RuntimeTask; type Nonce = u64; diff --git a/pallets/moonbeam-lazy-migrations/src/tests.rs b/pallets/moonbeam-lazy-migrations/src/tests.rs index 6d135c9a88..ff460d1fa1 100644 --- a/pallets/moonbeam-lazy-migrations/src/tests.rs +++ b/pallets/moonbeam-lazy-migrations/src/tests.rs @@ -18,13 +18,14 @@ use { crate::{ mock::{ExtBuilder, LazyMigrations, RuntimeOrigin, Test}, - Error, + Error, StateMigrationStatus, StateMigrationStatusValue, MAX_ITEM_PROOF_SIZE, + PROOF_SIZE_BUFFER, }, - frame_support::{assert_noop, assert_ok}, + frame_support::{assert_noop, assert_ok, traits::Hooks, weights::Weight}, rlp::RlpStream, sp_core::{H160, H256}, sp_io::hashing::keccak_256, - sp_runtime::AccountId32, + sp_runtime::{traits::Bounded, AccountId32}, }; use pallet_evm::AddressMapping; @@ -307,3 +308,269 @@ fn test_create_contract_metadata_success_path() { ); }); } + +fn count_keys_and_data_without_code() -> (u64, u64) { + let mut keys: u64 = 0; + let mut data: u64 = 0; + + let mut current_key: Option> = Some(Default::default()); + while let Some(key) = current_key { + if key.as_slice() == sp_core::storage::well_known_keys::CODE { + current_key = sp_io::storage::next_key(&key); + continue; + } + keys += 1; + if let Some(_) = sp_io::storage::get(&key) { + data += 1; + } + current_key = sp_io::storage::next_key(&key); + } + + (keys, data) +} + +fn weight_for(read: u64, write: u64) -> Weight { + ::DbWeight::get().reads_writes(read, write) +} + +fn rem_weight_for_entries(num_entries: u64) -> Weight { + let proof = PROOF_SIZE_BUFFER + num_entries * MAX_ITEM_PROOF_SIZE; + Weight::from_parts(u64::max_value(), proof) +} + +#[test] +fn test_state_migration_baseline() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::NotStarted, 0) + ); + + let (keys, data) = count_keys_and_data_without_code(); + println!("Keys: {}, Data: {}", keys, data); + + let weight = LazyMigrations::on_idle(0, Weight::max_value()); + + // READS: 2 * keys + 2 (skipped and status) + // Next key requests = keys (we have first key as default which is not counted, and extra + // next_key request to check if we are done) + // + // 1 next key request for the skipped key ":code" + // Read requests = keys (we read each key once) + // 1 Read request for the StateMigrationStatusValue + + // WRITES: data + 1 (status) + // Write requests = data (we write each data once) + // 1 Write request for the StateMigrationStatusValue + assert_eq!(weight, weight_for(2 * keys + 2, data + 1)); + + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::Complete, keys) + ); + }) +} + +#[test] +fn test_state_migration_cannot_fit_any_item() { + ExtBuilder::default().build().execute_with(|| { + StateMigrationStatusValue::::put((StateMigrationStatus::NotStarted, 0)); + + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(0)); + + assert_eq!(weight, weight_for(0, 0)); + }) +} + +#[test] +fn test_state_migration_when_complete() { + ExtBuilder::default().build().execute_with(|| { + StateMigrationStatusValue::::put((StateMigrationStatus::Complete, 0)); + + let weight = LazyMigrations::on_idle(0, Weight::max_value()); + + // just reading the status of the migration + assert_eq!(weight, weight_for(1, 0)); + }) +} + +#[test] +fn test_state_migration_when_errored() { + ExtBuilder::default().build().execute_with(|| { + StateMigrationStatusValue::::put(( + StateMigrationStatus::Error("Error".as_bytes().to_vec().try_into().unwrap_or_default()), + 1, + )); + + let weight = LazyMigrations::on_idle(0, Weight::max_value()); + + // just reading the status of the migration + assert_eq!(weight, weight_for(1, 0)); + }) +} + +#[test] +fn test_state_migration_can_only_fit_one_item() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::NotStarted, 0) + ); + + let data = sp_io::storage::get(Default::default()); + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(1)); + + let reads = 2; // key read + status read + let writes = 1 + data.map(|_| 1).unwrap_or(0); + assert_eq!(weight, weight_for(reads, writes)); + + assert!(matches!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::Started(_), 1) + )); + + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(3)); + let reads = 3 + 3 + 1; // next key + key read + status + let writes = 1 + 3; // status write + key write + assert_eq!(weight, weight_for(reads, writes)); + }) +} + +#[test] +fn test_state_migration_can_only_fit_three_item() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::NotStarted, 0) + ); + + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(3)); + + // 2 next key requests (default key dons't need a next key request) + 1 status read + // 3 key reads. + // 1 status write + 2 key writes (default key doesn't have any data) + let reads = 6; + let writes = 3; + assert_eq!(weight, weight_for(reads, writes)); + + assert!(matches!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::Started(_), 3) + )); + }) +} + +#[test] +fn test_state_migration_can_fit_exactly_all_item() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::NotStarted, 0) + ); + + let (keys, data) = count_keys_and_data_without_code(); + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(keys)); + + // we deduct the extra next_key request to check if we are done. + // will know if we are done on the next call to on_idle + assert_eq!(weight, weight_for(2 * keys + 1, data + 1)); + + assert!(matches!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::Started(_), n) if n == keys, + )); + + // after calling on_idle status is added to the storage so we need to account for that + let (new_keys, new_data) = count_keys_and_data_without_code(); + let (diff_keys, diff_data) = (new_keys - keys, new_data - data); + + let weight = LazyMigrations::on_idle(0, rem_weight_for_entries(1 + diff_keys)); + // (next_key + read) for each new key + status + next_key to check if we are done + let reads = diff_keys * 2 + 2; + let writes = 1 + diff_data; // status + assert_eq!(weight, weight_for(reads, writes)); + + assert!(matches!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::Complete, n) if n == new_keys, + )); + }) +} + +#[test] +fn test_state_migration_will_migrate_10_000_items() { + ExtBuilder::default().build().execute_with(|| { + assert_eq!( + StateMigrationStatusValue::::get(), + (StateMigrationStatus::NotStarted, 0) + ); + + for i in 0..100 { + mock_contract_with_entries(i as u8, i as u64, 100); + } + + StateMigrationStatusValue::::put((StateMigrationStatus::NotStarted, 0)); + + let (keys, data) = count_keys_and_data_without_code(); + + // assuming we can only fit 100 items at a time + + let mut total_weight: Weight = Weight::zero(); + let num_of_on_idle_calls = 200; + let entries_per_on_idle = 100; + let needed_on_idle_calls = (keys as f64 / entries_per_on_idle as f64).ceil() as u64; + + // Reads: + // Read status => num_of_on_idle_calls + // Read keys => keys + // Next keys => keys - 1 + 1 skip + 1 done check + // + // Writes: + // Write status => needed_on_idle_calls + // Write keys => data + let expected_reads = (keys - 1 + 2) + keys + num_of_on_idle_calls; + let expected_writes = data + needed_on_idle_calls; + + println!("Keys: {}, Data: {}", keys, data); + println!("entries_per_on_idle: {}", entries_per_on_idle); + println!("num_of_on_idle_calls: {}", num_of_on_idle_calls); + println!("needed_on_idle_calls: {}", needed_on_idle_calls); + println!( + "Expected Reads: {}, Expected Writes: {}", + expected_reads, expected_writes + ); + + for i in 1..=num_of_on_idle_calls { + let weight = LazyMigrations::on_idle(i, rem_weight_for_entries(entries_per_on_idle)); + total_weight = total_weight.saturating_add(weight); + + let status = StateMigrationStatusValue::::get(); + if i < needed_on_idle_calls { + let migrated_so_far = i * entries_per_on_idle; + assert!( + matches!(status, (StateMigrationStatus::Started(_), n) if n == migrated_so_far), + "Status: {:?} at call: #{} doesn't match Started", + status, + i, + ); + assert!(weight.all_gte(weight_for(1, 0))); + } else { + assert!( + matches!(status, (StateMigrationStatus::Complete, n) if n == keys), + "Status: {:?} at call: {} doesn't match Complete", + status, + i, + ); + if i == needed_on_idle_calls { + // last call to on_idle + assert!(weight.all_gte(weight_for(1, 0))); + } else { + // extra calls to on_idle, just status update check + assert_eq!(weight, weight_for(1, 0)); + } + } + } + + assert_eq!(total_weight, weight_for(expected_reads, expected_writes)); + }) +} diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index 66f0f92507..fda2a59bf3 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -59,6 +59,7 @@ pallet-xcm = { workspace = true } sp-api = { workspace = true } sp-consensus-slots = { workspace = true } sp-core = { workspace = true } +sp-io = { workspace = true } sp-runtime = { workspace = true } sp-std = { workspace = true } sp-genesis-builder = { workspace = true } @@ -88,6 +89,9 @@ parity-scale-codec = { workspace = true } account = { workspace = true } +# Runtime Interfaces +cumulus-primitives-storage-weight-reclaim = { workspace = true, default-features = false } + [features] std = [ "cumulus-pallet-parachain-system/std", @@ -117,9 +121,11 @@ std = [ "precompile-utils/std", "sp-consensus-slots/std", "sp-core/std", + "sp-io/std", "sp-runtime/std", "sp-std/std", "sp-genesis-builder/std", + "cumulus-primitives-storage-weight-reclaim/std", "xcm-executor/std", "xcm-fee-payment-runtime-api/std", "xcm/std", diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index ef94f52b87..3bc8a26c88 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -48,6 +48,58 @@ where } } +// pub struct MigrateCodeToStateTrieV1(PhantomData); +// impl Migration for MigrateCodeToStateTrieV1 +// where +// Runtime: frame_system::Config, +// { +// fn friendly_name(&self) -> &str { +// "MM_MigrateCodeToStateTrieVersion1" +// } + +// fn migrate(&self, _available_weight: Weight) -> Weight { +// use cumulus_primitives_storage_weight_reclaim::get_proof_size; +// use sp_core::Get; + +// let proof_size_before: u64 = get_proof_size().unwrap_or(0); + +// let key = sp_core::storage::well_known_keys::CODE; +// let data = sp_io::storage::get(&key); +// if let Some(data) = data { +// sp_io::storage::set(&key, &data); +// } + +// let proof_size_after: u64 = get_proof_size().unwrap_or(0); +// let proof_size_diff = proof_size_after.saturating_sub(proof_size_before); + +// Weight::from_parts(0, proof_size_diff) +// .saturating_add(::DbWeight::get().reads_writes(1, 1)) +// } + +// #[cfg(feature = "try-runtime")] +// fn pre_upgrade(&self) -> Result, sp_runtime::DispatchError> { +// use parity_scale_codec::Encode; + +// let key = sp_core::storage::well_known_keys::CODE; +// let data = sp_io::storage::get(&key); +// Ok(Encode::encode(&data)) +// } + +// #[cfg(feature = "try-runtime")] +// fn post_upgrade(&self, state: Vec) -> Result<(), sp_runtime::DispatchError> { +// use frame_support::ensure; +// use parity_scale_codec::Encode; +// use sp_core::storage::StorageKey; + +// let key = StorageKey(sp_core::storage::well_known_keys::CODE.to_vec()); +// let data = sp_io::storage::get(key.as_ref()); + +// ensure!(Encode::encode(&data) == state, "Invalid state"); + +// Ok(()) +// } +// } + #[derive(parity_scale_codec::Decode, Eq, Ord, PartialEq, PartialOrd)] enum OldAssetType { Xcm(xcm::v3::Location), @@ -299,6 +351,7 @@ where // completed in runtime 2900 // Box::new(remove_pallet_democracy), // Box::new(remove_collectives_addresses), + // Box::new(MigrateCodeToStateTrieV1::(Default::default())), // completed in runtime 3200 Box::new(MigrateXcmFeesAssetsMeatdata::(Default::default())), // permanent migrations diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index fe4204b6f1..7b6c21664e 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -188,6 +188,7 @@ pub mod opaque { /// The spec_version is composed of 2x2 digits. The first 2 digits represent major changes /// that can't be skipped, such as data migration upgrades. The last 2 digits represent minor /// changes which can be skipped. +#[cfg(feature = "runtime-benchmarks")] #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("moonbase"), @@ -200,6 +201,21 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { state_version: 0, }; +/// We need to duplicate this because the `runtime_version` macro is conflicting with the +/// conditional compilation at the state_version field. +#[cfg(not(feature = "runtime-benchmarks"))] +#[sp_version::runtime_version] +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: create_runtime_str!("moonbase"), + impl_name: create_runtime_str!("moonbase"), + authoring_version: 4, + spec_version: 3300, + impl_version: 0, + apis: RUNTIME_API_VERSIONS, + transaction_version: 3, + state_version: 1, +}; + /// The version information used to identify this runtime when compiled natively. #[cfg(feature = "std")] pub fn native_version() -> NativeVersion { diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 6fb184398a..16a5847413 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -181,6 +181,7 @@ pub mod opaque { /// The spec_version is composed of 2x2 digits. The first 2 digits represent major changes /// that can't be skipped, such as data migration upgrades. The last 2 digits represent minor /// changes which can be skipped. +#[cfg(feature = "runtime-benchmarks")] #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("moonbeam"), @@ -193,6 +194,21 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { state_version: 0, }; +/// We need to duplicate this because the `runtime_version` macro is conflicting with the +/// conditional compilation at the state_version field. +#[cfg(not(feature = "runtime-benchmarks"))] +#[sp_version::runtime_version] +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: create_runtime_str!("moonbeam"), + impl_name: create_runtime_str!("moonbeam"), + authoring_version: 3, + spec_version: 3300, + impl_version: 0, + apis: RUNTIME_API_VERSIONS, + transaction_version: 3, + state_version: 1, +}; + /// The version information used to identify this runtime when compiled natively. #[cfg(feature = "std")] pub fn native_version() -> NativeVersion { diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index 1aadd64b94..288cbc6310 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -183,6 +183,7 @@ pub mod opaque { /// The spec_version is composed of 2x2 digits. The first 2 digits represent major changes /// that can't be skipped, such as data migration upgrades. The last 2 digits represent minor /// changes which can be skipped. +#[cfg(feature = "runtime-benchmarks")] #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("moonriver"), @@ -195,6 +196,21 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { state_version: 0, }; +/// We need to duplicate this because the `runtime_version` macro is conflicting with the +/// conditional compilation at the state_version field. +#[cfg(not(feature = "runtime-benchmarks"))] +#[sp_version::runtime_version] +pub const VERSION: RuntimeVersion = RuntimeVersion { + spec_name: create_runtime_str!("moonriver"), + impl_name: create_runtime_str!("moonriver"), + authoring_version: 3, + spec_version: 3300, + impl_version: 0, + apis: RUNTIME_API_VERSIONS, + transaction_version: 3, + state_version: 1, +}; + /// The version information used to identify this runtime when compiled natively. #[cfg(feature = "std")] pub fn native_version() -> NativeVersion { diff --git a/test/configs/localZombie.json b/test/configs/localZombie.json index e58260a105..e900f42e76 100644 --- a/test/configs/localZombie.json +++ b/test/configs/localZombie.json @@ -48,7 +48,7 @@ "collator": { "name": "alith", "ws_port": 33345, - "command": "../target/release/moonbeam", + "command": "tmp/moonbeam_rt", "args": [ "--no-hardware-benchmarks", "--force-authoring", diff --git a/test/configs/zombieAlphanet.json b/test/configs/zombieAlphanet.json index ee79f422a2..ac00eda010 100644 --- a/test/configs/zombieAlphanet.json +++ b/test/configs/zombieAlphanet.json @@ -49,7 +49,7 @@ "collator": { "name": "alith", "ws_port": 33345, - "command": "../target/release/moonbeam", + "command": "tmp/moonbeam_rt", "args": [ "--no-hardware-benchmarks", "--force-authoring", diff --git a/test/configs/zombieAlphanetRpc.json b/test/configs/zombieAlphanetRpc.json index 9ded9d0983..4636b2d47e 100644 --- a/test/configs/zombieAlphanetRpc.json +++ b/test/configs/zombieAlphanetRpc.json @@ -49,7 +49,7 @@ { "name": "alith", "ws_port": 33345, - "command": "../target/release/moonbeam", + "command": "tmp/moonbeam_rt", "args": [ "--no-hardware-benchmarks", "--force-authoring", diff --git a/test/configs/zombieMoonbeam.json b/test/configs/zombieMoonbeam.json index 772fd5f715..6d147486b4 100644 --- a/test/configs/zombieMoonbeam.json +++ b/test/configs/zombieMoonbeam.json @@ -46,7 +46,7 @@ "chain_spec_path": "tmp/moonbeam-modified-raw-spec.json", "collator": { "name": "alith", - "command": "../target/release/moonbeam", + "command": "tmp/moonbeam_rt", "ws_port": 33345, "args": [ "--no-hardware-benchmarks", diff --git a/test/configs/zombieMoonbeamRpc.json b/test/configs/zombieMoonbeamRpc.json index 4aaf1239e6..0fe6584ebb 100644 --- a/test/configs/zombieMoonbeamRpc.json +++ b/test/configs/zombieMoonbeamRpc.json @@ -48,7 +48,7 @@ "collators": [ { "name": "alith", - "command": "../target/release/moonbeam", + "command": "tmp/moonbeam_rt", "ws_port": 33345, "args": [ "--no-hardware-benchmarks", diff --git a/test/scripts/prepare-chainspecs-for-zombie.sh b/test/scripts/prepare-chainspecs-for-zombie.sh new file mode 100755 index 0000000000..dfdd71af00 --- /dev/null +++ b/test/scripts/prepare-chainspecs-for-zombie.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# First argument is the chain name: moonbeam or moonbase +CHAIN=$1 + +LATEST_RUNTIME_RELEASE=$(curl -s https://api.github.com/repos/moonbeam-foundation/moonbeam/releases | jq -r '.[] | select(.name | test("runtime";"i")) | .tag_name' | head -n 1 | tr -d '[:blank:]') && [[ ! -z "${LATEST_RUNTIME_RELEASE}" ]] +ENDPOINT="https://api.github.com/repos/moonbeam-foundation/moonbeam/git/refs/tags/$LATEST_RUNTIME_RELEASE" +RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" $ENDPOINT) +TYPE=$(echo $RESPONSE | jq -r '.object.type') + if [[ $TYPE == "commit" ]] + then + LATEST_RT_SHA8=$(echo $RESPONSE | jq -r '.object.sha' | cut -c -8) + elif [[ $TYPE == "tag" ]] + then + URL=$(echo $RESPONSE | jq -r '.object.url') + TAG_RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" $URL) + TAG_RESPONSE_CLEAN=$(echo $TAG_RESPONSE | tr -d '\000-\037') + LATEST_RT_SHA8=$(echo $TAG_RESPONSE_CLEAN | jq -r '.object.sha' | cut -c -8) + fi +DOCKER_TAG="moonbeamfoundation/moonbeam:sha-$LATEST_RT_SHA8" + +echo $DOCKER_TAG + +docker rm -f moonbeam_container 2> /dev/null | true +docker create --name moonbeam_container $DOCKER_TAG bash +docker cp moonbeam_container:moonbeam/moonbeam tmp/moonbeam_rt +docker rm -f moonbeam_container + +chmod uog+x tmp/moonbeam_rt +chmod uog+x ../target/release/moonbeam +echo "Building plain Moonbase specs..." +tmp/moonbeam_rt build-spec --chain $CHAIN-local > tmp/$CHAIN\-plain-spec.json +pnpm tsx scripts/modify-plain-specs.ts process tmp/$CHAIN\-plain-spec.json tmp/$CHAIN\-modified-spec.json +tmp/moonbeam_rt build-spec --chain tmp/$CHAIN\-modified-spec.json --raw > tmp/$CHAIN\-raw-spec.json +pnpm tsx scripts/preapprove-rt-rawspec.ts process tmp/$CHAIN\-raw-spec.json tmp/$CHAIN\-modified-raw-spec.json ../target/release/wbuild/$CHAIN\-runtime/$CHAIN\_runtime.compact.compressed.wasm + +echo "Done preparing chainspecs for Zombienet tests! ✅" diff --git a/test/suites/dev/moonbase/test-pov/test-evm-over-pov.ts b/test/suites/dev/moonbase/test-pov/test-evm-over-pov.ts index cd78db2cc6..aa520b0c95 100644 --- a/test/suites/dev/moonbase/test-pov/test-evm-over-pov.ts +++ b/test/suites/dev/moonbase/test-pov/test-evm-over-pov.ts @@ -14,7 +14,7 @@ describeSuite({ let contracts: HeavyContract[]; let callData: `0x${string}`; const MAX_CONTRACTS = 20; - const EXPECTED_POV_ROUGH = 500_000; // bytes + const EXPECTED_POV_ROUGH = 40_000; // bytes beforeAll(async () => { const { contractAddress, abi } = await deployCreateCompiledContract(context, "CallForwarder"); @@ -52,8 +52,8 @@ describeSuite({ const { result, block } = await context.createBlock(rawSigned); log(`block.proofSize: ${block.proofSize} (successful: ${result?.successful})`); - expect(block.proofSize).toBeGreaterThanOrEqual(EXPECTED_POV_ROUGH / 1.1); - expect(block.proofSize).toBeLessThanOrEqual(EXPECTED_POV_ROUGH * 1.1); + expect(block.proofSize).toBeGreaterThanOrEqual(EXPECTED_POV_ROUGH / 1.2); + expect(block.proofSize).toBeLessThanOrEqual(EXPECTED_POV_ROUGH * 1.2); expect(result?.successful).to.equal(true); }, }); @@ -72,8 +72,8 @@ describeSuite({ const { result, block } = await context.createBlock(rawSigned); log(`block.proof_size: ${block.proofSize} (successful: ${result?.successful})`); - expect(block.proofSize).to.be.at.least(EXPECTED_POV_ROUGH / 1.1); - expect(block.proofSize).to.be.at.most(EXPECTED_POV_ROUGH * 1.1); + expect(block.proofSize).to.be.at.least(EXPECTED_POV_ROUGH / 1.2); + expect(block.proofSize).to.be.at.most(EXPECTED_POV_ROUGH * 1.2); expect(result?.successful).to.equal(true); }, }); @@ -96,9 +96,9 @@ describeSuite({ log(`block.proof_size: ${block.proofSize} (successful: ${result?.successful})`); // The block still contain the failed (out of gas) transaction so the PoV is still included // in the block. - // 1M Gas allows ~62k of PoV, so we verify we are within range. - expect(block.proofSize).to.be.at.least(50_000); - expect(block.proofSize).to.be.at.most(100_000); + // 1M Gas allows ~38k of PoV, so we verify we are within range. + expect(block.proofSize).to.be.at.least(30_000); + expect(block.proofSize).to.be.at.most(50_000); expect(result?.successful).to.equal(true); expectEVMResult(result!.events, "Error", "OutOfGas"); }, diff --git a/test/suites/dev/moonbase/test-pov/test-evm-over-pov2.ts b/test/suites/dev/moonbase/test-pov/test-evm-over-pov2.ts index 30055bc3b9..a6be04ca34 100644 --- a/test/suites/dev/moonbase/test-pov/test-evm-over-pov2.ts +++ b/test/suites/dev/moonbase/test-pov/test-evm-over-pov2.ts @@ -63,8 +63,8 @@ describeSuite({ const { result, block } = await context.createBlock(rawSigned); log(`block.proofSize: ${block.proofSize} (successful: ${result?.successful})`); - expect(block.proofSize).toBeGreaterThanOrEqual(MAX_ETH_POV_PER_TX - 20_000n); - expect(block.proofSize).toBeLessThanOrEqual(MAX_ETH_POV_PER_TX + emptyBlockProofSize); + expect(block.proofSize).toBeGreaterThanOrEqual(30_000); + expect(block.proofSize).toBeLessThanOrEqual(50_000n + emptyBlockProofSize); expect(result?.successful).to.equal(true); }, }); diff --git a/test/suites/dev/moonbase/test-pov/test-precompile-over-pov.ts b/test/suites/dev/moonbase/test-pov/test-precompile-over-pov.ts index ee96cab715..50502c7b61 100644 --- a/test/suites/dev/moonbase/test-pov/test-precompile-over-pov.ts +++ b/test/suites/dev/moonbase/test-pov/test-precompile-over-pov.ts @@ -18,7 +18,7 @@ describeSuite({ testCases: ({ context, log, it }) => { let contracts: HeavyContract[]; const MAX_CONTRACTS = 50; - const EXPECTED_POV_ROUGH = 1_000_000; // bytes + const EXPECTED_POV_ROUGH = 55_000; // bytes let batchAbi: Abi; let proxyAbi: Abi; let proxyAddress: `0x${string}`; @@ -72,8 +72,8 @@ describeSuite({ // With 1M gas we are allowed to use ~62kb of POV, so verify the range. // The tx is still included in the block because it contains the failed tx, // so POV is included in the block as well. - expect(block.proofSize).to.be.at.least(50_000); - expect(block.proofSize).to.be.at.most(100_000); + expect(block.proofSize).to.be.at.least(35_000); + expect(block.proofSize).to.be.at.most(70_000); expect(result?.successful).to.equal(true); expectEVMResult(result!.events, "Error", "OutOfGas"); }, diff --git a/test/suites/dev/moonbase/test-pov/test-precompile-over-pov2.ts b/test/suites/dev/moonbase/test-pov/test-precompile-over-pov2.ts index f93d0f6dbc..536cbd5360 100644 --- a/test/suites/dev/moonbase/test-pov/test-precompile-over-pov2.ts +++ b/test/suites/dev/moonbase/test-pov/test-precompile-over-pov2.ts @@ -73,8 +73,8 @@ describeSuite({ }); const { result, block } = await context.createBlock(rawSigned); - expect(block.proofSize).to.be.at.least(Number(MAX_ETH_POV_PER_TX - 20_000n)); - expect(block.proofSize).to.be.at.most(Number(MAX_ETH_POV_PER_TX + emptyBlockProofSize)); + expect(block.proofSize).to.be.at.least(Number(30_000)); + expect(block.proofSize).to.be.at.most(Number(50_000n + emptyBlockProofSize)); expect(result?.successful).to.equal(true); }, }); diff --git a/test/suites/dev/moonbase/test-pov/test-xcm-to-evm-pov.ts b/test/suites/dev/moonbase/test-pov/test-xcm-to-evm-pov.ts index 22940a5a6a..fc11d6d155 100644 --- a/test/suites/dev/moonbase/test-pov/test-xcm-to-evm-pov.ts +++ b/test/suites/dev/moonbase/test-pov/test-xcm-to-evm-pov.ts @@ -22,7 +22,7 @@ describeSuite({ let proxyAddress: `0x${string}`; const MAX_CONTRACTS = 15; let contracts: HeavyContract[]; - const EXPECTED_POV_ROUGH = 350_000; // bytes + const EXPECTED_POV_ROUGH = 43_000; // bytes let balancesPalletIndex: number; let STORAGE_READ_COST: bigint; @@ -146,8 +146,8 @@ describeSuite({ // With 500k gas we are allowed to use ~150k of POV, so verify the range. // The tx is still included in the block because it contains the failed tx, // so POV is included in the block as well. - expect(block.proofSize).to.be.at.least(130_000); - expect(block.proofSize).to.be.at.most(190_000); + expect(block.proofSize).to.be.at.least(30_000); + expect(block.proofSize).to.be.at.most(45_000); // Check the evm tx was not executed because of OutOfGas error const ethEvents = (await context.polkadotJs().query.system.events()).filter(({ event }) => diff --git a/test/suites/dev/moonbase/test-xcm-v3/test-mock-hrmp-transact-ethereum-12.ts b/test/suites/dev/moonbase/test-xcm-v3/test-mock-hrmp-transact-ethereum-12.ts index 37b7127cbf..ed610bdeb2 100644 --- a/test/suites/dev/moonbase/test-xcm-v3/test-mock-hrmp-transact-ethereum-12.ts +++ b/test/suites/dev/moonbase/test-xcm-v3/test-mock-hrmp-transact-ethereum-12.ts @@ -82,6 +82,12 @@ describeSuite({ let expectedTransferredAmount = 0n; let expectedTransferredAmountPlusFees = 0n; + // Just to make sure lazy state trie migration is done + // probably not needed after migration is done + for (let i = 0; i < 10; i++) { + await context.createBlock(); + } + const targetXcmWeight = 500_000n * 25000n + STORAGE_READ_COST + 4_250_000_000n; const targetXcmFee = targetXcmWeight * 50_000n; diff --git a/test/suites/smoke/test-state-v1-migration.ts b/test/suites/smoke/test-state-v1-migration.ts new file mode 100644 index 0000000000..023d4ec8cf --- /dev/null +++ b/test/suites/smoke/test-state-v1-migration.ts @@ -0,0 +1,27 @@ +import "@moonbeam-network/api-augment/moonbase"; +import { ApiPromise } from "@polkadot/api"; +import { beforeAll, describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "S27", + title: `State V1 Migration status should not be in an error state`, + foundationMethods: "read_only", + testCases: ({ context, it, log }) => { + let paraApi: ApiPromise; + + beforeAll(async function () { + paraApi = context.polkadotJs("para"); + }); + + it({ + id: "C100", + title: "Migration status should not be in an error state", + test: async function (context) { + const stateMigrationStatus = + await paraApi.query.moonbeamLazyMigrations.stateMigrationStatusValue(); + const isError = stateMigrationStatus.toString().toLowerCase().includes("error"); + expect(isError).to.be.false; + }, + }); + }, +});