From 8d2ad09fdcf8bce99dd846824f8fe75bbac0a4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Mon, 19 Sep 2022 16:43:20 +0200 Subject: [PATCH] WIP: Nested LazyMap validation and testing --- .../storage_api/collections/lazy_map.rs | 401 ++++++---- .../storage_api/collections/lazy_vec.rs | 353 ++++----- .../src/ledger/storage_api/collections/mod.rs | 118 ++- shared/src/types/storage.rs | 5 + .../collections/nested_lazy_map.txt | 7 + tests/src/storage_api/collections/lazy_map.rs | 9 +- tests/src/storage_api/collections/lazy_vec.rs | 5 +- tests/src/storage_api/collections/mod.rs | 1 + .../collections/nested_lazy_map.rs | 723 ++++++++++++++++++ 9 files changed, 1290 insertions(+), 332 deletions(-) create mode 100644 tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt create mode 100644 tests/src/storage_api/collections/nested_lazy_map.rs diff --git a/shared/src/ledger/storage_api/collections/lazy_map.rs b/shared/src/ledger/storage_api/collections/lazy_map.rs index 3801848b8f..686f9d336e 100644 --- a/shared/src/ledger/storage_api/collections/lazy_map.rs +++ b/shared/src/ledger/storage_api/collections/lazy_map.rs @@ -1,9 +1,11 @@ //! Lazy map. +use std::collections::HashMap; +use std::fmt::Debug; +use std::hash::Hash; use std::marker::PhantomData; use borsh::{BorshDeserialize, BorshSerialize}; -use derivative::Derivative; use thiserror::Error; use super::super::Result; @@ -30,14 +32,18 @@ pub const DATA_SUBKEY: &str = "data"; /// This is different from [`super::LazyHashMap`], which hashes borsh encoded /// key. #[derive(Debug)] -pub struct LazyMap { +pub struct LazyMap { key: storage::Key, phantom_k: PhantomData, phantom_v: PhantomData, + phantom_son: PhantomData, } +/// A `LazyMap` with another `LazyCollection` inside it's value `V` +pub type NestedMap = LazyMap; + /// Possible sub-keys of a [`LazyMap`] -#[derive(Debug)] +#[derive(Clone, Debug)] pub enum SubKey { /// Data sub-key, further sub-keyed by its literal map key Data(K), @@ -45,16 +51,14 @@ pub enum SubKey { /// Possible sub-keys of a [`LazyMap`], together with their [`validation::Data`] /// that contains prior and posterior state. -#[derive(Debug)] +#[derive(Clone, Debug)] pub enum SubKeyWithData { /// Data sub-key, further sub-keyed by its literal map key Data(K, Data), } -/// Possible actions that can modify a [`LazyMap`]. This roughly corresponds to -/// the methods that have `StorageWrite` access. -/// TODO: In a nested collection, `V` may be an action inside the nested -/// collection. +/// Possible actions that can modify a simple (not nested) [`LazyMap`]. This +/// roughly corresponds to the methods that have `StorageWrite` access. #[derive(Clone, Debug)] pub enum Action { /// Insert or update a value `V` at key `K` in a [`LazyMap`]. @@ -72,14 +76,23 @@ pub enum Action { }, } -/// TODO: In a nested collection, `V` may be an action inside the nested -/// collection. -#[derive(Debug)] -pub enum Nested { - /// Insert or update a value `V` at key `K` in a [`LazyMap`]. - Insert(K, V), - /// Remove a value `V` at key `K` from a [`LazyMap`]. - Remove(K, V), +/// Possible actions that can modify a nested [`LazyMap`]. +#[derive(Clone, Debug)] +pub enum NestedAction { + /// Nested collection action `A` at key `K` + At(K, A), +} + +/// Possible sub-keys of a nested [`LazyMap`] +#[derive(Clone, Debug)] +pub enum NestedSubKey { + /// Data sub-key + Data { + /// Literal map key + key: K, + /// Sub-key in the nested collection + nested_sub_key: S, + }, } #[allow(missing_docs)] @@ -87,57 +100,228 @@ pub enum Nested { pub enum ValidationError { #[error("Storage error in reading key {0}")] StorageError(storage::Key), - // #[error("Incorrect difference in LazyVec's length")] - // InvalidLenDiff, - // #[error("An empty LazyVec must be deleted from storage")] - // EmptyVecShouldBeDeleted, - // #[error("Push at a wrong index. Got {got}, expected {expected}.")] - // UnexpectedPushIndex { got: Index, expected: Index }, - // #[error("Pop at a wrong index. Got {got}, expected {expected}.")] - // UnexpectedPopIndex { got: Index, expected: Index }, - // #[error( - // "Update (combination of pop and push) at a wrong index. Got {got}, - // \ expected {expected}." - // )] - // UnexpectedUpdateIndex { got: Index, expected: Index }, - // #[error("An index has overflown its representation: {0}")] - // IndexOverflow(>::Error), - // #[error("Unexpected underflow in `{0} - {0}`")] - // UnexpectedUnderflow(Index, Index), #[error("Invalid storage key {0}")] InvalidSubKey(storage::Key), + #[error("Invalid nested storage key {0}")] + InvalidNestedSubKey(storage::Key), } /// [`LazyMap`] validation result pub type ValidationResult = std::result::Result; -/// [`LazyMap`] validation builder from storage changes. The changes can be -/// accumulated with `LazyMap::validate()` and then turned into a list -/// of valid actions on the map with `ValidationBuilder::build()`. -#[derive(Debug, Derivative)] -// https://mcarton.github.io/rust-derivative/latest/Default.html#custom-bound -#[derivative(Default(bound = ""))] -pub struct ValidationBuilder { - /// The accumulator of found changes under the vector - pub changes: Vec>, +impl LazyCollection for LazyMap +where + K: storage::KeySeg + Clone + Hash + Eq + Debug, + V: LazyCollection + Debug, +{ + type Action = NestedAction::Action>; + type SubKey = NestedSubKey::SubKey>; + type SubKeyWithData = + NestedSubKey::SubKeyWithData>; + type Value = ::Value; + + fn open(key: storage::Key) -> Self { + Self { + key, + phantom_k: PhantomData, + phantom_v: PhantomData, + phantom_son: PhantomData, + } + } + + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result> { + let suffix = match key.split_prefix(&self.key) { + None => { + // not matching prefix, irrelevant + return Ok(None); + } + Some(None) => { + // no suffix, invalid + return Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(); + } + Some(Some(suffix)) => suffix, + }; + + // Match the suffix against expected sub-keys + match &suffix.segments[..2] { + [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] + if sub_a == DATA_SUBKEY => + { + if let Ok(key_in_kv) = storage::KeySeg::parse(sub_b.clone()) { + let nested = self.at(&key_in_kv).is_valid_sub_key(key)?; + match nested { + Some(nested_sub_key) => Ok(Some(NestedSubKey::Data { + key: key_in_kv, + nested_sub_key, + })), + None => Err(ValidationError::InvalidNestedSubKey( + key.clone(), + )) + .into_storage_result(), + } + } else { + Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result() + } + } + _ => Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(), + } + } + + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>, + { + let NestedSubKey::Data { + key, + // In here, we just have a nested sub-key without data + nested_sub_key, + } = sub_key; + // Try to read data from the nested collection + let nested_data = ::read_sub_key_data( + env, + storage_key, + nested_sub_key, + )?; + // If found, transform it back into a `NestedSubKey`, but with + // `nested_sub_key` replaced with the one we read + Ok(nested_data.map(|nested_sub_key| NestedSubKey::Data { + key, + nested_sub_key, + })) + } + + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { + // We have to group the nested sub-keys by the key from this map + let mut grouped_by_key: HashMap< + K, + Vec<::SubKeyWithData>, + > = HashMap::new(); + for NestedSubKey::Data { + key, + nested_sub_key, + } in keys + { + grouped_by_key + .entry(key) + .or_insert_with(Vec::new) + .push(nested_sub_key); + } + + // Recurse for each sub-keys group + let mut actions = vec![]; + for (key, sub_keys) in grouped_by_key { + let nested_actions = + ::validate_changed_sub_keys(sub_keys)?; + actions.extend( + nested_actions + .into_iter() + .map(|action| NestedAction::At(key.clone(), action)), + ); + } + Ok(actions) + } } -impl LazyCollection for LazyMap +impl LazyCollection for LazyMap where - K: storage::KeySeg, + K: storage::KeySeg + Debug, + V: BorshDeserialize + BorshSerialize + 'static + Debug, { + type Action = Action; + type SubKey = SubKey; + type SubKeyWithData = SubKeyWithData; + type Value = V; + /// Create or use an existing map with the given storage `key`. fn open(key: storage::Key) -> Self { Self { key, phantom_k: PhantomData, phantom_v: PhantomData, + phantom_son: PhantomData, } } + + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result> { + let suffix = match key.split_prefix(&self.key) { + None => { + // not matching prefix, irrelevant + return Ok(None); + } + Some(None) => { + // no suffix, invalid + return Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(); + } + Some(Some(suffix)) => suffix, + }; + + // Match the suffix against expected sub-keys + match &suffix.segments[..] { + [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] + if sub_a == DATA_SUBKEY => + { + if let Ok(key_in_kv) = storage::KeySeg::parse(sub_b.clone()) { + Ok(Some(SubKey::Data(key_in_kv))) + } else { + Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result() + } + } + _ => Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(), + } + } + + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>, + { + let SubKey::Data(key) = sub_key; + let data = validation::read_data(env, storage_key)?; + Ok(data.map(|data| SubKeyWithData::Data(key, data))) + } + + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { + Ok(keys + .into_iter() + .map(|change| { + let SubKeyWithData::Data(key, data) = change; + match data { + Data::Add { post } => Action::Insert(key, post), + Data::Update { pre, post } => { + Action::Update { key, pre, post } + } + Data::Delete { pre } => Action::Remove(key, pre), + } + }) + .collect()) + } } // Generic `LazyMap` methods that require no bounds on values `V` -impl LazyMap +impl LazyMap where K: storage::KeySeg, { @@ -162,10 +346,10 @@ where } // `LazyMap` methods with nested `LazyCollection`s `V` -impl LazyMap +impl LazyMap where - K: storage::KeySeg, - V: LazyCollection, + K: storage::KeySeg + Clone + Hash + Eq + Debug, + V: LazyCollection + Debug, { /// Get a nested collection at given key `key`. If there is no nested /// collection at the given key, a new empty one will be provided. The @@ -173,10 +357,39 @@ where pub fn at(&self, key: &K) -> V { V::open(self.get_data_key(key)) } + + /// An iterator visiting all key-value elements, where the values are from + /// the inner-most collection. The iterator element type is `Result<_>`, + /// because iterator's call to `next` may fail with e.g. out of gas or + /// data decoding error. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded maps to avoid gas usage increasing with the length of the + /// map. + pub fn iter<'iter>( + &'iter self, + storage: &'iter impl StorageRead<'iter>, + ) -> Result< + impl Iterator< + Item = Result<( + ::SubKey, + ::Value, + )>, + > + 'iter, + > { + let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; + Ok(iter.map(|key_val_res| { + let (key, val) = key_val_res?; + let sub_key = LazyCollection::is_valid_sub_key(self, &key)? + .ok_or(ReadError::UnexpectedlyEmptyStorageKey) + .into_storage_result()?; + Ok((sub_key, val)) + })) + } } // `LazyMap` methods with borsh encoded values `V` -impl LazyMap +impl LazyMap where K: storage::KeySeg, V: BorshDeserialize + BorshSerialize + 'static, @@ -299,96 +512,6 @@ where ) -> Result<()> { storage.write(storage_key, val) } - - /// Check if the given storage key is a valid LazyMap sub-key and if so - /// return which one - pub fn is_valid_sub_key( - &self, - key: &storage::Key, - ) -> storage_api::Result>> { - let suffix = match key.split_prefix(&self.key) { - None => { - // not matching prefix, irrelevant - return Ok(None); - } - Some(None) => { - // no suffix, invalid - return Err(ValidationError::InvalidSubKey(key.clone())) - .into_storage_result(); - } - Some(Some(suffix)) => suffix, - }; - - // Match the suffix against expected sub-keys - match &suffix.segments[..] { - [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] - if sub_a == DATA_SUBKEY => - { - if let Ok(key_in_kv) = storage::KeySeg::parse(sub_b.clone()) { - Ok(Some(SubKey::Data(key_in_kv))) - } else { - Err(ValidationError::InvalidSubKey(key.clone())) - .into_storage_result() - } - } - _ => Err(ValidationError::InvalidSubKey(key.clone())) - .into_storage_result(), - } - } - - /// Accumulate storage changes inside a [`ValidationBuilder`]. This is - /// typically done by the validity predicate while looping through the - /// changed keys. If the resulting `builder` is not `None`, one must - /// call `fn build()` on it to get the validation result. - /// This function will return `Ok(true)` if the storage key is a valid - /// sub-key of this collection, `Ok(false)` if the storage key doesn't match - /// the prefix of this collection, or fail with - /// [`ValidationError::InvalidSubKey`] if the prefix matches this - /// collection, but the key itself is not recognized. - pub fn accumulate( - &self, - env: &ENV, - builder: &mut Option>, - key_changed: &storage::Key, - ) -> storage_api::Result - where - ENV: for<'a> VpEnv<'a>, - { - if let Some(sub) = self.is_valid_sub_key(key_changed)? { - let SubKey::Data(key) = sub; - let data = validation::read_data(env, key_changed)?; - let change = data.map(|data| SubKeyWithData::Data(key, data)); - if let Some(change) = change { - let builder = - builder.get_or_insert(ValidationBuilder::default()); - builder.changes.push(change); - } - return Ok(true); - } - Ok(false) - } -} - -impl ValidationBuilder -where - K: storage::KeySeg + Ord + Clone, -{ - /// Build a list of actions from storage changes. - pub fn build(self) -> Vec> { - self.changes - .into_iter() - .map(|change| { - let SubKeyWithData::Data(key, data) = change; - match data { - Data::Add { post } => Action::Insert(key, post), - Data::Update { pre, post } => { - Action::Update { key, pre, post } - } - Data::Delete { pre } => Action::Remove(key, pre), - } - }) - .collect() - } } #[cfg(test)] diff --git a/shared/src/ledger/storage_api/collections/lazy_vec.rs b/shared/src/ledger/storage_api/collections/lazy_vec.rs index d86f33ec08..fd61bef804 100644 --- a/shared/src/ledger/storage_api/collections/lazy_vec.rs +++ b/shared/src/ledger/storage_api/collections/lazy_vec.rs @@ -1,10 +1,10 @@ //! Lazy dynamically-sized vector. use std::collections::BTreeSet; +use std::fmt::Debug; use std::marker::PhantomData; use borsh::{BorshDeserialize, BorshSerialize}; -use derivative::Derivative; use thiserror::Error; use super::super::Result; @@ -110,18 +110,15 @@ pub enum UpdateError { /// [`LazyVec`] validation result pub type ValidationResult = std::result::Result; -/// [`LazyVec`] validation builder from storage changes. The changes can be -/// accumulated with `LazyVec::validate()` and then turned into a list -/// of valid actions on the vector with `ValidationBuilder::build()`. -#[derive(Debug, Derivative)] -// https://mcarton.github.io/rust-derivative/latest/Default.html#custom-bound -#[derivative(Default(bound = ""))] -pub struct ValidationBuilder { - /// The accumulator of found changes under the vector - pub changes: Vec>, -} +impl LazyCollection for LazyVec +where + T: BorshSerialize + BorshDeserialize + 'static + Debug, +{ + type Action = Action; + type SubKey = SubKey; + type SubKeyWithData = SubKeyWithData; + type Value = T; -impl LazyCollection for LazyVec { /// Create or use an existing vector with the given storage `key`. fn open(key: storage::Key) -> Self { Self { @@ -129,131 +126,10 @@ impl LazyCollection for LazyVec { phantom: PhantomData, } } -} - -// Generic `LazyVec` methods that require no bounds on values `T` -impl LazyVec { - /// Reads the number of elements in the vector. - #[allow(clippy::len_without_is_empty)] - pub fn len(&self, storage: &S) -> Result - where - S: for<'iter> StorageRead<'iter>, - { - let len = storage.read(&self.get_len_key())?; - Ok(len.unwrap_or_default()) - } - - /// Returns `true` if the vector contains no elements. - pub fn is_empty(&self, storage: &S) -> Result - where - S: for<'iter> StorageRead<'iter>, - { - Ok(self.len(storage)? == 0) - } - - /// Get the prefix of set's elements storage - fn get_data_prefix(&self) -> storage::Key { - self.key.push(&DATA_SUBKEY.to_owned()).unwrap() - } - - /// Get the sub-key of vector's elements storage - fn get_data_key(&self, index: Index) -> storage::Key { - self.get_data_prefix().push(&index).unwrap() - } - - /// Get the sub-key of vector's length storage - fn get_len_key(&self) -> storage::Key { - self.key.push(&LEN_SUBKEY.to_owned()).unwrap() - } -} - -// `LazyVec` methods with borsh encoded values `T` -impl LazyVec -where - T: BorshSerialize + BorshDeserialize + 'static, -{ - /// Appends an element to the back of a collection. - pub fn push(&self, storage: &mut S, val: T) -> Result<()> - where - S: StorageWrite + for<'iter> StorageRead<'iter>, - { - let len = self.len(storage)?; - let data_key = self.get_data_key(len); - storage.write(&data_key, val)?; - storage.write(&self.get_len_key(), len + 1) - } - - /// Removes the last element from a vector and returns it, or `Ok(None)` if - /// it is empty. - /// - /// Note that an empty vector is completely removed from storage. - pub fn pop(&self, storage: &mut S) -> Result> - where - S: StorageWrite + for<'iter> StorageRead<'iter>, - { - let len = self.len(storage)?; - if len == 0 { - Ok(None) - } else { - let index = len - 1; - let data_key = self.get_data_key(index); - if len == 1 { - storage.delete(&self.get_len_key())?; - } else { - storage.write(&self.get_len_key(), index)?; - } - let popped_val = storage.read(&data_key)?; - storage.delete(&data_key)?; - Ok(popped_val) - } - } - - /// Update an element at the given index. - /// - /// The index must be smaller than the length of the vector, otherwise this - /// will fail with `UpdateError::InvalidIndex`. - pub fn update(&self, storage: &mut S, index: Index, val: T) -> Result<()> - where - S: StorageWrite + for<'iter> StorageRead<'iter>, - { - let len = self.len(storage)?; - if index >= len { - return Err(UpdateError::InvalidIndex { index, len }) - .into_storage_result(); - } - let data_key = self.get_data_key(index); - storage.write(&data_key, val) - } - - /// Read an element at the index or `Ok(None)` if out of bounds. - pub fn get(&self, storage: &S, index: Index) -> Result> - where - S: for<'iter> StorageRead<'iter>, - { - storage.read(&self.get_data_key(index)) - } - - /// An iterator visiting all elements. The iterator element type is - /// `Result`, because iterator's call to `next` may fail with e.g. out of - /// gas or data decoding error. - /// - /// Note that this function shouldn't be used in transactions and VPs code - /// on unbounded sets to avoid gas usage increasing with the length of the - /// set. - pub fn iter<'iter>( - &self, - storage: &'iter impl StorageRead<'iter>, - ) -> Result> + 'iter> { - let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; - Ok(iter.map(|key_val_res| { - let (_key, val) = key_val_res?; - Ok(val) - })) - } /// Check if the given storage key is a valid LazyVec sub-key and if so /// return which one - pub fn is_valid_sub_key( + fn is_valid_sub_key( &self, key: &storage::Key, ) -> storage_api::Result> { @@ -290,50 +166,27 @@ where } } - /// Accumulate storage changes inside a [`ValidationBuilder`]. This is - /// typically done by the validity predicate while looping through the - /// changed keys. If the resulting `builder` is not `None`, one must - /// call `fn build()` on it to get the validation result. - /// This function will return `Ok(true)` if the storage key is a valid - /// sub-key of this collection, `Ok(false)` if the storage key doesn't match - /// the prefix of this collection, or fail with - /// [`ValidationError::InvalidSubKey`] if the prefix matches this - /// collection, but the key itself is not recognized. - pub fn accumulate( - &self, + fn read_sub_key_data( env: &ENV, - builder: &mut Option>, - key_changed: &storage::Key, - ) -> storage_api::Result + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> where ENV: for<'a> VpEnv<'a>, { - if let Some(sub) = self.is_valid_sub_key(key_changed)? { - let change = match sub { - SubKey::Len => { - let data = validation::read_data(env, key_changed)?; - data.map(SubKeyWithData::Len) - } - SubKey::Data(index) => { - let data = validation::read_data(env, key_changed)?; - data.map(|data| SubKeyWithData::Data(index, data)) - } - }; - if let Some(change) = change { - let builder = - builder.get_or_insert(ValidationBuilder::default()); - builder.changes.push(change); + let change = match sub_key { + SubKey::Len => { + let data = validation::read_data(env, storage_key)?; + data.map(SubKeyWithData::Len) } - return Ok(true); - } - Ok(false) + SubKey::Data(index) => { + let data = validation::read_data(env, storage_key)?; + data.map(|data| SubKeyWithData::Data(index, data)) + } + }; + Ok(change) } -} -impl ValidationBuilder { - /// Validate storage changes and if valid, build from them a list of - /// actions. - /// /// The validation rules for a [`LazyVec`] are: /// - A difference in the vector's length must correspond to the /// difference in how many elements were pushed versus how many elements @@ -342,7 +195,9 @@ impl ValidationBuilder { /// - In addition, we check that indices of any changes are within an /// expected range (i.e. the vectors indices should always be /// monotonically increasing from zero) - pub fn build(self) -> ValidationResult>> { + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { let mut actions = vec![]; // We need to accumulate some values for what's changed @@ -353,14 +208,15 @@ impl ValidationBuilder { let mut updated = BTreeSet::::default(); let mut deleted = BTreeSet::::default(); - for change in self.changes { - match change { + for key in keys { + match key { SubKeyWithData::Len(data) => match data { Data::Add { post } => { if post == 0 { return Err( ValidationError::EmptyVecShouldBeDeleted, - ); + ) + .into_storage_result(); } post_gt_pre = true; len_diff = post; @@ -369,7 +225,8 @@ impl ValidationBuilder { if post == 0 { return Err( ValidationError::EmptyVecShouldBeDeleted, - ); + ) + .into_storage_result(); } if post > pre { post_gt_pre = true; @@ -403,11 +260,13 @@ impl ValidationBuilder { let added_len: u64 = added .len() .try_into() - .map_err(ValidationError::IndexOverflow)?; + .map_err(ValidationError::IndexOverflow) + .into_storage_result()?; let deleted_len: u64 = deleted .len() .try_into() - .map_err(ValidationError::IndexOverflow)?; + .map_err(ValidationError::IndexOverflow) + .into_storage_result()?; if len_diff != 0 && !(if post_gt_pre { @@ -416,7 +275,7 @@ impl ValidationBuilder { added_len + len_diff == deleted_len }) { - return Err(ValidationError::InvalidLenDiff); + return Err(ValidationError::InvalidLenDiff).into_storage_result(); } let mut last_added = Option::None; @@ -430,7 +289,8 @@ impl ValidationBuilder { return Err(ValidationError::UnexpectedPushIndex { got: index, expected, - }); + }) + .into_storage_result(); } } else if index != len_pre { // The first addition must be at the pre length value. @@ -440,7 +300,8 @@ impl ValidationBuilder { return Err(ValidationError::UnexpectedPushIndex { got: index, expected: len_pre, - }); + }) + .into_storage_result(); } last_added = Some(index); } @@ -456,7 +317,8 @@ impl ValidationBuilder { return Err(ValidationError::UnexpectedPopIndex { got: index, expected, - }); + }) + .into_storage_result(); } } last_deleted = Some(index); @@ -469,7 +331,8 @@ impl ValidationBuilder { return Err(ValidationError::UnexpectedPopIndex { got: index, expected: len_pre, - }); + }) + .into_storage_result(); } } } @@ -482,7 +345,8 @@ impl ValidationBuilder { return Err(ValidationError::UnexpectedUpdateIndex { got: index, max, - }); + }) + .into_storage_result(); } } @@ -490,6 +354,127 @@ impl ValidationBuilder { } } +// Generic `LazyVec` methods that require no bounds on values `T` +impl LazyVec { + /// Reads the number of elements in the vector. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + let len = storage.read(&self.get_len_key())?; + Ok(len.unwrap_or_default()) + } + + /// Returns `true` if the vector contains no elements. + pub fn is_empty(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + Ok(self.len(storage)? == 0) + } + + /// Get the prefix of set's elements storage + fn get_data_prefix(&self) -> storage::Key { + self.key.push(&DATA_SUBKEY.to_owned()).unwrap() + } + + /// Get the sub-key of vector's elements storage + fn get_data_key(&self, index: Index) -> storage::Key { + self.get_data_prefix().push(&index).unwrap() + } + + /// Get the sub-key of vector's length storage + fn get_len_key(&self) -> storage::Key { + self.key.push(&LEN_SUBKEY.to_owned()).unwrap() + } +} + +// `LazyVec` methods with borsh encoded values `T` +impl LazyVec +where + T: BorshSerialize + BorshDeserialize + 'static, +{ + /// Appends an element to the back of a collection. + pub fn push(&self, storage: &mut S, val: T) -> Result<()> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + let data_key = self.get_data_key(len); + storage.write(&data_key, val)?; + storage.write(&self.get_len_key(), len + 1) + } + + /// Removes the last element from a vector and returns it, or `Ok(None)` if + /// it is empty. + /// + /// Note that an empty vector is completely removed from storage. + pub fn pop(&self, storage: &mut S) -> Result> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + if len == 0 { + Ok(None) + } else { + let index = len - 1; + let data_key = self.get_data_key(index); + if len == 1 { + storage.delete(&self.get_len_key())?; + } else { + storage.write(&self.get_len_key(), index)?; + } + let popped_val = storage.read(&data_key)?; + storage.delete(&data_key)?; + Ok(popped_val) + } + } + + /// Update an element at the given index. + /// + /// The index must be smaller than the length of the vector, otherwise this + /// will fail with `UpdateError::InvalidIndex`. + pub fn update(&self, storage: &mut S, index: Index, val: T) -> Result<()> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + if index >= len { + return Err(UpdateError::InvalidIndex { index, len }) + .into_storage_result(); + } + let data_key = self.get_data_key(index); + storage.write(&data_key, val) + } + + /// Read an element at the index or `Ok(None)` if out of bounds. + pub fn get(&self, storage: &S, index: Index) -> Result> + where + S: for<'iter> StorageRead<'iter>, + { + storage.read(&self.get_data_key(index)) + } + + /// An iterator visiting all elements. The iterator element type is + /// `Result`, because iterator's call to `next` may fail with e.g. out of + /// gas or data decoding error. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded sets to avoid gas usage increasing with the length of the + /// set. + pub fn iter<'iter>( + &self, + storage: &'iter impl StorageRead<'iter>, + ) -> Result> + 'iter> { + let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; + Ok(iter.map(|key_val_res| { + let (_key, val) = key_val_res?; + Ok(val) + })) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/shared/src/ledger/storage_api/collections/mod.rs b/shared/src/ledger/storage_api/collections/mod.rs index b3e1b4af0c..b77b207c7f 100644 --- a/shared/src/ledger/storage_api/collections/mod.rs +++ b/shared/src/ledger/storage_api/collections/mod.rs @@ -7,21 +7,27 @@ //! just receive the storage sub-keys that have experienced changes without //! having to check any of the unchanged elements. +use std::fmt::Debug; + +use borsh::BorshDeserialize; +use derivative::Derivative; use thiserror::Error; mod hasher; -pub mod lazy_hashmap; -pub mod lazy_hashset; +// pub mod lazy_hashmap; +// pub mod lazy_hashset; pub mod lazy_map; -pub mod lazy_set; +// pub mod lazy_set; pub mod lazy_vec; -pub use lazy_hashmap::LazyHashMap; -pub use lazy_hashset::LazyHashSet; +// pub use lazy_hashmap::LazyHashMap; +// pub use lazy_hashset::LazyHashSet; pub use lazy_map::LazyMap; -pub use lazy_set::LazySet; +// pub use lazy_set::LazySet; pub use lazy_vec::LazyVec; +use crate::ledger::storage_api; +use crate::ledger::vp_env::VpEnv; use crate::types::storage; #[allow(missing_docs)] @@ -31,6 +37,14 @@ pub enum ReadError { UnexpectedlyEmptyStorageKey, } +/// Simple lazy collection with borsh deserializable elements +#[derive(Debug)] +pub struct Simple; + +/// Lazy collection with a nested lazy collection +#[derive(Debug)] +pub struct Nested; + /// A lazy collection of storage values is a handler with some storage prefix /// that is given to its `fn new()`. The values are typically nested under this /// prefix and they can be changed individually (e.g. without reading in the @@ -40,6 +54,98 @@ pub enum ReadError { /// /// An empty collection must be deleted from storage. pub trait LazyCollection { + /// Actions on the collection determined from changed storage keys by + /// `Self::validate` + type Action; + + /// Possible sub-keys in the collection + type SubKey: Debug; + + /// Possible sub-keys together with the data read from storage + type SubKeyWithData: Debug; + + /// A type of a value in the inner-most collection + type Value: BorshDeserialize; + /// Create or use an existing vector with the given storage `key`. fn open(key: storage::Key) -> Self; + + /// Check if the given storage key is a valid LazyVec sub-key and if so + /// return which one. Returns: + /// - `Ok(Some(_))` if it's a valid sub-key + /// - `Ok(None)` if it's not a sub-key + /// - `Err(_)` if it's an invalid sub-key + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result>; + + /// Try to read and decode the data for each change storage key in prior and + /// posterior state. If there is no value in neither prior or posterior + /// state (which is a possible state when transaction e.g. writes and then + /// deletes one storage key, but it is treated as a no-op as it doesn't + /// affect result of validation), returns `Ok(None)`. + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>; + + /// Validate changed sub-keys associated with their data and return back + /// a vector of `Self::Action`s, if the changes are valid + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result>; + + /// Accumulate storage changes inside a `ValidationBuilder`. This is + /// typically done by the validity predicate while looping through the + /// changed keys. If the resulting `builder` is not `None`, one must + /// call `fn build()` on it to get the validation result. + /// This function will return `Ok(true)` if the storage key is a valid + /// sub-key of this collection, `Ok(false)` if the storage key doesn't match + /// the prefix of this collection, or fail with + /// [`ValidationError::InvalidSubKey`] if the prefix matches this + /// collection, but the key itself is not recognized. + fn accumulate( + &self, + env: &ENV, + builder: &mut Option>, + key_changed: &storage::Key, + ) -> storage_api::Result + where + ENV: for<'a> VpEnv<'a>, + { + if let Some(sub) = self.is_valid_sub_key(key_changed)? { + let change = Self::read_sub_key_data(env, key_changed, sub)?; + if let Some(change) = change { + let builder = + builder.get_or_insert(ValidationBuilder::default()); + builder.changes.push(change); + } + return Ok(true); + } + Ok(false) + } + + /// Execute validation on the validation builder, to be called when + /// `accumulate` instantiates the builder to `Some(_)`, after all the + /// changes storage keys have been processed. + fn validate( + builder: ValidationBuilder, + ) -> storage_api::Result> { + Self::validate_changed_sub_keys(builder.changes) + } +} + +/// Validation builder from storage changes. The changes can +/// be accumulated with `LazyCollection::accumulate()` and then turned into a +/// list of valid actions on the collection with `LazyCollection::validate()`. +#[derive(Debug, Derivative)] +// https://mcarton.github.io/rust-derivative/latest/Default.html#custom-bound +#[derivative(Default(bound = ""))] +pub struct ValidationBuilder { + /// The accumulator of found changes under the vector + pub changes: Vec, } diff --git a/shared/src/types/storage.rs b/shared/src/types/storage.rs index 1c3f6b1313..c9f87908fe 100644 --- a/shared/src/types/storage.rs +++ b/shared/src/types/storage.rs @@ -285,6 +285,11 @@ impl Key { self.len() == 0 } + /// Returns the first segment of the key, or `None` if it is empty. + pub fn first(&self) -> Option<&DbKeySeg> { + self.segments.first() + } + /// Returns the last segment of the key, or `None` if it is empty. pub fn last(&self) -> Option<&DbKeySeg> { self.segments.last() diff --git a/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt b/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt new file mode 100644 index 0000000000..d587a9680e --- /dev/null +++ b/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b5ce7502439712f95a4b50de0d5455e0a6788cc95dbd535e749d5717da0ee8e1 # shrinks to (initial_state, transitions) = (AbstractLazyMapState { valid_transitions: [], committed_transitions: [] }, [Insert((22253647846329582, -2060910714, -85), TestVal { x: 16862967849328560500, y: true })]) diff --git a/tests/src/storage_api/collections/lazy_map.rs b/tests/src/storage_api/collections/lazy_map.rs index c7a309ab4d..afff09bbf1 100644 --- a/tests/src/storage_api/collections/lazy_map.rs +++ b/tests/src/storage_api/collections/lazy_map.rs @@ -137,7 +137,9 @@ mod tests { Transition::Update(key, val) ), 3 => arb_existing_map_key().prop_map(Transition::Remove), - 5 => (arb_map_key().prop_filter("insert on non-existing keys only", move |key| !keys.contains(&key)), arb_map_val()).prop_map(|(key, val)| Transition::Insert(key, val)) + 5 => (arb_map_key().prop_filter("insert on non-existing keys only", + move |key| !keys.contains(key)), arb_map_val()) + .prop_map(|(key, val)| Transition::Insert(key, val)) ] .boxed() } @@ -426,7 +428,10 @@ mod tests { validation_builder.is_some(), "If some keys were changed, the builder must get filled in" ); - let actions = validation_builder.unwrap().build(); + let actions = LazyMap::::validate( + validation_builder.unwrap(), + ) + .unwrap(); let mut actions_to_check = actions.clone(); // Check that every transition has a corresponding action from diff --git a/tests/src/storage_api/collections/lazy_vec.rs b/tests/src/storage_api/collections/lazy_vec.rs index 20ee80592d..65e08b4ca7 100644 --- a/tests/src/storage_api/collections/lazy_vec.rs +++ b/tests/src/storage_api/collections/lazy_vec.rs @@ -418,7 +418,10 @@ mod tests { validation_builder.is_some(), "If some keys were changed, the builder must get filled in" ); - let actions = validation_builder.unwrap().build().expect( + let actions = LazyVec::::validate( + validation_builder.unwrap(), + ) + .expect( "With valid transitions only, validation should always \ pass", ); diff --git a/tests/src/storage_api/collections/mod.rs b/tests/src/storage_api/collections/mod.rs index fc7c5832ce..f39b880c09 100644 --- a/tests/src/storage_api/collections/mod.rs +++ b/tests/src/storage_api/collections/mod.rs @@ -1,2 +1,3 @@ mod lazy_map; mod lazy_vec; +mod nested_lazy_map; diff --git a/tests/src/storage_api/collections/nested_lazy_map.rs b/tests/src/storage_api/collections/nested_lazy_map.rs new file mode 100644 index 0000000000..80b066c18f --- /dev/null +++ b/tests/src/storage_api/collections/nested_lazy_map.rs @@ -0,0 +1,723 @@ +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::convert::TryInto; + + use borsh::{BorshDeserialize, BorshSerialize}; + use namada::types::address::{self, Address}; + use namada::types::storage; + use namada_tx_prelude::storage::KeySeg; + use namada_tx_prelude::storage_api::collections::lazy_map::{ + NestedMap, NestedSubKey, SubKey, + }; + use namada_tx_prelude::storage_api::collections::{ + lazy_map, LazyCollection, LazyMap, + }; + use proptest::prelude::*; + use proptest::prop_state_machine; + use proptest::state_machine::{AbstractStateMachine, StateMachineTest}; + use proptest::test_runner::Config; + use test_log::test; + + use crate::tx::tx_host_env; + use crate::vp::vp_host_env; + + prop_state_machine! { + #![proptest_config(Config { + // Instead of the default 256, we only run 5 because otherwise it + // takes too long and it's preferable to crank up the number of + // transitions instead, to allow each case to run for more epochs as + // some issues only manifest once the model progresses further. + // Additionally, more cases will be explored every time this test is + // executed in the CI. + cases: 5, + .. Config::default() + })] + #[test] + fn nested_lazy_map_api_state_machine_test(sequential 1..100 => ConcreteLazyMapState); + } + + /// Some borsh-serializable type with arbitrary fields to be used inside + /// LazyMap state machine test + #[derive( + Clone, + Debug, + BorshSerialize, + BorshDeserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + struct TestVal { + x: u64, + y: bool, + } + + type KeyOuter = u64; + type KeyMiddle = i32; + type KeyInner = i8; + + type NestedTestMap = + NestedMap>>; + + type NestedEagerMap = + BTreeMap>>; + + /// A `StateMachineTest` implemented on this struct manipulates it with + /// `Transition`s, which are also being accumulated into + /// `current_transitions`. It then: + /// + /// - checks its state against an in-memory `std::collections::HashMap` + /// - runs validation and checks that the `LazyMap::Action`s reported from + /// validation match with transitions that were applied + /// + /// Additionally, one of the transitions is to commit a block and/or + /// transaction, during which the currently accumulated state changes are + /// persisted, or promoted from transaction write log to block's write log. + #[derive(Debug)] + struct ConcreteLazyMapState { + /// Address is used to prefix the storage key of the `lazy_map` in + /// order to simulate a transaction and a validity predicate + /// check from changes on the `lazy_map` + address: Address, + /// In the test, we apply the same transitions on the `lazy_map` as on + /// `eager_map` to check that `lazy_map`'s state is consistent with + /// `eager_map`. + eager_map: NestedEagerMap, + /// Handle to a lazy map with nested lazy collections + lazy_map: NestedTestMap, + /// Valid LazyMap changes in the current transaction + current_transitions: Vec, + } + + #[derive(Clone, Debug, Default)] + struct AbstractLazyMapState { + /// Valid LazyMap changes in the current transaction + valid_transitions: Vec, + /// Valid LazyMap changes committed to storage + committed_transitions: Vec, + } + + /// Possible transitions that can modify a [`NestedTestMap`]. + /// This roughly corresponds to the methods that have `StorageWrite` + /// access and is very similar to [`Action`] + #[derive(Clone, Debug)] + enum Transition { + /// Commit all valid transitions in the current transaction + CommitTx, + /// Commit all valid transitions in the current transaction and also + /// commit the current block + CommitTxAndBlock, + /// Insert a key-val into a [`LazyMap`] + Insert(Key, TestVal), + /// Remove a key-val from a [`LazyMap`] + Remove(Key), + /// Update a value at key from pre to post state in a + /// [`LazyMap`] + Update(Key, TestVal), + } + + /// A key for transition + type Key = (KeyOuter, KeyMiddle, KeyInner); + + impl AbstractStateMachine for AbstractLazyMapState { + type State = Self; + type Transition = Transition; + + fn init_state() -> BoxedStrategy { + Just(Self::default()).boxed() + } + + // Apply a random transition to the state + fn transitions(state: &Self::State) -> BoxedStrategy { + let length = state.len(); + if length == 0 { + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_map_key(), arb_map_val()).prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } else { + let keys = state.find_existing_keys(); + let arb_existing_map_key = + || proptest::sample::select(keys.clone()); + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_existing_map_key(), arb_map_val()).prop_map(|(key, val)| + Transition::Update(key, val)), + 3 => arb_existing_map_key().prop_map(Transition::Remove), + 5 => (arb_map_key().prop_filter( + "insert on non-existing keys only", + move |key| !keys.contains(key)), arb_map_val()) + .prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } + } + + fn apply_abstract( + mut state: Self::State, + transition: &Self::Transition, + ) -> Self::State { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + let valid_actions_to_commit = + std::mem::take(&mut state.valid_transitions); + state + .committed_transitions + .extend(valid_actions_to_commit.into_iter()); + } + _ => state.valid_transitions.push(transition.clone()), + } + state + } + + fn preconditions( + state: &Self::State, + transition: &Self::Transition, + ) -> bool { + let length = state.len(); + // Ensure that the remove or update transitions are not applied + // to an empty state + if length == 0 + && matches!( + transition, + Transition::Remove(_) | Transition::Update(_, _) + ) + { + return false; + } + match transition { + Transition::Update(key, _) | Transition::Remove(key) => { + let keys = state.find_existing_keys(); + // Ensure that the update/remove key is an existing one + keys.contains(key) + } + Transition::Insert(key, _) => { + let keys = state.find_existing_keys(); + // Ensure that the insert key is not an existing one + !keys.contains(key) + } + _ => true, + } + } + } + + impl StateMachineTest for ConcreteLazyMapState { + type Abstract = AbstractLazyMapState; + type ConcreteState = Self; + + fn init_test( + _initial_state: ::State, + ) -> Self::ConcreteState { + // Init transaction env in which we'll be applying the transitions + tx_host_env::init(); + + // The lazy_map's path must be prefixed by the address to be able + // to trigger a validity predicate on it + let address = address::testing::established_address_1(); + tx_host_env::with(|env| env.spawn_accounts([&address])); + let lazy_map_prefix: storage::Key = address.to_db_key().into(); + + Self { + address, + eager_map: BTreeMap::new(), + lazy_map: NestedTestMap::open( + lazy_map_prefix.push(&"arbitrary".to_string()).unwrap(), + ), + current_transitions: vec![], + } + } + + fn apply_concrete( + mut state: Self::ConcreteState, + transition: ::Transition, + ) -> Self::ConcreteState { + // Apply transitions in transaction env + let ctx = tx_host_env::ctx(); + + // Persist the transitions in the current tx, or clear previous ones + // if we're committing a tx + match &transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + state.current_transitions = vec![]; + } + _ => { + state.current_transitions.push(transition.clone()); + } + } + + // Transition application on lazy map and post-conditions: + match &transition { + Transition::CommitTx => { + // commit the tx without committing the block + tx_host_env::with(|env| env.write_log.commit_tx()); + } + Transition::CommitTxAndBlock => { + // commit the tx and the block + tx_host_env::commit_tx_and_block(); + } + Transition::Insert( + (key_outer, key_middle, key_inner), + value, + ) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + inner.insert(ctx, *key_inner, value.clone()).unwrap(); + + // Post-conditions: + let stored_value = + inner.get(ctx, key_inner).unwrap().unwrap(); + assert_eq!( + &stored_value, value, + "the new item must be added to the back" + ); + + state.assert_validation_accepted(); + } + Transition::Remove((key_outer, key_middle, key_inner)) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + let removed = + inner.remove(ctx, key_inner).unwrap().unwrap(); + + // Post-conditions: + assert_eq!( + &removed, + state + .eager_map + .get(key_outer) + .unwrap() + .get(key_middle) + .unwrap() + .get(key_inner) + .unwrap(), + "removed element matches the value in eager map \ + before it's updated" + ); + + state.assert_validation_accepted(); + } + Transition::Update( + (key_outer, key_middle, key_inner), + value, + ) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + let old_val = inner.get(ctx, key_inner).unwrap().unwrap(); + + inner.insert(ctx, *key_inner, value.clone()).unwrap(); + + // Post-conditions: + let new_val = inner.get(ctx, key_inner).unwrap().unwrap(); + assert_eq!( + &old_val, + state + .eager_map + .get(key_outer) + .unwrap() + .get(key_middle) + .unwrap() + .get(key_inner) + .unwrap(), + "old value must match the value at the same key in \ + the eager map before it's updated" + ); + assert_eq!( + &new_val, value, + "new value must match that which was passed into the \ + Transition::Update" + ); + + state.assert_validation_accepted(); + } + } + + // Apply transition in the eager map for comparison + apply_transition_on_eager_map(&mut state.eager_map, &transition); + + // Global post-conditions: + + // All items in eager map must be present in lazy map + for (key_outer, middle) in state.eager_map.iter() { + for (key_middle, inner) in middle { + for (key_inner, expected_item) in inner { + let got = state + .lazy_map + .at(key_outer) + .at(key_middle) + .get(ctx, key_inner) + .unwrap() + .expect( + "The expected item must be present in lazy map", + ); + assert_eq!( + expected_item, &got, + "at key {key_outer}, {key_middle} {key_inner}" + ); + } + } + } + + // All items in lazy map must be present in eager map + for key_val in state.lazy_map.iter(ctx).unwrap() { + let ( + NestedSubKey::Data { + key: key_outer, + nested_sub_key: + NestedSubKey::Data { + key: key_middle, + nested_sub_key: SubKey::Data(key_inner), + }, + }, + expected_val, + ) = key_val.unwrap(); + let got = state + .eager_map + .get(&key_outer) + .unwrap() + .get(&key_middle) + .unwrap() + .get(&key_inner) + .expect("The expected item must be present in eager map"); + assert_eq!( + &expected_val, got, + "at key {key_outer}, {key_middle} {key_inner})" + ); + } + + state + } + } + + impl AbstractLazyMapState { + /// Find the length of the map from the applied transitions + fn len(&self) -> u64 { + (map_len_diff_from_transitions(self.committed_transitions.iter()) + + map_len_diff_from_transitions(self.valid_transitions.iter())) + .try_into() + .expect( + "It shouldn't be possible to underflow length from all \ + transactions applied in abstract state", + ) + } + + /// Build an eager map from the committed and current transitions + fn eager_map(&self) -> NestedEagerMap { + let mut eager_map = BTreeMap::new(); + for transition in &self.committed_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + for transition in &self.valid_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + eager_map + } + + /// Find the keys currently present in the map + fn find_existing_keys(&self) -> Vec { + let outer_map = self.eager_map(); + outer_map + .into_iter() + .fold(vec![], |acc, (outer, middle_map)| { + middle_map.into_iter().fold( + acc, + |mut acc, (middle, inner_map)| { + acc.extend( + inner_map + .into_iter() + .map(|(inner, _)| (outer, middle, inner)), + ); + acc + }, + ) + }) + } + } + + /// Find the difference in length of the map from the applied transitions + fn map_len_diff_from_transitions<'a>( + transitions: impl Iterator, + ) -> i64 { + let mut insert_count: i64 = 0; + let mut remove_count: i64 = 0; + + for trans in transitions { + match trans { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Update(_, _) => {} + Transition::Insert(_, _) => insert_count += 1, + Transition::Remove(_) => remove_count += 1, + } + } + insert_count - remove_count + } + + impl ConcreteLazyMapState { + fn assert_validation_accepted(&self) { + // Init the VP env from tx env in which we applied the map + // transitions + let tx_env = tx_host_env::take(); + vp_host_env::init_from_tx(self.address.clone(), tx_env, |_| {}); + + // Simulate a validity predicate run using the lazy map's validation + // helpers + let changed_keys = + vp_host_env::with(|env| env.all_touched_storage_keys()); + + let mut validation_builder = None; + + // Push followed by pop is a no-op, in which case we'd still see the + // changed keys for these actions, but they wouldn't affect the + // validation result and they never get persisted, but we'd still + // them as changed key here. To guard against this case, + // we check that `map_len_from_transitions` is not empty. + let map_len_diff = + map_len_diff_from_transitions(self.current_transitions.iter()); + + // To help debug validation issues... + dbg!( + &self.current_transitions, + &changed_keys + .iter() + .map(storage::Key::to_string) + .collect::>() + ); + + for key in &changed_keys { + let is_sub_key = self + .lazy_map + .accumulate( + vp_host_env::ctx(), + &mut validation_builder, + key, + ) + .unwrap(); + + assert!( + is_sub_key, + "We're only modifying the lazy_map's keys here. Key: \ + \"{key}\", map length diff {map_len_diff}" + ); + } + if !changed_keys.is_empty() && map_len_diff != 0 { + assert!( + validation_builder.is_some(), + "If some keys were changed, the builder must get filled in" + ); + let actions = + NestedTestMap::validate(validation_builder.unwrap()) + .unwrap(); + let mut actions_to_check = actions.clone(); + + // Check that every transition has a corresponding action from + // validation. We drop the found actions to check that all + // actions are matched too. + let current_transitions = + normalize_transitions(&self.current_transitions); + for transition in ¤t_transitions { + use lazy_map::Action; + use lazy_map::NestedAction::At; + + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + } + Transition::Insert(expected_key, expected_val) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Insert(key_inner, val), + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key + && expected_val == val + { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Remove(expected_key) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Remove(key_inner, _val), + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Update(expected_key, value) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Update { + key: key_inner, + pre: _, + post, + }, + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key && post == value { + actions_to_check.remove(ix); + break; + } + } + } + } + } + } + + assert!( + actions_to_check.is_empty(), + "All the actions reported from validation {actions:#?} \ + should have been matched with SM transitions \ + {current_transitions:#?}, but these actions didn't \ + match: {actions_to_check:#?}", + ) + } + + // Put the tx_env back before checking the result + tx_host_env::set_from_vp_env(vp_host_env::take()); + } + } + + /// Generate an arbitrary `TestKey` + fn arb_map_key() -> impl Strategy { + (any::(), any::(), any::()) + } + + /// Generate an arbitrary `TestVal` + fn arb_map_val() -> impl Strategy { + (any::(), any::()).prop_map(|(x, y)| TestVal { x, y }) + } + + /// Apply `Transition` on an eager `Map`. + fn apply_transition_on_eager_map( + map: &mut NestedEagerMap, + transition: &Transition, + ) { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => {} + Transition::Insert((key_outer, key_middle, key_inner), value) + | Transition::Update((key_outer, key_middle, key_inner), value) => { + let middle = + map.entry(*key_outer).or_insert_with(Default::default); + let inner = + middle.entry(*key_middle).or_insert_with(Default::default); + inner.insert(*key_inner, value.clone()); + } + Transition::Remove((key_outer, key_middle, key_inner)) => { + let middle = + map.entry(*key_outer).or_insert(Default::default()); + let inner = + middle.entry(*key_middle).or_insert_with(Default::default); + let _popped = inner.remove(key_inner); + } + } + } + + /// Normalize transitions: + /// - remove(key) + insert(key, val) -> update(key, val) + /// - insert(key, val) + update(key, new_val) -> insert(key, new_val) + /// - update(key, val) + update(key, new_val) -> update(key, new_val) + /// + /// Note that the normalizable transitions pairs do not have to be directly + /// next to each other, but their order does matter. + fn normalize_transitions(transitions: &[Transition]) -> Vec { + let mut collapsed = vec![]; + 'outer: for transition in transitions { + match transition { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Remove(_) => collapsed.push(transition.clone()), + Transition::Insert(key, val) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Remove(remove_key) = + collapsed_transition + { + if key == remove_key { + // remove(key) + insert(key, val) -> update(key, + // val) + + // Replace the Remove with an Update instead of + // inserting the Insert + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, val.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + Transition::Update(key, value) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Insert(insert_key, _) = + collapsed_transition + { + if key == insert_key { + // insert(key, val) + update(key, new_val) -> + // insert(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Insert(*key, value.clone()); + continue 'outer; + } + } else if let Transition::Update(update_key, _) = + collapsed_transition + { + if key == update_key { + // update(key, val) + update(key, new_val) -> + // update(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, value.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + } + } + collapsed + } +}