Switch to use storage::KeySeg and add LazySet

Also refactored the iterators implementation to take advantage of
changes from #335. Note that this requires `'static` lifetime bound on
the types of the collections' elements, which means we cannot use
non-static references, but we wouldn't do that anyway.

shared/src/ledger/storage_api/collections/lazy_hashmap.rs

@@ -1,4 +1,4 @@
-//! Lazy hash map
+//! Lazy hash map.
 
 use std::marker::PhantomData;
 
@@ -12,7 +12,22 @@ use crate::types::storage;
 /// Subkey corresponding to the data elements of the LazyMap
 pub const DATA_SUBKEY: &str = "data";
 
-/// LazyHashmap ! fill in !
+/// Lazy hash map.
+///
+/// This can be used as an alternative to `std::collections::HashMap` and
+/// `BTreeMap`. In the lazy map, the elements do not reside in memory but are
+/// instead read and written to storage sub-keys of the storage `key` given to
+/// construct the map.
+///
+/// In the [`LazyHashMap`], the type of key `K` can be anything that
+/// [`BorshSerialize`] and [`BorshDeserialize`] and a hex string of sha256 hash
+/// over the borsh encoded keys are used as storage key segments.
+///
+/// This is different from [`super::LazyMap`], which uses [`storage::KeySeg`]
+/// trait.
+///
+/// Additionally, [`LazyHashMap`] also writes the unhashed values into the
+/// storage together with the values (using an internal `KeyVal` type).
 pub struct LazyHashMap<K, V> {
     key: storage::Key,
     phantom_k: PhantomData<K>,
@@ -26,10 +41,10 @@ struct KeyVal<K, V> {
     val: V,
 }
 
-impl<K, V> LazyMap<K, V>
+impl<K, V> LazyHashMap<K, V>
 where
-    K: BorshDeserialize + BorshSerialize,
-    V: BorshDeserialize + BorshSerialize,
+    K: BorshDeserialize + BorshSerialize + 'static,
+    V: BorshDeserialize + BorshSerialize + 'static,
 {
     /// Create or use an existing map with the given storage `key`.
     pub fn new(key: storage::Key) -> Self {
@@ -85,7 +100,7 @@ where
         key: &K,
     ) -> Result<Option<V>> {
         let res = self.get_key_val(storage, key)?;
-        Ok(res.map(|elem| elem.1))
+        Ok(res.map(|(_key, val)| val))
     }
 
     /// Returns the key-value corresponding to the key, if any.
@@ -120,23 +135,12 @@ where
         &self,
         storage: &'a impl StorageRead,
     ) -> Result<impl Iterator<Item = Result<(K, V)>> + 'a> {
-        let iter = storage.iter_prefix(&self.get_data_prefix())?;
-        let iter = itertools::unfold(iter, |iter| {
-            match storage.iter_next(iter) {
-                Ok(Some((_key, value))) => {
-                    match KeyVal::<K, V>::try_from_slice(&value[..]) {
-                        Ok(KeyVal { key, val }) => Some(Ok((key, val))),
-                        Err(err) => Some(Err(storage_api::Error::new(err))),
-                    }
-                }
-                Ok(None) => None,
-                Err(err) => {
-                    // Propagate errors into Iterator's Item
-                    Some(Err(err))
-                }
-            }
-        });
-        Ok(iter)
+        let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?;
+        Ok(iter.map(|key_val_res| {
+            let (_key, val) = key_val_res?;
+            let KeyVal { key, val } = val;
+            Ok((key, val))
+        }))
     }
 
     /// Reads a key-value from storage

shared/src/ledger/storage_api/collections/lazy_hashset.rs

@@ -0,0 +1,119 @@
+//! Lazy hash set.
+
+use std::marker::PhantomData;
+
+use borsh::{BorshDeserialize, BorshSerialize};
+
+use super::super::Result;
+use super::hasher::hash_for_storage_key;
+use crate::ledger::storage_api::{self, StorageRead, StorageWrite};
+use crate::types::storage;
+
+/// Subkey corresponding to the data elements of the LazySet
+pub const DATA_SUBKEY: &str = "data";
+
+/// Lazy hash set.
+///
+/// This can be used as an alternative to `std::collections::HashSet` and
+/// `BTreeSet`. In the lazy set, the elements do not reside in memory but are
+/// instead read and written to storage sub-keys of the storage `key` given to
+/// construct the set.
+///
+/// In the [`LazyHashSet`], the type of value `T` can be anything that
+/// [`BorshSerialize`] and [`BorshDeserialize`] and a hex string of sha256 hash
+/// over the borsh encoded values are used as storage key segments.
+///
+/// This is different from [`super::LazySet`], which uses [`storage::KeySeg`]
+/// trait.
+///
+/// Additionally, [`LazyHashSet`] also writes the unhashed values into the
+/// storage.
+pub struct LazyHashSet<T> {
+    key: storage::Key,
+    phantom: PhantomData<T>,
+}
+
+impl<T> LazyHashSet<T>
+where
+    T: BorshSerialize + BorshDeserialize + 'static,
+{
+    /// Create or use an existing set with the given storage `key`.
+    pub fn new(key: storage::Key) -> Self {
+        Self {
+            key,
+            phantom: PhantomData,
+        }
+    }
+
+    /// Adds a value to the set. If the set did not have this value present,
+    /// `Ok(true)` is returned, `Ok(false)` otherwise.
+    pub fn insert<S>(&self, storage: &mut S, val: &T) -> Result<bool>
+    where
+        S: StorageWrite + StorageRead,
+    {
+        if self.contains(storage, val)? {
+            Ok(false)
+        } else {
+            let data_key = self.get_data_key(val);
+            storage.write(&data_key, &val)?;
+            Ok(true)
+        }
+    }
+
+    /// Removes a value from the set. Returns whether the value was present in
+    /// the set.
+    pub fn remove<S>(&self, storage: &mut S, val: &T) -> Result<bool>
+    where
+        S: StorageWrite + StorageRead,
+    {
+        let data_key = self.get_data_key(val);
+        let value: Option<T> = storage.read(&data_key)?;
+        storage.delete(&data_key)?;
+        Ok(value.is_some())
+    }
+
+    /// Returns whether the set contains a value.
+    pub fn contains(
+        &self,
+        storage: &impl StorageRead,
+        val: &T,
+    ) -> Result<bool> {
+        let value: Option<T> = storage.read(&self.get_data_key(val))?;
+        Ok(value.is_some())
+    }
+
+    /// Returns whether the set contains no elements.
+    pub fn is_empty(&self, storage: &impl StorageRead) -> Result<bool> {
+        let mut iter = storage.iter_prefix(&self.get_data_prefix())?;
+        Ok(storage.iter_next(&mut iter)?.is_none())
+    }
+
+    /// An iterator visiting all elements. The iterator element type is
+    /// `Result<T>`, because iterator's call to `next` may fail with e.g. out of
+    /// gas or data decoding error.
+    ///
+    /// Note that this function shouldn't be used in transactions and VPs code
+    /// on unbounded sets to avoid gas usage increasing with the length of the
+    /// set.
+    pub fn iter<'a>(
+        &self,
+        storage: &'a impl StorageRead,
+    ) -> Result<impl Iterator<Item = Result<T>> + 'a> {
+        let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?;
+        Ok(iter.map(|key_val_res| {
+            let (_key, val) = key_val_res?;
+            Ok(val)
+        }))
+    }
+
+    /// Get the prefix of set's elements storage
+    fn get_data_prefix(&self) -> storage::Key {
+        self.key.push(&DATA_SUBKEY.to_owned()).unwrap()
+    }
+
+    /// Get the sub-key of a given element
+    fn get_data_key(&self, val: &T) -> storage::Key {
+        let hash_str = hash_for_storage_key(val);
+        self.get_data_prefix().push(&hash_str).unwrap()
+    }
+}

shared/src/ledger/storage_api/collections/lazy_map.rs

@@ -1,18 +1,30 @@
-//! Lazy hash map
+//! Lazy map.
 
-use std::fmt::Display;
 use std::marker::PhantomData;
 
 use borsh::{BorshDeserialize, BorshSerialize};
 
 use super::super::Result;
-use crate::ledger::storage_api::{self, StorageRead, StorageWrite};
-use crate::types::storage;
+use super::ReadError;
+use crate::ledger::storage_api::{self, ResultExt, StorageRead, StorageWrite};
+use crate::types::storage::{self, KeySeg};
 
 /// Subkey corresponding to the data elements of the LazyMap
 pub const DATA_SUBKEY: &str = "data";
 
-/// LazyMap ! fill in !
+/// Lazy map.
+///
+/// This can be used as an alternative to `std::collections::HashMap` and
+/// `BTreeMap`. In the lazy map, the elements do not reside in memory but are
+/// instead read and written to storage sub-keys of the storage `key` used to
+/// construct the map.
+///
+/// In the [`LazyMap`], the type of key `K` can be anything that implements
+/// [`storage::KeySeg`] and this trait is used to turn the keys into key
+/// segments.
+///
+/// This is different from [`super::LazyHashMap`], which hashes borsh encoded
+/// key.
 pub struct LazyMap<K, V> {
     key: storage::Key,
     phantom_k: PhantomData<K>,
@@ -21,8 +33,8 @@ pub struct LazyMap<K, V> {
 
 impl<K, V> LazyMap<K, V>
 where
-    K: BorshDeserialize + BorshSerialize + Display,
-    V: BorshDeserialize + BorshSerialize,
+    K: storage::KeySeg,
+    V: BorshDeserialize + BorshSerialize + 'static,
 {
     /// Create or use an existing map with the given storage `key`.
     pub fn new(key: storage::Key) -> Self {
@@ -94,24 +106,17 @@ where
     pub fn iter<'a>(
         &self,
         storage: &'a impl StorageRead,
-    ) -> Result<impl Iterator<Item = Result<V>> + 'a> {
-        let iter = storage.iter_prefix(&self.get_data_prefix())?;
-        let iter = itertools::unfold(iter, |iter| {
-            match storage.iter_next(iter) {
-                Ok(Some((_key, value))) => {
-                    match V::try_from_slice(&value[..]) {
-                        Ok(decoded_value) => Some(Ok(decoded_value)),
-                        Err(err) => Some(Err(storage_api::Error::new(err))),
-                    }
-                }
-                Ok(None) => None,
-                Err(err) => {
-                    // Propagate errors into Iterator's Item
-                    Some(Err(err))
-                }
-            }
-        });
-        Ok(iter)
+    ) -> Result<impl Iterator<Item = Result<(K, V)>> + 'a> {
+        let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?;
+        Ok(iter.map(|key_val_res| {
+            let (key, val) = key_val_res?;
+            let last_key_seg = key
+                .last()
+                .ok_or(ReadError::UnexpectedlyEmptyStorageKey)
+                .into_storage_result()?;
+            let key = K::parse(last_key_seg.raw()).into_storage_result()?;
+            Ok((key, val))
+        }))
     }
 
     /// Reads a value from storage
@@ -139,6 +144,7 @@ where
 
     /// Get the sub-key of a given element
     fn get_data_key(&self, key: &K) -> storage::Key {
-        self.get_data_prefix().push(&key.to_string()).unwrap()
+        let key_str = key.to_db_key();
+        self.get_data_prefix().push(&key_str).unwrap()
     }
 }