Merk hash scheme migration

src/idstore-export/Cargo.toml

@@ -2,6 +2,9 @@
 name = "idstore-export"
 version = "0.1.1" # managed by release.sh
 edition = "2021"
+readme = "README.md"
+keywords = ["cli", "web3", "blockchain", "tendermint", "proto", "crypto", "liftedinit"]
+categories = ["command-line-utilities"]
 description = "Export IdStore credentials from storage."
 license-file = "../../LICENSE"
 homepage = "https://liftedinit.org/"
@@ -15,7 +18,9 @@ doc = false
 [dependencies]
 base64 = "0.21.0"
 clap = { version = "3.2.23", features = ["derive"] }
-merk = { git = "https://github.com/liftedinit/merk.git", rev = "857bf81963d9282ab03438da5013e1f816bd9da1" }
+derive_more = "0.99"
+merk_v1 = { package = "merk", git = "https://github.com/liftedinit/merk.git", rev = "857bf81963d9282ab03438da5013e1f816bd9da1" }
+merk_v2 = { package = "merk", git = "https://github.com/MavenRain/merk.git", branch = "merk-hash-migration" }
 serde = "1.0.137"
 serde_derive = "1.0.137"
 serde_json = "1.0.81"

src/idstore-export/src/main.rs

@@ -1,16 +1,20 @@
 extern crate core;
 
-use base64::{engine::general_purpose, Engine as _};
-use clap::Parser;
-use merk::rocksdb;
-use merk::rocksdb::{IteratorMode, ReadOptions};
-use merk::tree::Tree;
-use std::collections::BTreeMap;
-use std::path::PathBuf;
+use {
+    base64::{engine::general_purpose, Engine as _},
+    clap::Parser,
+    derive_more::{From, TryInto},
+    merk_v2::rocksdb::{self, IteratorMode, ReadOptions},
+    merk_v2::tree::Tree,
+    std::collections::BTreeMap,
+    std::path::PathBuf,
+};
 
 pub(crate) const IDSTORE_ROOT: &[u8] = b"/idstore/";
 pub(crate) const IDSTORE_SEED_ROOT: &[u8] = b"/config/idstore_seed";
 
+type InnerStorage = merk_v2::Merk;
+
 #[derive(Parser)]
 struct Opts {
     /// The RocksDB store to load.
@@ -23,18 +27,25 @@ struct JsonRoot {
     id_store_keys: BTreeMap<String, String>,
 }
 
-fn main() {
+#[derive(Debug, From, TryInto)]
+enum Error {
+    Merk(merk_v2::Error),
+    Rocks(merk_v2::rocksdb::Error),
+    Serde(serde_json::Error),
+}
+
+fn main() -> Result<(), Error> {
     let Opts { store } = Opts::parse();
 
-    let merk = merk::Merk::open(store).expect("Could not open the store.");
+    let merk = InnerStorage::open(store)?;
 
     let mut opts = ReadOptions::default();
     opts.set_iterate_range(rocksdb::PrefixRange(IDSTORE_ROOT));
     let it = merk.iter_opt(IteratorMode::Start, opts);
 
     let mut idstore = BTreeMap::new();
     for item in it {
-        let (key, value) = item.expect("Error while reading the DB");
+        let (key, value) = item?;
         let new_v = Tree::decode(key.to_vec(), value.as_ref());
         let value = new_v.value().to_vec();
 
@@ -45,19 +56,14 @@ fn main() {
     }
 
     let root = JsonRoot {
-        id_store_seed: merk
-            .get(IDSTORE_SEED_ROOT)
-            .expect("Could not read seed")
-            .map_or(0u64, |x| {
-                let mut bytes = [0u8; 8];
-                bytes.copy_from_slice(x.as_slice());
-                u64::from_be_bytes(bytes)
-            }),
+        id_store_seed: merk.get(IDSTORE_SEED_ROOT)?.map_or(0u64, |x| {
+            let mut bytes = [0u8; 8];
+            bytes.copy_from_slice(x.as_slice());
+            u64::from_be_bytes(bytes)
+        }),
         id_store_keys: idstore,
     };
 
-    println!(
-        "{}",
-        serde_json::to_string_pretty(&root).expect("Could not serialize")
-    );
+    println!("{}", serde_json::to_string_pretty(&root)?);
+    Ok(())
 }

src/ledger-db/Cargo.toml

@@ -2,6 +2,9 @@
 name = "ledger-db"
 version = "0.1.1" # managed by release.sh
 edition = "2021"
+readme = "README.md"
+keywords = ["cli", "web3", "blockchain", "tendermint", "proto", "crypto", "liftedinit"]
+categories = ["command-line-utilities"]
 description = "Ledger storage debugging CLI."
 license-file = "../../LICENSE"
 homepage = "https://liftedinit.org/"
@@ -14,8 +17,10 @@ doc = false
 
 [dependencies]
 clap = { version = "3.2.23", features = ["derive"] }
+derive_more = "0.99"
 hex = "0.4.3"
-merk = { git = "https://github.com/liftedinit/merk.git", rev = "857bf81963d9282ab03438da5013e1f816bd9da1" }
+merk_v1 = { package = "merk", git = "https://github.com/liftedinit/merk.git", rev = "857bf81963d9282ab03438da5013e1f816bd9da1" }
+merk_v2 = { package = "merk", git = "https://github.com/MavenRain/merk.git", branch = "merk-hash-migration" }
 many-modules = { path = "../many-modules", version = "0.1.1" } # managed by release.sh
 many-types = { path = "../many-types", version = "0.1.1" } # managed by release.sh
 minicbor = "0.19.1"

src/ledger-db/src/main.rs

@@ -1,24 +1,37 @@
-use clap::Parser;
-use many_types::ledger::TokenAmount;
-use merk::rocksdb::{IteratorMode, ReadOptions};
-use merk::tree::Tree;
-use std::path::PathBuf;
+use {
+    clap::Parser,
+    derive_more::{From, TryInto},
+    many_types::ledger::TokenAmount,
+    merk_v1::rocksdb::{IteratorMode, ReadOptions},
+    merk_v1::tree::Tree,
+    std::path::PathBuf,
+};
 
 #[derive(Parser)]
 struct Opts {
     /// The RocksDB store to load.
     store: PathBuf,
 }
 
-fn main() {
+type InnerStorage = merk_v2::Merk;
+
+#[derive(Debug, From, TryInto)]
+enum Error {
+    Decode(minicbor::decode::Error),
+    EndOfIterator,
+    Merk(merk_v2::Error),
+    Rocks(merk_v2::rocksdb::Error),
+}
+
+fn main() -> Result<(), Error> {
     let Opts { store } = Opts::parse();
 
-    let merk = merk::Merk::open(store).expect("Could not open the store.");
+    let merk = InnerStorage::open(store)?;
 
     let it = merk.iter_opt(IteratorMode::Start, ReadOptions::default());
 
     for kv_result in it {
-        let (k, v) = kv_result.unwrap();
+        let (k, v) = kv_result?;
         let new_v = Tree::decode(k.to_vec(), v.as_ref());
 
         let k: Vec<u8> = k.into();
@@ -27,14 +40,17 @@ fn main() {
         // Try to "smartly" decode the key.
         if k.starts_with(b"/events/") {
             let k = hex::encode(&k[8..]);
-            let log = minicbor::decode::<many_modules::events::EventLog>(v).unwrap();
+            let log = minicbor::decode::<many_modules::events::EventLog>(v)?;
             println!("event {k} => {log:?}",)
         } else if k.starts_with(b"/balances/") {
             let k = &k[10..];
             // This should be utf8.
             let k = String::from_utf8_lossy(k);
             let mut it = k.split('/');
-            let (id, symbol) = (it.next().unwrap(), it.next().unwrap());
+            let (id, symbol) = it
+                .next()
+                .and_then(|id| it.next().map(|symbol| (id, symbol)))
+                .ok_or(Error::EndOfIterator)?;
             let t = TokenAmount::from(v.to_vec());
             println!("balance {id} => {t} {symbol}");
         } else if k.starts_with(b"/multisig/") {
@@ -47,4 +63,5 @@ fn main() {
             println!("unknown 0x {} => {}", hex::encode(k), hex::encode(v));
         }
     }
+    Ok(())
 }

src/many-abci/build.rs

@@ -5,4 +5,4 @@ fn main() -> Result<(), Box<dyn Error>> {
     // Emit the instructions
     EmitBuilder::builder().git_sha(false).emit()?;
     Ok(())
-}
+}

src/many-abci/src/abci_app.rs

@@ -1,17 +1,21 @@
-use crate::migration::error_code::LEGACY_ERROR_CODE_TRIGGER;
-use crate::migration::{AbciAppMigrations, MIGRATIONS};
-use coset::{CborSerializable, CoseSign1};
-use many_client::client::blocking::{block_on, ManyClient};
-use many_error::{ManyError, ManyErrorCode};
-use many_identity::{Address, AnonymousIdentity};
-use many_migration::MigrationConfig;
-use many_modules::abci_backend::{AbciBlock, AbciCommitInfo, AbciInfo};
-use many_protocol::ResponseMessage;
-use reqwest::{IntoUrl, Url};
-use std::sync::{Arc, RwLock};
-use tendermint_abci::Application;
-use tendermint_proto::abci::*;
-use tracing::{debug, error};
+use {
+    crate::migration::{error_code::LEGACY_ERROR_CODE_TRIGGER, AbciAppMigrations, MIGRATIONS},
+    coset::{CborSerializable, CoseSign1},
+    many_client::client::blocking::{block_on, ManyClient},
+    many_error::{ManyError, ManyErrorCode},
+    many_identity::{Address, AnonymousIdentity},
+    many_migration::MigrationConfig,
+    many_modules::abci_backend::{AbciBlock, AbciCommitInfo, AbciInfo},
+    many_protocol::ResponseMessage,
+    reqwest::{IntoUrl, Url},
+    std::{
+        path::PathBuf,
+        sync::{Arc, RwLock},
+    },
+    tendermint_abci::Application,
+    tendermint_proto::abci::*,
+    tracing::{debug, error},
+};
 
 lazy_static::lazy_static!(
     static ref EPOCH: many_types::Timestamp = many_types::Timestamp::new(0).unwrap();
@@ -161,7 +165,7 @@ impl Application for AbciApp {
             if let Ok(mut m) = self.migrations.write() {
                 // Since it's impossible to truly handle error here, and
                 // we don't actually want to panic, just ignore any errors.
-                let _ = m.update_at_height(&mut (), height);
+                let _ = m.update_at_height(&mut (), height, [""].iter().collect::<PathBuf>());
             } else {
                 error!("Migration: Could not acquire migration lock...");
             }

src/many-abci/src/migration.rs

@@ -1,12 +1,15 @@
-use linkme::distributed_slice;
-use many_error::ManyError;
-use many_migration::{InnerMigration, MigrationSet};
+use {
+    linkme::distributed_slice,
+    many_error::ManyError,
+    many_migration::{InnerMigration, MigrationSet},
+    std::path::PathBuf,
+};
 
 pub mod error_code;
 
-pub type AbciAppMigrations = MigrationSet<'static, ()>;
+pub type AbciAppMigrations = MigrationSet<'static, (), PathBuf>;
 
 // This is the global migration registry
 // Doesn't contain any metadata
 #[distributed_slice]
-pub static MIGRATIONS: [InnerMigration<(), ManyError>] = [..];
+pub static MIGRATIONS: [InnerMigration<(), ManyError, PathBuf>] = [..];

src/many-abci/src/migration/error_code.rs

@@ -1,11 +1,12 @@
-use crate::migration::MIGRATIONS;
-use linkme::distributed_slice;
-use many_error::ManyError;
-use many_migration::InnerMigration;
+use {
+    crate::migration::MIGRATIONS, linkme::distributed_slice, many_error::ManyError,
+    many_migration::InnerMigration, std::path::PathBuf,
+};
 
 #[distributed_slice(MIGRATIONS)]
-pub static LEGACY_ERROR_CODE_TRIGGER: InnerMigration<(), ManyError> = InnerMigration::new_trigger(
-    true,
-    "LegacyErrorCode",
-    "Trigger a legacy bug in attribute-specific error codes decoding in ABCI.",
-);
+pub static LEGACY_ERROR_CODE_TRIGGER: InnerMigration<(), ManyError, PathBuf> =
+    InnerMigration::new_trigger(
+        true,
+        "LegacyErrorCode",
+        "Trigger a legacy bug in attribute-specific error codes decoding in ABCI.",
+    );

src/many-client-macros/src/lib.rs

@@ -36,42 +36,47 @@ pub fn many_client(attr: TokenStream, input: TokenStream) -> TokenStream {
 
     let input_trait = parse_macro_input!(input as ItemTrait);
 
-    let methods_vec = input_trait.items.iter().map(|func| {
-        let namespace = namespace.clone();
-        let func = func.to_token_stream();
-        let method: syn::TraitItemFn =
-            parse2(func)?;
-        let mut method = method.sig;
-        method.asyncness = parse_quote! { async };
-        let mut args_iter = method.inputs.iter();
-        let _self_arg = args_iter.next().ok_or_else(|| syn::Error::new(method.span(), "Should have a &self argument"))?;
-        let args_param = args_iter.next();
-        let args_var = if let Some(FnArg::Typed(args)) = args_param {
-            let args = &args.pat;
-            quote! { #args }
-        } else {
-            quote! { () }
-        };
-        let server_method = if let Some(namespace) = namespace {
-            format!("{}.{}", namespace.value(), method.ident)
-        } else {
-            format!("{}", method.ident)
-        };
-        let server_method: LitStr = parse_quote! { #server_method };
-        let q = quote! {
-            pub #method {
-                let response = self.0.call_(#server_method, #args_var).await?;
-                minicbor::decode(&response).map_err(many_server::ManyError::deserialization_error)
+    let methods_vec = input_trait
+        .items
+        .iter()
+        .map(|func| {
+            let namespace = namespace.clone();
+            let func = func.to_token_stream();
+            let method: syn::TraitItemFn = parse2(func)?;
+            let mut method = method.sig;
+            method.asyncness = parse_quote! { async };
+            let mut args_iter = method.inputs.iter();
+            let _self_arg = args_iter
+                .next()
+                .ok_or_else(|| syn::Error::new(method.span(), "Should have a &self argument"))?;
+            let args_param = args_iter.next();
+            let args_var = if let Some(FnArg::Typed(args)) = args_param {
+                let args = &args.pat;
+                quote! { #args }
+            } else {
+                quote! { () }
+            };
+            let server_method = if let Some(namespace) = namespace {
+                format!("{}.{}", namespace.value(), method.ident)
+            } else {
+                format!("{}", method.ident)
+            };
+            let server_method: LitStr = parse_quote! { #server_method };
+            let q = quote! {
+                pub #method {
+                    let response = self.0.call_(#server_method, #args_var).await?;
+                    minicbor::decode(&response).map_err(many_server::ManyError::deserialization_error)
+                }
+            };
+            Ok(q.into_token_stream())
+        })
+        .try_fold(vec![], |mut acc, curr: syn::Result<TokenStream2>| {
+            match curr {
+                Ok(c) => acc.push(c),
+                Err(e) => return Err(e),
             }
-        };
-        Ok(q.into_token_stream())
-    }).try_fold(vec![], |mut acc, curr: syn::Result<TokenStream2>| {
-        match curr {
-            Ok(c) => acc.push(c),
-            Err(e) => return Err(e)
-        }
-        Ok(acc)
-    });
+            Ok(acc)
+        });
     let methods_vec = match methods_vec {
         Ok(v) => v,
         Err(e) => return e.to_compile_error().into(),

src/many-client/Cargo.toml

@@ -20,6 +20,7 @@ ecdsa = "0.16.0"
 ed25519 = { version = "2.2.0", features = [ "std" ] }
 ed25519-dalek = { version = "1.0.1", features = [ "std" ] }
 fixed = "1.16.0"
+futures-util = "0.3"
 hex = "0.4.3"
 many-client-macros = { path = "../many-client-macros", version = "0.1.1" } # managed by release.sh
 many-identity = { path = "../many-identity", version = "0.1.1" } # managed by release.sh