Use parity-common to make substrate generic over hasher and trie enco…

…ding codec (#297)

* Genric over hasher

* WIP start adding NodeCodec

* Add codec to TrieBackend

* Typechecks

* Fix error type

* Cleanup

* Tests build (and fail)

* Fix tests: don't use MemoryDB::default()

* Lockfile

* Address grumbles

* Teach environmental! about generics

* Add Finder artifacts

* whitespace

* Add a toy impl of Hasher and plug it in to Externalities

* Use `uint` and `fixed-hash` from `parity-common`
Remove unused U512
Add test to ensure H256 impls heapsizeof

* lock file updates

* Make hashes Encodable/Decodable

* lock file updates

* Impl FromIterator for TestExternalities so we can collect() and use map!

* Use rustc-hex from crates
Use rlp from master so dependencies do not mess up the scope

* Fix tests in runtime-io

* lockfile shenanigans

* Add a BlakeHasher impl

* Use BlakeHasher in runtime-io

* lockfile updates

* ws

* Add a Blake2/RLP-flavoured NodeCodec

* Use Blake-flavoured Hasher and NodeCodec

* lockfile

* Implement PartialEq and Default for TestExternalities

* Add note about limitations of environmental!

* Make it compile, but this is probably broken

* Derive Debug so tests in executor can work

* Make executor use BlakeHasher

* ws

* WIP make client generic

* typechecks

* cleanup

* client tests pass

* Fix client/db

* cleanup

* Fix network

* Fix rpc

* Fix service

* Make TestExternalities work better au lieu d'un HashMap

* Fix tests in council

* Fix tests in contract

* Fix tests in council

* Fix democracy

* Add comment about odd-looking reexports in tests

* Don't need to load branch

* Fix staking

* Fix session

* Some polkadot fixes and lockfile

* Fix executive

* fixup lockfile

* Fix polkadot/api

* Fix polkadot/service

* Fix polkadot/runtime tests

* Fix tests in test-runtime

* Test fixes

* Fix missing component in the `std` feature

* Use PhantomData and Result from core

* Fix paths
Use core

* load heapsize on wasm

* implement `HeapSizeOf` for wasm

* Add toy impl of `blake2_256` for no_std

* lockfile

* Use kvdb* from parity-common and fix errors

* rebuilt lockfile

* Add dummy impl of `on_advance_round` for rhododendron::Context

* Fix build after merge

* Add HeapSizeOf bound where needed

* Sort out dependencies for no_std

* Add HeapSizeOf bound where needed

* use temp branch pending PR merges

* Remove unneeded tests

* Lock file and wasm artifacts

* lockfile

* Use magic commit for libp2p

* Cleanup

* Implement blake2_256 for no_std

* Back on parity-common master

* missing type params

* Update Cargo.lock

* whitespace

.gitignore

@@ -1,4 +1,4 @@
-/target/
+**/target/
 **/*.rs.bk
 *.swp
 .wasm-binaries

demo/executor/src/lib.rs

@@ -47,7 +47,7 @@ mod tests {
 	use keyring::Keyring;
 	use runtime_support::{Hashable, StorageValue, StorageMap};
 	use state_machine::{CodeExecutor, TestExternalities};
-	use primitives::twox_128;
+	use primitives::{twox_128, BlakeHasher};
 	use demo_primitives::{Hash, BlockNumber, AccountId};
 	use runtime_primitives::traits::Header as HeaderT;
 	use runtime_primitives::{ApplyOutcome, ApplyError, ApplyResult, MaybeUnsigned};
@@ -100,7 +100,7 @@ mod tests {
 
 	#[test]
 	fn panic_execution_with_foreign_code_gives_error() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
@@ -119,7 +119,7 @@ mod tests {
 
 	#[test]
 	fn bad_extrinsic_with_native_equivalent_code_gives_error() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
@@ -138,7 +138,7 @@ mod tests {
 
 	#[test]
 	fn successful_execution_with_native_equivalent_code_gives_ok() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
@@ -161,7 +161,7 @@ mod tests {
 
 	#[test]
 	fn successful_execution_with_foreign_code_gives_ok() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
@@ -182,7 +182,7 @@ mod tests {
 		});
 	}
 
-	fn new_test_ext() -> TestExternalities {
+	fn new_test_ext() -> TestExternalities<BlakeHasher> {
 		use keyring::Keyring::*;
 		let three = [3u8; 32].into();
 		GenesisConfig {
@@ -212,7 +212,7 @@ mod tests {
 			democracy: Some(Default::default()),
 			council: Some(Default::default()),
 			timestamp: Some(Default::default()),
-		}.build_storage().unwrap()
+		}.build_storage().unwrap().into()
 	}
 
 	fn construct_block(number: BlockNumber, parent_hash: Hash, state_root: Hash, extrinsics: Vec<BareExtrinsic>) -> (Vec<u8>, Hash) {
@@ -247,7 +247,7 @@ mod tests {
 		construct_block(
 			1,
 			[69u8; 32].into(),
-			hex!("b97d52254fc967bb94bed485de6a738e9fad05decfda3453711677b8becf6d0a").into(),
+			hex!("3437bf4b182ab17bb322af5c67e55f6be487a77084ad2b4e27ddac7242e4ad21").into(),
 			vec![BareExtrinsic {
 				signed: alice(),
 				index: 0,
@@ -260,7 +260,7 @@ mod tests {
 		construct_block(
 			2,
 			block1().1,
-			hex!("a1f018d2faa339f72f5ee29050b4670d971e2e271cc06c41ee9cbe1f4c6feec9").into(),
+			hex!("741fcb660e6fa9f625fbcd993b49f6c1cc4040f5e0cc8727afdedf11fd3c464b").into(),
 			vec![
 				BareExtrinsic {
 					signed: bob(),
@@ -280,7 +280,7 @@ mod tests {
 		construct_block(
 			1,
 			[69u8; 32].into(),
-			hex!("41d07010f49aa29b2c9aca542cbaa6f59aafd3dda53cdf711c51ddb7d386912e").into(),
+			hex!("2c7231a9c210a7aa4bea169d944bc4aaacd517862b244b8021236ffa7f697991").into(),
 			vec![BareExtrinsic {
 				signed: alice(),
 				index: 0,
@@ -353,7 +353,7 @@ mod tests {
 
 	#[test]
 	fn panic_execution_gives_error() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![69u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![70u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
@@ -373,7 +373,7 @@ mod tests {
 
 	#[test]
 	fn successful_execution_gives_ok() {
-		let mut t: TestExternalities = map![
+		let mut t: TestExternalities<BlakeHasher> = map![
 			twox_128(&<staking::FreeBalance<Concrete>>::key_for(alice())).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0],
 			twox_128(<staking::TransactionBaseFee<Concrete>>::key()).to_vec() => vec![0u8; 8],
 			twox_128(<staking::TransactionByteFee<Concrete>>::key()).to_vec() => vec![0u8; 8],

polkadot/api/src/full.rs

@@ -27,6 +27,7 @@ use runtime::Address;
 use runtime_primitives::traits::AuxLookup;
 use primitives::{AccountId, Block, Header, BlockId, Hash, Index, SessionKey, Timestamp, UncheckedExtrinsic};
 use primitives::parachain::{CandidateReceipt, DutyRoster, Id as ParaId};
+use substrate_primitives::{BlakeHasher, BlakeRlpCodec};
 
 use {BlockBuilder, PolkadotApi, LocalPolkadotApi, ErrorKind, Error, Result};
 
@@ -57,7 +58,7 @@ macro_rules! with_runtime {
 	}}
 }
 
-impl<B: LocalBackend<Block>> BlockBuilder for ClientBlockBuilder<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block> {
+impl<B: LocalBackend<Block, BlakeHasher, BlakeRlpCodec>> BlockBuilder for ClientBlockBuilder<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block, BlakeHasher, BlakeRlpCodec> {
 	fn push_extrinsic(&mut self, extrinsic: UncheckedExtrinsic) -> Result<()> {
 		self.push(extrinsic).map_err(Into::into)
 	}
@@ -68,8 +69,8 @@ impl<B: LocalBackend<Block>> BlockBuilder for ClientBlockBuilder<B, LocalCallExe
 	}
 }
 
-impl<B: LocalBackend<Block>> PolkadotApi for Client<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block> {
-	type BlockBuilder = ClientBlockBuilder<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block>;
+impl<B: LocalBackend<Block, BlakeHasher, BlakeRlpCodec>> PolkadotApi for Client<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block> {
+	type BlockBuilder = ClientBlockBuilder<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block, BlakeHasher, BlakeRlpCodec>;
 
 	fn session_keys(&self, at: &BlockId) -> Result<Vec<SessionKey>> {
 		with_runtime!(self, at, ::runtime::Consensus::authorities)
@@ -155,7 +156,7 @@ impl<B: LocalBackend<Block>> PolkadotApi for Client<B, LocalCallExecutor<B, Nati
 	}
 }
 
-impl<B: LocalBackend<Block>> LocalPolkadotApi for Client<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block>
+impl<B: LocalBackend<Block, BlakeHasher, BlakeRlpCodec>> LocalPolkadotApi for Client<B, LocalCallExecutor<B, NativeExecutor<LocalDispatch>>, Block>
 {}
 
 #[cfg(test)]
@@ -181,7 +182,7 @@ mod tests {
 		]
 	}
 
-	fn client() -> Client<InMemory<Block>, LocalCallExecutor<InMemory<Block>, NativeExecutor<LocalDispatch>>, Block> {
+	fn client() -> Client<InMemory<Block, BlakeHasher, BlakeRlpCodec>, LocalCallExecutor<InMemory<Block, BlakeHasher, BlakeRlpCodec>, NativeExecutor<LocalDispatch>>, Block> {
 		let genesis_config = GenesisConfig {
 			consensus: Some(ConsensusConfig {
 				code: LocalDispatch::native_equivalent().to_vec(),

polkadot/api/src/light.rs

@@ -24,6 +24,7 @@ use primitives::{AccountId, Block, BlockId, Hash, Index, SessionKey, Timestamp,
 use runtime::Address;
 use primitives::parachain::{CandidateReceipt, DutyRoster, Id as ParaId};
 use {PolkadotApi, BlockBuilder, RemotePolkadotApi, Result, ErrorKind};
+use substrate_primitives::{BlakeHasher, BlakeRlpCodec};
 
 /// Light block builder. TODO: make this work (efficiently)
 #[derive(Clone, Copy)]
@@ -40,9 +41,9 @@ impl BlockBuilder for LightBlockBuilder {
 }
 
 /// Remote polkadot API implementation.
-pub struct RemotePolkadotApiWrapper<B: Backend<Block>, E: CallExecutor<Block>>(pub Arc<Client<B, E, Block>>);
+pub struct RemotePolkadotApiWrapper<B: Backend<Block, BlakeHasher, BlakeRlpCodec>, E: CallExecutor<Block, BlakeHasher, BlakeRlpCodec>>(pub Arc<Client<B, E, Block>>);
 
-impl<B: Backend<Block>, E: CallExecutor<Block>> PolkadotApi for RemotePolkadotApiWrapper<B, E> {
+impl<B: Backend<Block, BlakeHasher, BlakeRlpCodec>, E: CallExecutor<Block, BlakeHasher, BlakeRlpCodec>> PolkadotApi for RemotePolkadotApiWrapper<B, E> {
 	type BlockBuilder = LightBlockBuilder;
 
 	fn session_keys(&self, at: &BlockId) -> Result<Vec<SessionKey>> {
@@ -101,4 +102,4 @@ impl<B: Backend<Block>, E: CallExecutor<Block>> PolkadotApi for RemotePolkadotAp
 	}
 }
 
-impl<B: RemoteBackend<Block>, E: CallExecutor<Block>> RemotePolkadotApi for RemotePolkadotApiWrapper<B, E> {}
+impl<B: RemoteBackend<Block, BlakeHasher, BlakeRlpCodec>, E: CallExecutor<Block, BlakeHasher, BlakeRlpCodec>> RemotePolkadotApi for RemotePolkadotApiWrapper<B, E> {}

polkadot/availability-store/Cargo.toml

@@ -10,6 +10,6 @@ parking_lot = "0.4"
 log = "0.3"
 substrate-codec = { path = "../../substrate/codec" }
 substrate-primitives = { path = "../../substrate/primitives" }
-kvdb = { git = "https://github.com/paritytech/parity.git" }
-kvdb-rocksdb = { git = "https://github.com/paritytech/parity.git" }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity.git" }
+kvdb = { git = "https://github.com/paritytech/parity-common.git" }
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common.git" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common.git" }

polkadot/availability-store/src/lib.rs

@@ -66,20 +66,6 @@ pub struct Data {
 	pub extrinsic: Option<Extrinsic>,
 }
 
-fn extract_io_err(err: ::kvdb::Error) -> io::Error {
-	match err {
-		::kvdb::Error(::kvdb::ErrorKind::Io(io_err), _) => io_err,
-		::kvdb::Error(::kvdb::ErrorKind::Msg(msg), _) => io::Error::new(
-			io::ErrorKind::Other,
-			msg,
-		),
-		x => io::Error::new(
-			io::ErrorKind::Other,
-			format!("Unexpected error variant: {:?}", x), // only necessary because of nonexaustive match.
-		)
-	}
-}
-
 fn block_data_key(relay_parent: &Hash, candidate_hash: &Hash) -> Vec<u8> {
 	(relay_parent, candidate_hash, 0i8).encode()
 }
@@ -99,14 +85,13 @@ impl Store {
 	pub fn new(config: Config) -> io::Result<Self> {
 		let mut db_config = DatabaseConfig::with_columns(Some(columns::NUM_COLUMNS));
 		db_config.memory_budget = config.cache_size;
-		db_config.wal = true;
 
 		let path = config.path.to_str().ok_or_else(|| io::Error::new(
 			io::ErrorKind::Other,
 			format!("Bad database path: {:?}", config.path),
 		))?;
 
-		let db = Database::open(&db_config, &path).map_err(extract_io_err)?;
+		let db = Database::open(&db_config, &path)?;
 
 		Ok(Store {
 			inner: Arc::new(db),
@@ -151,7 +136,7 @@ impl Store {
 			);
 		}
 
-		self.inner.write(tx).map_err(extract_io_err)
+		self.inner.write(tx)
 	}
 
 	/// Note that a set of candidates have been included in a finalized block with given hash and parent hash.
@@ -175,7 +160,7 @@ impl Store {
 			}
 		}
 
-		self.inner.write(tx).map_err(extract_io_err)
+		self.inner.write(tx)
 	}
 
 	/// Query block data.

polkadot/executor/Cargo.toml

@@ -6,4 +6,5 @@ description = "Polkadot node implementation in Rust."
 
 [dependencies]
 substrate-executor = { path = "../../substrate/executor" }
+substrate-primitives = { path = "../../substrate/primitives" }
 polkadot-runtime = { path = "../runtime" }

polkadot/executor/src/lib.rs

@@ -19,5 +19,6 @@
 
 extern crate polkadot_runtime;
 #[macro_use] extern crate substrate_executor;
+extern crate substrate_primitives as primitives;
 
 native_executor_instance!(pub Executor, polkadot_runtime::api::dispatch, polkadot_runtime::VERSION, include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm"));

polkadot/runtime/src/parachains.rs

@@ -30,7 +30,10 @@ use substrate_runtime_support::dispatch::Result;
 use rstd::marker::PhantomData;
 
 #[cfg(any(feature = "std", test))]
-use {runtime_io, runtime_primitives};
+use runtime_primitives;
+
+#[cfg(any(feature = "std", test))]
+use std::collections::HashMap;
 
 pub trait Trait: system::Trait<Hash = ::primitives::Hash> + session::Trait {
 	/// The position of the set_heads call in the block.
@@ -214,8 +217,7 @@ impl<T: Trait> Default for GenesisConfig<T> {
 #[cfg(any(feature = "std", test))]
 impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
 {
-	fn build_storage(mut self) -> ::std::result::Result<runtime_io::TestExternalities, String> {
-		use std::collections::HashMap;
+	fn build_storage(mut self) -> ::std::result::Result<HashMap<Vec<u8>, Vec<u8>>, String> {
 		use codec::Encode;
 
 		self.parachains.sort_unstable_by_key(|&(ref id, _, _)| id.clone());
@@ -235,15 +237,15 @@ impl<T: Trait> runtime_primitives::BuildStorage for GenesisConfig<T>
 			map.insert(head_key, genesis.encode());
 		}
 
-		Ok(map.into())
+		Ok(map)
 	}
 }
 
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use runtime_io::with_externalities;
-	use substrate_primitives::H256;
+	use runtime_io::{TestExternalities, with_externalities};
+	use substrate_primitives::{H256, BlakeHasher};
 	use runtime_primitives::BuildStorage;
 	use runtime_primitives::traits::{HasPublicAux, Identity, BlakeTwo256};
 	use runtime_primitives::testing::{Digest, Header};
@@ -283,7 +285,7 @@ mod tests {
 
 	type Parachains = Module<Test>;
 
-	fn new_test_ext(parachains: Vec<(Id, Vec<u8>, Vec<u8>)>) -> runtime_io::TestExternalities {
+	fn new_test_ext(parachains: Vec<(Id, Vec<u8>, Vec<u8>)>) -> TestExternalities<BlakeHasher> {
 		let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap();
 		t.extend(consensus::GenesisConfig::<Test>{
 			code: vec![],
@@ -298,7 +300,7 @@ mod tests {
 			parachains: parachains,
 			phantom: PhantomData,
 		}.build_storage().unwrap());
-		t
+		t.into()
 	}
 
 	#[test]