feat(consensus): joint overlord and chain (nervosnetwork#32)

* refactor consensus engine

* refactor consensus provider

* cargo clippy

* cargo clippy

* add unit test for fixed full transitions

* cargo clippy

* cargo fmt

* cargo clippy

* cargo update

* impl crypto for overlord

* develop update epoch

* develop init

clippy.toml

@@ -1 +1 @@
-too-many-arguments-threshold = 15
+too-many-arguments-threshold = 10

common/crypto/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+ophelia-bls12381 = { git = "https://github.com/zeroqn/ophelia.git", features = [ "generate" ] }
 ophelia-secp256k1 = { git = "https://github.com/zeroqn/ophelia.git" }
 ophelia = { git = "https://github.com/zeroqn/ophelia.git" }
 

common/crypto/src/lib.rs

@@ -1,4 +1,7 @@
 pub use ophelia::{Crypto, CryptoError, PrivateKey, PublicKey, Signature};
+pub use ophelia_bls12381::{
+    BLS12381PrivateKey, BLS12381PublicKey, BLS12381Signature, BLS12381Threshold, BLS12381,
+};
 pub use ophelia_secp256k1::{
     Secp256k1, Secp256k1PrivateKey, Secp256k1PublicKey, Secp256k1Signature,
 };

core/consensus/Cargo.toml

@@ -21,4 +21,9 @@ common-crypto = { path = "../../common/crypto"}
 core-mempool = { path = "../../core/mempool"}
 core-storage = { path = "../../core/storage"}
 core-network = { path = "../../core/network"}
-protocol = { path = "../../protocol" }
+protocol = { path = "../../protocol" }
+
+[dev-dependencies]
+futures-preview = "0.3.0-alpha.18"
+num-traits = "0.2"
+rand = "0.7"

core/consensus/src/adapter.rs

@@ -1,15 +1,13 @@
-use std::collections::HashSet;
 use std::marker::PhantomData;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use bytes::Bytes;
 
 use protocol::traits::{
     ConsensusAdapter, Context, Gossip, MemPool, MemPoolAdapter, MessageTarget, MixedTxHashes,
     Priority, Storage, StorageAdapter,
 };
-use protocol::types::{Epoch, Hash, Proof, Receipt, SignedTransaction, UserAddress, Validator};
+use protocol::types::{Epoch, Hash, Proof, Receipt, SignedTransaction, Validator};
 use protocol::ProtocolResult;
 
 // use crate::ConsensusError;
@@ -21,11 +19,6 @@ pub struct OverlordConsensusAdapter<
     MA: MemPoolAdapter,
     SA: StorageAdapter,
 > {
-    address:        UserAddress,
-    proof:          Option<Proof>,
-    cycle_limit:    u64,
-    exemption_hash: HashSet<Bytes>,
-
     network:         Arc<G>,
     mempool:         Arc<M>,
     storage:         Arc<S>,
@@ -97,6 +90,10 @@ where
         self.storage.insert_receipts(receipts).await
     }
 
+    async fn save_proof(&self, _ctx: Context, proof: Proof) -> ProtocolResult<()> {
+        self.storage.update_latest_proof(proof).await
+    }
+
     async fn save_signed_txs(
         &self,
         _ctx: Context,
@@ -123,36 +120,13 @@ where
     MA: MemPoolAdapter + 'static,
     SA: StorageAdapter + 'static,
 {
-    pub fn new(
-        address: UserAddress,
-        cycle_limit: u64,
-        network: Arc<G>,
-        mempool: Arc<M>,
-        storage: Arc<S>,
-    ) -> Self {
+    pub fn new(network: Arc<G>, mempool: Arc<M>, storage: Arc<S>) -> Self {
         OverlordConsensusAdapter {
-            address,
-            proof: None,
-            cycle_limit,
-            exemption_hash: HashSet::new(),
-
             mempool_adapter: PhantomData,
             storage_adapter: PhantomData,
             network,
             mempool,
             storage,
         }
     }
-
-    pub fn is_hold_proof(&self) -> bool {
-        self.proof.is_some()
-    }
-
-    pub fn get_proof(&self) -> Option<Proof> {
-        self.proof.clone()
-    }
-
-    pub fn update_proof(&mut self, proof: Proof) {
-        self.proof = Some(proof);
-    }
 }

core/consensus/src/consensus.rs

@@ -3,62 +3,54 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use creep::Context;
-use overlord::types::{AggregatedVote, OverlordMsg, SignedProposal, SignedVote};
-use overlord::{Codec, Consensus as OverlordConsensus, Crypto, Overlord, OverlordHandler};
+use overlord::types::{AggregatedVote, Node, OverlordMsg, SignedProposal, SignedVote, Status};
+use overlord::{Overlord, OverlordHandler};
 use rlp::decode;
 
-use protocol::traits::{
-    Consensus, ConsensusAdapter, Gossip, MemPool, MemPoolAdapter, Storage, StorageAdapter,
-};
-use protocol::types::{Epoch, Proof, SignedTransaction, UserAddress};
+use common_crypto::{Secp256k1PrivateKey, Secp256k1PublicKey};
+
+use protocol::traits::{Consensus, Gossip, MemPool, MemPoolAdapter, Storage, StorageAdapter};
+use protocol::types::{Epoch, Hash, Proof, SignedTransaction, UserAddress, Validator};
 use protocol::ProtocolResult;
 
 use crate::adapter::OverlordConsensusAdapter;
+use crate::engine::ConsensusEngine;
+use crate::fixed_types::{FixedPill, FixedSignedTxs};
+use crate::util::OverlordCrypto;
 use crate::{ConsensusError, MsgType};
 
+pub type OverlordRuntime<G, M, S, MA, SA> =
+    Overlord<FixedPill, FixedSignedTxs, ConsensusEngine<G, M, S, MA, SA>, OverlordCrypto>;
+
 /// Provide consensus
 pub struct ConsensusProvider<
-    CA: ConsensusAdapter,
-    MA: MemPoolAdapter,
-    SA: StorageAdapter,
-    G: Gossip,
+    MA: MemPoolAdapter + 'static,
+    SA: StorageAdapter + 'static,
+    G: Gossip + Send + Sync,
     M: MemPool<MA>,
     S: Storage<SA>,
-    C: Codec,
-    E: Codec,
-    F: OverlordConsensus<C, E>,
-    T: Crypto,
 > {
     /// Overlord consensus protocol instance.
-    overlord: Option<Overlord<C, E, F, T>>,
+    overlord: Option<OverlordRuntime<G, M, S, MA, SA>>,
     /// An overlord consensus protocol handler.
-    handler: OverlordHandler<C>,
-    /// Supply necessary functions from other modules.
-    adapter: CA,
-    /// **TODO** to be changed into `Engine`
-    overlord_adapter: OverlordConsensusAdapter<G, M, S, MA, SA>,
+    handler: OverlordHandler<FixedPill>,
 
     mempool_adapter: PhantomData<MA>,
     storage_adapter: PhantomData<SA>,
 }
 
 #[async_trait]
-impl<CA, MA, SA, G, M, S, C, E, F, T> Consensus<CA>
-    for ConsensusProvider<CA, MA, SA, G, M, S, C, E, F, T>
+impl<MA, SA, G, M, S> Consensus<OverlordConsensusAdapter<G, M, S, MA, SA>>
+    for ConsensusProvider<MA, SA, G, M, S>
 where
-    CA: ConsensusAdapter + 'static,
     MA: MemPoolAdapter + 'static,
     SA: StorageAdapter + 'static,
     G: Gossip + Send + Sync,
     M: MemPool<MA> + 'static,
     S: Storage<SA> + 'static,
-    C: Codec + Send + Sync + 'static,
-    E: Codec + Send + Sync + 'static,
-    F: OverlordConsensus<C, E> + 'static,
-    T: Crypto + Send + Sync + 'static,
 {
     async fn set_proposal(&self, ctx: Context, proposal: Vec<u8>) -> ProtocolResult<()> {
-        let signed_proposal: SignedProposal<C> =
+        let signed_proposal: SignedProposal<FixedPill> =
             decode(&proposal).map_err(|_| ConsensusError::DecodeErr(MsgType::SignedProposal))?;
         self.handler
             .send_msg(ctx, OverlordMsg::SignedProposal(signed_proposal))
@@ -95,51 +87,75 @@ where
     }
 }
 
-impl<CA, MA, SA, G, M, S, C, E, F, T> ConsensusProvider<CA, MA, SA, G, M, S, C, E, F, T>
+impl<MA, SA, G, M, S> ConsensusProvider<MA, SA, G, M, S>
 where
-    CA: ConsensusAdapter + 'static,
     MA: MemPoolAdapter + 'static,
     SA: StorageAdapter + 'static,
-    G: Gossip + Send + Sync,
+    G: Gossip + Send + Sync + 'static,
     M: MemPool<MA> + 'static,
     S: Storage<SA> + 'static,
-    C: Codec + Send + Sync + 'static,
-    E: Codec + Send + Sync + 'static,
-    F: OverlordConsensus<C, E> + 'static,
-    T: Crypto + Send + Sync + 'static,
 {
     pub fn new(
+        init_epoch_id: u64,
+        chain_id: Hash,
         address: UserAddress,
         cycle_limit: u64,
-        overlord_adapter: F,
-        crypto: T,
-        consensus_adapter: CA,
+        validators: Vec<Validator>,
+        pub_key: Secp256k1PublicKey,
+        priv_key: Secp256k1PrivateKey,
         gossip_network: Arc<G>,
         menpool_adapter: Arc<M>,
         storage_adapter: Arc<S>,
     ) -> Self {
-        let mut overlord = Overlord::new(address.as_bytes(), overlord_adapter, crypto);
+        let engine = ConsensusEngine::new(
+            chain_id,
+            address.clone(),
+            cycle_limit,
+            validators.clone(),
+            gossip_network,
+            menpool_adapter,
+            storage_adapter,
+        );
+
+        let crypto = OverlordCrypto::new(pub_key, priv_key);
+
+        let mut overlord = Overlord::new(address.as_bytes(), engine, crypto);
         let overlord_handler = overlord.take_handler();
+        overlord_handler
+            .send_msg(
+                Context::new(),
+                OverlordMsg::RichStatus(handle_genesis(init_epoch_id, validators)),
+            )
+            .unwrap();
 
         ConsensusProvider {
-            overlord:         Some(overlord),
-            handler:          overlord_handler,
-            adapter:          consensus_adapter,
-            overlord_adapter: OverlordConsensusAdapter::new(
-                address,
-                cycle_limit,
-                gossip_network,
-                menpool_adapter,
-                storage_adapter,
-            ),
-
+            overlord:        Some(overlord),
+            handler:         overlord_handler,
             mempool_adapter: PhantomData,
             storage_adapter: PhantomData,
         }
     }
 
-    pub fn take_overlord(&mut self) -> Overlord<C, E, F, T> {
+    pub fn take_overlord(&mut self) -> OverlordRuntime<G, M, S, MA, SA> {
         assert!(self.overlord.is_some());
         self.overlord.take().unwrap()
     }
 }
+
+fn handle_genesis(epoch_id: u64, validators: Vec<Validator>) -> Status {
+    let mut authority_list = validators
+        .into_iter()
+        .map(|v| Node {
+            address:        v.address.as_bytes(),
+            propose_weight: v.propose_weight,
+            vote_weight:    v.vote_weight,
+        })
+        .collect::<Vec<_>>();
+
+    authority_list.sort();
+    Status {
+        epoch_id,
+        interval: None,
+        authority_list,
+    }
+}