From aa7ae3fc5172e62e137ba6199bb4e0db38224b4c Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 10 Jun 2020 22:01:11 -0400 Subject: [PATCH 01/50] add availability bitfield types to primitives --- primitives/src/parachain.rs | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 6fcb696fc956..916ffa1a1a63 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -661,6 +661,73 @@ impl FeeSchedule { } } +/// A bitfield concerning availability of backed candidates. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct AvailabilityBitfield(pub BitVec); + +impl From> for AvailabilityBitfield { + fn from(inner: BitVec) -> Self { + AvailabilityBitfield(inner) + } +} + +impl AvailabilityBitfield { + /// Encodes the signing payload into the given buffer. + pub fn encode_signing_payload_into(&self, signing_context: &SigningContext, buf: &mut Vec) { + self.0.encode_to(buf); + signing_context.encode_to(buf); + } + + /// Encodes the signing payload into a fresh byte-vector. + pub fn encode_signing_payload(&self, signing_context: &SigningContext) -> Vec { + let mut v = Vec::new(); + self.encode_signing_payload_into(signing_context, &mut v); + v + } +} + +/// A bitfield signed by a particular validator about the availability of pending candidates. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct SignedAvailabilityBitfield { + /// The index of the validator in the current set. + pub validator_index: ValidatorIndex, + /// The bitfield itself, with one bit per core. Only occupied cores may have the `1` bit set. + pub bitfield: AvailabilityBitfield, + /// The signature by the validator on the bitfield's signing payload. The context of the signature + /// should be apparent when checking the signature. + pub signature: ValidatorSignature, +} + +/// Check a signature on an availability bitfield. Provide the bitfield, the validator who signed it, +/// the signature, the signing context, and an optional buffer in which to encode. +/// +/// If the buffer is provided, it is assumed to be empty. +pub fn check_availability_bitfield_signature( + bitfield: &AvailabilityBitfield, + validator: &ValidatorId, + signature: &ValidatorSignature, + signing_context: &SigningContext, + payload_encode_buf: Option<&mut Vec>, +) -> Result<(),()> { + use runtime_primitives::traits::AppVerify; + + let mut v = Vec::new(); + let payload_encode_buf = payload_encode_buf.unwrap_or(&mut v); + + bitfield.encode_signing_payload_into(signing_context, payload_encode_buf); + + if signature.verify(&payload_encode_buf[..], validator) { + Ok(()) + } else { + Err(()) + } +} + +/// A set of signed availability bitfields. Should be sorted by validator index, ascending. +pub struct Bitfields(pub Vec); + sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. #[api_version(3)] From 2c62c23f95a493eb1160b5b1922deca7e2df3490 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 10 Jun 2020 22:16:13 -0400 Subject: [PATCH 02/50] begin inclusion module --- primitives/src/parachain.rs | 2 +- runtime/parachains/src/inclusion.rs | 126 ++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 916ffa1a1a63..403719e85cd9 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -726,7 +726,7 @@ pub fn check_availability_bitfield_signature( } /// A set of signed availability bitfields. Should be sorted by validator index, ascending. -pub struct Bitfields(pub Vec); +pub struct SignedAvailabilityBitfields(pub Vec); sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 1f45de2df705..b84f4ffa643b 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -13,3 +13,129 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . + +//! The inclusion module is responsible for inclusion and availability of scheduled parachains +//! and parathreads. +//! +//! It is responsible for carrying candidates from being backable to being backed, and then from backed +//! to included. + +use sp_std::prelude::*; +use primitives::{ + parachain::{ + ValidatorId, AbridgedCandidateReceipt, ValidatorIndex, Id as ParaId, + AvailabilityBitfield as AvailabilityBitfield, SignedAvailabilityBitfields, + }, +}; +use frame_support::{ + decl_storage, decl_module, decl_error, + dispatch::DispatchResult, + weights::{DispatchClass, Weight}, +}; +use codec::{Encode, Decode}; +use system::ensure_root; +use bitvec::vec::BitVec; + +use crate::{configuration, paras, scheduler::CoreIndex}; + +/// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding +/// for any backed candidates referred to by a `1` bit available. +#[derive(Encode, Decode)] +#[cfg_attr(test, derive(Debug))] +pub struct AvailabilityBitfieldRecord { + bitfield: StorageBitVec, // one bit per core. + submitted_at: N, // for accounting, as meaning of bits may change over time. +} + +/// A backed candidate pending availability. +#[derive(Encode, Decode)] +#[cfg_attr(test, derive(Debug))] +pub struct CandidatePendingAvailability { + /// The availability core this is assigned to. + core: CoreIndex, + /// The candidate receipt itself. + receipt: AbridgedCandidateReceipt, + /// The received availability votes. One bit per validator. + availability_votes: bitvec::vec::BitVec, + /// The block number of the relay-parent of the receipt. + relay_parent_number: N, + /// The block number of the relay-chain block this was backed in. + backed_in_number: N, +} + +pub trait Trait: system::Trait + paras::Trait + configuration::Trait { } + +decl_storage! { + trait Store for Module as ParaInclusion { + /// The latest bitfield for each validator, referred to by their index in the validator set. + AvailabilityBitfields: map hasher(twox_64_concat) ValidatorIndex + => Option>; + + /// Candidates pending availability by `ParaId`. + PendingAvailability: map hasher(twox_64_concat) ParaId + => Option> + } +} + +decl_error! { + pub enum Error for Module { } +} + +decl_module! { + /// The parachain-candidate inclusion module. + pub struct Module for enum Call where origin: ::Origin { + type Error = Error; + } +} + +impl Module { + + /// Process a set of incoming bitfields. + pub(crate) fn process_bitfields( + signed_bitfields: SignedAvailabilityBitfields, + core_lookup: impl Fn(CoreIndex) -> Option, + ) -> DispatchResult { + let n_validators = unimplemented!(); // TODO [now]: at least one module needs to store all validators + let config = >::config(); + let parachains = >::parachains(); + + let n_bits = parachains.len() + config.parathread_cores as usize; + + // do sanity checks on the bitfields: + // 1. no more than one bitfield per validator + // 2. bitfields are ascending by validator index. + // 3. each bitfield has exactly `n_bits` + // 4. signature is valid. + { + let mut last_index = None; + let mut payload_encode_buf = Vec::new(); + + for signed_bitfield in &signed_bitfields.0 { + let signing_context = unimplemented!(); // TODO [now]. + + if signed_bitfield.bitfield.len() != n_bits { + // TODO [now] return "malformed bitfield" + } + + if last_index.map_or(false, |last| last >= signed_bitfield.validator_index) { + // TODO [now] return "bitfield duplicate or out of order" + } + + if let Err(()) = primitives::parachain::check_availability_bitfield_signature( + &signed_bitfield.bitfield, + &unimplemented!(), // TODO [now] get validator public key by index. + &signed_bitfield.signature, + &signing_context, + Some(&mut payload_encode_buf), + ) { + // TODO [now] return "invalid bitfield signature" + } + + last_index = signed_bitfield.validator_index; + payload_encode_buf.clear(); + } + } + + Ok(()) + } +} From 08f3bad5ffe5f5123828320c0980c192d1306296 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 10:55:28 -0400 Subject: [PATCH 03/50] use GitHub issue link for limitation --- runtime/parachains/src/scheduler.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 4f0554407774..04efb3c28a46 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -615,8 +615,8 @@ impl Module { /// timeouts, i.e. only within `max(config.chain_availability_period, config.thread_availability_period)` /// of the last rotation would this return `Some`. /// - /// This really should not be a box, but is working around a compiler limitation described here: - /// https://users.rust-lang.org/t/cannot-unify-associated-type-in-impl-fn-with-concrete-type/44129 + /// This really should not be a box, but is working around a compiler limitation filed here: + /// https://github.com/rust-lang/rust/issues/73226 /// which prevents us from testing the code if using `impl Trait`. #[allow(unused)] pub(crate) fn availability_timeout_predicate() -> Option bool>> { From c817204adf24343b2146e1a80590a448a678f062 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 11:09:35 -0400 Subject: [PATCH 04/50] fix some compiler errors --- runtime/parachains/src/inclusion.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index b84f4ffa643b..4e3754cfdd1c 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -43,7 +43,7 @@ use crate::{configuration, paras, scheduler::CoreIndex}; #[derive(Encode, Decode)] #[cfg_attr(test, derive(Debug))] pub struct AvailabilityBitfieldRecord { - bitfield: StorageBitVec, // one bit per core. + bitfield: AvailabilityBitfield, // one bit per core. submitted_at: N, // for accounting, as meaning of bits may change over time. } @@ -113,7 +113,7 @@ impl Module { for signed_bitfield in &signed_bitfields.0 { let signing_context = unimplemented!(); // TODO [now]. - if signed_bitfield.bitfield.len() != n_bits { + if signed_bitfield.bitfield.0.len() != n_bits { // TODO [now] return "malformed bitfield" } @@ -131,7 +131,7 @@ impl Module { // TODO [now] return "invalid bitfield signature" } - last_index = signed_bitfield.validator_index; + last_index = Some(signed_bitfield.validator_index); payload_encode_buf.clear(); } } From ee1c9c00d19cb2c00b7a41fe4565d313371f377a Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 11:19:25 -0400 Subject: [PATCH 05/50] integrate validators into initializer --- runtime/parachains/src/initializer.rs | 6 ++++++ runtime/parachains/src/mock.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/runtime/parachains/src/initializer.rs b/runtime/parachains/src/initializer.rs index 2b86d19654cd..f2d76df8a97d 100644 --- a/runtime/parachains/src/initializer.rs +++ b/runtime/parachains/src/initializer.rs @@ -60,6 +60,9 @@ decl_storage! { /// them writes to the trie and one does not. This confusion makes `Option<()>` more suitable for /// the semantics of this variable. HasInitialized: Option<()>; + + /// The current validators, by their parachain session keys. + Validators get(fn validators) config(validators): Vec; } } @@ -122,6 +125,9 @@ impl Module { buf }; + // place validators before calling session handlers for any modules. + Validators::put(&validators); + // We can't pass the new config into the thing that determines the new config, // so we don't pass the `SessionChangeNotification` into this module. configuration::Module::::initializer_on_new_session(&validators, &queued); diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs index 79a6b5bcc442..22bd58c7b2b6 100644 --- a/runtime/parachains/src/mock.rs +++ b/runtime/parachains/src/mock.rs @@ -123,6 +123,7 @@ pub fn new_test_ext(state: GenesisConfig) -> TestExternalities { #[derive(Default)] pub struct GenesisConfig { + pub initializer: crate::initializer::GenesisConfig, pub system: system::GenesisConfig, pub configuration: crate::configuration::GenesisConfig, pub paras: crate::paras::GenesisConfig, From 7b7306a10d6492af0fb320d04a59f6a412e8e984 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 12:12:18 -0400 Subject: [PATCH 06/50] add generic signing context --- primitives/src/parachain.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 403719e85cd9..2a8906813764 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -616,6 +616,15 @@ pub struct SigningContext { pub parent_hash: Hash, } +/// A type returned by runtime with current session index and a parent hash. +#[derive(Clone, Eq, PartialEq, Default, Decode, Encode, RuntimeDebug)] +pub struct GenericSigningContext { + /// Current session index. + pub session_index: sp_staking::SessionIndex, + /// Hash of the parent. + pub parent_hash: H, +} + /// An attested candidate. This is submitted to the relay chain by a block author. #[derive(Clone, PartialEq, Decode, Encode, RuntimeDebug)] pub struct AttestedCandidate { From 6b3969ea1771c512d69619f683635648b8f66b9d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 12:32:36 -0400 Subject: [PATCH 07/50] make signing-context more generic --- primitives/src/parachain.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 2a8906813764..256e00320e60 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -606,19 +606,9 @@ pub enum ValidityAttestation { #[codec(index = "2")] Explicit(ValidatorSignature), } - -/// A type returned by runtime with current session index and a parent hash. -#[derive(Clone, Eq, PartialEq, Default, Decode, Encode, RuntimeDebug)] -pub struct SigningContext { - /// Current session index. - pub session_index: sp_staking::SessionIndex, - /// Hash of the parent. - pub parent_hash: Hash, -} - /// A type returned by runtime with current session index and a parent hash. #[derive(Clone, Eq, PartialEq, Default, Decode, Encode, RuntimeDebug)] -pub struct GenericSigningContext { +pub struct SigningContext { /// Current session index. pub session_index: sp_staking::SessionIndex, /// Hash of the parent. @@ -683,13 +673,21 @@ impl From> for AvailabilityBitfield { impl AvailabilityBitfield { /// Encodes the signing payload into the given buffer. - pub fn encode_signing_payload_into(&self, signing_context: &SigningContext, buf: &mut Vec) { + pub fn encode_signing_payload_into( + &self, + signing_context: &SigningContext, + buf: &mut Vec, + ) { self.0.encode_to(buf); signing_context.encode_to(buf); } /// Encodes the signing payload into a fresh byte-vector. - pub fn encode_signing_payload(&self, signing_context: &SigningContext) -> Vec { + pub fn encode_signing_payload( + &self, + signing_context: + &SigningContext, + ) -> Vec { let mut v = Vec::new(); self.encode_signing_payload_into(signing_context, &mut v); v @@ -713,11 +711,11 @@ pub struct SignedAvailabilityBitfield { /// the signature, the signing context, and an optional buffer in which to encode. /// /// If the buffer is provided, it is assumed to be empty. -pub fn check_availability_bitfield_signature( +pub fn check_availability_bitfield_signature( bitfield: &AvailabilityBitfield, validator: &ValidatorId, signature: &ValidatorSignature, - signing_context: &SigningContext, + signing_context: &SigningContext, payload_encode_buf: Option<&mut Vec>, ) -> Result<(),()> { use runtime_primitives::traits::AppVerify; From 8a014229dd03997377e083656d90fc46b6377d91 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 12:33:49 -0400 Subject: [PATCH 08/50] fix issues with inclusion module --- runtime/parachains/src/inclusion.rs | 58 ++++++++++++++++++++------- runtime/parachains/src/initializer.rs | 42 ++++++++++++------- runtime/parachains/src/mock.rs | 6 ++- 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 4e3754cfdd1c..e489bbcf66cf 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -24,17 +24,18 @@ use sp_std::prelude::*; use primitives::{ parachain::{ ValidatorId, AbridgedCandidateReceipt, ValidatorIndex, Id as ParaId, - AvailabilityBitfield as AvailabilityBitfield, SignedAvailabilityBitfields, + AvailabilityBitfield as AvailabilityBitfield, SignedAvailabilityBitfields, SigningContext, }, }; use frame_support::{ decl_storage, decl_module, decl_error, - dispatch::DispatchResult, + dispatch::DispatchResult, ensure, weights::{DispatchClass, Weight}, }; use codec::{Encode, Decode}; use system::ensure_root; use bitvec::vec::BitVec; +use sp_staking::SessionIndex; use crate::{configuration, paras, scheduler::CoreIndex}; @@ -73,12 +74,27 @@ decl_storage! { /// Candidates pending availability by `ParaId`. PendingAvailability: map hasher(twox_64_concat) ParaId - => Option> + => Option>; + + /// The current validators, by their parachain session keys. + Validators get(fn validators) config(validators): Vec; + + /// The current session index. + CurrentSessionIndex: SessionIndex; } } decl_error! { - pub enum Error for Module { } + pub enum Error for Module { + /// Availability bitfield has unexpected size. + WrongBitfieldSize, + /// Multiple bitfields submitted by same validator or validators out of order by index. + BitfieldDuplicateOrUnordered, + /// Validator index out of bounds. + ValidatorIndexOutOfBounds, + /// Invalid signature + InvalidBitfieldSignature, + } } decl_module! { @@ -95,10 +111,12 @@ impl Module { signed_bitfields: SignedAvailabilityBitfields, core_lookup: impl Fn(CoreIndex) -> Option, ) -> DispatchResult { - let n_validators = unimplemented!(); // TODO [now]: at least one module needs to store all validators + let validators = Validators::get(); + let session_index = CurrentSessionIndex::get(); let config = >::config(); let parachains = >::parachains(); + let n_validators = validators.len(); let n_bits = parachains.len() + config.parathread_cores as usize; // do sanity checks on the bitfields: @@ -110,25 +128,37 @@ impl Module { let mut last_index = None; let mut payload_encode_buf = Vec::new(); + let signing_context = SigningContext { + parent_hash: >::parent_hash(), + session_index, + }; + for signed_bitfield in &signed_bitfields.0 { - let signing_context = unimplemented!(); // TODO [now]. + ensure!( + signed_bitfield.bitfield.0.len() == n_bits, + Error::::WrongBitfieldSize, + ); - if signed_bitfield.bitfield.0.len() != n_bits { - // TODO [now] return "malformed bitfield" - } + ensure!( + last_index.map_or(true, |last| last < signed_bitfield.validator_index), + Error::::BitfieldDuplicateOrUnordered, + ); - if last_index.map_or(false, |last| last >= signed_bitfield.validator_index) { - // TODO [now] return "bitfield duplicate or out of order" - } + ensure!( + signed_bitfield.validator_index < validators.len() as _, + Error::::ValidatorIndexOutOfBounds, + ); + + let validator_public = &validators[signed_bitfield.validator_index as usize]; if let Err(()) = primitives::parachain::check_availability_bitfield_signature( &signed_bitfield.bitfield, - &unimplemented!(), // TODO [now] get validator public key by index. + validator_public, &signed_bitfield.signature, &signing_context, Some(&mut payload_encode_buf), ) { - // TODO [now] return "invalid bitfield signature" + Err(Error::::InvalidBitfieldSignature)?; } last_index = Some(signed_bitfield.validator_index); diff --git a/runtime/parachains/src/initializer.rs b/runtime/parachains/src/initializer.rs index f2d76df8a97d..681824f04c69 100644 --- a/runtime/parachains/src/initializer.rs +++ b/runtime/parachains/src/initializer.rs @@ -27,7 +27,7 @@ use primitives::{ use frame_support::{ decl_storage, decl_module, decl_error, traits::Randomness, }; -use crate::{configuration::{self, HostConfiguration}, paras, scheduler}; +use crate::{configuration::{self, HostConfiguration}, paras, scheduler, inclusion}; /// Information about a session change that has just occurred. #[derive(Default, Clone)] @@ -42,9 +42,13 @@ pub struct SessionChangeNotification { pub new_config: HostConfiguration, /// A secure random seed for the session, gathered from BABE. pub random_seed: [u8; 32], + /// New session index. + pub session_index: sp_staking::SessionIndex, } -pub trait Trait: system::Trait + configuration::Trait + paras::Trait + scheduler::Trait { +pub trait Trait: + system::Trait + configuration::Trait + paras::Trait + scheduler::Trait + inclusion::Trait +{ /// A randomness beacon. type Randomness: Randomness; } @@ -60,9 +64,6 @@ decl_storage! { /// them writes to the trie and one does not. This confusion makes `Option<()>` more suitable for /// the semantics of this variable. HasInitialized: Option<()>; - - /// The current validators, by their parachain session keys. - Validators get(fn validators) config(validators): Vec; } } @@ -104,16 +105,25 @@ decl_module! { impl Module { /// Should be called when a new session occurs. Forwards the session notification to all - /// wrapped modules. + /// wrapped modules. If `queued` is `None`, the `validators` are considered queued. /// /// Panics if the modules have already been initialized. - fn on_new_session<'a, I: 'a>(_changed: bool, validators: I, queued: I) + fn on_new_session<'a, I: 'a>( + _changed: bool, + session_index: sp_staking::SessionIndex, + validators: I, + queued: Option, + ) where I: Iterator { assert!(HasInitialized::get().is_none()); let validators: Vec<_> = validators.map(|(_, v)| v).collect(); - let queued: Vec<_> = queued.map(|(_, v)| v).collect(); + let queued: Vec<_> = if let Some(queued) = queued { + queued.map(|(_, v)| v).collect() + } else { + validators.clone() + }; let prev_config = >::config(); @@ -125,9 +135,6 @@ impl Module { buf }; - // place validators before calling session handlers for any modules. - Validators::put(&validators); - // We can't pass the new config into the thing that determines the new config, // so we don't pass the `SessionChangeNotification` into this module. configuration::Module::::initializer_on_new_session(&validators, &queued); @@ -140,6 +147,7 @@ impl Module { prev_config, new_config, random_seed, + session_index, }; paras::Module::::initializer_on_new_session(¬ification); @@ -151,7 +159,7 @@ impl sp_runtime::BoundToRuntimeAppPublic for Module { type Public = ValidatorId; } -impl session::OneSessionHandler for Module { +impl session::OneSessionHandler for Module { type Key = ValidatorId; fn on_genesis_session<'a, I: 'a>(_validators: I) @@ -163,7 +171,8 @@ impl session::OneSessionHandler for Module { fn on_new_session<'a, I: 'a>(changed: bool, validators: I, queued: I) where I: Iterator { - >::on_new_session(changed, validators, queued); + let session_index = >::current_index(); + >::on_new_session(changed, session_index, validators, Some(queued)); } fn on_disabled(_i: usize) { } @@ -181,7 +190,12 @@ mod tests { fn panics_if_session_changes_after_on_initialize() { new_test_ext(Default::default()).execute_with(|| { Initializer::on_initialize(1); - Initializer::on_new_session(false, Vec::new().into_iter(), Vec::new().into_iter()); + Initializer::on_new_session( + false, + 1, + Vec::new().into_iter(), + Some(Vec::new().into_iter()), + ); }); } diff --git a/runtime/parachains/src/mock.rs b/runtime/parachains/src/mock.rs index 22bd58c7b2b6..b350d6c3a2cb 100644 --- a/runtime/parachains/src/mock.rs +++ b/runtime/parachains/src/mock.rs @@ -98,6 +98,8 @@ impl crate::paras::Trait for Test { } impl crate::scheduler::Trait for Test { } +impl crate::inclusion::Trait for Test { } + pub type System = system::Module; /// Mocked initializer. @@ -112,6 +114,9 @@ pub type Paras = crate::paras::Module; /// Mocked scheduler. pub type Scheduler = crate::scheduler::Module; +/// Mocked inclusion module. +pub type Inclusion = crate::inclusion::Module; + /// Create a new set of test externalities. pub fn new_test_ext(state: GenesisConfig) -> TestExternalities { let mut t = state.system.build_storage::().unwrap(); @@ -123,7 +128,6 @@ pub fn new_test_ext(state: GenesisConfig) -> TestExternalities { #[derive(Default)] pub struct GenesisConfig { - pub initializer: crate::initializer::GenesisConfig, pub system: system::GenesisConfig, pub configuration: crate::configuration::GenesisConfig, pub paras: crate::paras::GenesisConfig, From 795bc5414c64c24b3ffc6086fdb46d7969241d02 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 12:34:01 -0400 Subject: [PATCH 09/50] add TODO --- runtime/parachains/src/inclusion.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index e489bbcf66cf..1dca5377d64f 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -166,6 +166,8 @@ impl Module { } } + // TODO [now] actually apply bitfields + Ok(()) } } From 221b99e155a636d188354e385553917fa8a1b2ba Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 13:10:44 -0400 Subject: [PATCH 10/50] guide: add validators and session index to inclusion --- roadmap/implementors-guide/src/runtime/inclusion.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index 944cd1def2f2..0e4663cb0727 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -28,6 +28,12 @@ Storage Layout: bitfields: map ValidatorIndex => AvailabilityBitfield; /// Candidates pending availability. PendingAvailability: map ParaId => CandidatePendingAvailability; + +/// The current validators, by their parachain session keys. +Validators: Vec; + +/// The current session index. +CurrentSessionIndex: SessionIndex; ``` > TODO: `CandidateReceipt` and `AbridgedCandidateReceipt` can contain code upgrades which make them very large. the code entries should be split into a different storage map with infrequent access patterns @@ -36,6 +42,8 @@ PendingAvailability: map ParaId => CandidatePendingAvailability; 1. Clear out all candidates pending availability. 1. Clear out all validator bitfields. +1. Update `Validators` with the validators from the session change notification. +1. Update `CurrentSessionIndex` with the session index from the session change notification. ## Routines From 412d88c67e5eda8518df57c435d4305a2ab4e1b5 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 13:11:28 -0400 Subject: [PATCH 11/50] guide: add session index to change notification --- roadmap/implementors-guide/src/runtime/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roadmap/implementors-guide/src/runtime/README.md b/roadmap/implementors-guide/src/runtime/README.md index 2b25b1cf2035..7febb2e76001 100644 --- a/roadmap/implementors-guide/src/runtime/README.md +++ b/roadmap/implementors-guide/src/runtime/README.md @@ -48,6 +48,8 @@ struct SessionChangeNotification { new_config: HostConfiguration, // A secure randomn seed for the session, gathered from BABE. random_seed: [u8; 32], + // The session index of the beginning session. + session_index: SessionIndex, } ``` From c5131b8e8bab157ea2fc5f73e5ed340d5ddee76a Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 13:22:04 -0400 Subject: [PATCH 12/50] implement session change logic --- runtime/parachains/src/inclusion.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 1dca5377d64f..41a5fc6e65b3 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -28,8 +28,7 @@ use primitives::{ }, }; use frame_support::{ - decl_storage, decl_module, decl_error, - dispatch::DispatchResult, ensure, + decl_storage, decl_module, decl_error, ensure, dispatch::DispatchResult, IterableStorageMap, weights::{DispatchClass, Weight}, }; use codec::{Encode, Decode}; @@ -106,6 +105,19 @@ decl_module! { impl Module { + /// Handle an incoming session change. + pub(crate) fn initializer_on_new_session( + notification: crate::initializer::SessionChangeNotification + ) { + // unlike most drain methods, drained elements are not cleared on `Drop` of the iterator + // and require consumption. + for _ in >::drain() { } + for _ in >::drain() { } + + Validators::set(notification.validators.clone()); // substrate forces us to clone, stupidly. + CurrentSessionIndex::set(notification.session_index); + } + /// Process a set of incoming bitfields. pub(crate) fn process_bitfields( signed_bitfields: SignedAvailabilityBitfields, @@ -168,6 +180,9 @@ impl Module { // TODO [now] actually apply bitfields + // TODO: pass available candidates onwards to validity module once implemented. + // https://github.com/paritytech/polkadot/issues/1251 + Ok(()) } } From 7e702cd689b8af4ebbe61105766ff34ddbbb5f65 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 15:24:43 -0400 Subject: [PATCH 13/50] add BackedCandidate type --- primitives/src/parachain.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 256e00320e60..5d8875c9f12d 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -735,6 +735,21 @@ pub fn check_availability_bitfield_signature( /// A set of signed availability bitfields. Should be sorted by validator index, ascending. pub struct SignedAvailabilityBitfields(pub Vec); +/// A backed (or backable, depending on context) candidate. +// TODO: yes, this is roughly the same as AttestedCandidate. +// After https://github.com/paritytech/polkadot/issues/1250 +// they should be unified to this type. +#[derive(PartialEq, Eq, Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct BackedCandidate { + /// The candidate referred to. + pub candidate: AbridgedCandidateReceipt, + /// The validity votes themselves, expressed as signatures. + pub validity_votes: Vec, + /// The indices of the validators within the group, expressed as a bitfield. + pub validator_indices: BitVec, +} + sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. #[api_version(3)] From 48c0467cd47a9bcc2011ad48a7239bc57e0fe7ee Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 21:23:00 -0400 Subject: [PATCH 14/50] guide: refine inclusion pipeline --- roadmap/implementors-guide/src/runtime/inclusion.md | 8 +++++--- .../implementors-guide/src/runtime/inclusioninherent.md | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index 0e4663cb0727..f377acb03577 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -58,10 +58,12 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number. 1. > TODO: pass it onwards to `Validity` module. 1. Return a list of freed cores consisting of the cores where candidates have become available. -* `process_candidates(BackedCandidates, scheduled: Vec)`: - 1. check that each candidate corresponds to a scheduled core and that they are ordered in ascending order by `ParaId`. +* `process_candidates(BackedCandidates, scheduled: Vec, group_on: Fn(GroupIndex) -> Option>)`: + 1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`. + 1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates. + 1. check that there is no candidate pending availability for any scheduled `ParaId`. 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of the currently scheduled upgrade, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. - 1. check the backing of the candidate using the signatures and the bitfields. + 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_on` lookup. 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was successfully backed for, sorted ascending by CoreIndex. * `enact_candidate(relay_parent_number: BlockNumber, AbridgedCandidateReceipt)`: diff --git a/roadmap/implementors-guide/src/runtime/inclusioninherent.md b/roadmap/implementors-guide/src/runtime/inclusioninherent.md index bd5ecc375a93..1031581995d4 100644 --- a/roadmap/implementors-guide/src/runtime/inclusioninherent.md +++ b/roadmap/implementors-guide/src/runtime/inclusioninherent.md @@ -17,8 +17,9 @@ Included: Option<()>, ## Entry Points * `inclusion`: This entry-point accepts two parameters: [`Bitfields`](/type-definitions.html#signed-availability-bitfield) and [`BackedCandidates`](/type-definitions.html#backed-candidate). - 1. The `Bitfields` are first forwarded to the `process_bitfields` routine, returning a set of freed cores. Provide a `Scheduler::core_para` as a core-lookup to the `process_bitfields` routine. Annotate each of these freed cores with `FreedReason::Concluded`. + 1. The `Bitfields` are first forwarded to the `Inclusion::process_bitfields` routine, returning a set of freed cores. Provide a `Scheduler::core_para` as a core-lookup to the `process_bitfields` routine. Annotate each of these freed cores with `FreedReason::Concluded`. 1. If `Scheduler::availability_timeout_predicate` is `Some`, invoke `Inclusion::collect_pending` using it, and add timed-out cores to the free cores, annotated with `FreedReason::TimedOut`. 1. Invoke `Scheduler::schedule(freed)` - 1. Call `Scheduler::occupied` for all scheduled cores where a backed candidate was submitted. + 1. Invoke the `Inclusion::process_candidates` routine with the parameters `(backed_candidates, Scheduler::scheduled(), Scheduler::group_validators)`. + 1. Call `Scheduler::occupied` using the return value of the `Inclusion::process_candidates` call above, first sorting the list of assigned core indices. 1. If all of the above succeeds, set `Included` to `Some(())`. From d1d7fa9d686562d4336cea25da7aa4123e34772d Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 21:25:46 -0400 Subject: [PATCH 15/50] guide: rename group_on to group_validators --- roadmap/implementors-guide/src/runtime/inclusion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index f377acb03577..6883fac2f378 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -58,12 +58,12 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number. 1. > TODO: pass it onwards to `Validity` module. 1. Return a list of freed cores consisting of the cores where candidates have become available. -* `process_candidates(BackedCandidates, scheduled: Vec, group_on: Fn(GroupIndex) -> Option>)`: +* `process_candidates(BackedCandidates, scheduled: Vec, group_validators: Fn(GroupIndex) -> Option>)`: 1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`. 1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates. 1. check that there is no candidate pending availability for any scheduled `ParaId`. 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of the currently scheduled upgrade, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. - 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_on` lookup. + 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup. 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was successfully backed for, sorted ascending by CoreIndex. * `enact_candidate(relay_parent_number: BlockNumber, AbridgedCandidateReceipt)`: From 80613a4a36845cf2fcf79d32e62d6bfe475540bf Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 21:31:53 -0400 Subject: [PATCH 16/50] guide: add check about collator for parathread --- roadmap/implementors-guide/src/runtime/inclusion.md | 1 + 1 file changed, 1 insertion(+) diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index 6883fac2f378..113848a3e409 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -62,6 +62,7 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`. 1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates. 1. check that there is no candidate pending availability for any scheduled `ParaId`. + 1. If the core assignment includes a specific collator, ensure the backed candidate is issued by that collator. 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of the currently scheduled upgrade, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup. 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. From 4a2ee633e19dd87bfb4e4f71e1c901367dde047f Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 21:55:24 -0400 Subject: [PATCH 17/50] guide: add last_code_upgrade to paras and use in inclusion --- roadmap/implementors-guide/src/runtime/inclusion.md | 2 +- roadmap/implementors-guide/src/runtime/paras.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index 113848a3e409..21420e323fde 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -63,7 +63,7 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates. 1. check that there is no candidate pending availability for any scheduled `ParaId`. 1. If the core assignment includes a specific collator, ensure the backed candidate is issued by that collator. - 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of the currently scheduled upgrade, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. + 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of `Paras::last_code_upgrade(para_id, true)`, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup. 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was successfully backed for, sorted ascending by CoreIndex. diff --git a/roadmap/implementors-guide/src/runtime/paras.md b/roadmap/implementors-guide/src/runtime/paras.md index d22021d68116..e80c2d102d2f 100644 --- a/roadmap/implementors-guide/src/runtime/paras.md +++ b/roadmap/implementors-guide/src/runtime/paras.md @@ -111,6 +111,8 @@ OutgoingParas: Vec; * `validation_code_at(ParaId, at: BlockNumber, assume_intermediate: Option)`: Fetches the validation code to be used when validating a block in the context of the given relay-chain height. A second block number parameter may be used to tell the lookup to proceed as if an intermediate parablock has been included at the given relay-chain height. This may return past, current, or (with certain choices of `assume_intermediate`) future code. `assume_intermediate`, if provided, must be before `at`. If the validation code has been pruned, this will return `None`. * `is_parathread(ParaId) -> bool`: Returns true if the para ID references any live parathread. +* `last_code_upgrade(id: ParaId, include_future: bool) -> Option`: The block number of the last scheduled upgrade of the requested para. Includes future upgrades if the flag is set. This is the `expected_at` number, not the `activated_at` number. + ## Finalization No finalization routine runs for this module. From c8be591fd9e424c258455275fb74e46396fe02dc Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 21:57:30 -0400 Subject: [PATCH 18/50] implement Paras::last_code_upgrade --- runtime/parachains/src/paras.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs index 9d8c74ca0932..73ec5573e277 100644 --- a/runtime/parachains/src/paras.rs +++ b/runtime/parachains/src/paras.rs @@ -526,6 +526,18 @@ impl Module { pub(crate) fn is_parathread(id: ParaId) -> bool { Parathreads::get(&id).is_some() } + + /// The block number of the last scheduled upgrade of the requested para. Includes future upgrades + /// if the flag is set. This is the `expected_at` number, not the `activated_at` number. + pub(crate) fn last_code_upgrade(id: ParaId, include_future: bool) -> Option { + if include_future { + if let Some(at) = Self::future_code_upgrade_at(id) { + return Some(at); + } + } + + Self::past_code_meta(&id).most_recent_change() + } } #[cfg(test)] From ab81eef6b966c16d6a5ab12cff00f092d0aa9e39 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 22:09:12 -0400 Subject: [PATCH 19/50] implement most checks in process_candidates --- runtime/parachains/src/inclusion.rs | 134 +++++++++++++++++++++++++++- runtime/parachains/src/scheduler.rs | 1 - 2 files changed, 130 insertions(+), 5 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 41a5fc6e65b3..35c0d14f1309 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -25,6 +25,7 @@ use primitives::{ parachain::{ ValidatorId, AbridgedCandidateReceipt, ValidatorIndex, Id as ParaId, AvailabilityBitfield as AvailabilityBitfield, SignedAvailabilityBitfields, SigningContext, + BackedCandidate, }, }; use frame_support::{ @@ -35,8 +36,9 @@ use codec::{Encode, Decode}; use system::ensure_root; use bitvec::vec::BitVec; use sp_staking::SessionIndex; +use sp_runtime::{DispatchError, traits::{One, Saturating}}; -use crate::{configuration, paras, scheduler::CoreIndex}; +use crate::{configuration, paras, scheduler::{CoreIndex, GroupIndex, CoreAssignment}}; /// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding /// for any backed candidates referred to by a `1` bit available. @@ -93,6 +95,18 @@ decl_error! { ValidatorIndexOutOfBounds, /// Invalid signature InvalidBitfieldSignature, + /// Candidate submitted but para not scheduled. + UnscheduledCandidate, + /// Candidate scheduled despite pending candidate already existing for the para. + CandidateScheduledBeforeParaFree, + /// Candidate included with the wrong collator. + WrongCollator, + /// Scheduled cores out of order. + ScheduledOutOfOrder, + /// Code upgrade prematurely. + PrematureCodeUpgrade, + /// Candidate not in parent context. + CandidateNotInParentContext, } } @@ -118,11 +132,12 @@ impl Module { CurrentSessionIndex::set(notification.session_index); } - /// Process a set of incoming bitfields. + /// Process a set of incoming bitfields. Return a vec of cores freed by candidates + /// becoming available. pub(crate) fn process_bitfields( signed_bitfields: SignedAvailabilityBitfields, core_lookup: impl Fn(CoreIndex) -> Option, - ) -> DispatchResult { + ) -> Result, DispatchError> { let validators = Validators::get(); let session_index = CurrentSessionIndex::get(); let config = >::config(); @@ -183,6 +198,117 @@ impl Module { // TODO: pass available candidates onwards to validity module once implemented. // https://github.com/paritytech/polkadot/issues/1251 - Ok(()) + Ok(Vec::new()) + } + + /// Process candidates that have been backed. Provide a set of candidates and scheduled cores. + /// + /// Both should be sorted ascending by core index, and the candidates should be a subset of + /// scheduled cores. If these conditions are not met, the execution of the function fails. + pub(crate) fn process_candidates( + candidates: Vec, + scheduled: Vec, + group_validators: impl Fn(GroupIndex) -> Option>, + ) + -> Result, DispatchError> + { + ensure!( + candidates.len() <= scheduled.len(), + Error::::UnscheduledCandidate, + ); + + if scheduled.is_empty() { return Ok(Vec::new()) } + + let mut indices_in_scheduled = { + let mut skip = 0; + let mut indices = Vec::with_capacity(candidates.len()); + let mut last_core = None; + + let mut check_assignment_in_order = |assignment: &CoreAssignment| -> DispatchResult { + ensure!( + last_core.map_or(true, |core| assignment.core > core), + Error::::ScheduledOutOfOrder, + ); + + last_core = Some(assignment.core); + Ok(()) + }; + + 'a: + for candidate in &candidates { + for (i, assignment) in scheduled[skip..].iter().enumerate() { + check_assignment_in_order(assignment)?; + + if candidate.candidate.parachain_index == assignment.para_id { + // account for already skipped. + let index_in_scheduled = i + skip; + skip = index_in_scheduled; + + indices.push(index_in_scheduled); + continue 'a; + } + } + + // end of loop reached means that the candidate didn't appear in the non-traversed + // section of the `scheduled` slice. either it was not scheduled or didn't appear in + // `candidates` in the correct order. + ensure!( + false, + Error::::UnscheduledCandidate, + ); + } + + // check remainder of scheduled cores, if any. + for assignment in scheduled[skip..].iter() { + check_assignment_in_order(assignment)?; + } + + indices + }; + + let core_assignments = indices_in_scheduled.iter().cloned().map(|i| &scheduled[i]); + let config = >::config(); + let now = >::block_number(); + let parent_hash = >::parent_hash(); + let validators = Validators::get(); + + for (candidate, core_assignment) in candidates.into_iter().zip(core_assignments) { + let para_id = core_assignment.para_id; + + // we require that the candidate is in the context of the parent block. + // TODO [now] + // ensure!( + // candidate.candidate.relay_parent == parent_hash, + // Error::::CandidateNotInParentContext, + // ); + + let relay_parent_number = now - One::one(); + + ensure!( + >::get(&core_assignment.para_id).is_none(), + Error::::CandidateScheduledBeforeParaFree, + ); + + if let Some(required_collator) = core_assignment.required_collator() { + ensure!( + required_collator == &candidate.candidate.collator, + Error::::WrongCollator, + ); + } + + let code_upgrade_allowed = >::last_code_upgrade(para_id, true).map_or( + true, + |last| relay_parent_number.saturating_sub(last) + >= config.validation_upgrade_frequency, + ); + + ensure!(code_upgrade_allowed, Error::::PrematureCodeUpgrade); + + // TODO [now]: signature check. needs localvalidationdata / globalvalidationschedule? + + // TODO [now]: place into pending candidate storage. + } + + Ok(Vec::new()) } } diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 04efb3c28a46..ef8c3e4285fd 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -145,7 +145,6 @@ pub struct CoreAssignment { impl CoreAssignment { /// Get the ID of a collator who is required to collate this block. - #[allow(unused)] pub(crate) fn required_collator(&self) -> Option<&CollatorId> { match self.kind { AssignmentKind::Parachain => None, From d4585a13d02a520add2bad67844c74bb54682362 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 22:17:03 -0400 Subject: [PATCH 20/50] make candidate receipt structs more generic --- primitives/src/parachain.rs | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 5d8875c9f12d..f0639537de22 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -176,20 +176,20 @@ pub struct DutyRoster { /// These are global parameters that apply to all parachain candidates in a block. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct GlobalValidationSchedule { +pub struct GlobalValidationSchedule { /// The maximum code size permitted, in bytes. pub max_code_size: u32, /// The maximum head-data size permitted, in bytes. pub max_head_data_size: u32, /// The relay-chain block number this is in the context of. - pub block_number: BlockNumber, + pub block_number: N, } /// Extra data that is needed along with the other fields in a `CandidateReceipt` /// to fully validate the candidate. These fields are parachain-specific. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct LocalValidationData { +pub struct LocalValidationData { /// The parent head-data. pub parent_head: HeadData, /// The balance of the parachain at the moment of validation. @@ -205,19 +205,19 @@ pub struct LocalValidationData { /// height. This may be equal to the current perceived relay-chain block height, in /// which case the code upgrade should be applied at the end of the signaling /// block. - pub code_upgrade_allowed: Option, + pub code_upgrade_allowed: Option, } /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct CandidateCommitments { +pub struct CandidateCommitments { /// Fees paid from the chain to the relay chain validators. pub fees: Balance, /// Messages destined to be interpreted by the Relay chain itself. pub upward_messages: Vec, /// The root of a block's erasure encoding Merkle tree. - pub erasure_root: Hash, + pub erasure_root: H, /// New validation code. pub new_validation_code: Option, } @@ -258,12 +258,12 @@ fn check_collator_signature( /// All data pertaining to the execution of a parachain candidate. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct CandidateReceipt { +pub struct CandidateReceipt { /// The ID of the parachain this is a candidate for. pub parachain_index: Id, /// The hash of the relay-chain block this should be executed in /// the context of. - pub relay_parent: Hash, + pub relay_parent: H, /// The head-data pub head_data: HeadData, /// The collator's relay-chain account ID @@ -271,13 +271,13 @@ pub struct CandidateReceipt { /// Signature on blake2-256 of the block data by collator. pub signature: CollatorSignature, /// The hash of the PoV-block. - pub pov_block_hash: Hash, + pub pov_block_hash: H, /// The global validation schedule. - pub global_validation: GlobalValidationSchedule, + pub global_validation: GlobalValidationSchedule, /// The local validation data. - pub local_validation: LocalValidationData, + pub local_validation: LocalValidationData, /// Commitments made as a result of validation. - pub commitments: CandidateCommitments, + pub commitments: CandidateCommitments, } impl CandidateReceipt { @@ -345,11 +345,11 @@ impl Ord for CandidateReceipt { /// is necessary for validation of the parachain candidate. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct OmittedValidationData { +pub struct OmittedValidationData { /// The global validation schedule. - pub global_validation: GlobalValidationSchedule, + pub global_validation: GlobalValidationSchedule, /// The local validation data. - pub local_validation: LocalValidationData, + pub local_validation: LocalValidationData, } /// An abridged candidate-receipt. @@ -359,14 +359,14 @@ pub struct OmittedValidationData { /// be re-generated from relay-chain state. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug, Default))] -pub struct AbridgedCandidateReceipt { +pub struct AbridgedCandidateReceipt { /// The ID of the parachain this is a candidate for. pub parachain_index: Id, /// The hash of the relay-chain block this should be executed in /// the context of. // NOTE: the fact that the hash includes this value means that code depends // on this for deduplication. Removing this field is likely to break things. - pub relay_parent: Hash, + pub relay_parent: H, /// The head-data pub head_data: HeadData, /// The collator's relay-chain account ID @@ -374,9 +374,9 @@ pub struct AbridgedCandidateReceipt { /// Signature on blake2-256 of the block data by collator. pub signature: CollatorSignature, /// The hash of the pov-block. - pub pov_block_hash: Hash, + pub pov_block_hash: H, /// Commitments made as a result of validation. - pub commitments: CandidateCommitments, + pub commitments: CandidateCommitments, } impl AbridgedCandidateReceipt { From 5ce3860fd66439375e58f88d927a3640eba39894 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 22:17:50 -0400 Subject: [PATCH 21/50] make BackedCandidate struct more generic --- primitives/src/parachain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index f0639537de22..54589cc86e05 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -741,9 +741,9 @@ pub struct SignedAvailabilityBitfields(pub Vec); // they should be unified to this type. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] -pub struct BackedCandidate { +pub struct BackedCandidate { /// The candidate referred to. - pub candidate: AbridgedCandidateReceipt, + pub candidate: AbridgedCandidateReceipt, /// The validity votes themselves, expressed as signatures. pub validity_votes: Vec, /// The indices of the validators within the group, expressed as a bitfield. From cf2882041d72fc335ad02e2b4bd05a53e2495737 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 22:18:24 -0400 Subject: [PATCH 22/50] use hash param, not block number --- primitives/src/parachain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 54589cc86e05..bc0f668adb9e 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -741,9 +741,9 @@ pub struct SignedAvailabilityBitfields(pub Vec); // they should be unified to this type. #[derive(PartialEq, Eq, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] -pub struct BackedCandidate { +pub struct BackedCandidate { /// The candidate referred to. - pub candidate: AbridgedCandidateReceipt, + pub candidate: AbridgedCandidateReceipt, /// The validity votes themselves, expressed as signatures. pub validity_votes: Vec, /// The indices of the validators within the group, expressed as a bitfield. From b4bbf4cb9d10035941b975c52a2cdff89f358cdf Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 11 Jun 2020 22:19:05 -0400 Subject: [PATCH 23/50] check that candidate is in context of the parent block --- runtime/parachains/src/inclusion.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 35c0d14f1309..930a00926c1b 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -206,7 +206,7 @@ impl Module { /// Both should be sorted ascending by core index, and the candidates should be a subset of /// scheduled cores. If these conditions are not met, the execution of the function fails. pub(crate) fn process_candidates( - candidates: Vec, + candidates: Vec>, scheduled: Vec, group_validators: impl Fn(GroupIndex) -> Option>, ) @@ -276,11 +276,10 @@ impl Module { let para_id = core_assignment.para_id; // we require that the candidate is in the context of the parent block. - // TODO [now] - // ensure!( - // candidate.candidate.relay_parent == parent_hash, - // Error::::CandidateNotInParentContext, - // ); + ensure!( + candidate.candidate.relay_parent == parent_hash, + Error::::CandidateNotInParentContext, + ); let relay_parent_number = now - One::one(); From 976c289ccc8c095c39f6917dd59179438ae59de9 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 12:52:54 -0400 Subject: [PATCH 24/50] include inclusion module in initializer --- runtime/parachains/src/inclusion.rs | 8 +++++++- runtime/parachains/src/initializer.rs | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 930a00926c1b..6bf063cb0713 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -119,9 +119,15 @@ decl_module! { impl Module { + /// Block initialization logic, called by initializer. + pub(crate) fn initializer_initialize(_now: T::BlockNumber) -> Weight { 0 } + + /// Block finalization logic, called by initializer. + pub(crate) fn initializer_finalize() { } + /// Handle an incoming session change. pub(crate) fn initializer_on_new_session( - notification: crate::initializer::SessionChangeNotification + notification: &crate::initializer::SessionChangeNotification ) { // unlike most drain methods, drained elements are not cleared on `Drop` of the iterator // and require consumption. diff --git a/runtime/parachains/src/initializer.rs b/runtime/parachains/src/initializer.rs index 681824f04c69..5668a216f061 100644 --- a/runtime/parachains/src/initializer.rs +++ b/runtime/parachains/src/initializer.rs @@ -85,7 +85,8 @@ decl_module! { // - Validity let total_weight = configuration::Module::::initializer_initialize(now) + paras::Module::::initializer_initialize(now) + - scheduler::Module::::initializer_initialize(now); + scheduler::Module::::initializer_initialize(now) + + inclusion::Module::::initializer_initialize(now); HasInitialized::set(Some(())); @@ -95,6 +96,7 @@ decl_module! { fn on_finalize() { // reverse initialization order. + inclusion::Module::::initializer_finalize(); scheduler::Module::::initializer_finalize(); paras::Module::::initializer_finalize(); configuration::Module::::initializer_finalize(); @@ -152,6 +154,7 @@ impl Module { paras::Module::::initializer_on_new_session(¬ification); scheduler::Module::::initializer_on_new_session(¬ification); + inclusion::Module::::initializer_on_new_session(¬ification); } } From b37a08ccfa56571cc06252a4090ae242bd66d1e6 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 13:04:31 -0400 Subject: [PATCH 25/50] implement enact-candidate --- runtime/parachains/src/inclusion.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 6bf063cb0713..ad8b05969416 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -31,6 +31,7 @@ use primitives::{ use frame_support::{ decl_storage, decl_module, decl_error, ensure, dispatch::DispatchResult, IterableStorageMap, weights::{DispatchClass, Weight}, + traits::Get, }; use codec::{Encode, Decode}; use system::ensure_root; @@ -316,4 +317,28 @@ impl Module { Ok(Vec::new()) } + + fn enact_candidate( + relay_parent_number: T::BlockNumber, + receipt: AbridgedCandidateReceipt, + ) -> Weight { + let commitments = receipt.commitments; + let config = >::config(); + + // initial weight is config read. + let mut weight = T::DbWeight::get().reads_writes(1, 0); + if let Some(new_code) = commitments.new_validation_code { + weight += >::schedule_code_upgrade( + receipt.parachain_index, + new_code, + relay_parent_number + config.validation_upgrade_delay, + ); + } + + weight + >::note_new_head( + receipt.parachain_index, + receipt.head_data, + relay_parent_number, + ) + } } From 7ca35276122957f67aca0ee1cb44bca1381bb98e Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 18:41:32 -0400 Subject: [PATCH 26/50] check that only occupied cores have bits set --- runtime/parachains/src/inclusion.rs | 20 +++++++++++++++++--- runtime/parachains/src/scheduler.rs | 6 ++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index ad8b05969416..c92b42c928c1 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -35,7 +35,7 @@ use frame_support::{ }; use codec::{Encode, Decode}; use system::ensure_root; -use bitvec::vec::BitVec; +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use sp_staking::SessionIndex; use sp_runtime::{DispatchError, traits::{One, Saturating}}; @@ -59,7 +59,7 @@ pub struct CandidatePendingAvailability { /// The candidate receipt itself. receipt: AbridgedCandidateReceipt, /// The received availability votes. One bit per validator. - availability_votes: bitvec::vec::BitVec, + availability_votes: BitVec, /// The block number of the relay-parent of the receipt. relay_parent_number: N, /// The block number of the relay-chain block this was backed in. @@ -108,6 +108,8 @@ decl_error! { PrematureCodeUpgrade, /// Candidate not in parent context. CandidateNotInParentContext, + /// The bitfield contains a bit relating to an unassigned availability core. + UnoccupiedBitInBitfield, } } @@ -153,6 +155,11 @@ impl Module { let n_validators = validators.len(); let n_bits = parachains.len() + config.parathread_cores as usize; + let assigned_paras: Vec<_> = + (0..n_bits).map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32))).collect(); + + let occupied_bitmask: BitVec = assigned_paras.iter().map(|p| p.is_some()).collect(); + // do sanity checks on the bitfields: // 1. no more than one bitfield per validator // 2. bitfields are ascending by validator index. @@ -183,6 +190,11 @@ impl Module { Error::::ValidatorIndexOutOfBounds, ); + ensure!( + occupied_bitmask.clone() & signed_bitfield.bitfield.0.clone() == signed_bitfield.bitfield.0, + Error::::UnoccupiedBitInBitfield, + ); + let validator_public = &validators[signed_bitfield.validator_index as usize]; if let Err(()) = primitives::parachain::check_availability_bitfield_signature( @@ -200,7 +212,9 @@ impl Module { } } - // TODO [now] actually apply bitfields + for signed_bitfield in &signed_bitfields.0 { + + } // TODO: pass available candidates onwards to validity module once implemented. // https://github.com/paritytech/polkadot/issues/1251 diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index ef8c3e4285fd..15830b5e7b01 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -57,6 +57,12 @@ use crate::{configuration, paras, initializer::SessionChangeNotification}; #[cfg_attr(test, derive(Debug))] pub struct CoreIndex(u32); +impl From for CoreIndex { + fn from(i: u32) -> CoreIndex { + CoreIndex(i) + } +} + /// The unique (during session) index of a validator group. #[derive(Encode, Decode, Default, Clone, Copy)] #[cfg_attr(test, derive(PartialEq, Debug))] From 383db96373c744c1bae567aaf83e1dee3e482e29 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 19:48:39 -0400 Subject: [PATCH 27/50] finish implementing bitfield processing --- runtime/parachains/src/inclusion.rs | 71 +++++++++++++++++++++++++---- runtime/parachains/src/scheduler.rs | 7 +++ 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index c92b42c928c1..657cbf60ff56 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -53,11 +53,11 @@ pub struct AvailabilityBitfieldRecord { /// A backed candidate pending availability. #[derive(Encode, Decode)] #[cfg_attr(test, derive(Debug))] -pub struct CandidatePendingAvailability { +pub struct CandidatePendingAvailability { /// The availability core this is assigned to. core: CoreIndex, /// The candidate receipt itself. - receipt: AbridgedCandidateReceipt, + receipt: AbridgedCandidateReceipt, /// The received availability votes. One bit per validator. availability_votes: BitVec, /// The block number of the relay-parent of the receipt. @@ -76,7 +76,7 @@ decl_storage! { /// Candidates pending availability by `ParaId`. PendingAvailability: map hasher(twox_64_concat) ParaId - => Option>; + => Option>; /// The current validators, by their parachain session keys. Validators get(fn validators) config(validators): Vec; @@ -155,10 +155,10 @@ impl Module { let n_validators = validators.len(); let n_bits = parachains.len() + config.parathread_cores as usize; - let assigned_paras: Vec<_> = - (0..n_bits).map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32))).collect(); - - let occupied_bitmask: BitVec = assigned_paras.iter().map(|p| p.is_some()).collect(); + let mut assigned_paras_record: Vec<_> = (0..n_bits) + .map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32))) + .map(|core_para| core_para.map(|p| (p, PendingAvailability::::get(&p)))) + .collect(); // do sanity checks on the bitfields: // 1. no more than one bitfield per validator @@ -166,6 +166,10 @@ impl Module { // 3. each bitfield has exactly `n_bits` // 4. signature is valid. { + let occupied_bitmask: BitVec = assigned_paras_record.iter() + .map(|p| p.is_some()) + .collect(); + let mut last_index = None; let mut payload_encode_buf = Vec::new(); @@ -212,14 +216,61 @@ impl Module { } } - for signed_bitfield in &signed_bitfields.0 { + let now = >::block_number(); + for signed_bitfield in signed_bitfields.0 { + for (bit_idx, _) + in signed_bitfield.bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av) + { + let record = assigned_paras_record[bit_idx] + .as_mut() + .expect("validator bitfields checked not to contain bits corresponding to unoccupied cores; qed"); + + // defensive check - this is constructed by loading the availability bitfield record, + // which is always `Some` if the core is occupied - that's why we're here. + let val_idx = signed_bitfield.validator_index as usize; + if let Some(mut bit) = record.1.as_mut() + .and_then(|r| r.availability_votes.get_mut(val_idx)) + { + *bit = true; + } + } + + let record = AvailabilityBitfieldRecord { + bitfield: signed_bitfield.bitfield, + submitted_at: now, + }; + + >::insert(&signed_bitfield.validator_index, record); + } + + let threshold = { + let mut threshold = (validators.len() * 2) / 3; + threshold += (validators.len() * 2) % 3; + threshold + }; + + let mut freed_cores = Vec::with_capacity(n_bits); + for (para_id, pending_availability) in assigned_paras_record.into_iter() + .filter_map(|x| x) + .filter_map(|(id, p)| p.map(|p| (id, p))) + { + if pending_availability.availability_votes.count_ones() >= threshold { + >::remove(¶_id); + Self::enact_candidate( + pending_availability.relay_parent_number, + pending_availability.receipt, + ); + freed_cores.push(pending_availability.core); + } else { + >::insert(¶_id, &pending_availability); + } } // TODO: pass available candidates onwards to validity module once implemented. // https://github.com/paritytech/polkadot/issues/1251 - Ok(Vec::new()) + Ok(freed_cores) } /// Process candidates that have been backed. Provide a set of candidates and scheduled cores. @@ -334,7 +385,7 @@ impl Module { fn enact_candidate( relay_parent_number: T::BlockNumber, - receipt: AbridgedCandidateReceipt, + receipt: AbridgedCandidateReceipt, ) -> Weight { let commitments = receipt.commitments; let config = >::config(); diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 15830b5e7b01..0d49ba36dd6c 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -63,6 +63,13 @@ impl From for CoreIndex { } } +impl CoreIndex { + /// Convert into inner u32. + fn into_inner(self) -> u32 { + self.0 + } +} + /// The unique (during session) index of a validator group. #[derive(Encode, Decode, Default, Clone, Copy)] #[cfg_attr(test, derive(PartialEq, Debug))] From f5372b7deaae3a610e641a7c2724853642d256b8 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 20:16:46 -0400 Subject: [PATCH 28/50] restructure consistency checks on candidates --- runtime/parachains/src/inclusion.rs | 96 +++++++++++++++-------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 657cbf60ff56..697da0c63c25 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -110,6 +110,8 @@ decl_error! { CandidateNotInParentContext, /// The bitfield contains a bit relating to an unassigned availability core. UnoccupiedBitInBitfield, + /// Invalid group index in core assignment. + InvalidGroupIndex, } } @@ -284,17 +286,21 @@ impl Module { ) -> Result, DispatchError> { - ensure!( - candidates.len() <= scheduled.len(), - Error::::UnscheduledCandidate, - ); + ensure!(candidates.len() <= scheduled.len(), Error::::UnscheduledCandidate); - if scheduled.is_empty() { return Ok(Vec::new()) } + if scheduled.is_empty() { + return Ok(Vec::new()); + } + + let parent_hash = >::parent_hash(); + let config = >::config(); + let now = >::block_number(); - let mut indices_in_scheduled = { + let mut assignment_meta = { let mut skip = 0; - let mut indices = Vec::with_capacity(candidates.len()); + let mut assignment_meta = Vec::with_capacity(candidates.len()); let mut last_core = None; + let relay_parent_number = now - One::one(); let mut check_assignment_in_order = |assignment: &CoreAssignment| -> DispatchResult { ensure!( @@ -308,15 +314,45 @@ impl Module { 'a: for candidate in &candidates { + let para_id = candidate.candidate.parachain_index; + + // we require that the candidate is in the context of the parent block. + ensure!( + candidate.candidate.relay_parent == parent_hash, + Error::::CandidateNotInParentContext, + ); + + let code_upgrade_allowed = >::last_code_upgrade(para_id, true) + .map_or( + true, + |last| relay_parent_number.saturating_sub(last) + >= config.validation_upgrade_frequency, + ); + + ensure!(code_upgrade_allowed, Error::::PrematureCodeUpgrade); for (i, assignment) in scheduled[skip..].iter().enumerate() { check_assignment_in_order(assignment)?; if candidate.candidate.parachain_index == assignment.para_id { + if let Some(required_collator) = assignment.required_collator() { + ensure!( + required_collator == &candidate.candidate.collator, + Error::::WrongCollator, + ); + } + + ensure!( + >::get(&assignment.para_id).is_none(), + Error::::CandidateScheduledBeforeParaFree, + ); + // account for already skipped. - let index_in_scheduled = i + skip; - skip = index_in_scheduled; + skip = i + skip; - indices.push(index_in_scheduled); + let group_vals = group_validators(assignment.group_idx) + .ok_or_else(|| Error::::InvalidGroupIndex)?; + + assignment_meta.push((assignment.core, group_vals)); continue 'a; } } @@ -335,46 +371,14 @@ impl Module { check_assignment_in_order(assignment)?; } - indices + assignment_meta }; - let core_assignments = indices_in_scheduled.iter().cloned().map(|i| &scheduled[i]); - let config = >::config(); - let now = >::block_number(); - let parent_hash = >::parent_hash(); let validators = Validators::get(); - for (candidate, core_assignment) in candidates.into_iter().zip(core_assignments) { - let para_id = core_assignment.para_id; - - // we require that the candidate is in the context of the parent block. - ensure!( - candidate.candidate.relay_parent == parent_hash, - Error::::CandidateNotInParentContext, - ); - - let relay_parent_number = now - One::one(); - - ensure!( - >::get(&core_assignment.para_id).is_none(), - Error::::CandidateScheduledBeforeParaFree, - ); - - if let Some(required_collator) = core_assignment.required_collator() { - ensure!( - required_collator == &candidate.candidate.collator, - Error::::WrongCollator, - ); - } - - let code_upgrade_allowed = >::last_code_upgrade(para_id, true).map_or( - true, - |last| relay_parent_number.saturating_sub(last) - >= config.validation_upgrade_frequency, - ); - - ensure!(code_upgrade_allowed, Error::::PrematureCodeUpgrade); - + for (candidate, (core_index, group_validators)) + in candidates.into_iter().zip(assignment_meta) + { // TODO [now]: signature check. needs localvalidationdata / globalvalidationschedule? // TODO [now]: place into pending candidate storage. From 3a1959119b7f307bf7c82000d570cb1dfae9aea2 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 20:27:30 -0400 Subject: [PATCH 29/50] make some more primitives generic --- primitives/src/parachain.rs | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index bc0f668adb9e..09bdaed0b31e 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -223,10 +223,10 @@ pub struct CandidateCommitments { } /// Get a collator signature payload on a relay-parent, block-data combo. -pub fn collator_signature_payload( - relay_parent: &Hash, +pub fn collator_signature_payload>( + relay_parent: &H, parachain_index: &Id, - pov_block_hash: &Hash, + pov_block_hash: &H, ) -> [u8; 68] { // 32-byte hash length is protected in a test below. let mut payload = [0u8; 68]; @@ -238,10 +238,10 @@ pub fn collator_signature_payload( payload } -fn check_collator_signature( - relay_parent: &Hash, +fn check_collator_signature>( + relay_parent: &H, parachain_index: &Id, - pov_block_hash: &Hash, + pov_block_hash: &H, collator: &CollatorId, signature: &CollatorSignature, ) -> Result<(),()> { @@ -280,7 +280,7 @@ pub struct CandidateReceipt { pub commitments: CandidateCommitments, } -impl CandidateReceipt { +impl, N> CandidateReceipt { /// Check integrity vs. provided block data. pub fn check_signature(&self) -> Result<(), ()> { check_collator_signature( @@ -294,7 +294,7 @@ impl CandidateReceipt { /// Abridge this `CandidateReceipt`, splitting it into an `AbridgedCandidateReceipt` /// and its omitted component. - pub fn abridge(self) -> (AbridgedCandidateReceipt, OmittedValidationData) { + pub fn abridge(self) -> (AbridgedCandidateReceipt, OmittedValidationData) { let CandidateReceipt { parachain_index, relay_parent, @@ -379,7 +379,18 @@ pub struct AbridgedCandidateReceipt { pub commitments: CandidateCommitments, } -impl AbridgedCandidateReceipt { +impl + Encode> AbridgedCandidateReceipt { + /// Check integrity vs. provided block data. + pub fn check_signature(&self) -> Result<(), ()> { + check_collator_signature( + &self.relay_parent, + &self.parachain_index, + &self.pov_block_hash, + &self.collator, + &self.signature, + ) + } + /// Compute the hash of the abridged candidate receipt. /// /// This is often used as the canonical hash of the receipt, rather than @@ -391,10 +402,12 @@ impl AbridgedCandidateReceipt { use runtime_primitives::traits::{BlakeTwo256, Hash}; BlakeTwo256::hash_of(self) } +} +impl AbridgedCandidateReceipt { /// Combine the abridged candidate receipt with the omitted data, /// forming a full `CandidateReceipt`. - pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt { + pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt { let AbridgedCandidateReceipt { parachain_index, relay_parent, @@ -606,6 +619,7 @@ pub enum ValidityAttestation { #[codec(index = "2")] Explicit(ValidatorSignature), } + /// A type returned by runtime with current session index and a parent hash. #[derive(Clone, Eq, PartialEq, Default, Decode, Encode, RuntimeDebug)] pub struct SigningContext { From b2e2754dbb0d9e9e815a42e0210fcb7150e1bb99 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 20:50:47 -0400 Subject: [PATCH 30/50] signature checking logic for backed candidates --- primitives/src/parachain.rs | 81 +++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 09bdaed0b31e..72eafe266e1a 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -620,6 +620,35 @@ pub enum ValidityAttestation { Explicit(ValidatorSignature), } +impl ValidityAttestation { + /// Get a reference to the signature. + pub fn signature(&self) -> &ValidatorSignature { + match *self { + ValidityAttestation::Implicit(ref sig) => sig, + ValidityAttestation::Explicit(ref sig) => sig, + } + } + + /// Produce the underlying signed payload of the attestation, given the hash of the candidate, + /// which should be known in context. + pub fn signed_payload( + &self, + candidate_hash: Hash, + signing_context: &SigningContext, + ) -> Vec { + match *self { + ValidityAttestation::Implicit(_) => ( + Statement::Candidate(candidate_hash), + signing_context, + ).encode(), + ValidityAttestation::Explicit(_) => ( + Statement::Valid(candidate_hash), + signing_context, + ).encode(), + } + } +} + /// A type returned by runtime with current session index and a parent hash. #[derive(Clone, Eq, PartialEq, Default, Decode, Encode, RuntimeDebug)] pub struct SigningContext { @@ -764,6 +793,58 @@ pub struct BackedCandidate { pub validator_indices: BitVec, } +/// Verify the backing of the given candidate. +/// +/// Provide a lookup from the index of a validator within the group assigned to this para, +/// as opposed to the index of the validator within the overall validator set, as well as +/// the number of validators in the group. +/// +/// Also provide the signing context. +/// +/// Returns either an error, indicating that one of the signatures was invalid or that the index +/// was out-of-bounds, or the number of signatures checked. +pub fn check_candidate_backing + Encode>( + backed: &BackedCandidate, + signing_context: &SigningContext, + group_len: usize, + validator_lookup: impl Fn(usize) -> Option, +) -> Result { + use runtime_primitives::traits::AppVerify; + + if backed.validator_indices.len() != group_len { + return Err(()) + } + + if backed.validity_votes.len() > group_len { + return Err(()) + } + + // this is known, even in runtime, to be blake2-256. + let hash: Hash = backed.candidate.hash(); + + let mut signed = 0; + for ((val_in_group_idx, _), attestation) in backed.validator_indices.iter().enumerate() + .filter(|(_, signed)| **signed) + .zip(backed.validity_votes.iter()) + { + let validator_id = validator_lookup(val_in_group_idx).ok_or(())?; + let payload = attestation.signed_payload(hash.clone(), signing_context); + let sig = attestation.signature(); + + if sig.verify(&payload[..], &validator_id) { + signed += 1; + } else { + return Err(()) + } + } + + if signed != backed.validity_votes.len() { + return Err(()) + } + + Ok(signed) +} + sp_api::decl_runtime_apis! { /// The API for querying the state of parachains on-chain. #[api_version(3)] From 7a746567f48818c9ff88c2e2d83832581c28dedd Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 21:14:35 -0400 Subject: [PATCH 31/50] finish implementing process_candidates --- runtime/parachains/src/inclusion.rs | 87 ++++++++++++++++++++++------- runtime/parachains/src/scheduler.rs | 7 --- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 697da0c63c25..2335176b3509 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -112,6 +112,10 @@ decl_error! { UnoccupiedBitInBitfield, /// Invalid group index in core assignment. InvalidGroupIndex, + /// Insufficient (non-majority) backing. + InsufficientBacking, + /// Invalid (bad signature, unknown validator, etc.) backing. + InvalidBacking, } } @@ -154,7 +158,6 @@ impl Module { let config = >::config(); let parachains = >::parachains(); - let n_validators = validators.len(); let n_bits = parachains.len() + config.parathread_cores as usize; let mut assigned_paras_record: Vec<_> = (0..n_bits) @@ -292,15 +295,17 @@ impl Module { return Ok(Vec::new()); } + let validators = Validators::get(); let parent_hash = >::parent_hash(); let config = >::config(); let now = >::block_number(); + let relay_parent_number = now - One::one(); - let mut assignment_meta = { + // do all checks before writing storage. + let core_indices = { let mut skip = 0; - let mut assignment_meta = Vec::with_capacity(candidates.len()); + let mut core_indices = Vec::with_capacity(candidates.len()); let mut last_core = None; - let relay_parent_number = now - One::one(); let mut check_assignment_in_order = |assignment: &CoreAssignment| -> DispatchResult { ensure!( @@ -312,6 +317,21 @@ impl Module { Ok(()) }; + let signing_context = SigningContext { + parent_hash, + session_index: CurrentSessionIndex::get(), + }; + + // We combine an outer loop over candidates with an inner loop over the scheduled, + // where each iteration of the outer loop picks up at the position + // in scheduled just after the past iteration left off. + // + // If the candidates appear in the same order as they appear in `scheduled`, + // then they should always be found. If the end of `scheduled` is reached, + // then the candidate was either not scheduled or out-of-order. + // + // In the meantime, we do certain sanity checks on the candidates and on the scheduled + // list. 'a: for candidate in &candidates { let para_id = candidate.candidate.parachain_index; @@ -325,8 +345,8 @@ impl Module { let code_upgrade_allowed = >::last_code_upgrade(para_id, true) .map_or( true, - |last| relay_parent_number.saturating_sub(last) - >= config.validation_upgrade_frequency, + |last| last <= relay_parent_number && + relay_parent_number.saturating_sub(last) >= config.validation_upgrade_frequency, ); ensure!(code_upgrade_allowed, Error::::PrematureCodeUpgrade); @@ -346,13 +366,34 @@ impl Module { Error::::CandidateScheduledBeforeParaFree, ); - // account for already skipped. - skip = i + skip; + // account for already skipped, and then skip this one. + skip = i + skip + 1; let group_vals = group_validators(assignment.group_idx) .ok_or_else(|| Error::::InvalidGroupIndex)?; - assignment_meta.push((assignment.core, group_vals)); + // check the signatures in the backing and that it is a majority. + { + let maybe_amount_validated + = primitives::parachain::check_candidate_backing( + &candidate, + &signing_context, + group_vals.len(), + |idx| group_vals.get(idx) + .and_then(|i| validators.get(*i as usize)) + .map(|v| v.clone()), + ); + + match maybe_amount_validated { + Ok(amount_validated) => ensure!( + amount_validated * 2 > group_vals.len(), + Error::::InsufficientBacking, + ), + Err(()) => { Err(Error::::InvalidBacking)?; } + } + } + + core_indices.push(assignment.core); continue 'a; } } @@ -364,27 +405,33 @@ impl Module { false, Error::::UnscheduledCandidate, ); - } + }; // check remainder of scheduled cores, if any. for assignment in scheduled[skip..].iter() { check_assignment_in_order(assignment)?; } - assignment_meta + core_indices }; - let validators = Validators::get(); - - for (candidate, (core_index, group_validators)) - in candidates.into_iter().zip(assignment_meta) - { - // TODO [now]: signature check. needs localvalidationdata / globalvalidationschedule? - - // TODO [now]: place into pending candidate storage. + // one more sweep for actually writing to storage. + for (candidate, core) in candidates.into_iter().zip(core_indices.iter().cloned()) { + let para_id = candidate.candidate.parachain_index; + + // initialize all availability votes to 0. + let availability_votes: BitVec + = bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()]; + >::insert(¶_id, CandidatePendingAvailability { + core, + receipt: candidate.candidate, + availability_votes, + relay_parent_number, + backed_in_number: now, + }); } - Ok(Vec::new()) + Ok(core_indices) } fn enact_candidate( diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 0d49ba36dd6c..15830b5e7b01 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -63,13 +63,6 @@ impl From for CoreIndex { } } -impl CoreIndex { - /// Convert into inner u32. - fn into_inner(self) -> u32 { - self.0 - } -} - /// The unique (during session) index of a validator group. #[derive(Encode, Decode, Default, Clone, Copy)] #[cfg_attr(test, derive(PartialEq, Debug))] From 8f1774a125547d93f37fd775a0afc424b6505287 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 21:21:57 -0400 Subject: [PATCH 32/50] implement collect_pending --- runtime/parachains/src/inclusion.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 2335176b3509..3657a2cb7f20 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -457,4 +457,28 @@ impl Module { relay_parent_number, ) } + + /// Cleans up all paras pending availability that the predicate returns true for. + /// + /// The predicate accepts the index of the core and the block number the core has been occupied + /// since (i.e. the block number the candidate was backed at in this fork of the relay chain). + /// + /// Returns a vector of cleaned-up core IDs. + pub(crate) fn collect_pending(pred: impl Fn(CoreIndex, T::BlockNumber) -> bool) -> Vec { + let mut cleaned_up_ids = Vec::new(); + let mut cleaned_up_cores = Vec::new(); + + for (para_id, pending_record) in >::iter() { + if pred(pending_record.core, pending_record.backed_in_number) { + cleaned_up_ids.push(para_id); + cleaned_up_cores.push(pending_record.core); + } + } + + for para_id in cleaned_up_ids { + >::remove(¶_id); + } + + cleaned_up_cores + } } From 058e2f35c34799daf2770b7ae2ff4d70867904f5 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 21:34:39 -0400 Subject: [PATCH 33/50] add some trait implementations to primitives --- primitives/src/parachain.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 72eafe266e1a..ed1c77bda9c4 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -738,8 +738,7 @@ impl AvailabilityBitfield { } /// A bitfield signed by a particular validator about the availability of pending candidates. -#[derive(PartialEq, Eq, Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)] pub struct SignedAvailabilityBitfield { /// The index of the validator in the current set. pub validator_index: ValidatorIndex, @@ -776,14 +775,14 @@ pub fn check_availability_bitfield_signature( } /// A set of signed availability bitfields. Should be sorted by validator index, ascending. +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)] pub struct SignedAvailabilityBitfields(pub Vec); /// A backed (or backable, depending on context) candidate. // TODO: yes, this is roughly the same as AttestedCandidate. // After https://github.com/paritytech/polkadot/issues/1250 // they should be unified to this type. -#[derive(PartialEq, Eq, Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] +#[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug)] pub struct BackedCandidate { /// The candidate referred to. pub candidate: AbridgedCandidateReceipt, From 6b000bc6367d0d9ea471d53e14acfd3d0484ea64 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Fri, 12 Jun 2020 21:48:40 -0400 Subject: [PATCH 34/50] implement InclusionInherent and squash warnings --- runtime/parachains/src/inclusion.rs | 3 +- runtime/parachains/src/inclusion_inherent.rs | 120 +++++++++++++++++++ runtime/parachains/src/lib.rs | 1 + 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 runtime/parachains/src/inclusion_inherent.rs diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 3657a2cb7f20..bb871a103674 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -30,11 +30,10 @@ use primitives::{ }; use frame_support::{ decl_storage, decl_module, decl_error, ensure, dispatch::DispatchResult, IterableStorageMap, - weights::{DispatchClass, Weight}, + weights::Weight, traits::Get, }; use codec::{Encode, Decode}; -use system::ensure_root; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use sp_staking::SessionIndex; use sp_runtime::{DispatchError, traits::{One, Saturating}}; diff --git a/runtime/parachains/src/inclusion_inherent.rs b/runtime/parachains/src/inclusion_inherent.rs new file mode 100644 index 000000000000..46fe1fee469e --- /dev/null +++ b/runtime/parachains/src/inclusion_inherent.rs @@ -0,0 +1,120 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Provides glue code over the scheduler and inclusion modules, and accepting +//! one inherent per block that can include new para candidates and bitfields. +//! +//! Unlike other modules in this crate, it does not need to be initialized by the initializer, +//! as it has no initialization logic and its finalization logic depends only on the details of +//! this module. + +use sp_std::prelude::*; +use primitives::{ + parachain::{BackedCandidate, SignedAvailabilityBitfields}, +}; +use frame_support::{ + decl_storage, decl_module, decl_error, ensure, + dispatch::DispatchResult, + weights::{DispatchClass, Weight}, + traits::Get, +}; +use system::ensure_none; +use crate::{inclusion, scheduler::{self, FreedReason}}; + +pub trait Trait: inclusion::Trait + scheduler::Trait { } + +decl_storage! { + trait Store for Module as ParaInclusionInherent { + /// Whether the inclusion inherent was included within this block. + /// + /// The `Option<()>` is effectively a bool, but it never hits storage in the `None` variant + /// due to the guarantees of FRAME's storage APIs. + /// + /// If this is `None` at the end of the block, we panic and render the block invalid. + Included: Option<()>; + } +} + +decl_error! { + pub enum Error for Module { + /// Inclusion inherent called more than once per block. + TooManyInclusionInherents, + } +} + +decl_module! { + /// The inclusion inherent module. + pub struct Module for enum Call where origin: ::Origin { + type Error = Error; + + fn on_initialize() -> Weight { + T::DbWeight::get().reads_writes(1, 1) // in on_finalize. + } + + fn on_finalize() { + if Included::take().is_none() { + panic!("Bitfields and heads must be included every block"); + } + } + + /// Include backed candidates and bitfields. + #[weight = (1_000_000_000, DispatchClass::Mandatory)] + pub fn inclusion( + origin, + signed_bitfields: SignedAvailabilityBitfields, + backed_candidates: Vec>, + ) -> DispatchResult { + ensure_none(origin)?; + ensure!(!::exists(), Error::::TooManyInclusionInherents); + + // Process new availability bitfields, yielding any availability cores whose + // work has now concluded. + let freed_concluded = >::process_bitfields( + signed_bitfields, + >::core_para, + )?; + + // Handle timeouts for any availability core work. + let availability_pred = >::availability_timeout_predicate(); + let freed_timeout = if let Some(pred) = availability_pred { + >::collect_pending(pred) + } else { + Vec::new() + }; + + // Schedule paras again, given freed cores, and reasons for freeing. + let freed = freed_concluded.into_iter().map(|c| (c, FreedReason::Concluded)) + .chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut))); + + >::schedule(freed.collect()); + + // Process backed candidates according to scheduled cores. + let occupied = >::process_candidates( + backed_candidates, + >::scheduled(), + >::group_validators, + )?; + + // Note which of the scheduled cores were actually occupied by a backed candidate. + >::occupied(&occupied); + + // And track that we've finished processing the inherent for this block. + Included::set(Some(())); + + Ok(()) + } + } +} diff --git a/runtime/parachains/src/lib.rs b/runtime/parachains/src/lib.rs index 74c237fd2213..44554322e4a4 100644 --- a/runtime/parachains/src/lib.rs +++ b/runtime/parachains/src/lib.rs @@ -22,6 +22,7 @@ mod configuration; mod inclusion; +mod inclusion_inherent; mod initializer; mod paras; mod scheduler; From 11cf8a09c3efb6ae9d4a3bfe532bcf3e9597c4c7 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 18:20:55 -0400 Subject: [PATCH 35/50] test bitfield signing checks --- runtime/parachains/src/inclusion.rs | 296 +++++++++++++++++++++++++++- 1 file changed, 294 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index bb871a103674..db7b243b10fc 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -171,7 +171,9 @@ impl Module { // 4. signature is valid. { let occupied_bitmask: BitVec = assigned_paras_record.iter() - .map(|p| p.is_some()) + .map(|p| p.as_ref() + .map_or(false, |(_id, pending_availability)| pending_availability.is_some()) + ) .collect(); let mut last_index = None; @@ -194,7 +196,7 @@ impl Module { ); ensure!( - signed_bitfield.validator_index < validators.len() as _, + signed_bitfield.validator_index < validators.len() as ValidatorIndex, Error::::ValidatorIndexOutOfBounds, ); @@ -481,3 +483,293 @@ impl Module { cleaned_up_cores } } + +#[cfg(test)] +mod tests { + use super::*; + + use primitives::{BlockNumber, parachain::SignedAvailabilityBitfield}; + use frame_support::traits::{OnFinalize, OnInitialize}; + use keyring::Sr25519Keyring; + + use crate::mock::{ + new_test_ext, Configuration, Paras, System, Inclusion, + GenesisConfig as MockGenesisConfig, Test, + }; + use crate::initializer::SessionChangeNotification; + use crate::configuration::HostConfiguration; + use crate::paras::ParaGenesisArgs; + + fn default_config() -> HostConfiguration { + let mut config = HostConfiguration::default(); + config.parathread_cores = 1; + config + } + + fn genesis_config(paras: Vec<(ParaId, bool)>) -> MockGenesisConfig { + MockGenesisConfig { + paras: paras::GenesisConfig { + paras: paras.into_iter().map(|(id, is_chain)| (id, ParaGenesisArgs { + genesis_head: Vec::new().into(), + validation_code: Vec::new().into(), + parachain: is_chain, + })).collect(), + ..Default::default() + }, + configuration: configuration::GenesisConfig { + config: default_config(), + ..Default::default() + }, + ..Default::default() + } + } + + fn run_to_block( + to: BlockNumber, + new_session: impl Fn(BlockNumber) -> Option>, + ) { + while System::block_number() < to { + let b = System::block_number(); + + Inclusion::initializer_finalize(); + Paras::initializer_finalize(); + + System::on_finalize(b); + + System::on_initialize(b + 1); + System::set_block_number(b + 1); + + if let Some(notification) = new_session(b + 1) { + Paras::initializer_on_new_session(¬ification); + Inclusion::initializer_on_new_session(¬ification); + } + + Paras::initializer_initialize(b + 1); + Inclusion::initializer_initialize(b + 1); + } + } + + fn default_bitfield() -> AvailabilityBitfield { + let n_bits = Paras::parachains().len() + Configuration::config().parathread_cores as usize; + + AvailabilityBitfield(bitvec::bitvec![BitOrderLsb0, u8; 0; n_bits]) + } + + fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() + } + + fn sign_bitfield( + key: &Sr25519Keyring, + validator_index: ValidatorIndex, + bitfield: AvailabilityBitfield, + signing_context: &SigningContext, + ) + -> SignedAvailabilityBitfield + { + let payload = bitfield.encode_signing_payload(signing_context); + + SignedAvailabilityBitfield { + validator_index, + bitfield: bitfield, + signature: key.sign(&payload[..]).into(), + } + } + + #[test] + fn collect_pending_cleans_up_pending() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + new_test_ext(genesis_config(paras)).execute_with(|| { + >::insert(chain_a, CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: Default::default(), + availability_votes: Default::default(), + relay_parent_number: 0, + backed_in_number: 0, + }); + + >::insert(chain_b, CandidatePendingAvailability { + core: CoreIndex::from(1), + receipt: Default::default(), + availability_votes: Default::default(), + relay_parent_number: 0, + backed_in_number: 0, + }); + + run_to_block(5, |_| None); + + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + + Inclusion::collect_pending(|core, _since| core == CoreIndex::from(0)); + + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_some()); + }); + } + + // TODO [now]: signed bitfields are accepted + #[test] + fn bitfield_signing_checks() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + Validators::set(validator_public.clone()); + CurrentSessionIndex::set(5); + + let signing_context = SigningContext { + parent_hash: System::parent_hash(), + session_index: 5, + }; + + let core_lookup = |core| match core { + core if core == CoreIndex::from(0) => Some(chain_a), + core if core == CoreIndex::from(1) => Some(chain_b), + core if core == CoreIndex::from(2) => Some(thread_a), + _ => panic!("Core out of bounds for 2 parachains and 1 parathread core."), + }; + + // wrong number of bits. + { + let mut bare_bitfield = default_bitfield(); + bare_bitfield.0.push(false); + let signed = sign_bitfield( + &validators[0], + 0, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed]), + &core_lookup, + ).is_err()); + } + + // duplicate. + { + let bare_bitfield = default_bitfield(); + let signed = sign_bitfield( + &validators[0], + 0, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed.clone(), signed]), + &core_lookup, + ).is_err()); + } + + // out of order. + { + let bare_bitfield = default_bitfield(); + let signed_0 = sign_bitfield( + &validators[0], + 0, + bare_bitfield.clone(), + &signing_context, + ); + + let signed_1 = sign_bitfield( + &validators[1], + 1, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed_1, signed_0]), + &core_lookup, + ).is_err()); + } + + // non-pending bit set. + { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + let signed = sign_bitfield( + &validators[0], + 0, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed]), + &core_lookup, + ).is_err()); + } + + // empty bitfield signed: always OK, but kind of useless. + { + let bare_bitfield = default_bitfield(); + let signed = sign_bitfield( + &validators[0], + 0, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed]), + &core_lookup, + ).is_ok()); + } + + // bitfield signed with pending bit signed. + { + let mut bare_bitfield = default_bitfield(); + + assert_eq!(core_lookup(CoreIndex::from(0)), Some(chain_a)); + + >::insert(chain_a, CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: Default::default(), + availability_votes: Default::default(), + relay_parent_number: 0, + backed_in_number: 0, + }); + + *bare_bitfield.0.get_mut(0).unwrap() = true; + let signed = sign_bitfield( + &validators[0], + 0, + bare_bitfield, + &signing_context, + ); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(vec![signed]), + &core_lookup, + ).is_ok()); + } + }); + } + + // TODO [now]: duplicate bitfields are rejected + // TODO [now]: majority bitfields trigger availability + // TODO [now]: unscheduled candidates are rejected + // TODO [now]: candidates out of order are rejected + // TODO [now]: backed candidates are accepted + // TODO [now]: candidate with interfering code upgrade is rejected. + // TODO [now]: available candidate is enacted in paras + // TODO [now]: session change wipes everything. +} From d4fac4948312e50d2807389b1b767f48fcb341f9 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 18:57:21 -0400 Subject: [PATCH 36/50] rename parachain head to para_head --- runtime/parachains/src/paras.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs index 73ec5573e277..a859759a5c52 100644 --- a/runtime/parachains/src/paras.rs +++ b/runtime/parachains/src/paras.rs @@ -174,7 +174,7 @@ decl_storage! { /// All parathreads. Parathreads: map hasher(twox_64_concat) ParaId => Option<()>; /// The head-data of every registered para. - Heads get(fn parachain_head): map hasher(twox_64_concat) ParaId => Option; + Heads get(fn para_head): map hasher(twox_64_concat) ParaId => Option; /// The validation code of every live para. CurrentCode get(fn current_code): map hasher(twox_64_concat) ParaId => Option; /// Actual past code, indicated by the para id as well as the block number at which it became outdated. @@ -368,7 +368,7 @@ impl Module { ::PastCode::remove(&(para_id, pruned_repl_at)); } - meta.most_recent_change().is_none() && Self::parachain_head(¶_id).is_none() + meta.most_recent_change().is_none() && Self::para_head(¶_id).is_none() }); // This parachain has been removed and now the vestigial code From 58184ebc30ac660b45b600cf2352eaab4e462bc0 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 19:07:40 -0400 Subject: [PATCH 37/50] fix note_new_head bug in paras --- runtime/parachains/src/paras.rs | 40 ++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs index a859759a5c52..b9b2eabfe4fa 100644 --- a/runtime/parachains/src/paras.rs +++ b/runtime/parachains/src/paras.rs @@ -454,9 +454,9 @@ impl Module { new_head: HeadData, execution_context: T::BlockNumber, ) -> Weight { - if let Some(expected_at) = ::FutureCodeUpgrades::get(&id) { - Heads::insert(&id, new_head); + Heads::insert(&id, new_head); + if let Some(expected_at) = ::FutureCodeUpgrades::get(&id) { if expected_at <= execution_context { ::FutureCodeUpgrades::remove(&id); @@ -481,7 +481,7 @@ impl Module { T::DbWeight::get().reads_writes(1, 1 + 0) } } else { - T::DbWeight::get().reads_writes(1, 0) + T::DbWeight::get().reads_writes(1, 1) } } @@ -695,6 +695,40 @@ mod tests { }); } + #[test] + fn note_new_head_sets_head() { + let acceptance_period = 10; + let paras = vec![ + (0u32.into(), ParaGenesisArgs { + parachain: true, + genesis_head: Default::default(), + validation_code: Default::default(), + }), + ]; + + let genesis_config = MockGenesisConfig { + paras: GenesisConfig { paras, ..Default::default() }, + configuration: crate::configuration::GenesisConfig { + config: HostConfiguration { + acceptance_period, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }; + + new_test_ext(genesis_config).execute_with(|| { + let id_a = ParaId::from(0u32); + + assert_eq!(Paras::para_head(&id_a), Some(Default::default())); + + Paras::note_new_head(id_a, vec![1, 2, 3].into(), 0); + + assert_eq!(Paras::para_head(&id_a), Some(vec![1, 2, 3].into())); + }); + } + #[test] fn note_past_code_sets_up_pruning_correctly() { let acceptance_period = 10; From 9ed14c5b808940b02d58b0ea13be0d47dea798b3 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 19:08:01 -0400 Subject: [PATCH 38/50] test bitfield enactment in inclusion --- runtime/parachains/src/inclusion.rs | 142 ++++++++++++++++++++++++++-- 1 file changed, 135 insertions(+), 7 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index db7b243b10fc..e5f8ceae9ea7 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -555,6 +555,10 @@ mod tests { AvailabilityBitfield(bitvec::bitvec![BitOrderLsb0, u8; 0; n_bits]) } + fn default_availability_votes() -> BitVec { + bitvec::bitvec![BitOrderLsb0, u8; 0; Validators::get().len()] + } + fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { val_ids.iter().map(|v| v.public().into()).collect() } @@ -587,7 +591,7 @@ mod tests { >::insert(chain_a, CandidatePendingAvailability { core: CoreIndex::from(0), receipt: Default::default(), - availability_votes: Default::default(), + availability_votes: default_availability_votes(), relay_parent_number: 0, backed_in_number: 0, }); @@ -595,7 +599,7 @@ mod tests { >::insert(chain_b, CandidatePendingAvailability { core: CoreIndex::from(1), receipt: Default::default(), - availability_votes: Default::default(), + availability_votes: default_availability_votes(), relay_parent_number: 0, backed_in_number: 0, }); @@ -612,7 +616,6 @@ mod tests { }); } - // TODO [now]: signed bitfields are accepted #[test] fn bitfield_signing_checks() { let chain_a = ParaId::from(1); @@ -743,7 +746,7 @@ mod tests { >::insert(chain_a, CandidatePendingAvailability { core: CoreIndex::from(0), receipt: Default::default(), - availability_votes: Default::default(), + availability_votes: default_availability_votes(), relay_parent_number: 0, backed_in_number: 0, }); @@ -764,12 +767,137 @@ mod tests { }); } - // TODO [now]: duplicate bitfields are rejected - // TODO [now]: majority bitfields trigger availability + #[test] + fn supermajority_bitfields_trigger_availability() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + Validators::set(validator_public.clone()); + CurrentSessionIndex::set(5); + + let signing_context = SigningContext { + parent_hash: System::parent_hash(), + session_index: 5, + }; + + let core_lookup = |core| match core { + core if core == CoreIndex::from(0) => Some(chain_a), + core if core == CoreIndex::from(1) => Some(chain_b), + core if core == CoreIndex::from(2) => Some(thread_a), + _ => panic!("Core out of bounds for 2 parachains and 1 parathread core."), + }; + + >::insert(chain_a, CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: AbridgedCandidateReceipt { + parachain_index: chain_a, + head_data: vec![1, 2, 3, 4].into(), + ..Default::default() + }, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + }); + + >::insert(chain_b, CandidatePendingAvailability { + core: CoreIndex::from(1), + receipt: AbridgedCandidateReceipt { + parachain_index: chain_b, + head_data: vec![5, 6, 7, 8].into(), + ..Default::default() + }, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + }); + + // this bitfield signals that a and b are available. + let a_and_b_available = { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + *bare_bitfield.0.get_mut(1).unwrap() = true; + + bare_bitfield + }; + + // this bitfield signals that only a is available. + let a_available = { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + + bare_bitfield + }; + + let threshold = { + let mut threshold = (validators.len() * 2) / 3; + threshold += (validators.len() * 2) % 3; + threshold + }; + + // 4 of 5 first value >= 2/3 + assert_eq!(threshold, 4); + + let signed_bitfields = validators.iter().enumerate().filter_map(|(i, key)| { + let to_sign = if i < 3 { + a_and_b_available.clone() + } else if i < 4 { + a_available.clone() + } else { + // sign nothing. + return None + }; + + Some(sign_bitfield( + key, + i as ValidatorIndex, + to_sign, + &signing_context, + )) + }).collect(); + + assert!(Inclusion::process_bitfields( + SignedAvailabilityBitfields(signed_bitfields), + &core_lookup, + ).is_ok()); + + // chain A had 4 signing off, which is >= threshold. + // chain B has 3 signing off, which is < threshold. + assert!(>::get(&chain_a).is_none()); + assert_eq!( + >::get(&chain_b).unwrap().availability_votes, + { + // check that votes from first 3 were tracked. + + let mut votes = default_availability_votes(); + *votes.get_mut(0).unwrap() = true; + *votes.get_mut(1).unwrap() = true; + *votes.get_mut(2).unwrap() = true; + + votes + }, + ); + + // and check that chain head was enacted. + assert_eq!(Paras::para_head(&chain_a), Some(vec![1, 2, 3, 4].into())); + }); + } + // TODO [now]: unscheduled candidates are rejected // TODO [now]: candidates out of order are rejected // TODO [now]: backed candidates are accepted // TODO [now]: candidate with interfering code upgrade is rejected. // TODO [now]: available candidate is enacted in paras - // TODO [now]: session change wipes everything. + // TODO [now]: session change wipes everything and updates validators / session index. } From 4e5b648aaa19a77b33a6e3bd0682bc19448c560e Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 19:39:15 -0400 Subject: [PATCH 39/50] helpers for candidate checks --- primitives/src/parachain.rs | 9 + .../src/runtime/inclusion.md | 1 + runtime/parachains/src/inclusion.rs | 160 +++++++++++++++++- runtime/parachains/src/scheduler.rs | 6 + 4 files changed, 170 insertions(+), 6 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index ed1c77bda9c4..afc6bc6929f5 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -606,6 +606,15 @@ pub enum Statement { Invalid(Hash), } +impl Statement { + /// Produce a payload on this statement that is used for signing. + /// + /// It includes the context provided. + pub fn signing_payload(&self, context: &SigningContext) -> Vec { + (self, context).encode() + } +} + /// An either implicit or explicit attestation to the validity of a parachain /// candidate. #[derive(Clone, Eq, PartialEq, Decode, Encode, RuntimeDebug)] diff --git a/roadmap/implementors-guide/src/runtime/inclusion.md b/roadmap/implementors-guide/src/runtime/inclusion.md index 21420e323fde..36487117a8ad 100644 --- a/roadmap/implementors-guide/src/runtime/inclusion.md +++ b/roadmap/implementors-guide/src/runtime/inclusion.md @@ -64,6 +64,7 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. check that there is no candidate pending availability for any scheduled `ParaId`. 1. If the core assignment includes a specific collator, ensure the backed candidate is issued by that collator. 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_frequency` of `Paras::last_code_upgrade(para_id, true)`, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. + 1. Check the collator's signature on the pov block. 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup. 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was successfully backed for, sorted ascending by CoreIndex. diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index e5f8ceae9ea7..c364b770feba 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -115,6 +115,8 @@ decl_error! { InsufficientBacking, /// Invalid (bad signature, unknown validator, etc.) backing. InvalidBacking, + /// Collator did not sign PoV. + NotCollatorSigned, } } @@ -351,6 +353,11 @@ impl Module { ); ensure!(code_upgrade_allowed, Error::::PrematureCodeUpgrade); + ensure!( + candidate.candidate.check_signature().is_ok(), + Error::::NotCollatorSigned, + ); + for (i, assignment) in scheduled[skip..].iter().enumerate() { check_assignment_in_order(assignment)?; @@ -488,7 +495,8 @@ impl Module { mod tests { use super::*; - use primitives::{BlockNumber, parachain::SignedAvailabilityBitfield}; + use primitives::{BlockNumber, Hash}; + use primitives::parachain::{SignedAvailabilityBitfield, Statement, ValidityAttestation}; use frame_support::traits::{OnFinalize, OnInitialize}; use keyring::Sr25519Keyring; @@ -524,6 +532,80 @@ mod tests { } } + #[derive(Clone, Copy, PartialEq)] + enum BackingKind { + Unanimous, + Threshold, + Lacking, + } + + fn collator_sign_candidate( + collator: Sr25519Keyring, + candidate: &mut AbridgedCandidateReceipt, + ) { + candidate.collator = collator.public().into(); + + let payload = primitives::parachain::collator_signature_payload( + &candidate.relay_parent, + &candidate.parachain_index, + &candidate.pov_block_hash, + ); + + candidate.signature = collator.sign(&payload[..]).into(); + assert!(candidate.check_signature().is_ok()); + } + + fn back_candidate( + candidate: AbridgedCandidateReceipt, + validators: &[Sr25519Keyring], + group: &[ValidatorIndex], + signing_context: &SigningContext, + kind: BackingKind, + ) -> BackedCandidate { + let mut validator_indices = bitvec::bitvec![BitOrderLsb0, u8; 0; group.len()]; + let threshold = (group.len() / 2) + 1; + + let signing = match kind { + BackingKind::Unanimous => group.len(), + BackingKind::Threshold => threshold, + BackingKind::Lacking => threshold.saturating_sub(1), + }; + + let mut validity_votes = Vec::with_capacity(signing); + let candidate_hash = candidate.hash(); + let payload = Statement::Valid(candidate_hash).signing_payload(signing_context); + + for (idx_in_group, val_idx) in group.iter().enumerate().take(signing) { + let key: Sr25519Keyring = validators[*val_idx as usize]; + *validator_indices.get_mut(idx_in_group).unwrap() = true; + + validity_votes.push(ValidityAttestation::Explicit(key.sign(&payload[..]).into())); + } + + let backed = BackedCandidate { + candidate, + validity_votes, + validator_indices, + }; + + let should_pass = match kind { + BackingKind::Unanimous | BackingKind::Threshold => true, + BackingKind::Lacking => false, + }; + + assert_eq!( + primitives::parachain::check_candidate_backing( + &backed, + signing_context, + group.len(), + |i| Some(validators[i].public().into()), + ).is_ok(), + should_pass, + ); + + backed + } + fn run_to_block( to: BlockNumber, new_session: impl Fn(BlockNumber) -> Option>, @@ -617,7 +699,7 @@ mod tests { } #[test] - fn bitfield_signing_checks() { + fn bitfield_checks() { let chain_a = ParaId::from(1); let chain_b = ParaId::from(2); let thread_a = ParaId::from(3); @@ -894,10 +976,76 @@ mod tests { }); } - // TODO [now]: unscheduled candidates are rejected - // TODO [now]: candidates out of order are rejected - // TODO [now]: backed candidates are accepted + #[test] + fn candidate_checks() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + Validators::set(validator_public.clone()); + CurrentSessionIndex::set(5); + + run_to_block(5, |_| None); + + let signing_context = SigningContext { + parent_hash: System::parent_hash(), + session_index: 5, + }; + + let group_validators = |group_index: GroupIndex| match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![3, 4]), + group_index if group_index == GroupIndex::from(2) => Some(vec![5]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + }; + + // unscheduled candidate. + { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + } + + // candidates out of order. + { + + } + + // candidate not backed. + { + + } + + // candidate not in parent context. + { + + } + + // candidate has wrong collator. + { + + } + }); + } + // TODO [now]: candidate with interfering code upgrade is rejected. - // TODO [now]: available candidate is enacted in paras // TODO [now]: session change wipes everything and updates validators / session index. } diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 15830b5e7b01..8742a7b58a7b 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -68,6 +68,12 @@ impl From for CoreIndex { #[cfg_attr(test, derive(PartialEq, Debug))] pub struct GroupIndex(u32); +impl From for GroupIndex { + fn from(i: u32) -> GroupIndex { + GroupIndex(i) + } +} + /// A claim on authoring the next block for a given parathread. #[derive(Clone, Encode, Decode, Default)] #[cfg_attr(test, derive(PartialEq, Debug))] From 9340aba3460d983d753edf268ceb11f81a83380f Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 20:03:50 -0400 Subject: [PATCH 40/50] add test for most candidate checks --- runtime/parachains/src/inclusion.rs | 213 ++++++++++++++++++++++++++-- runtime/parachains/src/scheduler.rs | 2 +- 2 files changed, 203 insertions(+), 12 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index c364b770feba..be531dfabaea 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -496,7 +496,7 @@ mod tests { use super::*; use primitives::{BlockNumber, Hash}; - use primitives::parachain::{SignedAvailabilityBitfield, Statement, ValidityAttestation}; + use primitives::parachain::{SignedAvailabilityBitfield, Statement, ValidityAttestation, CollatorId}; use frame_support::traits::{OnFinalize, OnInitialize}; use keyring::Sr25519Keyring; @@ -507,6 +507,7 @@ mod tests { use crate::initializer::SessionChangeNotification; use crate::configuration::HostConfiguration; use crate::paras::ParaGenesisArgs; + use crate::scheduler::AssignmentKind; fn default_config() -> HostConfiguration { let mut config = HostConfiguration::default(); @@ -532,7 +533,7 @@ mod tests { } } - #[derive(Clone, Copy, PartialEq)] + #[derive(Debug, Clone, Copy, PartialEq)] enum BackingKind { Unanimous, Threshold, @@ -593,15 +594,18 @@ mod tests { BackingKind::Lacking => false, }; - assert_eq!( - primitives::parachain::check_candidate_backing( - &backed, - signing_context, - group.len(), - |i| Some(validators[i].public().into()), - ).is_ok(), - should_pass, - ); + let successfully_backed = primitives::parachain::check_candidate_backing( + &backed, + signing_context, + group.len(), + |i| Some(validators[group[i] as usize].public().into()), + ).ok().unwrap_or(0) * 2 > group.len(); + + if should_pass { + assert!(successfully_backed); + } else { + assert!(!successfully_backed); + } backed } @@ -1010,6 +1014,29 @@ mod tests { _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), }; + let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); + + let chain_a_assignment = CoreAssignment { + core: CoreIndex::from(0), + para_id: chain_b, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(0), + }; + + let chain_b_assignment = CoreAssignment { + core: CoreIndex::from(1), + para_id: chain_b, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(1), + }; + + let thread_a_assignment = CoreAssignment { + core: CoreIndex::from(2), + para_id: chain_b, + kind: AssignmentKind::Parathread(thread_collator.clone(), 0), + group_idx: GroupIndex::from(2), + }; + // unscheduled candidate. { let mut candidate = AbridgedCandidateReceipt { @@ -1022,30 +1049,194 @@ mod tests { Sr25519Keyring::One, &mut candidate, ); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![chain_b_assignment.clone()], + &group_validators, + ).is_err()); } // candidates out of order. { + let mut candidate_a = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + let mut candidate_b = AbridgedCandidateReceipt { + parachain_index: chain_b, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([2; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate_a, + ); + + collator_sign_candidate( + Sr25519Keyring::Two, + &mut candidate_b, + ); + + let backed_a = back_candidate( + candidate_a, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + let backed_b = back_candidate( + candidate_b, + &validators, + group_validators(GroupIndex::from(1)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed_b, backed_a], + vec![chain_a_assignment.clone(), chain_b_assignment.clone()], + &group_validators, + ).is_err()); } // candidate not backed. { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Lacking, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + ).is_err()); } // candidate not in parent context. { + let wrong_parent_hash = Hash::from([222; 32]); + assert!(System::parent_hash() != wrong_parent_hash); + + let mut candidate = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: wrong_parent_hash, + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + ).is_err()); } // candidate has wrong collator. { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: thread_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + assert!(CollatorId::from(Sr25519Keyring::One.public()) != thread_collator); + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![chain_a_assignment.clone(), chain_b_assignment.clone(), thread_a_assignment.clone()], + &group_validators, + ).is_err()); + } + + // candidate not well-signed by collator. + { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: thread_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + + assert_eq!(CollatorId::from(Sr25519Keyring::Two.public()), thread_collator); + collator_sign_candidate( + Sr25519Keyring::Two, + &mut candidate, + ); + + candidate.pov_block_hash = Hash::from([2; 32]); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![thread_a_assignment.clone()], + &group_validators, + ).is_err()); } }); } + // TODO [now]: candidate inclusion sets storage correctly. // TODO [now]: candidate with interfering code upgrade is rejected. // TODO [now]: session change wipes everything and updates validators / session index. } diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index 8742a7b58a7b..a82250ff04d7 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -142,7 +142,7 @@ pub enum AssignmentKind { } /// How a free core is scheduled to be assigned. -#[derive(Encode, Decode)] +#[derive(Clone, Encode, Decode)] #[cfg_attr(test, derive(PartialEq, Debug))] pub struct CoreAssignment { /// The core that is assigned. From cf44264cd1e8e1081f1e9df8e4b6c90fa3d842fd Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 20:32:35 -0400 Subject: [PATCH 41/50] add test for backing setting storage --- runtime/parachains/src/inclusion.rs | 262 +++++++++++++++++++++++++++- 1 file changed, 254 insertions(+), 8 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index be531dfabaea..24e26d3ef364 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -50,7 +50,7 @@ pub struct AvailabilityBitfieldRecord { } /// A backed candidate pending availability. -#[derive(Encode, Decode)] +#[derive(Encode, Decode, PartialEq)] #[cfg_attr(test, derive(Debug))] pub struct CandidatePendingAvailability { /// The availability core this is assigned to. @@ -496,7 +496,10 @@ mod tests { use super::*; use primitives::{BlockNumber, Hash}; - use primitives::parachain::{SignedAvailabilityBitfield, Statement, ValidityAttestation, CollatorId}; + use primitives::parachain::{ + SignedAvailabilityBitfield, Statement, ValidityAttestation, CollatorId, + CandidateCommitments, + }; use frame_support::traits::{OnFinalize, OnInitialize}; use keyring::Sr25519Keyring; @@ -1009,8 +1012,8 @@ mod tests { let group_validators = |group_index: GroupIndex| match group_index { group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![3, 4]), - group_index if group_index == GroupIndex::from(2) => Some(vec![5]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + group_index if group_index == GroupIndex::from(2) => Some(vec![4]), _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), }; @@ -1190,14 +1193,18 @@ mod tests { let backed = back_candidate( candidate, &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), + group_validators(GroupIndex::from(2)).unwrap().as_ref(), &signing_context, BackingKind::Threshold, ); assert!(Inclusion::process_candidates( vec![backed], - vec![chain_a_assignment.clone(), chain_b_assignment.clone(), thread_a_assignment.clone()], + vec![ + chain_a_assignment.clone(), + chain_b_assignment.clone(), + thread_a_assignment.clone(), + ], &group_validators, ).is_err()); } @@ -1219,6 +1226,35 @@ mod tests { candidate.pov_block_hash = Hash::from([2; 32]); + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![thread_a_assignment.clone()], + &group_validators, + ).is_err()); + } + + // para occupied - reject. + { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + let backed = back_candidate( candidate, &validators, @@ -1227,6 +1263,57 @@ mod tests { BackingKind::Threshold, ); + >::insert(&chain_a, CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: Default::default(), + availability_votes: default_availability_votes(), + relay_parent_number: 3, + backed_in_number: 4, + }); + + assert!(Inclusion::process_candidates( + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + ).is_err()); + + >::remove(&chain_a); + } + + // interfering code upgrade - reject + { + let mut candidate = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + commitments: CandidateCommitments { + new_validation_code: Some(vec![5, 6, 7, 8].into()), + ..Default::default() + }, + ..Default::default() + }; + + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate, + ); + + let backed = back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + Paras::schedule_code_upgrade( + chain_a, + vec![1, 2, 3, 4].into(), + 10, + ); + + assert_eq!(Paras::last_code_upgrade(chain_a, true), Some(10)); + assert!(Inclusion::process_candidates( vec![backed], vec![thread_a_assignment.clone()], @@ -1236,7 +1323,166 @@ mod tests { }); } - // TODO [now]: candidate inclusion sets storage correctly. - // TODO [now]: candidate with interfering code upgrade is rejected. + #[test] + fn backing_works() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + Validators::set(validator_public.clone()); + CurrentSessionIndex::set(5); + + run_to_block(5, |_| None); + + let signing_context = SigningContext { + parent_hash: System::parent_hash(), + session_index: 5, + }; + + let group_validators = |group_index: GroupIndex| match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + group_index if group_index == GroupIndex::from(2) => Some(vec![4]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + }; + + let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); + + let chain_a_assignment = CoreAssignment { + core: CoreIndex::from(0), + para_id: chain_a, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(0), + }; + + let chain_b_assignment = CoreAssignment { + core: CoreIndex::from(1), + para_id: chain_b, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(1), + }; + + let thread_a_assignment = CoreAssignment { + core: CoreIndex::from(2), + para_id: thread_a, + kind: AssignmentKind::Parathread(thread_collator.clone(), 0), + group_idx: GroupIndex::from(2), + }; + + let mut candidate_a = AbridgedCandidateReceipt { + parachain_index: chain_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([1; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate_a, + ); + + let mut candidate_b = AbridgedCandidateReceipt { + parachain_index: chain_b, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([2; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::One, + &mut candidate_b, + ); + + let mut candidate_c = AbridgedCandidateReceipt { + parachain_index: thread_a, + relay_parent: System::parent_hash(), + pov_block_hash: Hash::from([3; 32]), + ..Default::default() + }; + collator_sign_candidate( + Sr25519Keyring::Two, + &mut candidate_c, + ); + + let backed_a = back_candidate( + candidate_a.clone(), + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + let backed_b = back_candidate( + candidate_b.clone(), + &validators, + group_validators(GroupIndex::from(1)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + let backed_c = back_candidate( + candidate_c.clone(), + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &signing_context, + BackingKind::Threshold, + ); + + let occupied_cores = Inclusion::process_candidates( + vec![backed_a, backed_b, backed_c], + vec![ + chain_a_assignment.clone(), + chain_b_assignment.clone(), + thread_a_assignment.clone(), + ], + &group_validators, + ).expect("candidates scheduled, in order, and backed"); + + assert_eq!(occupied_cores, vec![CoreIndex::from(0), CoreIndex::from(1), CoreIndex::from(2)]); + + assert_eq!( + >::get(&chain_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: candidate_a, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + }) + ); + + assert_eq!( + >::get(&chain_b), + Some(CandidatePendingAvailability { + core: CoreIndex::from(1), + receipt: candidate_b, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + }) + ); + + assert_eq!( + >::get(&thread_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(2), + receipt: candidate_c, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + }) + ); + }); + } + // TODO [now]: session change wipes everything and updates validators / session index. } From bd88a203fc485f48b971751d4d1ea29b98d63566 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 20:54:27 -0400 Subject: [PATCH 42/50] test session change logic --- runtime/parachains/src/inclusion.rs | 110 +++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 24e26d3ef364..42fcff036a42 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -1484,5 +1484,113 @@ mod tests { }); } - // TODO [now]: session change wipes everything and updates validators / session index. + #[test] + fn session_change_wipes_and_updates_session_info() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + Validators::set(validator_public.clone()); + CurrentSessionIndex::set(5); + + let validators_new = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + ]; + + let validator_public_new = validator_pubkeys(&validators_new); + + run_to_block(10, |_| None); + + >::insert( + &0, + AvailabilityBitfieldRecord { + bitfield: default_bitfield(), + submitted_at: 9, + }, + ); + + >::insert( + &1, + AvailabilityBitfieldRecord { + bitfield: default_bitfield(), + submitted_at: 9, + }, + ); + + >::insert( + &4, + AvailabilityBitfieldRecord { + bitfield: default_bitfield(), + submitted_at: 9, + }, + ); + + >::insert(&chain_a, CandidatePendingAvailability { + core: CoreIndex::from(0), + receipt: Default::default(), + availability_votes: default_availability_votes(), + relay_parent_number: 5, + backed_in_number: 6, + }); + + >::insert(&chain_b, CandidatePendingAvailability { + core: CoreIndex::from(1), + receipt: Default::default(), + availability_votes: default_availability_votes(), + relay_parent_number: 6, + backed_in_number: 7, + }); + + run_to_block(11, |_| None); + + assert_eq!(Validators::get(), validator_public); + assert_eq!(CurrentSessionIndex::get(), 5); + + assert!(>::get(&0).is_some()); + assert!(>::get(&1).is_some()); + assert!(>::get(&4).is_some()); + + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + + run_to_block(12, |n| match n { + 12 => Some(SessionChangeNotification { + validators: validator_public_new.clone(), + queued: Vec::new(), + prev_config: default_config(), + new_config: default_config(), + random_seed: Default::default(), + session_index: 6, + }), + _ => None, + }); + + assert_eq!(Validators::get(), validator_public_new); + assert_eq!(CurrentSessionIndex::get(), 6); + + assert!(>::get(&0).is_none()); + assert!(>::get(&1).is_none()); + assert!(>::get(&4).is_none()); + + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_none()); + + assert!(>::iter().collect::>().is_empty()); + assert!(>::iter().collect::>().is_empty()); + + }); + } } From 17dd6b2eb96729cc27b2497dbe3d2b006094ca79 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Mon, 15 Jun 2020 20:56:51 -0400 Subject: [PATCH 43/50] remove extraneous type parameter --- primitives/src/parachain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index afc6bc6929f5..2309a6bc11ec 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -407,7 +407,7 @@ impl + Encode> AbridgedCandidateReceipt { impl AbridgedCandidateReceipt { /// Combine the abridged candidate receipt with the omitted data, /// forming a full `CandidateReceipt`. - pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt { + pub fn complete(self, omitted: OmittedValidationData) -> CandidateReceipt { let AbridgedCandidateReceipt { parachain_index, relay_parent, From dc9bff0900f698053e7d887bc6580d8a85652a16 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 14:51:22 -0400 Subject: [PATCH 44/50] remove some allow(unused)s --- runtime/parachains/src/paras.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime/parachains/src/paras.rs b/runtime/parachains/src/paras.rs index b9b2eabfe4fa..3bd55e277803 100644 --- a/runtime/parachains/src/paras.rs +++ b/runtime/parachains/src/paras.rs @@ -428,7 +428,6 @@ impl Module { /// with number >= `expected_at` /// /// If there is already a scheduled code upgrade for the para, this is a no-op. - #[allow(unused)] pub(crate) fn schedule_code_upgrade( id: ParaId, new_code: ValidationCode, @@ -448,7 +447,6 @@ impl Module { /// Note that a para has progressed to a new head, where the new head was executed in the context /// of a relay-chain block with given number. This will apply pending code upgrades based /// on the block number provided. - #[allow(unused)] pub(crate) fn note_new_head( id: ParaId, new_head: HeadData, From 4bf651924bf111db17704a8b3ccb5485d8a34bd5 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 14:51:55 -0400 Subject: [PATCH 45/50] extract threshold computation to const fn --- runtime/parachains/src/inclusion.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 42fcff036a42..fc6b4abc3e6a 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -251,11 +251,7 @@ impl Module { >::insert(&signed_bitfield.validator_index, record); } - let threshold = { - let mut threshold = (validators.len() * 2) / 3; - threshold += (validators.len() * 2) % 3; - threshold - }; + let threshold = availability_threshold(validators.len()); let mut freed_cores = Vec::with_capacity(n_bits); for (para_id, pending_availability) in assigned_paras_record.into_iter() @@ -491,6 +487,12 @@ impl Module { } } +const fn availability_threshold(n_validators: usize) -> usize { + let mut threshold = (n_validators * 2) / 3; + threshold += (n_validators * 2) % 3; + threshold +} + #[cfg(test)] mod tests { use super::*; @@ -929,11 +931,7 @@ mod tests { bare_bitfield }; - let threshold = { - let mut threshold = (validators.len() * 2) / 3; - threshold += (validators.len() * 2) % 3; - threshold - }; + let threshold = availability_threshold(validators.len()); // 4 of 5 first value >= 2/3 assert_eq!(threshold, 4); From b6c55a4cd70e085556436d63b22b0c115cd88d88 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 14:52:49 -0400 Subject: [PATCH 46/50] remove some more allow(unused)s --- runtime/parachains/src/scheduler.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/runtime/parachains/src/scheduler.rs b/runtime/parachains/src/scheduler.rs index a82250ff04d7..6539915946c8 100644 --- a/runtime/parachains/src/scheduler.rs +++ b/runtime/parachains/src/scheduler.rs @@ -164,7 +164,6 @@ impl CoreAssignment { } } - #[allow(unused)] fn to_core_occupied(&self) -> CoreOccupied { match self.kind { AssignmentKind::Parachain => CoreOccupied::Parachain, @@ -179,7 +178,6 @@ impl CoreAssignment { } /// Reasons a core might be freed -#[allow(unused)] pub enum FreedReason { /// The core's work concluded and the parablock assigned to it is considered available. Concluded, @@ -536,7 +534,6 @@ impl Module { /// /// Complexity: O(n) in the number of scheduled cores, which is capped at the number of total cores. /// This is efficient in the case that most scheduled cores are occupied. - #[allow(unused)] pub(crate) fn occupied(now_occupied: &[CoreIndex]) { if now_occupied.is_empty() { return } @@ -568,7 +565,6 @@ impl Module { /// Get the para (chain or thread) ID assigned to a particular core or index, if any. Core indices /// out of bounds will return `None`, as will indices of unassigned cores. - #[allow(unused)] pub(crate) fn core_para(core_index: CoreIndex) -> Option { let cores = AvailabilityCores::get(); match cores.get(core_index.0 as usize).and_then(|c| c.as_ref()) { @@ -582,7 +578,6 @@ impl Module { } /// Get the validators in the given group, if the group index is valid for this session. - #[allow(unused)] pub(crate) fn group_validators(group_index: GroupIndex) -> Option> { ValidatorGroups::get().get(group_index.0 as usize).map(|g| g.clone()) } @@ -629,7 +624,6 @@ impl Module { /// This really should not be a box, but is working around a compiler limitation filed here: /// https://github.com/rust-lang/rust/issues/73226 /// which prevents us from testing the code if using `impl Trait`. - #[allow(unused)] pub(crate) fn availability_timeout_predicate() -> Option bool>> { let now = >::block_number(); let config = >::config(); From 0dc9072ffc42dd372e0f1ca73cb79586b08dd628 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 14:53:25 -0400 Subject: [PATCH 47/50] improve doc --- runtime/parachains/src/inclusion.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index fc6b4abc3e6a..6dddfdec3d20 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -42,6 +42,9 @@ use crate::{configuration, paras, scheduler::{CoreIndex, GroupIndex, CoreAssignm /// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding /// for any backed candidates referred to by a `1` bit available. +/// +/// The bitfield's signature should be checked at the point of submission. Afterwards it can be +/// dropped. #[derive(Encode, Decode)] #[cfg_attr(test, derive(Debug))] pub struct AvailabilityBitfieldRecord { From 3c43c1a27c38caadbe43af6624887f0c9f23a6ee Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 14:55:16 -0400 Subject: [PATCH 48/50] add debug assertion --- runtime/parachains/src/inclusion.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index 6dddfdec3d20..a4472fa3a097 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -120,6 +120,8 @@ decl_error! { InvalidBacking, /// Collator did not sign PoV. NotCollatorSigned, + /// Internal error only returned when compiled with debug assertions. + InternalError, } } @@ -243,6 +245,8 @@ impl Module { .and_then(|r| r.availability_votes.get_mut(val_idx)) { *bit = true; + } else if cfg!(debug_assertions) { + ensure!(false, Error::::InternalError); } } From 2f7059b827a71764fe3d441a566f0bbfdb403be5 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 18:33:01 -0400 Subject: [PATCH 49/50] fix primitive test compilation --- primitives/src/parachain.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/src/parachain.rs b/primitives/src/parachain.rs index 2309a6bc11ec..4e458f3ab036 100644 --- a/primitives/src/parachain.rs +++ b/primitives/src/parachain.rs @@ -905,9 +905,9 @@ mod tests { assert_eq!(h.as_ref().len(), 32); let _payload = collator_signature_payload( - &[1; 32].into(), + &Hash::from([1; 32]), &5u32.into(), - &[2; 32].into(), + &Hash::from([2; 32]), ); } } From 1587db77cdeabf9dc3d3d2c725d6abcab6ff4b95 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 17 Jun 2020 19:48:50 -0400 Subject: [PATCH 50/50] tag unanimous variant as unused --- runtime/parachains/src/inclusion.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/parachains/src/inclusion.rs b/runtime/parachains/src/inclusion.rs index a4472fa3a097..5fba4cf26ca3 100644 --- a/runtime/parachains/src/inclusion.rs +++ b/runtime/parachains/src/inclusion.rs @@ -547,6 +547,7 @@ mod tests { #[derive(Debug, Clone, Copy, PartialEq)] enum BackingKind { + #[allow(unused)] Unanimous, Threshold, Lacking,