From 30a27f87b2cac9f46551902e4cd84884bbc3bdde Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 18 Nov 2021 11:23:45 +0100 Subject: [PATCH 01/14] Improve paras runtime `BenchBuilder` api --- runtime/parachains/src/builder.rs | 24 ++++++++++----- runtime/parachains/src/paras_inherent.rs | 29 +++++++++---------- .../src/paras_inherent/benchmarking.rs | 8 ++--- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 362d97ecc259..ef66c6e20bae 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -477,8 +477,11 @@ impl BenchBuilder { (start..last) .map(|seed| { - let session = - dispute_sessions.get(seed as usize).cloned().unwrap_or(self.target_session); + let dispute_session_idx = (seed - start) as usize; + let session = dispute_sessions + .get(dispute_session_idx) + .cloned() + .unwrap_or(self.target_session); let (para_id, core_idx, group_idx) = self.create_indexes(seed); let candidate_hash = CandidateHash(H256::from(byte32_slice_from(seed))); @@ -520,13 +523,18 @@ impl BenchBuilder { /// Build a scenario for testing or benchmarks. /// + /// Note that this only allows cores with backed candidates / becoming available to be + /// mutually exclusive of cores of with disputes. + /// /// - `backed_and_concluding_cores`: Map from core/para id/group index seed to number of /// validity votes. - /// - `dispute_sessions`: Session index of for each dispute. Index of slice corresponds to core. - /// The length of this must equal total cores used. Seed index for disputes starts at - /// `backed_and_concluding_cores.len()`, so `dispute_sessions` needs to be left padded by - /// `backed_and_concluding_cores.len()` values which effectively get ignored. - /// TODO we should fix this. + /// - `dispute_sessions`: Session index of for each dispute. Index of slice corresponds to a core, + /// which is offset by the number of entries for `backed_and_concluding_cores`. I.E. if + /// `backed_and_concluding_cores` cores has 3 entries, the first index of `dispute_sessions` + /// will correspond to core index 3. There must be one entry for each core with a dispute + /// statement set. + /// - `includes_code_upgrade`: Set to `Some` to include a code upgrade for all backed candidates. + /// The value within `Some` will be the byte length of the code. pub(crate) fn build( self, backed_and_concluding_cores: BTreeMap, @@ -540,7 +548,7 @@ impl BenchBuilder { // We don't allow a core to have both disputes and be marked fully available at this block. let cores = self.max_cores(); - let used_cores = dispute_sessions.len() as u32; + let used_cores = (dispute_sessions.len() + backed_and_concluding_cores.len()) as u32; assert!(used_cores <= cores); // NOTE: there is an n+2 session delay for these actions to take effect diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs index db0ed5ec9043..0e85b8220f5e 100644 --- a/runtime/parachains/src/paras_inherent.rs +++ b/runtime/parachains/src/paras_inherent.rs @@ -1063,7 +1063,10 @@ mod tests { }: TestConfig, ) -> Bench { BenchBuilder::::new() - .set_max_validators((dispute_sessions.len() as u32) * num_validators_per_core) + .set_max_validators( + (dispute_sessions.len() + backed_and_concluding.len()) as u32 * + num_validators_per_core, + ) .set_max_validators_per_core(num_validators_per_core) .set_dispute_statements(dispute_statements) .build(backed_and_concluding, dispute_sessions.as_slice(), includes_code_upgrade) @@ -1083,7 +1086,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![0, 0], + dispute_sessions: vec![], // No disputes backed_and_concluding, num_validators_per_core: 1, includes_code_upgrade: None, @@ -1217,12 +1220,12 @@ mod tests { new_test_ext(MockGenesisConfig::default()).execute_with(|| { // Create the inherent data for this block let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be fileld with disputesw + // No backed and concluding cores, so all cores will be filld with disputes. let backed_and_concluding = BTreeMap::new(); let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 6, includes_code_upgrade: None, @@ -1284,7 +1287,7 @@ mod tests { new_test_ext(MockGenesisConfig::default()).execute_with(|| { // Create the inherent data for this block let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be fileld with disputesw + // No backed and concluding cores, so all cores will be filled with disputes. let backed_and_concluding = BTreeMap::new(); let scenario = make_inherent_data(TestConfig { @@ -1335,8 +1338,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], + dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes backed_and_concluding, num_validators_per_core: 4, includes_code_upgrade: None, @@ -1414,8 +1416,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 4, includes_code_upgrade: None, @@ -1476,8 +1477,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, includes_code_upgrade: None, @@ -1557,8 +1557,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, includes_code_upgrade: None, @@ -1617,7 +1616,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, includes_code_upgrade: None, @@ -1688,7 +1687,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, includes_code_upgrade: None, diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs index d5aea6dde331..32b315e4a850 100644 --- a/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -29,7 +29,7 @@ benchmarks! { let v in 10..BenchBuilder::::fallback_max_validators(); let scenario = BenchBuilder::::new() - .build(Default::default(), &[2], None); + .build(Default::default(), &[], None); let mut benchmark = scenario.data.clone(); let dispute = benchmark.disputes.pop().unwrap(); @@ -60,7 +60,7 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed, &[1], None); + .build(cores_with_backed, &[], None); let mut benchmark = scenario.data.clone(); let bitfield = benchmark.bitfields.pop().unwrap(); @@ -97,7 +97,7 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed.clone(), &[1], None); + .build(cores_with_backed.clone(), &[], None); let mut benchmark = scenario.data.clone(); @@ -149,7 +149,7 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed.clone(), &[1], Some(v)); + .build(cores_with_backed.clone(), &[], Some(v)); let mut benchmark = scenario.data.clone(); From af0ee7ce7eb8e4d53bea69f56b30a711e1fafb53 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 18 Nov 2021 11:31:50 +0100 Subject: [PATCH 02/14] fix --- runtime/parachains/src/paras_inherent/benchmarking.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs index 32b315e4a850..4066f810db2a 100644 --- a/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -29,7 +29,7 @@ benchmarks! { let v in 10..BenchBuilder::::fallback_max_validators(); let scenario = BenchBuilder::::new() - .build(Default::default(), &[], None); + .build(Default::default(), &[2], None); let mut benchmark = scenario.data.clone(); let dispute = benchmark.disputes.pop().unwrap(); From 220dd26244cab72fbb6efea418c636a5e98c1357 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 18 Nov 2021 12:02:52 +0100 Subject: [PATCH 03/14] Improve doc comment --- runtime/parachains/src/builder.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index ef66c6e20bae..048f7e83dbdc 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -523,8 +523,10 @@ impl BenchBuilder { /// Build a scenario for testing or benchmarks. /// - /// Note that this only allows cores with backed candidates / becoming available to be - /// mutually exclusive of cores of with disputes. + /// Note that this only allows api only allow building scenarios where the + /// `backed_and_concluding_cores` are mutually exclusive with the cores for disputes. So + /// `backed_and_concluding_cores.len() + dispute_sessions.len()` must be less than the max + /// number of cores. /// /// - `backed_and_concluding_cores`: Map from core/para id/group index seed to number of /// validity votes. From c388e2b8389aa612f7d1324eb3bd9dcd96d0145f Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Thu, 18 Nov 2021 12:13:50 +0100 Subject: [PATCH 04/14] Some doc improvemens' --- runtime/parachains/src/builder.rs | 4 ++-- runtime/parachains/src/paras_inherent.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 048f7e83dbdc..7e2fdf686682 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -523,8 +523,8 @@ impl BenchBuilder { /// Build a scenario for testing or benchmarks. /// - /// Note that this only allows api only allow building scenarios where the - /// `backed_and_concluding_cores` are mutually exclusive with the cores for disputes. So + /// Note that this api only allows building scenarios where the `backed_and_concluding_cores` + /// are mutually exclusive with the cores for disputes. So /// `backed_and_concluding_cores.len() + dispute_sessions.len()` must be less than the max /// number of cores. /// diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs index 0e85b8220f5e..3c4347c59375 100644 --- a/runtime/parachains/src/paras_inherent.rs +++ b/runtime/parachains/src/paras_inherent.rs @@ -1292,7 +1292,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 6, includes_code_upgrade: None, @@ -1338,7 +1338,7 @@ mod tests { let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 4, includes_code_upgrade: None, From 4a4d03cab1b0ea53229094cc2707abaed05c6e51 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:21:13 -0800 Subject: [PATCH 05/14] Use setters and no params for build --- runtime/parachains/src/builder.rs | 126 +++++++++++++----- runtime/parachains/src/paras_inherent.rs | 9 +- .../src/paras_inherent/benchmarking.rs | 13 +- 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 33300e78bbce..41128b5ed303 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -63,13 +63,31 @@ fn byte32_slice_from(n: u32) -> [u8; 32] { /// Paras inherent `enter` benchmark scenario builder. pub(crate) struct BenchBuilder { + /// Active validators. Validators should be declared prior to all other setup. validators: Option>, + /// Starting block number; we expect it to get incremented on session setup. block_number: T::BlockNumber, + /// Starting session; we expect it to get incremented on session setup. session: SessionIndex, + /// Session we want the scenario to take place in. We roll to to this session. target_session: u32, + /// Optionally set the max validators per core; otherwise uses the configuration value. max_validators_per_core: Option, + /// Optionally set the max validators; otherwise uses the configuration value. max_validators: Option, + /// Optionally set the number of dispute statements for each candidate, dispute_statements: BTreeMap, + /// Session index of for each dispute. Index of slice corresponds to a core, + /// which is offset by the number of entries for `backed_and_concluding_cores`. I.E. if + /// `backed_and_concluding_cores` cores has 3 entries, the first index of `dispute_sessions` + /// will correspond to core index 3. There must be one entry for each core with a dispute + /// statement set. + dispute_sessions: Vec, + /// Map from core/para id/group index seed to number of validity votes. + backed_and_concluding_cores: BTreeMap, + /// Have every candidate include a code upgrade by setting this to `Some` where the interior + /// value is the byte length of the new code. + code_upgrade: Option, _phantom: sp_std::marker::PhantomData, } @@ -86,24 +104,40 @@ impl BenchBuilder { /// of the functions in this implementation. pub(crate) fn new() -> Self { BenchBuilder { - // Validators should be declared prior to all other setup. validators: None, - // Starting block number; we expect it to get incremented on session setup. block_number: Zero::zero(), - // Starting session; we expect it to get incremented on session setup. session: SessionIndex::from(0u32), - // Session we want the scenario to take place in. We roll to to this session. target_session: 2u32, - // Optionally set the max validators per core; otherwise uses the configuration value. max_validators_per_core: None, - // Optionally set the max validators; otherwise uses the configuration value. max_validators: None, - // Optionally set the number of dispute statements for each candidate, dispute_statements: BTreeMap::new(), + dispute_sessions: Default::default(), + backed_and_concluding_cores: Default::default(), + code_upgrade: None, _phantom: sp_std::marker::PhantomData::, } } + /// Set `self.dispute_sessions`. + pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: Vec) -> Self { + self.dispute_sessions = dispute_sessions; + self + } + + /// Set `self.backed_and_concluding_cores` + pub(crate) fn set_backed_and_concluding_cores( + mut self, + backed_and_concluding_cores: BTreeMap, + ) -> Self { + self.backed_and_concluding_cores = backed_and_concluding_cores; + self + } + + pub(crate) fn set_code_upgrade(mut self, code_upgrade: Option) -> Self { + self.code_upgrade = code_upgrade; + self + } + /// Mock header. pub(crate) fn header(block_number: T::BlockNumber) -> T::Header { T::Header::new( @@ -133,6 +167,7 @@ impl BenchBuilder { self.max_validators.unwrap_or(Self::fallback_max_validators()) } + /// Set the maximum number of active validators. #[cfg(not(feature = "runtime-benchmarks"))] pub(crate) fn set_max_validators(mut self, n: u32) -> Self { self.max_validators = Some(n); @@ -155,6 +190,7 @@ impl BenchBuilder { self } + /// Get the maximum number of validators per core. fn max_validators_per_core(&self) -> u32 { self.max_validators_per_core.unwrap_or(Self::fallback_max_validators_per_core()) } @@ -166,17 +202,18 @@ impl BenchBuilder { self } - /// Maximum number of cores we expect from this configuration. + /// Get the maximum number of cores we expect from this configuration. pub(crate) fn max_cores(&self) -> u32 { self.max_validators() / self.max_validators_per_core() } - /// Minimum number of validity votes in order for a backed candidate to be included. + /// Get the minimum number of validity votes in order for a backed candidate to be included. #[cfg(feature = "runtime-benchmarks")] pub(crate) fn fallback_min_validity_votes() -> u32 { (Self::fallback_max_validators() / 2) + 1 } + /// Create para id, core index, and grab the associated group index from the scheduler pallet. fn create_indexes(&self, seed: u32) -> (ParaId, CoreIndex, GroupIndex) { let para_id = ParaId::from(seed); let core_idx = CoreIndex(seed); @@ -186,6 +223,7 @@ impl BenchBuilder { (para_id, core_idx, group_idx) } + /// Create a mock of `CandidatePendingAvailability`. fn candidate_availability_mock( group_idx: GroupIndex, core_idx: CoreIndex, @@ -204,6 +242,11 @@ impl BenchBuilder { ) } + /// Add `CandidatePendingAvailability` and `CandidateCommitments` to the relevant storage items. + /// + /// NOTE: the default `CandidateCommitments` used does not include any data that would lead to + /// heavy code paths in `enact_candidate`. But enact_candidates does return a weight which will + /// get taken into account. fn add_availability( para_id: ParaId, core_idx: CoreIndex, @@ -217,14 +260,14 @@ impl BenchBuilder { candidate_hash, availability_votes, ); - // NOTE: commitments does not include any data that would lead to heavy code - // paths in `enact_candidate`. But enact_candidates does return a weight which will get - // taken into account. + let commitments = CandidateCommitments::::default(); inclusion::PendingAvailability::::insert(para_id, candidate_availability); inclusion::PendingAvailabilityCommitments::::insert(¶_id, commitments); } + /// Create an `AvailabilityBitfield` where `concluding` is a map where each key is a core index + /// that is concluding and `cores` is the total number of cores in the system. fn availability_bitvec(concluding: &BTreeMap, cores: u32) -> AvailabilityBitfield { let mut bitfields = bitvec::bitvec![bitvec::order::Lsb0, u8; 0; 0]; for i in 0..cores { @@ -238,6 +281,8 @@ impl BenchBuilder { bitfields.into() } + /// Run to block number `to`, calling `initializer` `on_initialize` and `on_finalize` along the + /// way. fn run_to_block(to: u32) { let to = to.into(); while frame_system::Pallet::::block_number() < to { @@ -250,6 +295,10 @@ impl BenchBuilder { } } + /// Schedule a parachain for each `cores`. + /// + /// Note that this must be called at least 2 sessions before the target session as there is a + /// n+2 session delay for the scheduled actions to take effect. fn setup_para_ids(cores: u32) { // make sure parachains exist prior to session change. for i in 0..cores { @@ -288,6 +337,7 @@ impl BenchBuilder { } } + /// Create a bitvec of `validators` length with all yes votes. fn validator_availability_votes_yes(validators: usize) -> BitVec { // every validator confirms availability. bitvec::bitvec![bitvec::order::Lsb0, u8; 1; validators as usize] @@ -338,6 +388,11 @@ impl BenchBuilder { self } + /// Create a `UncheckedSigned for each validator where each core in + /// `concluding_cores` is signed off by every validator as fully available. Additionally set up + /// storage such that each `concluding_cores` is pending becoming fully available so the + /// generated bitfields will be to the cores successfully being freed from the candidates being + /// marked as available. fn create_availability_bitfields( &self, concluding_cores: &BTreeMap, @@ -364,8 +419,7 @@ impl BenchBuilder { .collect(); for (seed, _) in concluding_cores.iter() { - // make sure the candidates that are concluding by becoming available are marked as - // pending availability. + // make sure the candidates that will be concluding are marked as pending availability. let (para_id, core_idx, group_idx) = self.create_indexes(seed.clone()); Self::add_availability( para_id, @@ -483,6 +537,8 @@ impl BenchBuilder { .collect() } + /// Fill cores `start..last` with dispute statement sets. The statement sets will have 3/4th of + /// votes be valid, and 1/4th of votes be invalid. fn create_disputes_with_no_spam( &self, start: u32, @@ -540,26 +596,23 @@ impl BenchBuilder { /// Build a scenario for testing or benchmarks. /// - /// Note that this api only allows building scenarios where the `backed_and_concluding_cores` + /// Note that this API only allows building scenarios where the `backed_and_concluding_cores` /// are mutually exclusive with the cores for disputes. So /// `backed_and_concluding_cores.len() + dispute_sessions.len()` must be less than the max /// number of cores. /// - /// - `backed_and_concluding_cores`: Map from core/para id/group index seed to number of + /// Some methods for configuring the builder: + /// + /// * `set_backed_and_concluding_cores`: Set a map from core/para id/group index seed to number of /// validity votes. - /// - `dispute_sessions`: Session index of for each dispute. Index of slice corresponds to a core, - /// which is offset by the number of entries for `backed_and_concluding_cores`. I.E. if - /// `backed_and_concluding_cores` cores has 3 entries, the first index of `dispute_sessions` - /// will correspond to core index 3. There must be one entry for each core with a dispute - /// statement set. - /// - `includes_code_upgrade`: Set to `Some` to include a code upgrade for all backed candidates. + /// * `set_dispute_sessions`: Set the session index of for each dispute. Index of slice + /// corresponds to a core, which is offset by the number of entries for + /// `backed_and_concluding_cores`. I.E. if `backed_and_concluding_cores` cores has 3 entries, + /// the first index of `dispute_sessions` will correspond to core index 3. There must be one + /// entry for each core with a dispute statement set. + /// *`set_code_upgrade`: Set to `Some` to include a code upgrade for all backed candidates. /// The value within `Some` will be the byte length of the code. - pub(crate) fn build( - self, - backed_and_concluding_cores: BTreeMap, - dispute_sessions: &[u32], - includes_code_upgrade: Option, - ) -> Bench { + pub(crate) fn build(self) -> Bench { // Make sure relevant storage is cleared. This is just to get the asserts to work when // running tests because it seems the storage is not cleared in between. inclusion::PendingAvailabilityCommitments::::remove_all(None); @@ -567,11 +620,12 @@ impl BenchBuilder { // We don't allow a core to have both disputes and be marked fully available at this block. let cores = self.max_cores(); - let used_cores = (dispute_sessions.len() + backed_and_concluding_cores.len()) as u32; + let used_cores = + (self.dispute_sessions.len() + self.backed_and_concluding_cores.len()) as u32; assert!(used_cores <= cores); - // NOTE: there is an n+2 session delay for these actions to take effect - // We are currently in Session 0, so these changes will take effect in Session 2 + // NOTE: there is an n+2 session delay for these actions to take effect. + // We are currently in Session 0, so these changes will take effect in Session 2. Self::setup_para_ids(used_cores); let validator_ids = Self::generate_validator_pairs(self.max_validators()); @@ -579,14 +633,14 @@ impl BenchBuilder { let builder = self.setup_session(target_session, validator_ids, used_cores); let bitfields = - builder.create_availability_bitfields(&backed_and_concluding_cores, used_cores); + builder.create_availability_bitfields(&builder.backed_and_concluding_cores, used_cores); let backed_candidates = - builder.create_backed_candidates(&backed_and_concluding_cores, includes_code_upgrade); + builder.create_backed_candidates(&builder.backed_and_concluding_cores, builder.code_upgrade); let disputes = builder.create_disputes_with_no_spam( - backed_and_concluding_cores.len() as u32, + builder.backed_and_concluding_cores.len() as u32, used_cores, - dispute_sessions, + builder.dispute_sessions.as_slice(), ); assert_eq!( @@ -595,7 +649,7 @@ impl BenchBuilder { ); assert_eq!(inclusion::PendingAvailability::::iter().count(), used_cores as usize,); - // Mark all the use cores as occupied. We expect that their are `backed_and_concluding_cores` + // Mark all the used cores as occupied. We expect that their are `backed_and_concluding_cores` // that are pending availability and that there are `used_cores - backed_and_concluding_cores ` // which are about to be disputed. scheduler::AvailabilityCores::::set(vec![ diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs index 3aeba810c4b9..a33961864f61 100644 --- a/runtime/parachains/src/paras_inherent.rs +++ b/runtime/parachains/src/paras_inherent.rs @@ -1152,7 +1152,7 @@ mod tests { dispute_sessions: Vec, backed_and_concluding: BTreeMap, num_validators_per_core: u32, - includes_code_upgrade: Option, + code_upgrade: Option, } fn make_inherent_data( @@ -1161,7 +1161,7 @@ mod tests { dispute_sessions, backed_and_concluding, num_validators_per_core, - includes_code_upgrade, + code_upgrade, }: TestConfig, ) -> Bench { BenchBuilder::::new() @@ -1171,7 +1171,10 @@ mod tests { ) .set_max_validators_per_core(num_validators_per_core) .set_dispute_statements(dispute_statements) - .build(backed_and_concluding, dispute_sessions.as_slice(), includes_code_upgrade) + .set_backed_and_concluding_cores(backed_and_concluding) + .set_dispute_sessions(backed_and_concluding) + .set_code_upgrade(code_upgrade) + .build() } #[test] diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs index 2e6a5213d76e..45940f2021cc 100644 --- a/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -29,7 +29,8 @@ benchmarks! { let v in 10..BenchBuilder::::fallback_max_validators(); let scenario = BenchBuilder::::new() - .build(Default::default(), &[2], None); + .set_dispute_sessions(vec![2]) + .build(); let mut benchmark = scenario.data.clone(); let dispute = benchmark.disputes.pop().unwrap(); @@ -60,7 +61,8 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed, &[], None); + .set_backed_and_concluding_cores(cores_with_backed) + .build(); let mut benchmark = scenario.data.clone(); let bitfield = benchmark.bitfields.pop().unwrap(); @@ -104,7 +106,8 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed.clone(), &[], None); + .set_backed_and_concluding_cores(cores_with_backed.clone()) + .build(); let mut benchmark = scenario.data.clone(); @@ -156,7 +159,9 @@ benchmarks! { .collect(); let scenario = BenchBuilder::::new() - .build(cores_with_backed.clone(), &[], Some(v)); + .set_backed_and_concluding_cores(cores_with_backed.clone()) + .set_code_upgrade(Some(v)) + .build(); let mut benchmark = scenario.data.clone(); From c067b71cdbf8420488c3e582c0d1f2792b1ab590 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:23:05 -0800 Subject: [PATCH 06/14] +nightly-2021-10-29 fmt --- runtime/parachains/src/builder.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 41128b5ed303..d17ac0604623 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -634,8 +634,8 @@ impl BenchBuilder { let bitfields = builder.create_availability_bitfields(&builder.backed_and_concluding_cores, used_cores); - let backed_candidates = - builder.create_backed_candidates(&builder.backed_and_concluding_cores, builder.code_upgrade); + let backed_candidates = builder + .create_backed_candidates(&builder.backed_and_concluding_cores, builder.code_upgrade); let disputes = builder.create_disputes_with_no_spam( builder.backed_and_concluding_cores.len() as u32, From 66e6517a096c738dc09f70d3ebf3b6623e9bd570 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:43:46 -0800 Subject: [PATCH 07/14] Clean up some comments --- runtime/parachains/src/builder.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index d17ac0604623..d1eee58a3c39 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -69,23 +69,23 @@ pub(crate) struct BenchBuilder { block_number: T::BlockNumber, /// Starting session; we expect it to get incremented on session setup. session: SessionIndex, - /// Session we want the scenario to take place in. We roll to to this session. + /// Session we want the scenario to take place in. We will roll to this session. target_session: u32, /// Optionally set the max validators per core; otherwise uses the configuration value. max_validators_per_core: Option, /// Optionally set the max validators; otherwise uses the configuration value. max_validators: Option, - /// Optionally set the number of dispute statements for each candidate, + /// Optionally set the number of dispute statements for each candidate. dispute_statements: BTreeMap, /// Session index of for each dispute. Index of slice corresponds to a core, /// which is offset by the number of entries for `backed_and_concluding_cores`. I.E. if - /// `backed_and_concluding_cores` cores has 3 entries, the first index of `dispute_sessions` + /// `backed_and_concluding_cores` has 3 entries, the first index of `dispute_sessions` /// will correspond to core index 3. There must be one entry for each core with a dispute /// statement set. dispute_sessions: Vec, - /// Map from core/para id/group index seed to number of validity votes. + /// Map from core seed to number of validity votes. backed_and_concluding_cores: BTreeMap, - /// Have every candidate include a code upgrade by setting this to `Some` where the interior + /// Make every candidate include a code upgrade by setting this to `Some` where the interior /// value is the byte length of the new code. code_upgrade: Option, _phantom: sp_std::marker::PhantomData, @@ -124,7 +124,7 @@ impl BenchBuilder { self } - /// Set `self.backed_and_concluding_cores` + /// Set `self.backed_and_concluding_cores`. pub(crate) fn set_backed_and_concluding_cores( mut self, backed_and_concluding_cores: BTreeMap, @@ -260,7 +260,6 @@ impl BenchBuilder { candidate_hash, availability_votes, ); - let commitments = CandidateCommitments::::default(); inclusion::PendingAvailability::::insert(para_id, candidate_availability); inclusion::PendingAvailabilityCommitments::::insert(¶_id, commitments); @@ -295,7 +294,7 @@ impl BenchBuilder { } } - /// Schedule a parachain for each `cores`. + /// Register `cores` count of parachains. /// /// Note that this must be called at least 2 sessions before the target session as there is a /// n+2 session delay for the scheduled actions to take effect. @@ -389,10 +388,9 @@ impl BenchBuilder { } /// Create a `UncheckedSigned for each validator where each core in - /// `concluding_cores` is signed off by every validator as fully available. Additionally set up - /// storage such that each `concluding_cores` is pending becoming fully available so the - /// generated bitfields will be to the cores successfully being freed from the candidates being - /// marked as available. + /// `concluding_cores` is fully available. Additionally set up storage such that each + /// `concluding_cores`is pending becoming fully available so the generated bitfields will be + /// to the cores successfully being freed from the candidates being marked as available. fn create_availability_bitfields( &self, concluding_cores: &BTreeMap, @@ -603,12 +601,12 @@ impl BenchBuilder { /// /// Some methods for configuring the builder: /// - /// * `set_backed_and_concluding_cores`: Set a map from core/para id/group index seed to number of - /// validity votes. + /// * `set_backed_and_concluding_cores`: Set a map from core/para id seed to number of validity + /// votes. /// * `set_dispute_sessions`: Set the session index of for each dispute. Index of slice /// corresponds to a core, which is offset by the number of entries for /// `backed_and_concluding_cores`. I.E. if `backed_and_concluding_cores` cores has 3 entries, - /// the first index of `dispute_sessions` will correspond to core index 3. There must be one + /// the first index of `dispute_sessions` will correspond to core index 3. There must be an /// entry for each core with a dispute statement set. /// *`set_code_upgrade`: Set to `Some` to include a code upgrade for all backed candidates. /// The value within `Some` will be the byte length of the code. From 0dbe497bcaa1bd51f1c8076b59bd2b165392eaee Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 29 Nov 2021 13:26:33 -0800 Subject: [PATCH 08/14] Simplify set_code_upgrade --- runtime/parachains/src/builder.rs | 4 +-- runtime/parachains/src/paras_inherent.rs | 32 +++++++++++-------- .../src/paras_inherent/benchmarking.rs | 2 +- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index d1eee58a3c39..59e3c8bc5f8f 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -133,8 +133,8 @@ impl BenchBuilder { self } - pub(crate) fn set_code_upgrade(mut self, code_upgrade: Option) -> Self { - self.code_upgrade = code_upgrade; + pub(crate) fn set_code_upgrade(mut self, code_upgrade: impl Into>) -> Self { + self.code_upgrade = code_upgrade.into(); self } diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs index a33961864f61..ba3a5ba38151 100644 --- a/runtime/parachains/src/paras_inherent.rs +++ b/runtime/parachains/src/paras_inherent.rs @@ -1164,7 +1164,7 @@ mod tests { code_upgrade, }: TestConfig, ) -> Bench { - BenchBuilder::::new() + let builder = BenchBuilder::::new() .set_max_validators( (dispute_sessions.len() + backed_and_concluding.len()) as u32 * num_validators_per_core, @@ -1172,9 +1172,13 @@ mod tests { .set_max_validators_per_core(num_validators_per_core) .set_dispute_statements(dispute_statements) .set_backed_and_concluding_cores(backed_and_concluding) - .set_dispute_sessions(backed_and_concluding) - .set_code_upgrade(code_upgrade) - .build() + .set_dispute_sessions(dispute_sessions); + + if let Some(code_size) = code_upgrade { + builder.set_code_upgrade(code_size).build() + } else { + builder.build() + } } #[test] @@ -1194,7 +1198,7 @@ mod tests { dispute_sessions: vec![], // No disputes backed_and_concluding, num_validators_per_core: 1, - includes_code_upgrade: None, + code_upgrade: None, }); // We expect the scenario to have cores 0 & 1 with pending availability. The backed @@ -1262,7 +1266,7 @@ mod tests { ], backed_and_concluding, num_validators_per_core: 5, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1333,7 +1337,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 6, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1400,7 +1404,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 6, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1446,7 +1450,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 4, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1524,7 +1528,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 4, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1585,7 +1589,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1665,7 +1669,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1724,7 +1728,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); @@ -1795,7 +1799,7 @@ mod tests { dispute_sessions: vec![2, 2, 1], // 3 cores with disputes backed_and_concluding, num_validators_per_core: 5, - includes_code_upgrade: None, + code_upgrade: None, }); let expected_para_inherent_data = scenario.data.clone(); diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs index 45940f2021cc..7c047e1966ca 100644 --- a/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -160,7 +160,7 @@ benchmarks! { let scenario = BenchBuilder::::new() .set_backed_and_concluding_cores(cores_with_backed.clone()) - .set_code_upgrade(Some(v)) + .set_code_upgrade(v) .build(); let mut benchmark = scenario.data.clone(); From de60558a336a75e1ba2fc0fef73dda293040098a Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:12:17 -0800 Subject: [PATCH 09/14] Accept a slice for set_dispute_sessions --- runtime/parachains/src/builder.rs | 4 ++-- runtime/parachains/src/paras_inherent.rs | 2 +- runtime/parachains/src/paras_inherent/benchmarking.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 59e3c8bc5f8f..ff2dcca515c2 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -119,8 +119,8 @@ impl BenchBuilder { } /// Set `self.dispute_sessions`. - pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: Vec) -> Self { - self.dispute_sessions = dispute_sessions; + pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: &[u32]) -> Self { + self.dispute_sessions = dispute_sessions.to_vec(); self } diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs index ba3a5ba38151..2a1c7e5eb05f 100644 --- a/runtime/parachains/src/paras_inherent.rs +++ b/runtime/parachains/src/paras_inherent.rs @@ -1172,7 +1172,7 @@ mod tests { .set_max_validators_per_core(num_validators_per_core) .set_dispute_statements(dispute_statements) .set_backed_and_concluding_cores(backed_and_concluding) - .set_dispute_sessions(dispute_sessions); + .set_dispute_sessions(&dispute_sessions[..]); if let Some(code_size) = code_upgrade { builder.set_code_upgrade(code_size).build() diff --git a/runtime/parachains/src/paras_inherent/benchmarking.rs b/runtime/parachains/src/paras_inherent/benchmarking.rs index 7c047e1966ca..0ef9062430bd 100644 --- a/runtime/parachains/src/paras_inherent/benchmarking.rs +++ b/runtime/parachains/src/paras_inherent/benchmarking.rs @@ -29,7 +29,7 @@ benchmarks! { let v in 10..BenchBuilder::::fallback_max_validators(); let scenario = BenchBuilder::::new() - .set_dispute_sessions(vec![2]) + .set_dispute_sessions(&[2]) .build(); let mut benchmark = scenario.data.clone(); From d81c612ccf277b405e4eb9f6eee8e662844217cc Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:45:08 -0800 Subject: [PATCH 10/14] Doc comments --- runtime/parachains/src/builder.rs | 33 ++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index ff2dcca515c2..8309f4c61e76 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -118,13 +118,19 @@ impl BenchBuilder { } } - /// Set `self.dispute_sessions`. + /// Set the session index for each dispute statement set (in other words, set the session the + /// the dispute statement set's relay chain block is from). Indexes of `dispute_sessions` + /// correspond to a core, which is offset by the number of entries for + /// `backed_and_concluding_cores`. I.E. if `backed_and_concluding_cores` cores has 3 entries, + /// the first index of `dispute_sessions` will correspond to core index 3. + /// + /// Note that there must be an entry for each core with a dispute statement set. pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: &[u32]) -> Self { self.dispute_sessions = dispute_sessions.to_vec(); self } - /// Set `self.backed_and_concluding_cores`. + /// Set a map from core/para id seed to number of validity votes. pub(crate) fn set_backed_and_concluding_cores( mut self, backed_and_concluding_cores: BTreeMap, @@ -133,6 +139,8 @@ impl BenchBuilder { self } + /// Set to include a code upgrade for all backed candidates. The value will be the byte length + /// of the code. pub(crate) fn set_code_upgrade(mut self, code_upgrade: impl Into>) -> Self { self.code_upgrade = code_upgrade.into(); self @@ -180,10 +188,11 @@ impl BenchBuilder { configuration::Pallet::::config().max_validators_per_core.unwrap_or(5) } - /// Specify a mapping of core index, para id, group index seed to the number of dispute statements for the - /// corresponding dispute statement set. Note that if the number of disputes is not specified it fallbacks - /// to having a dispute per every validator. Additionally, an entry is not guaranteed to have a dispute - it - /// must line up with the cores marked as disputed as defined in `Self::Build`. + /// Specify a mapping of core index/ para id eed to the number of dispute statements for the + /// corresponding dispute statement set. Note that if the number of disputes is not specified + /// it fallbacks to having a dispute per every validator. Additionally, an entry is not + /// guaranteed to have a dispute - it must line up with the cores marked as disputed as defined + /// in `Self::Build`. #[cfg(not(feature = "runtime-benchmarks"))] pub(crate) fn set_dispute_statements(mut self, m: BTreeMap) -> Self { self.dispute_statements = m; @@ -598,18 +607,6 @@ impl BenchBuilder { /// are mutually exclusive with the cores for disputes. So /// `backed_and_concluding_cores.len() + dispute_sessions.len()` must be less than the max /// number of cores. - /// - /// Some methods for configuring the builder: - /// - /// * `set_backed_and_concluding_cores`: Set a map from core/para id seed to number of validity - /// votes. - /// * `set_dispute_sessions`: Set the session index of for each dispute. Index of slice - /// corresponds to a core, which is offset by the number of entries for - /// `backed_and_concluding_cores`. I.E. if `backed_and_concluding_cores` cores has 3 entries, - /// the first index of `dispute_sessions` will correspond to core index 3. There must be an - /// entry for each core with a dispute statement set. - /// *`set_code_upgrade`: Set to `Some` to include a code upgrade for all backed candidates. - /// The value within `Some` will be the byte length of the code. pub(crate) fn build(self) -> Bench { // Make sure relevant storage is cleared. This is just to get the asserts to work when // running tests because it seems the storage is not cleared in between. From 46e7e03bbf7bf86715e6f3487e420206b715e81e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Mon, 29 Nov 2021 15:59:09 -0800 Subject: [PATCH 11/14] Spelling --- runtime/parachains/src/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 8309f4c61e76..5cb9c4f33e21 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -188,7 +188,7 @@ impl BenchBuilder { configuration::Pallet::::config().max_validators_per_core.unwrap_or(5) } - /// Specify a mapping of core index/ para id eed to the number of dispute statements for the + /// Specify a mapping of core index/ para id to the number of dispute statements for the /// corresponding dispute statement set. Note that if the number of disputes is not specified /// it fallbacks to having a dispute per every validator. Additionally, an entry is not /// guaranteed to have a dispute - it must line up with the cores marked as disputed as defined From e35c7bf3de51c4b2428a6c2ccc0e509f75b41673 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Nov 2021 12:38:23 +0100 Subject: [PATCH 12/14] use impl AsRef --- runtime/parachains/src/builder.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runtime/parachains/src/builder.rs b/runtime/parachains/src/builder.rs index 5cb9c4f33e21..09a39500eff7 100644 --- a/runtime/parachains/src/builder.rs +++ b/runtime/parachains/src/builder.rs @@ -125,8 +125,8 @@ impl BenchBuilder { /// the first index of `dispute_sessions` will correspond to core index 3. /// /// Note that there must be an entry for each core with a dispute statement set. - pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: &[u32]) -> Self { - self.dispute_sessions = dispute_sessions.to_vec(); + pub(crate) fn set_dispute_sessions(mut self, dispute_sessions: impl AsRef<[u32]>) -> Self { + self.dispute_sessions = dispute_sessions.as_ref().to_vec(); self } @@ -550,11 +550,12 @@ impl BenchBuilder { &self, start: u32, last: u32, - dispute_sessions: &[u32], + dispute_sessions: impl AsRef<[u32]>, ) -> Vec { let validators = self.validators.as_ref().expect("must have some validators prior to calling"); + let dispute_sessions = dispute_sessions.as_ref(); (start..last) .map(|seed| { let dispute_session_idx = (seed - start) as usize; From 9d85b6d5cf12699a4729a08e217491f045b43405 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 1 Dec 2021 18:46:12 -0800 Subject: [PATCH 13/14] Prepare for merge --- runtime/parachains/src/paras_inherent.rs | 2250 ----------------- runtime/parachains/src/paras_inherent/misc.rs | 40 + runtime/parachains/src/paras_inherent/mod.rs | 1022 ++++++++ .../parachains/src/paras_inherent/tests.rs | 1115 ++++++++ .../parachains/src/paras_inherent/weights.rs | 118 + 5 files changed, 2295 insertions(+), 2250 deletions(-) delete mode 100644 runtime/parachains/src/paras_inherent.rs create mode 100644 runtime/parachains/src/paras_inherent/misc.rs create mode 100644 runtime/parachains/src/paras_inherent/mod.rs create mode 100644 runtime/parachains/src/paras_inherent/tests.rs create mode 100644 runtime/parachains/src/paras_inherent/weights.rs diff --git a/runtime/parachains/src/paras_inherent.rs b/runtime/parachains/src/paras_inherent.rs deleted file mode 100644 index 2a1c7e5eb05f..000000000000 --- a/runtime/parachains/src/paras_inherent.rs +++ /dev/null @@ -1,2250 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Provides glue code over the scheduler and inclusion modules, and accepting -//! one inherent per block that can include new para candidates and bitfields. -//! -//! Unlike other modules in this crate, it does not need to be initialized by the initializer, -//! as it has no initialization logic and its finalization logic depends only on the details of -//! this module. - -use crate::{ - disputes::DisputesHandler, - inclusion, - inclusion::{CandidateCheckContext, FullCheck}, - initializer, - scheduler::{self, CoreAssignment, FreedReason}, - shared, ump, -}; -use bitvec::prelude::BitVec; -use frame_support::{ - inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent}, - pallet_prelude::*, - traits::Randomness, -}; -use frame_system::pallet_prelude::*; -use pallet_babe::{self, CurrentBlockRandomness}; -use primitives::v1::{ - BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet, - InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes, - SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield, - UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, - PARACHAINS_INHERENT_IDENTIFIER, -}; -use rand::{seq::SliceRandom, SeedableRng}; - -use scale_info::TypeInfo; -use sp_runtime::traits::{Header as HeaderT, One}; -use sp_std::{ - cmp::Ordering, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - prelude::*, - vec::Vec, -}; - -#[cfg(feature = "runtime-benchmarks")] -mod benchmarking; - -const LOG_TARGET: &str = "runtime::inclusion-inherent"; - -pub trait WeightInfo { - /// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the - /// weight of a single dispute statement set. - fn enter_variable_disputes(v: u32) -> Weight; - /// The weight of one bitfield. - fn enter_bitfields() -> Weight; - /// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight - /// of a single backed candidate. - fn enter_backed_candidates_variable(v: u32) -> Weight; - /// The weight of a single backed candidate with a code upgrade. - fn enter_backed_candidate_code_upgrade() -> Weight; -} - -pub struct TestWeightInfo; -// `WeightInfo` impl for unit and integration tests. Based off of the `max_block` weight for the -// mock. -#[cfg(not(feature = "runtime-benchmarks"))] -impl WeightInfo for TestWeightInfo { - fn enter_variable_disputes(v: u32) -> Weight { - // MAX Block Weight should fit 4 disputes - 80_000 * v as Weight + 80_000 - } - fn enter_bitfields() -> Weight { - // MAX Block Weight should fit 4 backed candidates - 40_000 as Weight - } - fn enter_backed_candidates_variable(v: u32) -> Weight { - // MAX Block Weight should fit 4 backed candidates - 40_000 * v as Weight + 40_000 - } - fn enter_backed_candidate_code_upgrade() -> Weight { - 0 - } -} -// To simplify benchmarks running as tests, we set all the weights to 0. `enter` will exit early -// when if the data causes it to be over weight, but we don't want that to block a benchmark from -// running as a test. -#[cfg(feature = "runtime-benchmarks")] -impl WeightInfo for TestWeightInfo { - fn enter_variable_disputes(_v: u32) -> Weight { - 0 - } - fn enter_bitfields() -> Weight { - 0 - } - fn enter_backed_candidates_variable(_v: u32) -> Weight { - 0 - } - fn enter_backed_candidate_code_upgrade() -> Weight { - 0 - } -} - -fn paras_inherent_total_weight( - backed_candidates: &[BackedCandidate<::Hash>], - bitfields: &[UncheckedSignedAvailabilityBitfield], - disputes: &[DisputeStatementSet], -) -> Weight { - backed_candidates_weight::(backed_candidates) - .saturating_add(signed_bitfields_weight::(bitfields.len())) - .saturating_add(dispute_statements_weight::(disputes)) -} - -fn dispute_statements_weight(disputes: &[DisputeStatementSet]) -> Weight { - disputes - .iter() - .map(|d| { - <::WeightInfo as WeightInfo>::enter_variable_disputes( - d.statements.len() as u32 - ) - }) - .fold(0, |acc, x| acc.saturating_add(x)) -} - -fn signed_bitfields_weight(bitfields_len: usize) -> Weight { - <::WeightInfo as WeightInfo>::enter_bitfields() - .saturating_mul(bitfields_len as Weight) -} - -fn backed_candidate_weight( - candidate: &BackedCandidate, -) -> Weight { - if candidate.candidate.commitments.new_validation_code.is_some() { - <::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade() - } else { - <::WeightInfo as WeightInfo>::enter_backed_candidates_variable( - candidate.validity_votes.len() as u32, - ) - } -} - -fn backed_candidates_weight( - candidates: &[BackedCandidate], -) -> Weight { - candidates - .iter() - .map(|c| backed_candidate_weight::(c)) - .fold(0, |acc, x| acc.saturating_add(x)) -} - -/// A helper trait to allow calling retain while getting access -/// to the index of the item in the `vec`. -trait IndexedRetain { - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` residing at - /// index `i` such that `f(i, &e)` returns `false`. This method - /// operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - fn indexed_retain(&mut self, f: impl FnMut(usize, &T) -> bool); -} - -impl IndexedRetain for Vec { - fn indexed_retain(&mut self, mut f: impl FnMut(usize, &T) -> bool) { - let mut idx = 0_usize; - self.retain(move |item| { - let ret = f(idx, item); - idx += 1_usize; - ret - }) - } -} - -/// A bitfield concerning concluded disputes for candidates -/// associated to the core index equivalent to the bit position. -#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] -pub(crate) struct DisputedBitfield(pub(crate) BitVec); - -impl From> for DisputedBitfield { - fn from(inner: BitVec) -> Self { - Self(inner) - } -} - -#[cfg(test)] -impl DisputedBitfield { - /// Create a new bitfield, where each bit is set to `false`. - pub fn zeros(n: usize) -> Self { - Self::from(BitVec::::repeat(false, n)) - } -} - -pub use pallet::*; - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - #[pallet::disable_frame_system_supertrait_check] - pub trait Config: - inclusion::Config + scheduler::Config + initializer::Config + pallet_babe::Config - { - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; - } - - #[pallet::error] - pub enum Error { - /// Inclusion inherent called more than once per block. - TooManyInclusionInherents, - /// The hash of the submitted parent header doesn't correspond to the saved block hash of - /// the parent. - InvalidParentHeader, - /// Disputed candidate that was concluded invalid. - CandidateConcludedInvalid, - /// The data given to the inherent will result in an overweight block. - InherentOverweight, - } - - /// Whether the paras inherent was included within this block. - /// - /// The `Option<()>` is effectively a `bool`, but it never hits storage in the `None` variant - /// due to the guarantees of FRAME's storage APIs. - /// - /// If this is `None` at the end of the block, we panic and render the block invalid. - #[pallet::storage] - pub(crate) type Included = StorageValue<_, ()>; - - /// Scraped on chain data for extracting resolved disputes as well as backing votes. - #[pallet::storage] - #[pallet::getter(fn on_chain_votes)] - pub(crate) type OnChainVotes = StorageValue<_, ScrapedOnChainVotes>; - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_initialize(_: T::BlockNumber) -> Weight { - T::DbWeight::get().reads_writes(1, 1) // in on_finalize. - } - - fn on_finalize(_: T::BlockNumber) { - if Included::::take().is_none() { - panic!("Bitfields and heads must be included every block"); - } - } - } - - #[pallet::inherent] - impl ProvideInherent for Pallet { - type Call = Call; - type Error = MakeFatalError<()>; - const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER; - - fn create_inherent(data: &InherentData) -> Option { - let inherent_data = Self::create_inherent_inner(data)?; - // Sanity check: session changes can invalidate an inherent, - // and we _really_ don't want that to happen. - // See - - // Calling `Self::enter` here is a safe-guard, to avoid any discrepancy between on-chain checks - // (`enter`) and the off-chain checks by the block author (this function). Once we are confident - // in all the logic in this module this check should be removed to optimize performance. - - let inherent_data = match Self::enter_inner(inherent_data.clone(), FullCheck::Skip) { - Ok(_) => inherent_data, - Err(err) => { - log::error!( - target: LOG_TARGET, - "dropping paras inherent data because they produced \ - an invalid paras inherent: {:?}", - err.error, - ); - - ParachainsInherentData { - bitfields: Vec::new(), - backed_candidates: Vec::new(), - disputes: Vec::new(), - parent_header: inherent_data.parent_header, - } - }, - }; - - Some(Call::enter { data: inherent_data }) - } - - fn is_inherent(call: &Self::Call) -> bool { - matches!(call, Call::enter { .. }) - } - } - - /// Collect all freed cores based on storage data. (i.e. append cores freed from timeouts to - /// the given `freed_concluded`). - /// - /// The parameter `freed_concluded` contains all core indicies that became - /// free due to candidate that became available. - pub(crate) fn collect_all_freed_cores( - freed_concluded: I, - ) -> BTreeMap - where - I: core::iter::IntoIterator, - T: Config, - { - // Handle timeouts for any availability core work. - let availability_pred = >::availability_timeout_predicate(); - let freed_timeout = if let Some(pred) = availability_pred { - >::collect_pending(pred) - } else { - Vec::new() - }; - - // Schedule paras again, given freed cores, and reasons for freeing. - let freed = freed_concluded - .into_iter() - .map(|(c, _hash)| (c, FreedReason::Concluded)) - .chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut))) - .collect::>(); - freed - } - - #[pallet::call] - impl Pallet { - /// Enter the paras inherent. This will process bitfields and backed candidates. - #[pallet::weight(( - paras_inherent_total_weight::( - data.backed_candidates.as_slice(), - data.bitfields.as_slice(), - data.disputes.as_slice(), - ), - DispatchClass::Mandatory, - ))] - pub fn enter( - origin: OriginFor, - data: ParachainsInherentData, - ) -> DispatchResultWithPostInfo { - ensure_none(origin)?; - - ensure!(!Included::::exists(), Error::::TooManyInclusionInherents); - Included::::set(Some(())); - - Self::enter_inner(data, FullCheck::Yes) - } - } -} - -impl Pallet { - pub(crate) fn enter_inner( - data: ParachainsInherentData, - full_check: FullCheck, - ) -> DispatchResultWithPostInfo { - let ParachainsInherentData { - bitfields: mut signed_bitfields, - mut backed_candidates, - parent_header, - mut disputes, - } = data; - - log::debug!( - target: LOG_TARGET, - "[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", - signed_bitfields.len(), - backed_candidates.len(), - disputes.len() - ); - - // Check that the submitted parent header indeed corresponds to the previous block hash. - let parent_hash = >::parent_hash(); - ensure!( - parent_header.hash().as_ref() == parent_hash.as_ref(), - Error::::InvalidParentHeader, - ); - - let now = >::block_number(); - - let mut candidate_weight = backed_candidates_weight::(&backed_candidates); - let mut bitfields_weight = signed_bitfields_weight::(signed_bitfields.len()); - let disputes_weight = dispute_statements_weight::(&disputes); - - let max_block_weight = ::BlockWeights::get().max_block; - - // Potentially trim inherent data to ensure processing will be within weight limits - let total_weight = { - if candidate_weight - .saturating_add(bitfields_weight) - .saturating_add(disputes_weight) > - max_block_weight - { - // if the total weight is over the max block weight, first try clearing backed - // candidates and bitfields. - backed_candidates.clear(); - candidate_weight = 0; - signed_bitfields.clear(); - bitfields_weight = 0; - } - - if disputes_weight > max_block_weight { - // if disputes are by themselves overweight already, trim the disputes. - debug_assert!(candidate_weight == 0 && bitfields_weight == 0); - - let entropy = compute_entropy::(parent_hash); - let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); - - let remaining_weight = - limit_disputes::(&mut disputes, max_block_weight, &mut rng); - max_block_weight.saturating_sub(remaining_weight) - } else { - candidate_weight - .saturating_add(bitfields_weight) - .saturating_add(disputes_weight) - } - }; - - let expected_bits = >::availability_cores().len(); - - // Handle disputes logic. - let current_session = >::session_index(); - let disputed_bitfield = { - let new_current_dispute_sets: Vec<_> = disputes - .iter() - .filter(|s| s.session == current_session) - .map(|s| (s.session, s.candidate_hash)) - .collect(); - - // Note that `provide_multi_dispute_data` will iterate, verify, and import each - // dispute; so the input here must be reasonably bounded. - let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?; - if T::DisputesHandler::is_frozen() { - // The relay chain we are currently on is invalid. Proceed no further on parachains. - return Ok(Some(dispute_statements_weight::(&disputes)).into()) - } - - let mut freed_disputed = if !new_current_dispute_sets.is_empty() { - let concluded_invalid_disputes = new_current_dispute_sets - .iter() - .filter(|(session, candidate)| { - T::DisputesHandler::concluded_invalid(*session, *candidate) - }) - .map(|(_, candidate)| *candidate) - .collect::>(); - - let freed_disputed = - >::collect_disputed(&concluded_invalid_disputes) - .into_iter() - .map(|core| (core, FreedReason::Concluded)) - .collect(); - freed_disputed - } else { - Vec::new() - }; - - // Create a bit index from the set of core indices where each index corresponds to - // a core index that was freed due to a dispute. - let disputed_bitfield = create_disputed_bitfield( - expected_bits, - freed_disputed.iter().map(|(core_index, _)| core_index), - ); - - if !freed_disputed.is_empty() { - // unstable sort is fine, because core indices are unique - // i.e. the same candidate can't occupy 2 cores at once. - freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index - >::free_cores(freed_disputed); - } - - disputed_bitfield - }; - - // Process new availability bitfields, yielding any availability cores whose - // work has now concluded. - let freed_concluded = >::process_bitfields( - expected_bits, - signed_bitfields, - disputed_bitfield, - >::core_para, - ); - - // Inform the disputes module of all included candidates. - for (_, candidate_hash) in &freed_concluded { - T::DisputesHandler::note_included(current_session, *candidate_hash, now); - } - - let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); - - >::clear(); - >::schedule(freed, now); - - let scheduled = >::scheduled(); - let backed_candidates = sanitize_backed_candidates::( - parent_hash, - backed_candidates, - move |_candidate_index: usize, backed_candidate: &BackedCandidate| -> bool { - ::DisputesHandler::concluded_invalid(current_session, backed_candidate.hash()) - // `fn process_candidates` does the verification checks - }, - &scheduled[..], - ); - - // Process backed candidates according to scheduled cores. - let parent_storage_root = parent_header.state_root().clone(); - let inclusion::ProcessedCandidates::<::Hash> { - core_indices: occupied, - candidate_receipt_with_backing_validator_indices, - } = >::process_candidates( - parent_storage_root, - backed_candidates, - scheduled, - >::group_validators, - full_check, - )?; - - // The number of disputes included in a block is - // limited by the weight as well as the number of candidate blocks. - OnChainVotes::::put(ScrapedOnChainVotes::<::Hash> { - session: current_session, - backing_validators_per_candidate: candidate_receipt_with_backing_validator_indices, - disputes, - }); - - // Note which of the scheduled cores were actually occupied by a backed candidate. - >::occupied(&occupied); - - // Give some time slice to dispatch pending upward messages. - // this is max config.ump_service_total_weight - let _ump_weight = >::process_pending_upward_messages(); - - Ok(Some(total_weight).into()) - } -} - -impl Pallet { - /// Create the `ParachainsInherentData` that gets passed to [`Self::enter`] in [`Self::create_inherent`]. - /// This code is pulled out of [`Self::create_inherent`] so it can be unit tested. - fn create_inherent_inner(data: &InherentData) -> Option> { - let ParachainsInherentData:: { - bitfields, - backed_candidates, - mut disputes, - parent_header, - } = match data.get_data(&Self::INHERENT_IDENTIFIER) { - Ok(Some(d)) => d, - Ok(None) => return None, - Err(_) => { - log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode"); - return None - }, - }; - - log::debug!( - target: LOG_TARGET, - "[create_inherent_inner] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", - bitfields.len(), - backed_candidates.len(), - disputes.len() - ); - - let parent_hash = >::parent_hash(); - - if parent_hash != parent_header.hash() { - log::warn!( - target: LOG_TARGET, - "ParachainsInherentData references a different parent header hash than frame" - ); - return None - } - - let current_session = >::session_index(); - let expected_bits = >::availability_cores().len(); - let validator_public = shared::Pallet::::active_validator_keys(); - - T::DisputesHandler::filter_multi_dispute_data(&mut disputes); - - let (mut backed_candidates, mut bitfields) = - frame_support::storage::with_transaction(|| { - // we don't care about fresh or not disputes - // this writes them to storage, so let's query it via those means - // if this fails for whatever reason, that's ok - let _ = - T::DisputesHandler::provide_multi_dispute_data(disputes.clone()).map_err(|e| { - log::warn!( - target: LOG_TARGET, - "MultiDisputesData failed to update: {:?}", - e - ); - e - }); - - // Contains the disputes that are concluded in the current session only, - // since these are the only ones that are relevant for the occupied cores - // and lightens the load on `collect_disputed` significantly. - // Cores can't be occupied with candidates of the previous sessions, and only - // things with new votes can have just concluded. We only need to collect - // cores with disputes that conclude just now, because disputes that - // concluded longer ago have already had any corresponding cores cleaned up. - let current_concluded_invalid_disputes = disputes - .iter() - .filter(|dss| dss.session == current_session) - .map(|dss| (dss.session, dss.candidate_hash)) - .filter(|(session, candidate)| { - ::DisputesHandler::concluded_invalid(*session, *candidate) - }) - .map(|(_session, candidate)| candidate) - .collect::>(); - - // All concluded invalid disputes, that are relevant for the set of candidates - // the inherent provided. - let concluded_invalid_disputes = backed_candidates - .iter() - .map(|backed_candidate| backed_candidate.hash()) - .filter(|candidate| { - ::DisputesHandler::concluded_invalid(current_session, *candidate) - }) - .collect::>(); - - let mut freed_disputed: Vec<_> = - >::collect_disputed(¤t_concluded_invalid_disputes) - .into_iter() - .map(|core| (core, FreedReason::Concluded)) - .collect(); - - let disputed_bitfield = - create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x, _)| x)); - - if !freed_disputed.is_empty() { - // unstable sort is fine, because core indices are unique - // i.e. the same candidate can't occupy 2 cores at once. - freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index - >::free_cores(freed_disputed.clone()); - } - - // The following 3 calls are equiv to a call to `process_bitfields` - // but we can retain access to `bitfields`. - let bitfields = sanitize_bitfields::( - bitfields, - disputed_bitfield, - expected_bits, - parent_hash, - current_session, - &validator_public[..], - FullCheck::Skip, - ); - - let freed_concluded = - >::update_pending_availability_and_get_freed_cores::< - _, - false, - >( - expected_bits, - &validator_public[..], - bitfields.clone(), - >::core_para, - ); - - let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); - - >::clear(); - let now = >::block_number(); - >::schedule(freed, now); - - let scheduled = >::scheduled(); - - let relay_parent_number = now - One::one(); - - let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); - let backed_candidates = sanitize_backed_candidates::( - parent_hash, - backed_candidates, - move |candidate_idx: usize, - backed_candidate: &BackedCandidate<::Hash>| - -> bool { - // never include a concluded-invalid candidate - concluded_invalid_disputes.contains(&backed_candidate.hash()) || - // Instead of checking the candidates with code upgrades twice - // move the checking up here and skip it in the training wheels fallback. - // That way we avoid possible duplicate checks while assuring all - // backed candidates fine to pass on. - check_ctx - .verify_backed_candidate(parent_hash, candidate_idx, backed_candidate) - .is_err() - }, - &scheduled[..], - ); - - frame_support::storage::TransactionOutcome::Rollback(( - // filtered backed candidates - backed_candidates, - // filtered bitfields - bitfields, - )) - }); - - let entropy = compute_entropy::(parent_hash); - let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); - - // Assure the maximum block weight is adhered. - let max_block_weight = ::BlockWeights::get().max_block; - let _consumed_weight = apply_weight_limit::( - &mut backed_candidates, - &mut bitfields, - &mut disputes, - max_block_weight, - &mut rng, - ); - - Some(ParachainsInherentData:: { - bitfields, - backed_candidates, - disputes, - parent_header, - }) - } -} - -/// Derive a bitfield from dispute -pub(super) fn create_disputed_bitfield<'a, I>( - expected_bits: usize, - freed_cores: I, -) -> DisputedBitfield -where - I: 'a + IntoIterator, -{ - let mut bitvec = BitVec::repeat(false, expected_bits); - for core_idx in freed_cores { - let core_idx = core_idx.0 as usize; - if core_idx < expected_bits { - bitvec.set(core_idx, true); - } - } - DisputedBitfield::from(bitvec) -} - -/// Select a random subset, with preference for certain indices. -/// -/// Adds random items to the set until all candidates -/// are tried or the remaining weight is depleted. -/// -/// Returns the weight of all selected items from `selectables` -/// as well as their indices in ascending order. -fn random_sel Weight>( - rng: &mut rand_chacha::ChaChaRng, - selectables: Vec, - mut preferred_indices: Vec, - weight_fn: F, - weight_limit: Weight, -) -> (Weight, Vec) { - if selectables.is_empty() { - return (0 as Weight, Vec::new()) - } - // all indices that are not part of the preferred set - let mut indices = (0..selectables.len()) - .into_iter() - .filter(|idx| !preferred_indices.contains(idx)) - .collect::>(); - let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1)); - - let mut weight_acc = 0 as Weight; - - preferred_indices.shuffle(rng); - for preferred_idx in preferred_indices { - // preferred indices originate from outside - if let Some(item) = selectables.get(preferred_idx) { - let updated = weight_acc.saturating_add(weight_fn(item)); - if updated > weight_limit { - continue - } - weight_acc = updated; - picked_indices.push(preferred_idx); - } - } - - indices.shuffle(rng); - for idx in indices { - let item = &selectables[idx]; - let updated = weight_acc.saturating_add(weight_fn(item)); - - if updated > weight_limit { - continue - } - weight_acc = updated; - - picked_indices.push(idx); - } - - // sorting indices, so the ordering is retained - // unstable sorting is fine, since there are no duplicates - picked_indices.sort_unstable(); - (weight_acc, picked_indices) -} - -/// Considers an upper threshold that the inherent data must not exceed. -/// -/// If there is sufficient space, all disputes, all bitfields and all candidates -/// will be included. -/// -/// Otherwise tries to include all disputes, and then tries to fill the remaining space with bitfields and then candidates. -/// -/// The selection process is random. For candidates, there is an exception for code upgrades as they are preferred. -/// And for disputes, local and older disputes are preferred (see `limit_disputes`). -/// for backed candidates, since with a increasing number of parachains their chances of -/// inclusion become slim. All backed candidates are checked beforehands in `fn create_inherent_inner` -/// which guarantees sanity. -fn apply_weight_limit( - candidates: &mut Vec::Hash>>, - bitfields: &mut UncheckedSignedAvailabilityBitfields, - disputes: &mut MultiDisputeStatementSet, - max_block_weight: Weight, - rng: &mut rand_chacha::ChaChaRng, -) -> Weight { - // include as many disputes as possible, always - let remaining_weight = limit_disputes::(disputes, max_block_weight, rng); - - let total_candidates_weight = backed_candidates_weight::(candidates.as_slice()); - - let total_bitfields_weight = signed_bitfields_weight::(bitfields.len()); - - let total = total_bitfields_weight.saturating_add(total_candidates_weight); - - // candidates + bitfields fit into the block - if remaining_weight >= total { - return total - } - - // Prefer code upgrades, they tend to be large and hence stand no chance to be picked - // late while maintaining the weight bounds - let preferred_indices = candidates - .iter() - .enumerate() - .filter_map(|(idx, candidate)| { - candidate.candidate.commitments.new_validation_code.as_ref().map(|_code| idx) - }) - .collect::>(); - - // There is weight remaining to be consumed by a subset of candidates - // which are going to be picked now. - if let Some(remaining_weight) = remaining_weight.checked_sub(total_bitfields_weight) { - let (acc_candidate_weight, indices) = - random_sel::::Hash>, _>( - rng, - candidates.clone(), - preferred_indices, - |c| backed_candidate_weight::(c), - remaining_weight, - ); - candidates.indexed_retain(|idx, _backed_candidate| indices.binary_search(&idx).is_ok()); - // pick all bitfields, and - // fill the remaining space with candidates - let total = acc_candidate_weight.saturating_add(total_bitfields_weight); - return total - } - - candidates.clear(); - - // insufficient space for even the bitfields alone, so only try to fit as many of those - // into the block and skip the candidates entirely - let (total, indices) = random_sel::( - rng, - bitfields.clone(), - vec![], - |_| <::WeightInfo as WeightInfo>::enter_bitfields(), - remaining_weight, - ); - - bitfields.indexed_retain(|idx, _bitfield| indices.binary_search(&idx).is_ok()); - - total -} - -/// Filter bitfields based on freed core indices, validity, and other sanity checks. -/// -/// Do sanity checks on the bitfields: -/// -/// 1. no more than one bitfield per validator -/// 2. bitfields are ascending by validator index. -/// 3. each bitfield has exactly `expected_bits` -/// 4. signature is valid -/// 5. remove any disputed core indices -/// -/// If any of those is not passed, the bitfield is dropped. -/// -/// While this function technically returns a set of unchecked bitfields, -/// they were actually checked and filtered to allow using it in both -/// cases, as `filtering` and `checking` stage. -/// -/// `full_check` determines if validator signatures are checked. If `::Yes`, -/// bitfields that have an invalid signature will be filtered out. -pub(crate) fn sanitize_bitfields( - unchecked_bitfields: UncheckedSignedAvailabilityBitfields, - disputed_bitfield: DisputedBitfield, - expected_bits: usize, - parent_hash: T::Hash, - session_index: SessionIndex, - validators: &[ValidatorId], - full_check: FullCheck, -) -> UncheckedSignedAvailabilityBitfields { - let mut bitfields = Vec::with_capacity(unchecked_bitfields.len()); - - let mut last_index: Option = None; - - if disputed_bitfield.0.len() != expected_bits { - // This is a system logic error that should never occur, but we want to handle it gracefully - // so we just drop all bitfields - log::error!(target: LOG_TARGET, "BUG: disputed_bitfield != expected_bits"); - return vec![] - } - - let all_zeros = BitVec::::repeat(false, expected_bits); - let signing_context = SigningContext { parent_hash, session_index }; - for unchecked_bitfield in unchecked_bitfields { - // Find and skip invalid bitfields. - if unchecked_bitfield.unchecked_payload().0.len() != expected_bits { - log::trace!( - target: LOG_TARGET, - "[{:?}] bad bitfield length: {} != {:?}", - full_check, - unchecked_bitfield.unchecked_payload().0.len(), - expected_bits, - ); - continue - } - - if unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != - all_zeros - { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield contains disputed cores: {:?}", - full_check, - unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() - ); - continue - } - - let validator_index = unchecked_bitfield.unchecked_validator_index(); - - if !last_index.map_or(true, |last_index: ValidatorIndex| last_index < validator_index) { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield validator index is not greater than last: !({:?} < {})", - full_check, - last_index.as_ref().map(|x| x.0), - validator_index.0 - ); - continue - } - - if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield validator index is out of bounds: {} >= {}", - full_check, - validator_index.0, - validators.len(), - ); - continue - } - - let validator_public = &validators[validator_index.0 as usize]; - - if let FullCheck::Yes = full_check { - if let Ok(signed_bitfield) = - unchecked_bitfield.try_into_checked(&signing_context, validator_public) - { - bitfields.push(signed_bitfield.into_unchecked()); - } else { - log::warn!(target: LOG_TARGET, "Invalid bitfield signature"); - }; - } else { - bitfields.push(unchecked_bitfield); - } - - last_index = Some(validator_index); - } - bitfields -} - -/// Filter out any candidates that have a concluded invalid dispute. -/// -/// `scheduled` follows the same naming scheme as provided in the -/// guide: Currently `free` but might become `occupied`. -/// For the filtering here the relevant part is only the current `free` -/// state. -/// -/// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate -/// is disputed, false otherwise -fn sanitize_backed_candidates< - T: crate::inclusion::Config, - F: FnMut(usize, &BackedCandidate) -> bool, ->( - relay_parent: T::Hash, - mut backed_candidates: Vec>, - mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, - scheduled: &[CoreAssignment], -) -> Vec> { - // Remove any candidates that were concluded invalid. - backed_candidates.indexed_retain(move |idx, backed_candidate| { - !candidate_has_concluded_invalid_dispute_or_is_invalid(idx, backed_candidate) - }); - - // Assure the backed candidate's `ParaId`'s core is free. - // This holds under the assumption that `Scheduler::schedule` is called _before_. - // Also checks the candidate references the correct relay parent. - let scheduled_paras_set = scheduled - .into_iter() - .map(|core_assignment| core_assignment.para_id) - .collect::>(); - backed_candidates.retain(|backed_candidate| { - let desc = backed_candidate.descriptor(); - desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id) - }); - - backed_candidates -} - -/// Derive entropy from babe provided per block randomness. -/// -/// In the odd case none is available, uses the `parent_hash` and -/// a const value, while emitting a warning. -fn compute_entropy(parent_hash: T::Hash) -> [u8; 32] { - const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject"; - let vrf_random = CurrentBlockRandomness::::random(&CANDIDATE_SEED_SUBJECT[..]).0; - let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone(); - if let Some(vrf_random) = vrf_random { - entropy.as_mut().copy_from_slice(vrf_random.as_ref()); - } else { - // in case there is no vrf randomness present, we utilize the relay parent - // as seed, it's better than a static value. - log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy"); - entropy.as_mut().copy_from_slice(parent_hash.as_ref()); - } - entropy -} - -/// Limit disputes in place. -/// -/// Returns the unused weight of `remaining_weight`. -fn limit_disputes( - disputes: &mut MultiDisputeStatementSet, - remaining_weight: Weight, - rng: &mut rand_chacha::ChaChaRng, -) -> Weight { - let mut remaining_weight = remaining_weight; - let disputes_weight = dispute_statements_weight::(&disputes); - if disputes_weight > remaining_weight { - // Sort the dispute statements according to the following prioritization: - // 1. Prioritize local disputes over remote disputes. - // 2. Prioritize older disputes over newer disputes. - disputes.sort_by(|a, b| { - let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash); - let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash); - match (a_local_block, b_local_block) { - // Prioritize local disputes over remote disputes. - (None, Some(_)) => Ordering::Greater, - (Some(_), None) => Ordering::Less, - // For local disputes, prioritize those that occur at an earlier height. - (Some(a_height), Some(b_height)) => a_height.cmp(&b_height), - // Prioritize earlier remote disputes using session as rough proxy. - (None, None) => a.session.cmp(&b.session), - } - }); - - // Since the disputes array is sorted, we may use binary search to find the beginning of - // remote disputes - let idx = disputes - .binary_search_by(|probe| { - if T::DisputesHandler::included_state(probe.session, probe.candidate_hash).is_some() - { - Ordering::Greater - } else { - Ordering::Less - } - }) - // The above predicate will never find an item and therefore we are guaranteed to obtain - // an error, which we can safely unwrap. QED. - .unwrap_err(); - - // Due to the binary search predicate above, the index computed will constitute the beginning - // of the remote disputes sub-array - let remote_disputes = disputes.split_off(idx); - - // Select disputes in-order until the remaining weight is attained - disputes.retain(|d| { - let dispute_weight = <::WeightInfo as WeightInfo>::enter_variable_disputes( - d.statements.len() as u32, - ); - if remaining_weight >= dispute_weight { - remaining_weight -= dispute_weight; - true - } else { - false - } - }); - - // Compute the statements length of all remote disputes - let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::>(); - - // Select remote disputes at random until the block is full - let (acc_remote_disputes_weight, indices) = random_sel::( - rng, - d, - vec![], - |v| <::WeightInfo as WeightInfo>::enter_variable_disputes(*v), - remaining_weight, - ); - - // Collect all remote disputes - let mut remote_disputes = - indices.into_iter().map(|idx| disputes[idx].clone()).collect::>(); - - // Construct the full list of selected disputes - disputes.append(&mut remote_disputes); - - // Update the remaining weight - remaining_weight = remaining_weight.saturating_sub(acc_remote_disputes_weight); - } - - remaining_weight -} - -#[cfg(test)] -mod tests { - use super::*; - - // In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl - // that uses 0 for all the weights. Because all the weights are 0, the tests that rely on - // weights for limiting data will fail, so we don't run them when using the benchmark feature. - #[cfg(not(feature = "runtime-benchmarks"))] - mod enter { - use super::*; - use crate::{ - builder::{Bench, BenchBuilder}, - mock::{new_test_ext, MockGenesisConfig, Test}, - }; - use frame_support::assert_ok; - use sp_std::collections::btree_map::BTreeMap; - - struct TestConfig { - dispute_statements: BTreeMap, - dispute_sessions: Vec, - backed_and_concluding: BTreeMap, - num_validators_per_core: u32, - code_upgrade: Option, - } - - fn make_inherent_data( - TestConfig { - dispute_statements, - dispute_sessions, - backed_and_concluding, - num_validators_per_core, - code_upgrade, - }: TestConfig, - ) -> Bench { - let builder = BenchBuilder::::new() - .set_max_validators( - (dispute_sessions.len() + backed_and_concluding.len()) as u32 * - num_validators_per_core, - ) - .set_max_validators_per_core(num_validators_per_core) - .set_dispute_statements(dispute_statements) - .set_backed_and_concluding_cores(backed_and_concluding) - .set_dispute_sessions(&dispute_sessions[..]); - - if let Some(code_size) = code_upgrade { - builder.set_code_upgrade(code_size).build() - } else { - builder.build() - } - } - - #[test] - // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via - // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and - // will not cause `enter` to early. - fn include_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - backed_and_concluding.insert(0, 1); - backed_and_concluding.insert(1, 1); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![], // No disputes - backed_and_concluding, - num_validators_per_core: 1, - code_upgrade: None, - }); - - // We expect the scenario to have cores 0 & 1 with pending availability. The backed - // candidates are also created for cores 0 & 1, so once the pending available - // become fully available those cores are marked as free and scheduled for the backed - // candidates. - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (2 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 2); - // * 1 backed candidate per core (2 cores) - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 0 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 0); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - assert_eq!( - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), - expected_para_inherent_data - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 backed candidates - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data - )); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 2 - ); - }); - } - - #[test] - // Ensure that disputes are filtered out if the session is in the future. - fn filter_multi_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![ - 1, 2, 3, /* Session 3 too new, will get filtered out */ - ], - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 15); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let multi_dispute_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Dispute for session that lies too far in the future should be filtered out - assert!(multi_dispute_inherent_data != expected_para_inherent_data); - - assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); - - // Assert that the first 2 disputes are included - assert_eq!( - &multi_dispute_inherent_data.disputes[..2], - &expected_para_inherent_data.disputes[..2], - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - multi_dispute_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know there - // where no backed candidates included - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - - #[test] - // Ensure that when dispute data establishes an over weight block that we adequately - // filter out disputes according to our prioritization rule - fn limit_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be filld with disputes. - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 6, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // Ensure that the included disputes are sorted by session - assert_eq!(limit_inherent_data.disputes.len(), 2); - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // Ensure that our inherent data did not included backed candidates as expected - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - - #[test] - // Ensure that when dispute data establishes an over weight block that we abort - // due to an over weight block - fn limit_dispute_data_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be filled with disputes. - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 6, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes, but there is still sufficient - // block weight to include a number of signed bitfields, the inherent data is filtered - // as expected - fn limit_dispute_data_ignore_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 4, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!( - limit_inherent_data.bitfields.len(), - expected_para_inherent_data.bitfields.len() - ); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that we abort if we encounter an over weight block for disputes + bitfields - fn limit_dispute_data_ignore_backed_candidates_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 4, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Ensure that calling enter with 3 disputes and 2 candidates is over weight - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are - // filtered to accommodate the block size and no backed candidates are included. - fn limit_bitfields() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Cap the number of statements per dispute to 20 in order to ensure we have enough - // space in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // Schedule 2 backed candidates - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates, - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!(limit_inherent_data.bitfields.len(), 20,); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_bitfields_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_1() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // * 1 bitfields - assert_eq!(limit_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(limit_inherent_data.backed_candidates.len(), 1); - // * 3 disputes. - assert_eq!(limit_inherent_data.disputes.len(), 2); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 1 - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_0() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - } - - fn default_header() -> primitives::v1::Header { - primitives::v1::Header { - parent_hash: Default::default(), - number: 0, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - - mod sanitizers { - use super::*; - - use crate::inclusion::tests::{ - back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, - }; - use bitvec::order::Lsb0; - use primitives::v1::{ - AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, - ValidatorIndex, - }; - - use crate::mock::Test; - use futures::executor::block_on; - use keyring::Sr25519Keyring; - use primitives::v0::PARACHAIN_KEY_TYPE_ID; - use sc_keystore::LocalKeystore; - use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - use std::sync::Arc; - - fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() - } - - #[test] - fn bitfields() { - let header = default_header(); - let parent_hash = header.hash(); - // 2 cores means two bits - let expected_bits = 2; - let session_index = SessionIndex::from(0_u32); - - let crypto_store = LocalKeystore::in_memory(); - let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*crypto_store, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - let unchecked_bitfields = [ - BitVec::::repeat(true, expected_bits), - BitVec::::repeat(true, expected_bits), - { - let mut bv = BitVec::::repeat(false, expected_bits); - bv.set(expected_bits - 1, true); - bv - }, - ] - .iter() - .enumerate() - .map(|(vi, ab)| { - let validator_index = ValidatorIndex::from(vi as u32); - block_on(SignedAvailabilityBitfield::sign( - &crypto_store, - AvailabilityBitfield::from(ab.clone()), - &signing_context, - validator_index, - &validator_public[vi], - )) - .unwrap() - .unwrap() - .into_unchecked() - }) - .collect::>(); - - let disputed_bitfield = DisputedBitfield::zeros(expected_bits); - - { - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip, - ), - unchecked_bitfields.clone() - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ), - unchecked_bitfields.clone() - ); - } - - // disputed bitfield is non-zero - { - let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); - // pretend the first core was freed by either a malicious validator - // or by resolved dispute - disputed_bitfield.0.set(0, true); - - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ) - .len(), - 1 - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - ) - .len(), - 1 - ); - } - - // bitfield size mismatch - { - assert!(sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits + 1, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ) - .is_empty()); - assert!(sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits + 1, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - ) - .is_empty()); - } - - // remove the last validator - { - let shortened = validator_public.len() - 2; - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Yes, - )[..], - &unchecked_bitfields[..shortened] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Skip, - )[..], - &unchecked_bitfields[..shortened] - ); - } - - // switch ordering of bitfields - { - let mut unchecked_bitfields = unchecked_bitfields.clone(); - let x = unchecked_bitfields.swap_remove(0); - unchecked_bitfields.push(x); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - } - - // check the validators signature - { - use primitives::v1::ValidatorSignature; - let mut unchecked_bitfields = unchecked_bitfields.clone(); - - // insert a bad signature for the last bitfield - let last_bit_idx = unchecked_bitfields.len() - 1; - unchecked_bitfields - .get_mut(last_bit_idx) - .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) - .expect("we are accessing a valid index"); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..last_bit_idx] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..] - ); - } - } - - #[test] - fn candidates() { - const RELAY_PARENT_NUM: u32 = 3; - - let header = default_header(); - let relay_parent = header.hash(); - let session_index = SessionIndex::from(0_u32); - - let keystore = LocalKeystore::in_memory(); - let keystore = Arc::new(keystore) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash: relay_parent, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - - let has_concluded_invalid = - |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; - - let scheduled = (0_usize..2) - .into_iter() - .map(|idx| { - let ca = CoreAssignment { - kind: scheduler::AssignmentKind::Parachain, - group_idx: GroupIndex::from(idx as u32), - para_id: ParaId::from(1_u32 + idx as u32), - core: CoreIndex::from(idx as u32), - }; - ca - }) - .collect::>(); - let scheduled = &scheduled[..]; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), - } - .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) - }; - - let backed_candidates = (0_usize..2) - .into_iter() - .map(|idx0| { - let idx1 = idx0 + 1; - let mut candidate = TestCandidateBuilder { - para_id: ParaId::from(idx1), - relay_parent, - pov_hash: Hash::repeat_byte(idx1 as u8), - persisted_validation_data_hash: [42u8; 32].into(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - backed - }) - .collect::>(); - - // happy path - assert_eq!( - sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ), - backed_candidates - ); - - // nothing is scheduled, so no paraids match, thus all backed candidates are skipped - { - let scheduled = &[][..]; - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .is_empty()); - } - - // relay parent mismatch - { - let relay_parent = Hash::repeat_byte(0xFA); - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .is_empty()); - } - - // candidates that have concluded as invalid are filtered out - { - // mark every second one as concluded invalid - let set = { - let mut set = std::collections::HashSet::new(); - for (idx, backed_candidate) in backed_candidates.iter().enumerate() { - if idx & 0x01 == 0 { - set.insert(backed_candidate.hash().clone()); - } - } - set - }; - let has_concluded_invalid = - |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); - assert_eq!( - sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .len(), - backed_candidates.len() / 2 - ); - } - } - } -} diff --git a/runtime/parachains/src/paras_inherent/misc.rs b/runtime/parachains/src/paras_inherent/misc.rs new file mode 100644 index 000000000000..51b1253e9483 --- /dev/null +++ b/runtime/parachains/src/paras_inherent/misc.rs @@ -0,0 +1,40 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use sp_std::vec::Vec; + +/// A helper trait to allow calling retain while getting access +/// to the index of the item in the `vec`. +pub trait IndexedRetain { + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` residing at + /// index `i` such that `f(i, &e)` returns `false`. This method + /// operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + fn indexed_retain(&mut self, f: impl FnMut(usize, &T) -> bool); +} + +impl IndexedRetain for Vec { + fn indexed_retain(&mut self, mut f: impl FnMut(usize, &T) -> bool) { + let mut idx = 0_usize; + self.retain(move |item| { + let ret = f(idx, item); + idx += 1_usize; + ret + }) + } +} diff --git a/runtime/parachains/src/paras_inherent/mod.rs b/runtime/parachains/src/paras_inherent/mod.rs new file mode 100644 index 000000000000..c0740826d720 --- /dev/null +++ b/runtime/parachains/src/paras_inherent/mod.rs @@ -0,0 +1,1022 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Provides glue code over the scheduler and inclusion modules, and accepting +//! one inherent per block that can include new para candidates and bitfields. +//! +//! Unlike other modules in this crate, it does not need to be initialized by the initializer, +//! as it has no initialization logic and its finalization logic depends only on the details of +//! this module. + +use crate::{ + disputes::DisputesHandler, + inclusion, + inclusion::{CandidateCheckContext, FullCheck}, + initializer, + scheduler::{self, CoreAssignment, FreedReason}, + shared, ump, +}; +use bitvec::prelude::BitVec; +use frame_support::{ + inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent}, + pallet_prelude::*, + traits::Randomness, +}; +use frame_system::pallet_prelude::*; +use pallet_babe::{self, CurrentBlockRandomness}; +use primitives::v1::{ + BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet, + InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes, + SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield, + UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, + PARACHAINS_INHERENT_IDENTIFIER, +}; +use rand::{seq::SliceRandom, SeedableRng}; + +use scale_info::TypeInfo; +use sp_runtime::traits::{Header as HeaderT, One}; +use sp_std::{ + cmp::Ordering, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + prelude::*, + vec::Vec, +}; + +mod misc; +mod weights; + +pub use self::{ + misc::IndexedRetain, + weights::{ + backed_candidate_weight, backed_candidates_weight, dispute_statements_weight, + paras_inherent_total_weight, signed_bitfields_weight, TestWeightInfo, WeightInfo, + }, +}; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + +#[cfg(test)] +mod tests; + +const LOG_TARGET: &str = "runtime::inclusion-inherent"; + +/// A bitfield concerning concluded disputes for candidates +/// associated to the core index equivalent to the bit position. +#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +pub(crate) struct DisputedBitfield(pub(crate) BitVec); + +impl From> for DisputedBitfield { + fn from(inner: BitVec) -> Self { + Self(inner) + } +} + +#[cfg(test)] +impl DisputedBitfield { + /// Create a new bitfield, where each bit is set to `false`. + pub fn zeros(n: usize) -> Self { + Self::from(BitVec::::repeat(false, n)) + } +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + #[pallet::disable_frame_system_supertrait_check] + pub trait Config: + inclusion::Config + scheduler::Config + initializer::Config + pallet_babe::Config + { + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::error] + pub enum Error { + /// Inclusion inherent called more than once per block. + TooManyInclusionInherents, + /// The hash of the submitted parent header doesn't correspond to the saved block hash of + /// the parent. + InvalidParentHeader, + /// Disputed candidate that was concluded invalid. + CandidateConcludedInvalid, + /// The data given to the inherent will result in an overweight block. + InherentOverweight, + } + + /// Whether the paras inherent was included within this block. + /// + /// The `Option<()>` is effectively a `bool`, but it never hits storage in the `None` variant + /// due to the guarantees of FRAME's storage APIs. + /// + /// If this is `None` at the end of the block, we panic and render the block invalid. + #[pallet::storage] + pub(crate) type Included = StorageValue<_, ()>; + + /// Scraped on chain data for extracting resolved disputes as well as backing votes. + #[pallet::storage] + #[pallet::getter(fn on_chain_votes)] + pub(crate) type OnChainVotes = StorageValue<_, ScrapedOnChainVotes>; + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_: T::BlockNumber) -> Weight { + T::DbWeight::get().reads_writes(1, 1) // in on_finalize. + } + + fn on_finalize(_: T::BlockNumber) { + if Included::::take().is_none() { + panic!("Bitfields and heads must be included every block"); + } + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + type Error = MakeFatalError<()>; + const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER; + + fn create_inherent(data: &InherentData) -> Option { + let inherent_data = Self::create_inherent_inner(data)?; + // Sanity check: session changes can invalidate an inherent, + // and we _really_ don't want that to happen. + // See + + // Calling `Self::enter` here is a safe-guard, to avoid any discrepancy between on-chain checks + // (`enter`) and the off-chain checks by the block author (this function). Once we are confident + // in all the logic in this module this check should be removed to optimize performance. + + let inherent_data = match Self::enter_inner(inherent_data.clone(), FullCheck::Skip) { + Ok(_) => inherent_data, + Err(err) => { + log::error!( + target: LOG_TARGET, + "dropping paras inherent data because they produced \ + an invalid paras inherent: {:?}", + err.error, + ); + + ParachainsInherentData { + bitfields: Vec::new(), + backed_candidates: Vec::new(), + disputes: Vec::new(), + parent_header: inherent_data.parent_header, + } + }, + }; + + Some(Call::enter { data: inherent_data }) + } + + fn is_inherent(call: &Self::Call) -> bool { + matches!(call, Call::enter { .. }) + } + } + + /// Collect all freed cores based on storage data. (i.e. append cores freed from timeouts to + /// the given `freed_concluded`). + /// + /// The parameter `freed_concluded` contains all core indicies that became + /// free due to candidate that became available. + pub(crate) fn collect_all_freed_cores( + freed_concluded: I, + ) -> BTreeMap + where + I: core::iter::IntoIterator, + T: Config, + { + // Handle timeouts for any availability core work. + let availability_pred = >::availability_timeout_predicate(); + let freed_timeout = if let Some(pred) = availability_pred { + >::collect_pending(pred) + } else { + Vec::new() + }; + + // Schedule paras again, given freed cores, and reasons for freeing. + let freed = freed_concluded + .into_iter() + .map(|(c, _hash)| (c, FreedReason::Concluded)) + .chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut))) + .collect::>(); + freed + } + + #[pallet::call] + impl Pallet { + /// Enter the paras inherent. This will process bitfields and backed candidates. + #[pallet::weight(( + paras_inherent_total_weight::( + data.backed_candidates.as_slice(), + data.bitfields.as_slice(), + data.disputes.as_slice(), + ), + DispatchClass::Mandatory, + ))] + pub fn enter( + origin: OriginFor, + data: ParachainsInherentData, + ) -> DispatchResultWithPostInfo { + ensure_none(origin)?; + + ensure!(!Included::::exists(), Error::::TooManyInclusionInherents); + Included::::set(Some(())); + + Self::enter_inner(data, FullCheck::Yes) + } + } +} + +impl Pallet { + pub(crate) fn enter_inner( + data: ParachainsInherentData, + full_check: FullCheck, + ) -> DispatchResultWithPostInfo { + let ParachainsInherentData { + bitfields: mut signed_bitfields, + mut backed_candidates, + parent_header, + mut disputes, + } = data; + + log::debug!( + target: LOG_TARGET, + "[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", + signed_bitfields.len(), + backed_candidates.len(), + disputes.len() + ); + + // Check that the submitted parent header indeed corresponds to the previous block hash. + let parent_hash = >::parent_hash(); + ensure!( + parent_header.hash().as_ref() == parent_hash.as_ref(), + Error::::InvalidParentHeader, + ); + + let now = >::block_number(); + + let mut candidate_weight = backed_candidates_weight::(&backed_candidates); + let mut bitfields_weight = signed_bitfields_weight::(signed_bitfields.len()); + let disputes_weight = dispute_statements_weight::(&disputes); + + let max_block_weight = ::BlockWeights::get().max_block; + + // Potentially trim inherent data to ensure processing will be within weight limits + let total_weight = { + if candidate_weight + .saturating_add(bitfields_weight) + .saturating_add(disputes_weight) > + max_block_weight + { + // if the total weight is over the max block weight, first try clearing backed + // candidates and bitfields. + backed_candidates.clear(); + candidate_weight = 0; + signed_bitfields.clear(); + bitfields_weight = 0; + } + + if disputes_weight > max_block_weight { + // if disputes are by themselves overweight already, trim the disputes. + debug_assert!(candidate_weight == 0 && bitfields_weight == 0); + + let entropy = compute_entropy::(parent_hash); + let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); + + let remaining_weight = + limit_disputes::(&mut disputes, max_block_weight, &mut rng); + max_block_weight.saturating_sub(remaining_weight) + } else { + candidate_weight + .saturating_add(bitfields_weight) + .saturating_add(disputes_weight) + } + }; + + let expected_bits = >::availability_cores().len(); + + // Handle disputes logic. + let current_session = >::session_index(); + let disputed_bitfield = { + let new_current_dispute_sets: Vec<_> = disputes + .iter() + .filter(|s| s.session == current_session) + .map(|s| (s.session, s.candidate_hash)) + .collect(); + + // Note that `provide_multi_dispute_data` will iterate, verify, and import each + // dispute; so the input here must be reasonably bounded. + let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?; + if T::DisputesHandler::is_frozen() { + // The relay chain we are currently on is invalid. Proceed no further on parachains. + return Ok(Some(dispute_statements_weight::(&disputes)).into()) + } + + let mut freed_disputed = if !new_current_dispute_sets.is_empty() { + let concluded_invalid_disputes = new_current_dispute_sets + .iter() + .filter(|(session, candidate)| { + T::DisputesHandler::concluded_invalid(*session, *candidate) + }) + .map(|(_, candidate)| *candidate) + .collect::>(); + + let freed_disputed = + >::collect_disputed(&concluded_invalid_disputes) + .into_iter() + .map(|core| (core, FreedReason::Concluded)) + .collect(); + freed_disputed + } else { + Vec::new() + }; + + // Create a bit index from the set of core indices where each index corresponds to + // a core index that was freed due to a dispute. + let disputed_bitfield = create_disputed_bitfield( + expected_bits, + freed_disputed.iter().map(|(core_index, _)| core_index), + ); + + if !freed_disputed.is_empty() { + // unstable sort is fine, because core indices are unique + // i.e. the same candidate can't occupy 2 cores at once. + freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index + >::free_cores(freed_disputed); + } + + disputed_bitfield + }; + + // Process new availability bitfields, yielding any availability cores whose + // work has now concluded. + let freed_concluded = >::process_bitfields( + expected_bits, + signed_bitfields, + disputed_bitfield, + >::core_para, + ); + + // Inform the disputes module of all included candidates. + for (_, candidate_hash) in &freed_concluded { + T::DisputesHandler::note_included(current_session, *candidate_hash, now); + } + + let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); + + >::clear(); + >::schedule(freed, now); + + let scheduled = >::scheduled(); + let backed_candidates = sanitize_backed_candidates::( + parent_hash, + backed_candidates, + move |_candidate_index: usize, backed_candidate: &BackedCandidate| -> bool { + ::DisputesHandler::concluded_invalid(current_session, backed_candidate.hash()) + // `fn process_candidates` does the verification checks + }, + &scheduled[..], + ); + + // Process backed candidates according to scheduled cores. + let parent_storage_root = parent_header.state_root().clone(); + let inclusion::ProcessedCandidates::<::Hash> { + core_indices: occupied, + candidate_receipt_with_backing_validator_indices, + } = >::process_candidates( + parent_storage_root, + backed_candidates, + scheduled, + >::group_validators, + full_check, + )?; + + // The number of disputes included in a block is + // limited by the weight as well as the number of candidate blocks. + OnChainVotes::::put(ScrapedOnChainVotes::<::Hash> { + session: current_session, + backing_validators_per_candidate: candidate_receipt_with_backing_validator_indices, + disputes, + }); + + // Note which of the scheduled cores were actually occupied by a backed candidate. + >::occupied(&occupied); + + // Give some time slice to dispatch pending upward messages. + // this is max config.ump_service_total_weight + let _ump_weight = >::process_pending_upward_messages(); + + Ok(Some(total_weight).into()) + } +} + +impl Pallet { + /// Create the `ParachainsInherentData` that gets passed to [`Self::enter`] in [`Self::create_inherent`]. + /// This code is pulled out of [`Self::create_inherent`] so it can be unit tested. + fn create_inherent_inner(data: &InherentData) -> Option> { + let ParachainsInherentData:: { + bitfields, + backed_candidates, + mut disputes, + parent_header, + } = match data.get_data(&Self::INHERENT_IDENTIFIER) { + Ok(Some(d)) => d, + Ok(None) => return None, + Err(_) => { + log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode"); + return None + }, + }; + + log::debug!( + target: LOG_TARGET, + "[create_inherent_inner] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", + bitfields.len(), + backed_candidates.len(), + disputes.len() + ); + + let parent_hash = >::parent_hash(); + + if parent_hash != parent_header.hash() { + log::warn!( + target: LOG_TARGET, + "ParachainsInherentData references a different parent header hash than frame" + ); + return None + } + + let current_session = >::session_index(); + let expected_bits = >::availability_cores().len(); + let validator_public = shared::Pallet::::active_validator_keys(); + + T::DisputesHandler::filter_multi_dispute_data(&mut disputes); + + let (mut backed_candidates, mut bitfields) = + frame_support::storage::with_transaction(|| { + // we don't care about fresh or not disputes + // this writes them to storage, so let's query it via those means + // if this fails for whatever reason, that's ok + let _ = + T::DisputesHandler::provide_multi_dispute_data(disputes.clone()).map_err(|e| { + log::warn!( + target: LOG_TARGET, + "MultiDisputesData failed to update: {:?}", + e + ); + e + }); + + // Contains the disputes that are concluded in the current session only, + // since these are the only ones that are relevant for the occupied cores + // and lightens the load on `collect_disputed` significantly. + // Cores can't be occupied with candidates of the previous sessions, and only + // things with new votes can have just concluded. We only need to collect + // cores with disputes that conclude just now, because disputes that + // concluded longer ago have already had any corresponding cores cleaned up. + let current_concluded_invalid_disputes = disputes + .iter() + .filter(|dss| dss.session == current_session) + .map(|dss| (dss.session, dss.candidate_hash)) + .filter(|(session, candidate)| { + ::DisputesHandler::concluded_invalid(*session, *candidate) + }) + .map(|(_session, candidate)| candidate) + .collect::>(); + + // All concluded invalid disputes, that are relevant for the set of candidates + // the inherent provided. + let concluded_invalid_disputes = backed_candidates + .iter() + .map(|backed_candidate| backed_candidate.hash()) + .filter(|candidate| { + ::DisputesHandler::concluded_invalid(current_session, *candidate) + }) + .collect::>(); + + let mut freed_disputed: Vec<_> = + >::collect_disputed(¤t_concluded_invalid_disputes) + .into_iter() + .map(|core| (core, FreedReason::Concluded)) + .collect(); + + let disputed_bitfield = + create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x, _)| x)); + + if !freed_disputed.is_empty() { + // unstable sort is fine, because core indices are unique + // i.e. the same candidate can't occupy 2 cores at once. + freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index + >::free_cores(freed_disputed.clone()); + } + + // The following 3 calls are equiv to a call to `process_bitfields` + // but we can retain access to `bitfields`. + let bitfields = sanitize_bitfields::( + bitfields, + disputed_bitfield, + expected_bits, + parent_hash, + current_session, + &validator_public[..], + FullCheck::Skip, + ); + + let freed_concluded = + >::update_pending_availability_and_get_freed_cores::< + _, + false, + >( + expected_bits, + &validator_public[..], + bitfields.clone(), + >::core_para, + ); + + let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); + + >::clear(); + let now = >::block_number(); + >::schedule(freed, now); + + let scheduled = >::scheduled(); + + let relay_parent_number = now - One::one(); + + let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); + let backed_candidates = sanitize_backed_candidates::( + parent_hash, + backed_candidates, + move |candidate_idx: usize, + backed_candidate: &BackedCandidate<::Hash>| + -> bool { + // never include a concluded-invalid candidate + concluded_invalid_disputes.contains(&backed_candidate.hash()) || + // Instead of checking the candidates with code upgrades twice + // move the checking up here and skip it in the training wheels fallback. + // That way we avoid possible duplicate checks while assuring all + // backed candidates fine to pass on. + check_ctx + .verify_backed_candidate(parent_hash, candidate_idx, backed_candidate) + .is_err() + }, + &scheduled[..], + ); + + frame_support::storage::TransactionOutcome::Rollback(( + // filtered backed candidates + backed_candidates, + // filtered bitfields + bitfields, + )) + }); + + let entropy = compute_entropy::(parent_hash); + let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); + + // Assure the maximum block weight is adhered. + let max_block_weight = ::BlockWeights::get().max_block; + let _consumed_weight = apply_weight_limit::( + &mut backed_candidates, + &mut bitfields, + &mut disputes, + max_block_weight, + &mut rng, + ); + + Some(ParachainsInherentData:: { + bitfields, + backed_candidates, + disputes, + parent_header, + }) + } +} + +/// Derive a bitfield from dispute +pub(super) fn create_disputed_bitfield<'a, I>( + expected_bits: usize, + freed_cores: I, +) -> DisputedBitfield +where + I: 'a + IntoIterator, +{ + let mut bitvec = BitVec::repeat(false, expected_bits); + for core_idx in freed_cores { + let core_idx = core_idx.0 as usize; + if core_idx < expected_bits { + bitvec.set(core_idx, true); + } + } + DisputedBitfield::from(bitvec) +} + +/// Select a random subset, with preference for certain indices. +/// +/// Adds random items to the set until all candidates +/// are tried or the remaining weight is depleted. +/// +/// Returns the weight of all selected items from `selectables` +/// as well as their indices in ascending order. +fn random_sel Weight>( + rng: &mut rand_chacha::ChaChaRng, + selectables: Vec, + mut preferred_indices: Vec, + weight_fn: F, + weight_limit: Weight, +) -> (Weight, Vec) { + if selectables.is_empty() { + return (0 as Weight, Vec::new()) + } + // all indices that are not part of the preferred set + let mut indices = (0..selectables.len()) + .into_iter() + .filter(|idx| !preferred_indices.contains(idx)) + .collect::>(); + let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1)); + + let mut weight_acc = 0 as Weight; + + preferred_indices.shuffle(rng); + for preferred_idx in preferred_indices { + // preferred indices originate from outside + if let Some(item) = selectables.get(preferred_idx) { + let updated = weight_acc.saturating_add(weight_fn(item)); + if updated > weight_limit { + continue + } + weight_acc = updated; + picked_indices.push(preferred_idx); + } + } + + indices.shuffle(rng); + for idx in indices { + let item = &selectables[idx]; + let updated = weight_acc.saturating_add(weight_fn(item)); + + if updated > weight_limit { + continue + } + weight_acc = updated; + + picked_indices.push(idx); + } + + // sorting indices, so the ordering is retained + // unstable sorting is fine, since there are no duplicates + picked_indices.sort_unstable(); + (weight_acc, picked_indices) +} + +/// Considers an upper threshold that the inherent data must not exceed. +/// +/// If there is sufficient space, all disputes, all bitfields and all candidates +/// will be included. +/// +/// Otherwise tries to include all disputes, and then tries to fill the remaining space with bitfields and then candidates. +/// +/// The selection process is random. For candidates, there is an exception for code upgrades as they are preferred. +/// And for disputes, local and older disputes are preferred (see `limit_disputes`). +/// for backed candidates, since with a increasing number of parachains their chances of +/// inclusion become slim. All backed candidates are checked beforehands in `fn create_inherent_inner` +/// which guarantees sanity. +fn apply_weight_limit( + candidates: &mut Vec::Hash>>, + bitfields: &mut UncheckedSignedAvailabilityBitfields, + disputes: &mut MultiDisputeStatementSet, + max_block_weight: Weight, + rng: &mut rand_chacha::ChaChaRng, +) -> Weight { + // include as many disputes as possible, always + let remaining_weight = limit_disputes::(disputes, max_block_weight, rng); + + let total_candidates_weight = backed_candidates_weight::(candidates.as_slice()); + + let total_bitfields_weight = signed_bitfields_weight::(bitfields.len()); + + let total = total_bitfields_weight.saturating_add(total_candidates_weight); + + // candidates + bitfields fit into the block + if remaining_weight >= total { + return total + } + + // Prefer code upgrades, they tend to be large and hence stand no chance to be picked + // late while maintaining the weight bounds + let preferred_indices = candidates + .iter() + .enumerate() + .filter_map(|(idx, candidate)| { + candidate.candidate.commitments.new_validation_code.as_ref().map(|_code| idx) + }) + .collect::>(); + + // There is weight remaining to be consumed by a subset of candidates + // which are going to be picked now. + if let Some(remaining_weight) = remaining_weight.checked_sub(total_bitfields_weight) { + let (acc_candidate_weight, indices) = + random_sel::::Hash>, _>( + rng, + candidates.clone(), + preferred_indices, + |c| backed_candidate_weight::(c), + remaining_weight, + ); + candidates.indexed_retain(|idx, _backed_candidate| indices.binary_search(&idx).is_ok()); + // pick all bitfields, and + // fill the remaining space with candidates + let total = acc_candidate_weight.saturating_add(total_bitfields_weight); + return total + } + + candidates.clear(); + + // insufficient space for even the bitfields alone, so only try to fit as many of those + // into the block and skip the candidates entirely + let (total, indices) = random_sel::( + rng, + bitfields.clone(), + vec![], + |_| <::WeightInfo as WeightInfo>::enter_bitfields(), + remaining_weight, + ); + + bitfields.indexed_retain(|idx, _bitfield| indices.binary_search(&idx).is_ok()); + + total +} + +/// Filter bitfields based on freed core indices, validity, and other sanity checks. +/// +/// Do sanity checks on the bitfields: +/// +/// 1. no more than one bitfield per validator +/// 2. bitfields are ascending by validator index. +/// 3. each bitfield has exactly `expected_bits` +/// 4. signature is valid +/// 5. remove any disputed core indices +/// +/// If any of those is not passed, the bitfield is dropped. +/// +/// While this function technically returns a set of unchecked bitfields, +/// they were actually checked and filtered to allow using it in both +/// cases, as `filtering` and `checking` stage. +/// +/// `full_check` determines if validator signatures are checked. If `::Yes`, +/// bitfields that have an invalid signature will be filtered out. +pub(crate) fn sanitize_bitfields( + unchecked_bitfields: UncheckedSignedAvailabilityBitfields, + disputed_bitfield: DisputedBitfield, + expected_bits: usize, + parent_hash: T::Hash, + session_index: SessionIndex, + validators: &[ValidatorId], + full_check: FullCheck, +) -> UncheckedSignedAvailabilityBitfields { + let mut bitfields = Vec::with_capacity(unchecked_bitfields.len()); + + let mut last_index: Option = None; + + if disputed_bitfield.0.len() != expected_bits { + // This is a system logic error that should never occur, but we want to handle it gracefully + // so we just drop all bitfields + log::error!(target: LOG_TARGET, "BUG: disputed_bitfield != expected_bits"); + return vec![] + } + + let all_zeros = BitVec::::repeat(false, expected_bits); + let signing_context = SigningContext { parent_hash, session_index }; + for unchecked_bitfield in unchecked_bitfields { + // Find and skip invalid bitfields. + if unchecked_bitfield.unchecked_payload().0.len() != expected_bits { + log::trace!( + target: LOG_TARGET, + "[{:?}] bad bitfield length: {} != {:?}", + full_check, + unchecked_bitfield.unchecked_payload().0.len(), + expected_bits, + ); + continue + } + + if unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != + all_zeros + { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield contains disputed cores: {:?}", + full_check, + unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() + ); + continue + } + + let validator_index = unchecked_bitfield.unchecked_validator_index(); + + if !last_index.map_or(true, |last_index: ValidatorIndex| last_index < validator_index) { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield validator index is not greater than last: !({:?} < {})", + full_check, + last_index.as_ref().map(|x| x.0), + validator_index.0 + ); + continue + } + + if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield validator index is out of bounds: {} >= {}", + full_check, + validator_index.0, + validators.len(), + ); + continue + } + + let validator_public = &validators[validator_index.0 as usize]; + + if let FullCheck::Yes = full_check { + if let Ok(signed_bitfield) = + unchecked_bitfield.try_into_checked(&signing_context, validator_public) + { + bitfields.push(signed_bitfield.into_unchecked()); + } else { + log::warn!(target: LOG_TARGET, "Invalid bitfield signature"); + }; + } else { + bitfields.push(unchecked_bitfield); + } + + last_index = Some(validator_index); + } + bitfields +} + +/// Filter out any candidates that have a concluded invalid dispute. +/// +/// `scheduled` follows the same naming scheme as provided in the +/// guide: Currently `free` but might become `occupied`. +/// For the filtering here the relevant part is only the current `free` +/// state. +/// +/// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate +/// is disputed, false otherwise +fn sanitize_backed_candidates< + T: crate::inclusion::Config, + F: FnMut(usize, &BackedCandidate) -> bool, +>( + relay_parent: T::Hash, + mut backed_candidates: Vec>, + mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, + scheduled: &[CoreAssignment], +) -> Vec> { + // Remove any candidates that were concluded invalid. + backed_candidates.indexed_retain(move |idx, backed_candidate| { + !candidate_has_concluded_invalid_dispute_or_is_invalid(idx, backed_candidate) + }); + + // Assure the backed candidate's `ParaId`'s core is free. + // This holds under the assumption that `Scheduler::schedule` is called _before_. + // Also checks the candidate references the correct relay parent. + let scheduled_paras_set = scheduled + .into_iter() + .map(|core_assignment| core_assignment.para_id) + .collect::>(); + backed_candidates.retain(|backed_candidate| { + let desc = backed_candidate.descriptor(); + desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id) + }); + + backed_candidates +} + +/// Derive entropy from babe provided per block randomness. +/// +/// In the odd case none is available, uses the `parent_hash` and +/// a const value, while emitting a warning. +fn compute_entropy(parent_hash: T::Hash) -> [u8; 32] { + const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject"; + let vrf_random = CurrentBlockRandomness::::random(&CANDIDATE_SEED_SUBJECT[..]).0; + let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone(); + if let Some(vrf_random) = vrf_random { + entropy.as_mut().copy_from_slice(vrf_random.as_ref()); + } else { + // in case there is no vrf randomness present, we utilize the relay parent + // as seed, it's better than a static value. + log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy"); + entropy.as_mut().copy_from_slice(parent_hash.as_ref()); + } + entropy +} + +/// Limit disputes in place. +/// +/// Returns the unused weight of `remaining_weight`. +fn limit_disputes( + disputes: &mut MultiDisputeStatementSet, + remaining_weight: Weight, + rng: &mut rand_chacha::ChaChaRng, +) -> Weight { + let mut remaining_weight = remaining_weight; + let disputes_weight = dispute_statements_weight::(&disputes); + if disputes_weight > remaining_weight { + // Sort the dispute statements according to the following prioritization: + // 1. Prioritize local disputes over remote disputes. + // 2. Prioritize older disputes over newer disputes. + disputes.sort_by(|a, b| { + let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash); + let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash); + match (a_local_block, b_local_block) { + // Prioritize local disputes over remote disputes. + (None, Some(_)) => Ordering::Greater, + (Some(_), None) => Ordering::Less, + // For local disputes, prioritize those that occur at an earlier height. + (Some(a_height), Some(b_height)) => a_height.cmp(&b_height), + // Prioritize earlier remote disputes using session as rough proxy. + (None, None) => a.session.cmp(&b.session), + } + }); + + // Since the disputes array is sorted, we may use binary search to find the beginning of + // remote disputes + let idx = disputes + .binary_search_by(|probe| { + if T::DisputesHandler::included_state(probe.session, probe.candidate_hash).is_some() + { + Ordering::Greater + } else { + Ordering::Less + } + }) + // The above predicate will never find an item and therefore we are guaranteed to obtain + // an error, which we can safely unwrap. QED. + .unwrap_err(); + + // Due to the binary search predicate above, the index computed will constitute the beginning + // of the remote disputes sub-array + let remote_disputes = disputes.split_off(idx); + + // Select disputes in-order until the remaining weight is attained + disputes.retain(|d| { + let dispute_weight = <::WeightInfo as WeightInfo>::enter_variable_disputes( + d.statements.len() as u32, + ); + if remaining_weight >= dispute_weight { + remaining_weight -= dispute_weight; + true + } else { + false + } + }); + + // Compute the statements length of all remote disputes + let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::>(); + + // Select remote disputes at random until the block is full + let (acc_remote_disputes_weight, indices) = random_sel::( + rng, + d, + vec![], + |v| <::WeightInfo as WeightInfo>::enter_variable_disputes(*v), + remaining_weight, + ); + + // Collect all remote disputes + let mut remote_disputes = + indices.into_iter().map(|idx| disputes[idx].clone()).collect::>(); + + // Construct the full list of selected disputes + disputes.append(&mut remote_disputes); + + // Update the remaining weight + remaining_weight = remaining_weight.saturating_sub(acc_remote_disputes_weight); + } + + remaining_weight +} diff --git a/runtime/parachains/src/paras_inherent/tests.rs b/runtime/parachains/src/paras_inherent/tests.rs new file mode 100644 index 000000000000..c57de4af4a4f --- /dev/null +++ b/runtime/parachains/src/paras_inherent/tests.rs @@ -0,0 +1,1115 @@ +use super::*; + + // In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl + // that uses 0 for all the weights. Because all the weights are 0, the tests that rely on + // weights for limiting data will fail, so we don't run them when using the benchmark feature. + #[cfg(not(feature = "runtime-benchmarks"))] + mod enter { + use super::*; + use crate::{ + builder::{Bench, BenchBuilder}, + mock::{new_test_ext, MockGenesisConfig, Test}, + }; + use frame_support::assert_ok; + use sp_std::collections::btree_map::BTreeMap; + + struct TestConfig { + dispute_statements: BTreeMap, + dispute_sessions: Vec, + backed_and_concluding: BTreeMap, + num_validators_per_core: u32, + code_upgrade: Option, + } + + fn make_inherent_data( + TestConfig { + dispute_statements, + dispute_sessions, + backed_and_concluding, + num_validators_per_core, + code_upgrade, + }: TestConfig, + ) -> Bench { + let builder = BenchBuilder::::new() + .set_max_validators( + (dispute_sessions.len() + backed_and_concluding.len()) as u32 * + num_validators_per_core, + ) + .set_max_validators_per_core(num_validators_per_core) + .set_dispute_statements(dispute_statements) + .set_backed_and_concluding_cores(backed_and_concluding) + .set_dispute_sessions(&dispute_sessions[..]); + + if let Some(code_size) = code_upgrade { + builder.set_code_upgrade(code_size).build() + } else { + builder.build() + } + } + + #[test] + // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via + // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and + // will not cause `enter` to early. + fn include_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![], // No disputes + backed_and_concluding, + num_validators_per_core: 1, + code_upgrade: None, + }); + + // We expect the scenario to have cores 0 & 1 with pending availability. The backed + // candidates are also created for cores 0 & 1, so once the pending available + // become fully available those cores are marked as free and scheduled for the backed + // candidates. + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 2); + // * 1 backed candidate per core (2 cores) + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 0 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 0); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + assert_eq!( + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), + expected_para_inherent_data + ); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 backed candidates + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data + )); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 2 + ); + }); + } + + #[test] + // Ensure that disputes are filtered out if the session is in the future. + fn filter_multi_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![ + 1, 2, 3, /* Session 3 too new, will get filtered out */ + ], + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 15); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let multi_dispute_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Dispute for session that lies too far in the future should be filtered out + assert!(multi_dispute_inherent_data != expected_para_inherent_data); + + assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); + + // Assert that the first 2 disputes are included + assert_eq!( + &multi_dispute_inherent_data.disputes[..2], + &expected_para_inherent_data.disputes[..2], + ); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + multi_dispute_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know there + // where no backed candidates included + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we adequately + // filter out disputes according to our prioritization rule + fn limit_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be filld with disputes. + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 6, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // Ensure that the included disputes are sorted by session + assert_eq!(limit_inherent_data.disputes.len(), 2); + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // Ensure that our inherent data did not included backed candidates as expected + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we abort + // due to an over weight block + fn limit_dispute_data_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be filled with disputes. + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 6, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes, but there is still sufficient + // block weight to include a number of signed bitfields, the inherent data is filtered + // as expected + fn limit_dispute_data_ignore_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 4, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!( + limit_inherent_data.bitfields.len(), + expected_para_inherent_data.bitfields.len() + ); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0, + ); + }); + } + + #[test] + // Ensure that we abort if we encounter an over weight block for disputes + bitfields + fn limit_dispute_data_ignore_backed_candidates_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 4, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Ensure that calling enter with 3 disputes and 2 candidates is over weight + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are + // filtered to accommodate the block size and no backed candidates are included. + fn limit_bitfields() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Cap the number of statements per dispute to 20 in order to ensure we have enough + // space in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // Schedule 2 backed candidates + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates, + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!(limit_inherent_data.bitfields.len(), 20,); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_bitfields_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_1() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // * 1 bitfields + assert_eq!(limit_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(limit_inherent_data.backed_candidates.len(), 1); + // * 3 disputes. + assert_eq!(limit_inherent_data.disputes.len(), 2); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 1 + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_0() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes() + .unwrap() + .backing_validators_per_candidate + .len(), + 0 + ); + }); + } + } + + fn default_header() -> primitives::v1::Header { + primitives::v1::Header { + parent_hash: Default::default(), + number: 0, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + } + } + + mod sanitizers { + use super::*; + + use crate::inclusion::tests::{ + back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, + }; + use bitvec::order::Lsb0; + use primitives::v1::{ + AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, + ValidatorIndex, + }; + + use crate::mock::Test; + use futures::executor::block_on; + use keyring::Sr25519Keyring; + use primitives::v0::PARACHAIN_KEY_TYPE_ID; + use sc_keystore::LocalKeystore; + use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; + use std::sync::Arc; + + fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() + } + + #[test] + fn bitfields() { + let header = default_header(); + let parent_hash = header.hash(); + // 2 cores means two bits + let expected_bits = 2; + let session_index = SessionIndex::from(0_u32); + + let crypto_store = LocalKeystore::in_memory(); + let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*crypto_store, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + let unchecked_bitfields = [ + BitVec::::repeat(true, expected_bits), + BitVec::::repeat(true, expected_bits), + { + let mut bv = BitVec::::repeat(false, expected_bits); + bv.set(expected_bits - 1, true); + bv + }, + ] + .iter() + .enumerate() + .map(|(vi, ab)| { + let validator_index = ValidatorIndex::from(vi as u32); + block_on(SignedAvailabilityBitfield::sign( + &crypto_store, + AvailabilityBitfield::from(ab.clone()), + &signing_context, + validator_index, + &validator_public[vi], + )) + .unwrap() + .unwrap() + .into_unchecked() + }) + .collect::>(); + + let disputed_bitfield = DisputedBitfield::zeros(expected_bits); + + { + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip, + ), + unchecked_bitfields.clone() + ); + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ), + unchecked_bitfields.clone() + ); + } + + // disputed bitfield is non-zero + { + let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); + // pretend the first core was freed by either a malicious validator + // or by resolved dispute + disputed_bitfield.0.set(0, true); + + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ) + .len(), + 1 + ); + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + ) + .len(), + 1 + ); + } + + // bitfield size mismatch + { + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ) + .is_empty()); + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + ) + .is_empty()); + } + + // remove the last validator + { + let shortened = validator_public.len() - 2; + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Yes, + )[..], + &unchecked_bitfields[..shortened] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Skip, + )[..], + &unchecked_bitfields[..shortened] + ); + } + + // switch ordering of bitfields + { + let mut unchecked_bitfields = unchecked_bitfields.clone(); + let x = unchecked_bitfields.swap_remove(0); + unchecked_bitfields.push(x); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); + } + + // check the validators signature + { + use primitives::v1::ValidatorSignature; + let mut unchecked_bitfields = unchecked_bitfields.clone(); + + // insert a bad signature for the last bitfield + let last_bit_idx = unchecked_bitfields.len() - 1; + unchecked_bitfields + .get_mut(last_bit_idx) + .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) + .expect("we are accessing a valid index"); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..last_bit_idx] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..] + ); + } + } + + #[test] + fn candidates() { + const RELAY_PARENT_NUM: u32 = 3; + + let header = default_header(); + let relay_parent = header.hash(); + let session_index = SessionIndex::from(0_u32); + + let keystore = LocalKeystore::in_memory(); + let keystore = Arc::new(keystore) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash: relay_parent, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + + let has_concluded_invalid = + |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; + + let scheduled = (0_usize..2) + .into_iter() + .map(|idx| { + let ca = CoreAssignment { + kind: scheduler::AssignmentKind::Parachain, + group_idx: GroupIndex::from(idx as u32), + para_id: ParaId::from(1_u32 + idx as u32), + core: CoreIndex::from(idx as u32), + }; + ca + }) + .collect::>(); + let scheduled = &scheduled[..]; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + } + .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) + }; + + let backed_candidates = (0_usize..2) + .into_iter() + .map(|idx0| { + let idx1 = idx0 + 1; + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(idx1), + relay_parent, + pov_hash: Hash::repeat_byte(idx1 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + backed + }) + .collect::>(); + + // happy path + assert_eq!( + sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ), + backed_candidates + ); + + // nothing is scheduled, so no paraids match, thus all backed candidates are skipped + { + let scheduled = &[][..]; + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // relay parent mismatch + { + let relay_parent = Hash::repeat_byte(0xFA); + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // candidates that have concluded as invalid are filtered out + { + // mark every second one as concluded invalid + let set = { + let mut set = std::collections::HashSet::new(); + for (idx, backed_candidate) in backed_candidates.iter().enumerate() { + if idx & 0x01 == 0 { + set.insert(backed_candidate.hash().clone()); + } + } + set + }; + let has_concluded_invalid = + |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); + assert_eq!( + sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .len(), + backed_candidates.len() / 2 + ); + } + } + } diff --git a/runtime/parachains/src/paras_inherent/weights.rs b/runtime/parachains/src/paras_inherent/weights.rs new file mode 100644 index 000000000000..06dcbe57d198 --- /dev/null +++ b/runtime/parachains/src/paras_inherent/weights.rs @@ -0,0 +1,118 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +use super::{ + BackedCandidate, Config, DisputeStatementSet, UncheckedSignedAvailabilityBitfield, Weight, +}; + +pub trait WeightInfo { + /// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the + /// weight of a single dispute statement set. + fn enter_variable_disputes(v: u32) -> Weight; + /// The weight of one bitfield. + fn enter_bitfields() -> Weight; + /// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight + /// of a single backed candidate. + fn enter_backed_candidates_variable(v: u32) -> Weight; + /// The weight of a single backed candidate with a code upgrade. + fn enter_backed_candidate_code_upgrade() -> Weight; +} + +pub struct TestWeightInfo; +// `WeightInfo` impl for unit and integration tests. Based off of the `max_block` weight for the +// mock. +#[cfg(not(feature = "runtime-benchmarks"))] +impl WeightInfo for TestWeightInfo { + fn enter_variable_disputes(v: u32) -> Weight { + // MAX Block Weight should fit 4 disputes + 80_000 * v as Weight + 80_000 + } + fn enter_bitfields() -> Weight { + // MAX Block Weight should fit 4 backed candidates + 40_000 as Weight + } + fn enter_backed_candidates_variable(v: u32) -> Weight { + // MAX Block Weight should fit 4 backed candidates + 40_000 * v as Weight + 40_000 + } + fn enter_backed_candidate_code_upgrade() -> Weight { + 0 + } +} +// To simplify benchmarks running as tests, we set all the weights to 0. `enter` will exit early +// when if the data causes it to be over weight, but we don't want that to block a benchmark from +// running as a test. +#[cfg(feature = "runtime-benchmarks")] +impl WeightInfo for TestWeightInfo { + fn enter_variable_disputes(_v: u32) -> Weight { + 0 + } + fn enter_bitfields() -> Weight { + 0 + } + fn enter_backed_candidates_variable(_v: u32) -> Weight { + 0 + } + fn enter_backed_candidate_code_upgrade() -> Weight { + 0 + } +} + +pub fn paras_inherent_total_weight( + backed_candidates: &[BackedCandidate<::Hash>], + bitfields: &[UncheckedSignedAvailabilityBitfield], + disputes: &[DisputeStatementSet], +) -> Weight { + backed_candidates_weight::(backed_candidates) + .saturating_add(signed_bitfields_weight::(bitfields.len())) + .saturating_add(dispute_statements_weight::(disputes)) +} + +pub fn dispute_statements_weight(disputes: &[DisputeStatementSet]) -> Weight { + disputes + .iter() + .map(|d| { + <::WeightInfo as WeightInfo>::enter_variable_disputes( + d.statements.len() as u32 + ) + }) + .fold(0, |acc, x| acc.saturating_add(x)) +} + +pub fn signed_bitfields_weight(bitfields_len: usize) -> Weight { + <::WeightInfo as WeightInfo>::enter_bitfields() + .saturating_mul(bitfields_len as Weight) +} + +pub fn backed_candidate_weight( + candidate: &BackedCandidate, +) -> Weight { + if candidate.candidate.commitments.new_validation_code.is_some() { + <::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade() + } else { + <::WeightInfo as WeightInfo>::enter_backed_candidates_variable( + candidate.validity_votes.len() as u32, + ) + } +} + +pub fn backed_candidates_weight( + candidates: &[BackedCandidate], +) -> Weight { + candidates + .iter() + .map(|c| backed_candidate_weight::(c)) + .fold(0, |acc, x| acc.saturating_add(x)) +} From bb2799e9f0b5e1d604a7a082e88952c23112d3c1 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 1 Dec 2021 18:48:29 -0800 Subject: [PATCH 14/14] fmt --- .../parachains/src/paras_inherent/tests.rs | 2063 ++++++++--------- 1 file changed, 1025 insertions(+), 1038 deletions(-) diff --git a/runtime/parachains/src/paras_inherent/tests.rs b/runtime/parachains/src/paras_inherent/tests.rs index c57de4af4a4f..e5b58b8287d5 100644 --- a/runtime/parachains/src/paras_inherent/tests.rs +++ b/runtime/parachains/src/paras_inherent/tests.rs @@ -1,1115 +1,1102 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use super::*; - // In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl - // that uses 0 for all the weights. Because all the weights are 0, the tests that rely on - // weights for limiting data will fail, so we don't run them when using the benchmark feature. - #[cfg(not(feature = "runtime-benchmarks"))] - mod enter { - use super::*; - use crate::{ - builder::{Bench, BenchBuilder}, - mock::{new_test_ext, MockGenesisConfig, Test}, - }; - use frame_support::assert_ok; - use sp_std::collections::btree_map::BTreeMap; - - struct TestConfig { - dispute_statements: BTreeMap, - dispute_sessions: Vec, - backed_and_concluding: BTreeMap, - num_validators_per_core: u32, - code_upgrade: Option, +// In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl +// that uses 0 for all the weights. Because all the weights are 0, the tests that rely on +// weights for limiting data will fail, so we don't run them when using the benchmark feature. +#[cfg(not(feature = "runtime-benchmarks"))] +mod enter { + use super::*; + use crate::{ + builder::{Bench, BenchBuilder}, + mock::{new_test_ext, MockGenesisConfig, Test}, + }; + use frame_support::assert_ok; + use sp_std::collections::btree_map::BTreeMap; + + struct TestConfig { + dispute_statements: BTreeMap, + dispute_sessions: Vec, + backed_and_concluding: BTreeMap, + num_validators_per_core: u32, + code_upgrade: Option, + } + + fn make_inherent_data( + TestConfig { + dispute_statements, + dispute_sessions, + backed_and_concluding, + num_validators_per_core, + code_upgrade, + }: TestConfig, + ) -> Bench { + let builder = BenchBuilder::::new() + .set_max_validators( + (dispute_sessions.len() + backed_and_concluding.len()) as u32 * + num_validators_per_core, + ) + .set_max_validators_per_core(num_validators_per_core) + .set_dispute_statements(dispute_statements) + .set_backed_and_concluding_cores(backed_and_concluding) + .set_dispute_sessions(&dispute_sessions[..]); + + if let Some(code_size) = code_upgrade { + builder.set_code_upgrade(code_size).build() + } else { + builder.build() } + } + + #[test] + // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via + // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and + // will not cause `enter` to early. + fn include_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let dispute_statements = BTreeMap::new(); - fn make_inherent_data( - TestConfig { + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + + let scenario = make_inherent_data(TestConfig { dispute_statements, - dispute_sessions, + dispute_sessions: vec![], // No disputes backed_and_concluding, - num_validators_per_core, - code_upgrade, - }: TestConfig, - ) -> Bench { - let builder = BenchBuilder::::new() - .set_max_validators( - (dispute_sessions.len() + backed_and_concluding.len()) as u32 * - num_validators_per_core, - ) - .set_max_validators_per_core(num_validators_per_core) - .set_dispute_statements(dispute_statements) - .set_backed_and_concluding_cores(backed_and_concluding) - .set_dispute_sessions(&dispute_sessions[..]); - - if let Some(code_size) = code_upgrade { - builder.set_code_upgrade(code_size).build() - } else { - builder.build() - } - } - - #[test] - // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via - // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and - // will not cause `enter` to early. - fn include_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - backed_and_concluding.insert(0, 1); - backed_and_concluding.insert(1, 1); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![], // No disputes - backed_and_concluding, - num_validators_per_core: 1, - code_upgrade: None, - }); - - // We expect the scenario to have cores 0 & 1 with pending availability. The backed - // candidates are also created for cores 0 & 1, so once the pending available - // become fully available those cores are marked as free and scheduled for the backed - // candidates. - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (2 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 2); - // * 1 backed candidate per core (2 cores) - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 0 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 0); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - assert_eq!( - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), - expected_para_inherent_data - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 backed candidates - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data - )); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 2 - ); + num_validators_per_core: 1, + code_upgrade: None, }); - } - #[test] - // Ensure that disputes are filtered out if the session is in the future. - fn filter_multi_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![ - 1, 2, 3, /* Session 3 too new, will get filtered out */ - ], - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 15); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let multi_dispute_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Dispute for session that lies too far in the future should be filtered out - assert!(multi_dispute_inherent_data != expected_para_inherent_data); - - assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); - - // Assert that the first 2 disputes are included - assert_eq!( - &multi_dispute_inherent_data.disputes[..2], - &expected_para_inherent_data.disputes[..2], - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - multi_dispute_inherent_data, - )); + // We expect the scenario to have cores 0 & 1 with pending availability. The backed + // candidates are also created for cores 0 & 1, so once the pending available + // become fully available those cores are marked as free and scheduled for the backed + // candidates. + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 2); + // * 1 backed candidate per core (2 cores) + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 0 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 0); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know there - // where no backed candidates included - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); - #[test] - // Ensure that when dispute data establishes an over weight block that we adequately - // filter out disputes according to our prioritization rule - fn limit_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be filld with disputes. - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 6, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // Ensure that the included disputes are sorted by session - assert_eq!(limit_inherent_data.disputes.len(), 2); - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); + // Nothing is filtered out (including the backed candidates.) + assert_eq!( + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), + expected_para_inherent_data + ); - assert_eq!( - // Ensure that our inherent data did not included backed candidates as expected - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); - #[test] - // Ensure that when dispute data establishes an over weight block that we abort - // due to an over weight block - fn limit_dispute_data_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be filled with disputes. - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 6, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 backed candidates + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data + )); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 2 + ); + }); + } + + #[test] + // Ensure that disputes are filtered out if the session is in the future. + fn filter_multi_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![1, 2, 3 /* Session 3 too new, will get filtered out */], + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, }); - } - #[test] - // Ensure that when a block is over weight due to disputes, but there is still sufficient - // block weight to include a number of signed bitfields, the inherent data is filtered - // as expected - fn limit_dispute_data_ignore_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 4, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!( - limit_inherent_data.bitfields.len(), - expected_para_inherent_data.bitfields.len() - ); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 15); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let multi_dispute_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Dispute for session that lies too far in the future should be filtered out + assert!(multi_dispute_inherent_data != expected_para_inherent_data); + + assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); + + // Assert that the first 2 disputes are included + assert_eq!( + &multi_dispute_inherent_data.disputes[..2], + &expected_para_inherent_data.disputes[..2], + ); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + multi_dispute_inherent_data, + )); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know there + // where no backed candidates included + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we adequately + // filter out disputes according to our prioritization rule + fn limit_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be filld with disputes. + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 6, + code_upgrade: None, }); - } - #[test] - // Ensure that we abort if we encounter an over weight block for disputes + bitfields - fn limit_dispute_data_ignore_backed_candidates_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 4, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Ensure that calling enter with 3 disputes and 2 candidates is over weight - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // Ensure that the included disputes are sorted by session + assert_eq!(limit_inherent_data.disputes.len(), 2); + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // Ensure that our inherent data did not included backed candidates as expected + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we abort + // due to an over weight block + fn limit_dispute_data_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be filled with disputes. + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 6, + code_upgrade: None, }); - } - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are - // filtered to accommodate the block size and no backed candidates are included. - fn limit_bitfields() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Cap the number of statements per dispute to 20 in order to ensure we have enough - // space in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // Schedule 2 backed candidates - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates, - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!(limit_inherent_data.bitfields.len(), 20,); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + }); + } - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); + #[test] + // Ensure that when a block is over weight due to disputes, but there is still sufficient + // block weight to include a number of signed bitfields, the inherent data is filtered + // as expected + fn limit_dispute_data_ignore_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 4, + code_upgrade: None, }); - } - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_bitfields_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!( + limit_inherent_data.bitfields.len(), + expected_para_inherent_data.bitfields.len() + ); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that we abort if we encounter an over weight block for disputes + bitfields + fn limit_dispute_data_ignore_backed_candidates_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 4, + code_upgrade: None, }); - } - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_1() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // * 1 bitfields - assert_eq!(limit_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(limit_inherent_data.backed_candidates.len(), 1); - // * 3 disputes. - assert_eq!(limit_inherent_data.disputes.len(), 2); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 1 - ); + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Ensure that calling enter with 3 disputes and 2 candidates is over weight + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are + // filtered to accommodate the block size and no backed candidates are included. + fn limit_bitfields() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Cap the number of statements per dispute to 20 in order to ensure we have enough + // space in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // Schedule 2 backed candidates + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, }); - } - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_0() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores with disputes - backed_and_concluding, - num_validators_per_core: 5, - code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates, + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!(limit_inherent_data.bitfields.len(), 20,); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_bitfields_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, }); - } + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); } - fn default_header() -> primitives::v1::Header { - primitives::v1::Header { - parent_hash: Default::default(), - number: 0, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_1() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // * 1 bitfields + assert_eq!(limit_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(limit_inherent_data.backed_candidates.len(), 1); + // * 3 disputes. + assert_eq!(limit_inherent_data.disputes.len(), 2); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 1 + ); + }); } - mod sanitizers { - use super::*; + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_0() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores with disputes + backed_and_concluding, + num_validators_per_core: 5, + code_upgrade: None, + }); - use crate::inclusion::tests::{ - back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, - }; - use bitvec::order::Lsb0; - use primitives::v1::{ - AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, - ValidatorIndex, - }; + let expected_para_inherent_data = scenario.data.clone(); - use crate::mock::Test; - use futures::executor::block_on; - use keyring::Sr25519Keyring; - use primitives::v0::PARACHAIN_KEY_TYPE_ID; - use sc_keystore::LocalKeystore; - use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - use std::sync::Arc; + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); - fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() - } + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); - #[test] - fn bitfields() { - let header = default_header(); - let parent_hash = header.hash(); - // 2 cores means two bits - let expected_bits = 2; - let session_index = SessionIndex::from(0_u32); - - let crypto_store = LocalKeystore::in_memory(); - let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*crypto_store, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - let unchecked_bitfields = [ - BitVec::::repeat(true, expected_bits), - BitVec::::repeat(true, expected_bits), - { - let mut bv = BitVec::::repeat(false, expected_bits); - bv.set(expected_bits - 1, true); - bv - }, - ] - .iter() - .enumerate() - .map(|(vi, ab)| { - let validator_index = ValidatorIndex::from(vi as u32); - block_on(SignedAvailabilityBitfield::sign( - &crypto_store, - AvailabilityBitfield::from(ab.clone()), - &signing_context, - validator_index, - &validator_public[vi], - )) - .unwrap() - .unwrap() - .into_unchecked() - }) - .collect::>(); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } +} + +fn default_header() -> primitives::v1::Header { + primitives::v1::Header { + parent_hash: Default::default(), + number: 0, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + } +} + +mod sanitizers { + use super::*; + + use crate::inclusion::tests::{ + back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, + }; + use bitvec::order::Lsb0; + use primitives::v1::{ + AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, + ValidatorIndex, + }; + + use crate::mock::Test; + use futures::executor::block_on; + use keyring::Sr25519Keyring; + use primitives::v0::PARACHAIN_KEY_TYPE_ID; + use sc_keystore::LocalKeystore; + use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; + use std::sync::Arc; + + fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() + } - let disputed_bitfield = DisputedBitfield::zeros(expected_bits); + #[test] + fn bitfields() { + let header = default_header(); + let parent_hash = header.hash(); + // 2 cores means two bits + let expected_bits = 2; + let session_index = SessionIndex::from(0_u32); + + let crypto_store = LocalKeystore::in_memory(); + let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*crypto_store, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + let unchecked_bitfields = [ + BitVec::::repeat(true, expected_bits), + BitVec::::repeat(true, expected_bits), { - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip, - ), - unchecked_bitfields.clone() - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ), - unchecked_bitfields.clone() - ); - } + let mut bv = BitVec::::repeat(false, expected_bits); + bv.set(expected_bits - 1, true); + bv + }, + ] + .iter() + .enumerate() + .map(|(vi, ab)| { + let validator_index = ValidatorIndex::from(vi as u32); + block_on(SignedAvailabilityBitfield::sign( + &crypto_store, + AvailabilityBitfield::from(ab.clone()), + &signing_context, + validator_index, + &validator_public[vi], + )) + .unwrap() + .unwrap() + .into_unchecked() + }) + .collect::>(); + + let disputed_bitfield = DisputedBitfield::zeros(expected_bits); + + { + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip, + ), + unchecked_bitfields.clone() + ); + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ), + unchecked_bitfields.clone() + ); + } - // disputed bitfield is non-zero - { - let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); - // pretend the first core was freed by either a malicious validator - // or by resolved dispute - disputed_bitfield.0.set(0, true); - - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ) - .len(), - 1 - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - ) - .len(), - 1 - ); - } + // disputed bitfield is non-zero + { + let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); + // pretend the first core was freed by either a malicious validator + // or by resolved dispute + disputed_bitfield.0.set(0, true); - // bitfield size mismatch - { - assert!(sanitize_bitfields::( + assert_eq!( + sanitize_bitfields::( unchecked_bitfields.clone(), disputed_bitfield.clone(), - expected_bits + 1, + expected_bits, parent_hash, session_index, &validator_public[..], FullCheck::Yes ) - .is_empty()); - assert!(sanitize_bitfields::( + .len(), + 1 + ); + assert_eq!( + sanitize_bitfields::( unchecked_bitfields.clone(), disputed_bitfield.clone(), - expected_bits + 1, + expected_bits, parent_hash, session_index, &validator_public[..], FullCheck::Skip ) - .is_empty()); - } + .len(), + 1 + ); + } - // remove the last validator - { - let shortened = validator_public.len() - 2; - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Yes, - )[..], - &unchecked_bitfields[..shortened] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Skip, - )[..], - &unchecked_bitfields[..shortened] - ); - } + // bitfield size mismatch + { + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ) + .is_empty()); + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + ) + .is_empty()); + } - // switch ordering of bitfields - { - let mut unchecked_bitfields = unchecked_bitfields.clone(); - let x = unchecked_bitfields.swap_remove(0); - unchecked_bitfields.push(x); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - } + // remove the last validator + { + let shortened = validator_public.len() - 2; + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Yes, + )[..], + &unchecked_bitfields[..shortened] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Skip, + )[..], + &unchecked_bitfields[..shortened] + ); + } - // check the validators signature - { - use primitives::v1::ValidatorSignature; - let mut unchecked_bitfields = unchecked_bitfields.clone(); - - // insert a bad signature for the last bitfield - let last_bit_idx = unchecked_bitfields.len() - 1; - unchecked_bitfields - .get_mut(last_bit_idx) - .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) - .expect("we are accessing a valid index"); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..last_bit_idx] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..] - ); - } + // switch ordering of bitfields + { + let mut unchecked_bitfields = unchecked_bitfields.clone(); + let x = unchecked_bitfields.swap_remove(0); + unchecked_bitfields.push(x); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); } - #[test] - fn candidates() { - const RELAY_PARENT_NUM: u32 = 3; - - let header = default_header(); - let relay_parent = header.hash(); - let session_index = SessionIndex::from(0_u32); - - let keystore = LocalKeystore::in_memory(); - let keystore = Arc::new(keystore) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash: relay_parent, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); + // check the validators signature + { + use primitives::v1::ValidatorSignature; + let mut unchecked_bitfields = unchecked_bitfields.clone(); + + // insert a bad signature for the last bitfield + let last_bit_idx = unchecked_bitfields.len() - 1; + unchecked_bitfields + .get_mut(last_bit_idx) + .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) + .expect("we are accessing a valid index"); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..last_bit_idx] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..] + ); + } + } + + #[test] + fn candidates() { + const RELAY_PARENT_NUM: u32 = 3; + + let header = default_header(); + let relay_parent = header.hash(); + let session_index = SessionIndex::from(0_u32); + + let keystore = LocalKeystore::in_memory(); + let keystore = Arc::new(keystore) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash: relay_parent, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + + let has_concluded_invalid = + |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; + + let scheduled = (0_usize..2) + .into_iter() + .map(|idx| { + let ca = CoreAssignment { + kind: scheduler::AssignmentKind::Parachain, + group_idx: GroupIndex::from(idx as u32), + para_id: ParaId::from(1_u32 + idx as u32), + core: CoreIndex::from(idx as u32), + }; + ca + }) + .collect::>(); + let scheduled = &scheduled[..]; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), } + .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) + }; - let has_concluded_invalid = - |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; - - let scheduled = (0_usize..2) - .into_iter() - .map(|idx| { - let ca = CoreAssignment { - kind: scheduler::AssignmentKind::Parachain, - group_idx: GroupIndex::from(idx as u32), - para_id: ParaId::from(1_u32 + idx as u32), - core: CoreIndex::from(idx as u32), - }; - ca - }) - .collect::>(); - let scheduled = &scheduled[..]; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + let backed_candidates = (0_usize..2) + .into_iter() + .map(|idx0| { + let idx1 = idx0 + 1; + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(idx1), + relay_parent, + pov_hash: Hash::repeat_byte(idx1 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() } - .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) - }; + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - let backed_candidates = (0_usize..2) - .into_iter() - .map(|idx0| { - let idx1 = idx0 + 1; - let mut candidate = TestCandidateBuilder { - para_id: ParaId::from(idx1), - relay_parent, - pov_hash: Hash::repeat_byte(idx1 as u8), - persisted_validation_data_hash: [42u8; 32].into(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + backed + }) + .collect::>(); + + // happy path + assert_eq!( + sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ), + backed_candidates + ); + + // nothing is scheduled, so no paraids match, thus all backed candidates are skipped + { + let scheduled = &[][..]; + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // relay parent mismatch + { + let relay_parent = Hash::repeat_byte(0xFA); + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // candidates that have concluded as invalid are filtered out + { + // mark every second one as concluded invalid + let set = { + let mut set = std::collections::HashSet::new(); + for (idx, backed_candidate) in backed_candidates.iter().enumerate() { + if idx & 0x01 == 0 { + set.insert(backed_candidate.hash().clone()); } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - backed - }) - .collect::>(); - - // happy path + } + set + }; + let has_concluded_invalid = + |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); assert_eq!( sanitize_backed_candidates::( relay_parent, backed_candidates.clone(), has_concluded_invalid, scheduled - ), - backed_candidates - ); - - // nothing is scheduled, so no paraids match, thus all backed candidates are skipped - { - let scheduled = &[][..]; - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled ) - .is_empty()); - } - - // relay parent mismatch - { - let relay_parent = Hash::repeat_byte(0xFA); - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .is_empty()); - } - - // candidates that have concluded as invalid are filtered out - { - // mark every second one as concluded invalid - let set = { - let mut set = std::collections::HashSet::new(); - for (idx, backed_candidate) in backed_candidates.iter().enumerate() { - if idx & 0x01 == 0 { - set.insert(backed_candidate.hash().clone()); - } - } - set - }; - let has_concluded_invalid = - |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); - assert_eq!( - sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .len(), - backed_candidates.len() / 2 - ); - } + .len(), + backed_candidates.len() / 2 + ); } } +}