Merge branch 'master' into wasmer1_errors

Cargo.toml

@@ -107,7 +107,7 @@ no_cache = ["neard/no_cache"]
 metric_recorder = ["neard/metric_recorder"]
 delay_detector = ["neard/delay_detector"]
 rosetta_rpc = ["neard/rosetta_rpc"]
-nightly_protocol = ["near-primitives/nightly_protocol", "near-jsonrpc/nightly_protocol", "testlib/nightly_protocol"]
+nightly_protocol = ["near-primitives/nightly_protocol", "near-jsonrpc/nightly_protocol", "testlib/nightly_protocol", "neard/nightly_protocol"]
 nightly_protocol_features = ["nightly_protocol", "neard/nightly_protocol_features", "protocol_feature_evm", "protocol_feature_block_header_v3", "protocol_feature_alt_bn128", "protocol_feature_access_key_nonce_range", "protocol_feature_add_account_versions", "protocol_feature_tx_size_limit", "testlib/nightly_protocol_features"]
 protocol_feature_forward_chunk_parts = ["neard/protocol_feature_forward_chunk_parts"]
 protocol_feature_evm = ["neard/protocol_feature_evm", "testlib/protocol_feature_evm", "runtime-params-estimator/protocol_feature_evm"]

chain/chain-primitives/src/error.rs

@@ -52,6 +52,12 @@ pub enum QueryError {
         block_height: near_primitives::types::BlockHeight,
         block_hash: near_primitives::hash::CryptoHash,
     },
+    #[error("The state of account {requested_account_id} is too large")]
+    TooLargeContractState {
+        requested_account_id: near_primitives::types::AccountId,
+        block_height: near_primitives::types::BlockHeight,
+        block_hash: near_primitives::hash::CryptoHash,
+    },
 }
 
 #[derive(Debug)]

chain/chain/Cargo.toml

@@ -8,6 +8,7 @@ edition = "2018"
 chrono = { version = "0.4.4", features = ["serde"] }
 failure = "0.1"
 failure_derive = "0.1"
+itertools = "0.10.0"
 lazy_static = "1.4"
 rand = "0.7"
 serde = { version = "1", features = [ "derive" ] }

chain/chain/src/chain.rs

@@ -5,6 +5,7 @@ use std::time::{Duration as TimeDuration, Instant};
 use borsh::BorshSerialize;
 use chrono::Duration;
 use chrono::Utc;
+use itertools::Itertools;
 use rand::rngs::StdRng;
 use rand::seq::SliceRandom;
 use rand::SeedableRng;
@@ -46,11 +47,14 @@ use near_primitives::syncing::{
     StateHeaderKey, StatePartKey,
 };
 use near_primitives::transaction::ExecutionOutcomeWithIdAndProof;
+use near_primitives::types::chunk_extra::ChunkExtra;
 use near_primitives::types::{
-    AccountId, Balance, BlockExtra, BlockHeight, BlockHeightDelta, ChunkExtra, EpochId, MerkleHash,
-    NumBlocks, ShardId, ValidatorStake,
+    AccountId, Balance, BlockExtra, BlockHeight, BlockHeightDelta, EpochId, MerkleHash, NumBlocks,
+    ShardId,
 };
 use near_primitives::unwrap_or_return;
+#[cfg(feature = "protocol_feature_block_header_v3")]
+use near_primitives::version::{ProtocolFeature, PROTOCOL_FEATURES_TO_VERSION_MAPPING};
 use near_primitives::views::{
     ExecutionOutcomeWithIdView, ExecutionStatusView, FinalExecutionOutcomeView,
     FinalExecutionOutcomeWithReceiptView, FinalExecutionStatus, LightClientBlockView,
@@ -357,8 +361,8 @@ impl Chain {
         self.doomslug_threshold_mode = DoomslugThresholdMode::NoApprovals
     }
 
-    pub fn compute_bp_hash_inner(bps: Vec<ValidatorStake>) -> Result<CryptoHash, Error> {
-        Ok(hash(&bps.try_to_vec()?))
+    pub fn compute_collection_hash<T: BorshSerialize>(elems: Vec<T>) -> Result<CryptoHash, Error> {
+        Ok(hash(&elems.try_to_vec()?))
     }
 
     pub fn compute_bp_hash(
@@ -367,7 +371,24 @@ impl Chain {
         last_known_hash: &CryptoHash,
     ) -> Result<CryptoHash, Error> {
         let bps = runtime_adapter.get_epoch_block_producers_ordered(&epoch_id, last_known_hash)?;
-        Chain::compute_bp_hash_inner(bps.iter().map(|(bp, _)| bp).cloned().collect::<Vec<_>>())
+        #[cfg(not(feature = "protocol_feature_block_header_v3"))]
+        {
+            let validator_stakes = bps.into_iter().map(|(bp, _)| bp).collect();
+            Chain::compute_collection_hash(validator_stakes)
+        }
+        #[cfg(feature = "protocol_feature_block_header_v3")]
+        {
+            let protocol_version = runtime_adapter.get_epoch_protocol_version(&epoch_id)?;
+            let block_header_v3_version =
+                PROTOCOL_FEATURES_TO_VERSION_MAPPING.get(&ProtocolFeature::BlockHeaderV3).unwrap();
+            if &protocol_version < block_header_v3_version {
+                let validator_stakes = bps.into_iter().map(|(bp, _)| bp.into_v1()).collect();
+                Chain::compute_collection_hash(validator_stakes)
+            } else {
+                let validator_stakes = bps.into_iter().map(|(bp, _)| bp).collect();
+                Chain::compute_collection_hash(validator_stakes)
+            }
+        }
     }
 
     /// Creates a light client block for the last final block from perspective of some other block
@@ -1062,8 +1083,7 @@ impl Chain {
                 let sum = block
                     .header()
                     .validator_proposals()
-                    .iter()
-                    .map(|validator_stake| (validator_stake.stake / NEAR_BASE) as i64)
+                    .map(|validator_stake| (validator_stake.stake() / NEAR_BASE) as i64)
                     .sum::<i64>();
                 near_metrics::set_gauge(&metrics::VALIDATOR_AMOUNT_STAKED, sum);
 
@@ -1435,6 +1455,8 @@ impl Chain {
                     prev_chunk_header.and_then(|prev_header| match prev_header {
                         ShardChunkHeader::V1(header) => Some(header),
                         ShardChunkHeader::V2(_) => None,
+                        #[cfg(feature = "protocol_feature_block_header_v3")]
+                        ShardChunkHeader::V3(_) => None,
                     });
                 ShardStateSyncResponseHeader::V1(ShardStateSyncResponseHeaderV1 {
                     chunk,
@@ -1680,7 +1702,7 @@ impl Chain {
         let chunk_inner = chunk.take_header().take_inner();
         if !self.runtime_adapter.validate_state_root_node(
             shard_state_header.state_root_node(),
-            &chunk_inner.prev_state_root,
+            chunk_inner.prev_state_root(),
         ) {
             byzantine_assert!(false);
             return Err(ErrorKind::Other(
@@ -1716,7 +1738,7 @@ impl Chain {
     ) -> Result<(), Error> {
         let shard_state_header = self.get_state_header(shard_id, sync_hash)?;
         let chunk = shard_state_header.take_chunk();
-        let state_root = chunk.take_header().take_inner().prev_state_root;
+        let state_root = *chunk.take_header().take_inner().prev_state_root();
         if !self.runtime_adapter.validate_state_part(&state_root, part_id, num_parts, data) {
             byzantine_assert!(false);
             return Err(ErrorKind::Other(
@@ -2678,16 +2700,16 @@ impl<'a> ChainUpdate<'a> {
             .runtime_adapter
             .apply_transactions_with_optional_storage_proof(
                 chunk_shard_id,
-                &prev_chunk_inner.prev_state_root,
+                prev_chunk_inner.prev_state_root(),
                 prev_chunk.height_included(),
                 prev_block.header().raw_timestamp(),
-                &prev_chunk_inner.prev_block_hash,
+                prev_chunk_inner.prev_block_hash(),
                 &prev_block.hash(),
                 &receipts,
                 prev_chunk.transactions(),
-                &prev_chunk_inner.validator_proposals,
+                prev_chunk_inner.validator_proposals(),
                 prev_block.header().gas_price(),
-                prev_chunk_inner.gas_limit,
+                prev_chunk_inner.gas_limit(),
                 &challenges_result,
                 *block.header().random_value(),
                 true,
@@ -2824,21 +2846,21 @@ impl<'a> ChainUpdate<'a> {
                     );
 
                     let chunk_inner = chunk.cloned_header().take_inner();
-                    let gas_limit = chunk_inner.gas_limit;
+                    let gas_limit = chunk_inner.gas_limit();
 
                     // Apply transactions and receipts.
                     let apply_result = self
                         .runtime_adapter
                         .apply_transactions(
                             shard_id,
-                            &chunk_inner.prev_state_root,
+                            chunk_inner.prev_state_root(),
                             chunk_header.height_included(),
                             block.header().raw_timestamp(),
                             &chunk_header.prev_block_hash(),
                             &block.hash(),
                             &receipts,
                             chunk.transactions(),
-                            &chunk_inner.validator_proposals,
+                            chunk_inner.validator_proposals(),
                             prev_block.header().gas_price(),
                             gas_limit,
                             &block.header().challenges_result(),
@@ -2885,23 +2907,23 @@ impl<'a> ChainUpdate<'a> {
                         .runtime_adapter
                         .apply_transactions(
                             shard_id,
-                            &new_extra.state_root,
+                            new_extra.state_root(),
                             block.header().height(),
                             block.header().raw_timestamp(),
                             &prev_block.hash(),
                             &block.hash(),
                             &[],
                             &[],
-                            &new_extra.validator_proposals,
+                            new_extra.validator_proposals(),
                             block.header().gas_price(),
-                            new_extra.gas_limit,
+                            new_extra.gas_limit(),
                             &block.header().challenges_result(),
                             *block.header().random_value(),
                         )
                         .map_err(|e| ErrorKind::Other(e.to_string()))?;
 
                     self.chain_store_update.save_trie_changes(apply_result.trie_changes);
-                    new_extra.state_root = apply_result.new_root;
+                    *new_extra.state_root_mut() = apply_result.new_root;
 
                     self.chain_store_update.save_chunk_extra(&block.hash(), shard_id, new_extra);
                 }
@@ -3067,15 +3089,28 @@ impl<'a> ChainUpdate<'a> {
         }
 
         // Verify that proposals from chunks match block header proposals.
-        let mut all_chunk_proposals = vec![];
-        for chunk in block.chunks().iter() {
-            if block.header().height() == chunk.height_included() {
-                all_chunk_proposals.extend(chunk.validator_proposals().iter().cloned());
+        let block_height = block.header().height();
+        for pair in block
+            .chunks()
+            .iter()
+            .filter(|chunk| block_height == chunk.height_included())
+            .flat_map(|chunk| chunk.validator_proposals())
+            .zip_longest(block.header().validator_proposals())
+        {
+            match pair {
+                itertools::EitherOrBoth::Both(cp, hp) => {
+                    if hp != cp {
+                        // Proposals differed!
+                        return Err(ErrorKind::InvalidValidatorProposals.into());
+                    }
+                }
+                _ => {
+                    // Can only occur if there were a different number of proposals in the header
+                    // and chunks
+                    return Err(ErrorKind::InvalidValidatorProposals.into());
+                }
             }
         }
-        if all_chunk_proposals.as_slice() != block.header().validator_proposals() {
-            return Err(ErrorKind::InvalidValidatorProposals.into());
-        }
 
         // If block checks out, record validator proposals for given block.
         let last_final_block = block.header().last_final_block();
@@ -3643,22 +3678,22 @@ impl<'a> ChainUpdate<'a> {
 
         let apply_result = self.runtime_adapter.apply_transactions(
             shard_id,
-            &chunk_extra.state_root,
+            chunk_extra.state_root(),
             block_header.height(),
             block_header.raw_timestamp(),
             &prev_block_header.hash(),
             &block_header.hash(),
             &[],
             &[],
-            &chunk_extra.validator_proposals,
+            chunk_extra.validator_proposals(),
             prev_block_header.gas_price(),
-            chunk_extra.gas_limit,
+            chunk_extra.gas_limit(),
             &block_header.challenges_result(),
             *block_header.random_value(),
         )?;
 
         self.chain_store_update.save_trie_changes(apply_result.trie_changes);
-        chunk_extra.state_root = apply_result.new_root;
+        *chunk_extra.state_root_mut() = apply_result.new_root;
 
         self.chain_store_update.save_chunk_extra(&block_header.hash(), shard_id, chunk_extra);
         Ok(true)
@@ -3714,7 +3749,7 @@ impl<'a> ChainUpdate<'a> {
             &head.last_block_hash,
             &block_producer,
         )?;
-        Ok(header.signature().verify(header.hash().as_ref(), &block_producer.public_key))
+        Ok(header.signature().verify(header.hash().as_ref(), block_producer.public_key()))
     }
 }
 

chain/chain/src/lightclient.rs

@@ -2,7 +2,8 @@ use near_chain_primitives::Error;
 use near_primitives::block::BlockHeader;
 use near_primitives::hash::{hash, CryptoHash};
 use near_primitives::types::EpochId;
-use near_primitives::views::{BlockHeaderInnerLiteView, LightClientBlockView, ValidatorStakeView};
+use near_primitives::views::validator_stake_view::ValidatorStakeView;
+use near_primitives::views::{BlockHeaderInnerLiteView, LightClientBlockView};
 
 use crate::{ChainStoreAccess, RuntimeAdapter};
 

chain/chain/src/store.rs

@@ -26,9 +26,10 @@ use near_primitives::transaction::{
     ExecutionOutcomeWithId, ExecutionOutcomeWithIdAndProof, SignedTransaction,
 };
 use near_primitives::trie_key::{trie_key_parsers, TrieKey};
+use near_primitives::types::chunk_extra::ChunkExtra;
 use near_primitives::types::{
-    AccountId, BlockExtra, BlockHeight, ChunkExtra, EpochId, GCCount, NumBlocks, ShardId,
-    StateChanges, StateChangesExt, StateChangesKinds, StateChangesKindsExt, StateChangesRequest,
+    AccountId, BlockExtra, BlockHeight, EpochId, GCCount, NumBlocks, ShardId, StateChanges,
+    StateChangesExt, StateChangesKinds, StateChangesKindsExt, StateChangesRequest,
 };
 use near_primitives::utils::{get_block_shard_id, index_to_bytes, to_timestamp};
 use near_primitives::views::LightClientBlockView;
@@ -2953,7 +2954,7 @@ mod tests {
     use near_crypto::KeyType;
     use near_primitives::block::{Block, Tip};
     #[cfg(feature = "expensive_tests")]
-    use near_primitives::epoch_manager::BlockInfo;
+    use near_primitives::epoch_manager::block_info::BlockInfo;
     use near_primitives::errors::InvalidTxError;
     use near_primitives::hash::hash;
     use near_primitives::types::{BlockHeight, EpochId, GCCount, NumBlocks};

chain/chain/src/store_validator.rs

@@ -10,12 +10,15 @@ use tracing::warn;
 use near_chain_configs::GenesisConfig;
 use near_primitives::block::{Block, BlockHeader};
 use near_primitives::borsh;
-use near_primitives::epoch_manager::{BlockInfo, EpochInfo, AGGREGATOR_KEY};
+use near_primitives::epoch_manager::block_info::BlockInfo;
+use near_primitives::epoch_manager::epoch_info::EpochInfo;
+use near_primitives::epoch_manager::AGGREGATOR_KEY;
 use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::{ChunkHash, ShardChunk, StateSyncInfo};
 use near_primitives::syncing::{ShardStateSyncResponseHeader, StateHeaderKey, StatePartKey};
 use near_primitives::transaction::ExecutionOutcomeWithIdAndProof;
-use near_primitives::types::{AccountId, BlockHeight, ChunkExtra, EpochId, GCCount, ShardId};
+use near_primitives::types::chunk_extra::ChunkExtra;
+use near_primitives::types::{AccountId, BlockHeight, EpochId, GCCount, ShardId};
 use near_primitives::utils::get_block_shard_id_rev;
 use near_store::{
     decode_value_with_rc, DBCol, Store, TrieChanges, NUM_COLS, SHOULD_COL_GC, SKIP_COL_GC,

chain/chain/src/store_validator/validate.rs

@@ -4,14 +4,16 @@ use borsh::BorshSerialize;
 use thiserror::Error;
 
 use near_primitives::block::{Block, BlockHeader, Tip};
-use near_primitives::epoch_manager::{BlockInfo, EpochInfo};
+use near_primitives::epoch_manager::block_info::BlockInfo;
+use near_primitives::epoch_manager::epoch_info::EpochInfo;
 use near_primitives::hash::CryptoHash;
 use near_primitives::sharding::{ChunkHash, ShardChunk, StateSyncInfo};
 use near_primitives::syncing::{
     get_num_state_parts, ShardStateSyncResponseHeader, StateHeaderKey, StatePartKey,
 };
 use near_primitives::transaction::{ExecutionOutcomeWithIdAndProof, SignedTransaction};
-use near_primitives::types::{BlockHeight, ChunkExtra, EpochId, ShardId};
+use near_primitives::types::chunk_extra::ChunkExtra;
+use near_primitives::types::{BlockHeight, EpochId, ShardId};
 use near_primitives::utils::{get_block_shard_id, index_to_bytes};
 use near_store::{
     ColBlock, ColBlockHeader, ColBlockHeight, ColBlockInfo, ColBlockMisc, ColBlockPerHeight,
@@ -570,17 +572,17 @@ pub(crate) fn trie_changes_chunk_extra_exists(
                 &get_block_shard_id(block.header().prev_hash(), *shard_id),
             ) {
                 check_discrepancy!(
-                    prev_chunk_extra.state_root,
-                    trie_changes.old_root,
+                    prev_chunk_extra.state_root(),
+                    &trie_changes.old_root,
                     "Prev State Root discrepancy, previous ChunkExtra {:?}",
                     prev_chunk_extra
                 );
             }
 
             // 7. State Roots should be equal
             check_discrepancy!(
-                chunk_extra.state_root,
-                new_root,
+                chunk_extra.state_root(),
+                &new_root,
                 "State Root discrepancy, ShardChunk {:?}",
                 chunk_header
             );