fix(sync): avoid unnecessary block hash fetching during block sync (#…

…3241)

Currently on every iteration of block sync we would recompute the hashes of blocks that we need to sync, which is extremely slow when the amount of blocks that we need to sync is large. This PR fixes it by caching the hashes and only update it when we are close to syncing all the blocks in the cache.

Test plan
-----------
* `test_block_sync`
* Run nightly tests
* Test it on testnet.

chain/chain/src/chain.rs

@@ -15,8 +15,7 @@ use crate::lightclient::get_epoch_block_producers_view;
 use crate::store::{ChainStore, ChainStoreAccess, ChainStoreUpdate, GCMode};
 use crate::types::{
     AcceptedBlock, ApplyTransactionResult, Block, BlockEconomicsConfig, BlockHeader,
-    BlockHeaderInfo, BlockStatus, BlockSyncResponse, ChainGenesis, Provenance, ReceiptList,
-    RuntimeAdapter,
+    BlockHeaderInfo, BlockStatus, ChainGenesis, Provenance, ReceiptList, RuntimeAdapter,
 };
 use crate::validate::{
     validate_challenge, validate_chunk_proofs, validate_chunk_with_chunk_extra,
@@ -827,57 +826,8 @@ impl Chain {
         chain_update.commit()
     }
 
-    /// Check if state download is required, otherwise return hashes of blocks to fetch.
-    /// Hashes are sorted increasingly by height.
-    pub fn check_state_needed(
-        &mut self,
-        block_fetch_horizon: BlockHeightDelta,
-        force_block_sync: bool,
-    ) -> Result<BlockSyncResponse, Error> {
-        let block_head = self.head()?;
-        let header_head = self.header_head()?;
-        let mut hashes = vec![];
-
-        // If latest block is up to date return early.
-        // No state download is required, neither any blocks need to be fetched.
-        if block_head.height >= header_head.height {
-            return Ok(BlockSyncResponse::None);
-        }
-
-        let next_epoch_id =
-            self.get_block_header(&block_head.last_block_hash)?.next_epoch_id().clone();
-
-        // Don't run State Sync if header head is not more than one epoch ahead.
-        if block_head.epoch_id != header_head.epoch_id && next_epoch_id != header_head.epoch_id {
-            if block_head.height < header_head.height.saturating_sub(block_fetch_horizon)
-                && !force_block_sync
-            {
-                // Epochs are different and we are too far from horizon, State Sync is needed
-                return Ok(BlockSyncResponse::StateNeeded);
-            }
-        }
-
-        // Find hashes of blocks to sync
-        let mut current = self.get_block_header(&header_head.last_block_hash).map(|h| h.clone());
-        while let Ok(header) = current {
-            if header.height() <= block_head.height {
-                if self.is_on_current_chain(&header).is_ok() {
-                    break;
-                }
-            }
-
-            hashes.push(*header.hash());
-            current = self.get_previous_header(&header).map(|h| h.clone());
-        }
-
-        // Sort hashes by height
-        hashes.reverse();
-
-        Ok(BlockSyncResponse::BlocksNeeded(hashes))
-    }
-
     /// Returns if given block header is on the current chain.
-    fn is_on_current_chain(&mut self, header: &BlockHeader) -> Result<(), Error> {
+    pub fn is_on_current_chain(&mut self, header: &BlockHeader) -> Result<(), Error> {
         let chain_header = self.get_header_by_height(header.height())?;
         if chain_header.hash() == header.hash() {
             Ok(())

chain/chain/src/types.rs

@@ -584,17 +584,6 @@ pub struct LatestKnown {
     pub seen: u64,
 }
 
-/// When running block sync response to know if the node needs to sync state,
-/// or the hashes from the blocks that are needed.
-pub enum BlockSyncResponse {
-    /// State is needed before we start fetching recent blocks.
-    StateNeeded,
-    /// We are up to date with state, list of block hashes that need to be fetched.
-    BlocksNeeded(Vec<CryptoHash>),
-    /// We are up to date, nothing is required.
-    None,
-}
-
 #[cfg(test)]
 mod tests {
     use chrono::Utc;

chain/client/Cargo.toml

@@ -22,6 +22,7 @@ lazy_static = "1.4"
 borsh = "0.7.0"
 reed-solomon-erasure = "4"
 num-rational = "0.2.4"
+linked-hash-map = "0.5.3"
 
 near-crypto = { path = "../../core/crypto" }
 near-primitives = { path = "../../core/primitives" }