Altair validator client and HTTP API (#2404)

## Proposed Changes

* Implement the validator client and HTTP API changes necessary to support Altair


Co-authored-by: realbigsean <[email protected]>
Co-authored-by: Michael Sproul <[email protected]>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -43,7 +43,7 @@ use crate::validator_pubkey_cache::ValidatorPubkeyCache;
 use crate::BeaconForkChoiceStore;
 use crate::BeaconSnapshot;
 use crate::{metrics, BeaconChainError};
-use eth2::types::{EventKind, SseBlock, SseChainReorg, SseFinalizedCheckpoint, SseHead};
+use eth2::types::{EventKind, SseBlock, SseChainReorg, SseFinalizedCheckpoint, SseHead, SyncDuty};
 use fork_choice::ForkChoice;
 use futures::channel::mpsc::Sender;
 use itertools::process_results;
@@ -1081,6 +1081,29 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(pubkey_cache.get_index(pubkey))
     }
 
+    /// Return the validator indices of all public keys fetched from an iterator.
+    ///
+    /// If any public key doesn't belong to a known validator then an error will be returned.
+    /// We could consider relaxing this by returning `Vec<Option<usize>>` in future.
+    pub fn validator_indices<'a>(
+        &self,
+        validator_pubkeys: impl Iterator<Item = &'a PublicKeyBytes>,
+    ) -> Result<Vec<u64>, Error> {
+        let pubkey_cache = self
+            .validator_pubkey_cache
+            .try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
+            .ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
+
+        validator_pubkeys
+            .map(|pubkey| {
+                pubkey_cache
+                    .get_index(pubkey)
+                    .map(|id| id as u64)
+                    .ok_or(Error::ValidatorPubkeyUnknown(*pubkey))
+            })
+            .collect()
+    }
+
     /// Returns the validator pubkey (if any) for the given validator index.
     ///
     /// ## Notes
@@ -1214,6 +1237,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .get_by_slot_and_root(slot, attestation_data_root)
     }
 
+    /// Return an aggregated `SyncCommitteeContribution` matching the given `root`.
+    pub fn get_aggregated_sync_committee_contribution(
+        &self,
+        sync_contribution_data: &SyncContributionData,
+    ) -> Option<SyncCommitteeContribution<T::EthSpec>> {
+        self.naive_sync_aggregation_pool
+            .read()
+            .get(sync_contribution_data)
+    }
+
     /// Produce an unaggregated `Attestation` that is valid for the given `slot` and `index`.
     ///
     /// The produced `Attestation` will not be valid until it has been signed by exactly one
@@ -1882,6 +1915,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(())
     }
 
+    /// Attempt to obtain sync committee duties from the head.
+    pub fn sync_committee_duties_from_head(
+        &self,
+        epoch: Epoch,
+        validator_indices: &[u64],
+    ) -> Result<Vec<Option<SyncDuty>>, Error> {
+        self.with_head(move |head| {
+            head.beacon_state
+                .get_sync_committee_duties(epoch, validator_indices, &self.spec)
+                .map_err(Error::SyncDutiesError)
+        })
+    }
+
     /// Attempt to verify and import a chain of blocks to `self`.
     ///
     /// The provided blocks _must_ each reference the previous block via `block.parent_root` (i.e.,
@@ -2624,6 +2670,22 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let proposer_index = state.get_beacon_proposer_index(state.slot(), &self.spec)? as u64;
         let voluntary_exits = self.op_pool.get_voluntary_exits(&state, &self.spec).into();
 
+        // Closure to fetch a sync aggregate in cases where it is required.
+        let get_sync_aggregate = || -> Result<SyncAggregate<_>, BlockProductionError> {
+            Ok(self
+                .op_pool
+                .get_sync_aggregate(&state)
+                .map_err(BlockProductionError::OpPoolError)?
+                .unwrap_or_else(|| {
+                    warn!(
+                        self.log,
+                        "Producing block with no sync contributions";
+                        "slot" => state.slot(),
+                    );
+                    SyncAggregate::new()
+                }))
+        };
+
         let inner_block = match state {
             BeaconState::Base(_) => BeaconBlock::Base(BeaconBlockBase {
                 slot,
@@ -2641,24 +2703,26 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     voluntary_exits,
                 },
             }),
-            BeaconState::Altair(_) => BeaconBlock::Altair(BeaconBlockAltair {
-                slot,
-                proposer_index,
-                parent_root,
-                state_root: Hash256::zero(),
-                body: BeaconBlockBodyAltair {
-                    randao_reveal,
-                    eth1_data,
-                    graffiti,
-                    proposer_slashings: proposer_slashings.into(),
-                    attester_slashings: attester_slashings.into(),
-                    attestations,
-                    deposits,
-                    voluntary_exits,
-                    // FIXME(altair): put a sync aggregate from the pool here (once implemented)
-                    sync_aggregate: SyncAggregate::new(),
-                },
-            }),
+            BeaconState::Altair(_) => {
+                let sync_aggregate = get_sync_aggregate()?;
+                BeaconBlock::Altair(BeaconBlockAltair {
+                    slot,
+                    proposer_index,
+                    parent_root,
+                    state_root: Hash256::zero(),
+                    body: BeaconBlockBodyAltair {
+                        randao_reveal,
+                        eth1_data,
+                        graffiti,
+                        proposer_slashings: proposer_slashings.into(),
+                        attester_slashings: attester_slashings.into(),
+                        attestations,
+                        deposits,
+                        voluntary_exits,
+                        sync_aggregate,
+                    },
+                })
+            }
         };
 
         let block = SignedBeaconBlock::from_block(

beacon_node/beacon_chain/src/errors.rs

@@ -87,6 +87,7 @@ pub enum BeaconChainError {
     DuplicateValidatorPublicKey,
     ValidatorPubkeyCacheFileError(String),
     ValidatorIndexUnknown(usize),
+    ValidatorPubkeyUnknown(PublicKeyBytes),
     OpPoolError(OpPoolError),
     NaiveAggregationError(NaiveAggregationError),
     ObservedAttestationsError(ObservedAttestationsError),
@@ -120,6 +121,7 @@ pub enum BeaconChainError {
         state_epoch: Epoch,
         shuffling_epoch: Epoch,
     },
+    SyncDutiesError(BeaconStateError),
     InconsistentForwardsIter {
         request_slot: Slot,
         slot: Slot,

beacon_node/beacon_chain/src/test_utils.rs

@@ -151,7 +151,7 @@ pub fn test_spec<E: EthSpec>() -> ChainSpec {
 pub struct BeaconChainHarness<T: BeaconChainTypes> {
     pub validator_keypairs: Vec<Keypair>,
 
-    pub chain: BeaconChain<T>,
+    pub chain: Arc<BeaconChain<T>>,
     pub spec: ChainSpec,
     pub data_dir: TempDir,
     pub shutdown_receiver: Receiver<ShutdownReason>,
@@ -229,6 +229,29 @@ impl<E: EthSpec> BeaconChainHarness<EphemeralHarnessType<E>> {
         target_aggregators_per_committee: u64,
         store_config: StoreConfig,
         chain_config: ChainConfig,
+    ) -> Self {
+        Self::new_with_mutator(
+            eth_spec_instance,
+            spec,
+            validator_keypairs,
+            target_aggregators_per_committee,
+            store_config,
+            chain_config,
+            |x| x,
+        )
+    }
+
+    /// Apply a function to beacon chain builder before building.
+    pub fn new_with_mutator(
+        eth_spec_instance: E,
+        spec: Option<ChainSpec>,
+        validator_keypairs: Vec<Keypair>,
+        target_aggregators_per_committee: u64,
+        store_config: StoreConfig,
+        chain_config: ChainConfig,
+        mutator: impl FnOnce(
+            BeaconChainBuilder<EphemeralHarnessType<E>>,
+        ) -> BeaconChainBuilder<EphemeralHarnessType<E>>,
     ) -> Self {
         let data_dir = tempdir().expect("should create temporary data_dir");
         let mut spec = spec.unwrap_or_else(test_spec::<E>);
@@ -240,7 +263,7 @@ impl<E: EthSpec> BeaconChainHarness<EphemeralHarnessType<E>> {
         let log = test_logger();
 
         let store = HotColdDB::open_ephemeral(store_config, spec.clone(), log.clone()).unwrap();
-        let chain = BeaconChainBuilder::new(eth_spec_instance)
+        let builder = BeaconChainBuilder::new(eth_spec_instance)
             .logger(log.clone())
             .custom_spec(spec.clone())
             .store(Arc::new(store))
@@ -260,13 +283,13 @@ impl<E: EthSpec> BeaconChainHarness<EphemeralHarnessType<E>> {
                 log.clone(),
                 1,
             )))
-            .monitor_validators(true, vec![], log)
-            .build()
-            .expect("should build");
+            .monitor_validators(true, vec![], log);
+
+        let chain = mutator(builder).build().expect("should build");
 
         Self {
             spec: chain.spec.clone(),
-            chain,
+            chain: Arc::new(chain),
             validator_keypairs,
             data_dir,
             shutdown_receiver,
@@ -311,7 +334,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
 
         Self {
             spec: chain.spec.clone(),
-            chain,
+            chain: Arc::new(chain),
             validator_keypairs,
             data_dir,
             shutdown_receiver,
@@ -353,7 +376,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
 
         Self {
             spec: chain.spec.clone(),
-            chain,
+            chain: Arc::new(chain),
             validator_keypairs,
             data_dir,
             shutdown_receiver,

beacon_node/beacon_chain/tests/block_verification.rs

@@ -4,7 +4,7 @@
 extern crate lazy_static;
 
 use beacon_chain::test_utils::{
-    AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
+    test_logger, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType,
 };
 use beacon_chain::{BeaconSnapshot, BlockError, ChainConfig, ChainSegmentResult};
 use slasher::{Config as SlasherConfig, Slasher};
@@ -830,17 +830,25 @@ fn block_gossip_verification() {
 
 #[test]
 fn verify_block_for_gossip_slashing_detection() {
-    let mut harness = get_harness(VALIDATOR_COUNT);
-
     let slasher_dir = tempdir().unwrap();
     let slasher = Arc::new(
         Slasher::open(
             SlasherConfig::new(slasher_dir.path().into()).for_testing(),
-            harness.logger().clone(),
+            test_logger(),
         )
         .unwrap(),
     );
-    harness.chain.slasher = Some(slasher.clone());
+
+    let harness = BeaconChainHarness::new_with_mutator(
+        MainnetEthSpec,
+        None,
+        KEYPAIRS.to_vec(),
+        1 << 32,
+        StoreConfig::default(),
+        ChainConfig::default(),
+        |builder| builder.slasher(slasher.clone()),
+    );
+    harness.advance_slot();
 
     let state = harness.get_current_state();
     let (block1, _) = harness.make_block(state.clone(), Slot::new(1));

beacon_node/http_api/Cargo.toml

@@ -3,6 +3,7 @@ name = "http_api"
 version = "0.1.0"
 authors = ["Paul Hauner <[email protected]>"]
 edition = "2018"
+autotests = false # using a single test binary compiles faster
 
 [dependencies]
 warp = { git = "https://github.com/paulhauner/warp ", branch = "cors-wildcard" }
@@ -35,3 +36,7 @@ store = { path = "../store" }
 environment = { path = "../../lighthouse/environment" }
 tree_hash = "0.1.1"
 sensitive_url = { path = "../../common/sensitive_url" }
+
+[[test]]
+name = "bn_http_api_tests"
+path = "tests/main.rs"