Try some things

common/eth2/src/lib.rs

@@ -121,6 +121,7 @@ impl fmt::Display for Error {
 pub struct Timeouts {
     pub attestation: Duration,
     pub attester_duties: Duration,
+    pub attestation_subscriptions: Duration,
     pub liveness: Duration,
     pub proposal: Duration,
     pub proposer_duties: Duration,
@@ -137,6 +138,7 @@ impl Timeouts {
         Timeouts {
             attestation: timeout,
             attester_duties: timeout,
+            attestation_subscriptions: timeout,
             liveness: timeout,
             proposal: timeout,
             proposer_duties: timeout,
@@ -2515,7 +2517,12 @@ impl BeaconNodeHttpClient {
             .push("validator")
             .push("beacon_committee_subscriptions");
 
-        self.post(path, &subscriptions).await?;
+        self.post_with_timeout(
+            path,
+            &subscriptions,
+            self.timeouts.attestation_subscriptions,
+        )
+        .await?;
 
         Ok(())
     }

validator_client/src/beacon_node_fallback.rs

@@ -184,6 +184,11 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
         spec: &ChainSpec,
         log: &Logger,
     ) -> Result<(), CandidateError> {
+        debug!(
+            log,
+            "Refresh started";
+            "endpoint" => %self.beacon_node,
+        );
         let previous_status = self.status(RequireSynced::Yes).await;
         let was_offline = matches!(previous_status, Err(CandidateError::Offline));
 
@@ -203,6 +208,12 @@ impl<E: EthSpec> CandidateBeaconNode<E> {
         // holding a write-lock whilst we check the online status of the node.
         *self.status.write().await = new_status;
 
+        debug!(
+            log,
+            "Refresh complete";
+            "endpoint" => %self.beacon_node,
+        );
+
         new_status
     }
 
@@ -628,6 +639,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
         // First pass: try `func` on all synced and ready candidates.
         //
         // This ensures that we always choose a synced node if it is available.
+        debug!(self.log, "Starting first batch of subscription requests");
         let mut first_batch_futures = vec![];
         for candidate in &self.candidates {
             match candidate.status(RequireSynced::Yes).await {
@@ -651,13 +663,15 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
         //
         // Due to async race-conditions, it is possible that we will send a request to a candidate
         // that has been set to an offline/unready status. This is acceptable.
+        debug!(self.log, "Starting second batch of subscription requests");
         let second_batch_results = if require_synced == false {
             futures::future::join_all(retry_unsynced.into_iter().map(run_on_candidate)).await
         } else {
             vec![]
         };
 
         // Third pass: try again, attempting to make non-ready clients become ready.
+        debug!(self.log, "Starting third batch of subscription requests");
         let mut third_batch_futures = vec![];
         let mut third_batch_results = vec![];
         for candidate in to_retry {
@@ -684,6 +698,7 @@ impl<T: SlotClock, E: EthSpec> BeaconNodeFallback<T, E> {
             }
         }
         third_batch_results.extend(futures::future::join_all(third_batch_futures).await);
+        debug!(self.log, "Completed all subscription requests");
 
         let mut results = first_batch_results;
         results.extend(second_batch_results);

validator_client/src/lib.rs

@@ -75,6 +75,7 @@ const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12);
 /// This can help ensure that proper endpoint fallback occurs.
 const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4;
 const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
+const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24;
 const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4;
 const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2;
 const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4;
@@ -323,6 +324,8 @@ impl<E: EthSpec> ProductionValidatorClient<E> {
                 Timeouts {
                     attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT,
                     attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT,
+                    attestation_subscriptions: slot_duration
+                        / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT,
                     liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT,
                     proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT,
                     proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT,