Config for web3signer keep-alive (#5007)

* Allow tweaking connection pool settings

* Build docker image

* Fix imports

* Merge tag 'v4.6.0' into web3signer-keep-alive

v4.6.0

* Delete temp docker build stuff

* Fix tests

* Merge remote-tracking branch 'origin/unstable' into web3signer-keep-alive

* Update CLI text

book/src/help_vc.md

@@ -209,4 +209,9 @@ OPTIONS:
         --validators-dir <VALIDATORS_DIR>
             The directory which contains the validator keystores, deposit data for each validator along with the common
             slashing protection database and the validator_definitions.yml
+        --web3-signer-keep-alive-timeout <MILLIS>
+            Keep-alive timeout for each web3signer connection. Set to 'null' to never timeout [default: 90000]
+
+        --web3-signer-max-idle-connections <COUNT>
+            Maximum number of idle connections to maintain per web3signer host. Default is unlimited.
 ```

testing/web3signer_tests/src/lib.rs

@@ -301,10 +301,12 @@ mod tests {
             let log = environment::null_logger().unwrap();
             let validator_dir = TempDir::new().unwrap();
 
+            let config = validator_client::Config::default();
             let validator_definitions = ValidatorDefinitions::from(validator_definitions);
             let initialized_validators = InitializedValidators::from_definitions(
                 validator_definitions,
                 validator_dir.path().into(),
+                config.clone(),
                 log.clone(),
             )
             .await
@@ -331,7 +333,6 @@ mod tests {
 
             let slot_clock =
                 TestingSlotClock::new(Slot::new(0), Duration::from_secs(0), Duration::from_secs(1));
-            let config = validator_client::Config::default();
 
             let validator_store = ValidatorStore::<_, E>::new(
                 initialized_validators,

validator_client/src/cli.rs

@@ -367,4 +367,24 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     constructed by builders, regardless of payload value.")
                 .takes_value(false),
         )
+        /*
+         * Experimental/development options.
+         */
+        .arg(
+            Arg::with_name("web3-signer-keep-alive-timeout")
+                .long("web3-signer-keep-alive-timeout")
+                .value_name("MILLIS")
+                .default_value("90000")
+                .help("Keep-alive timeout for each web3signer connection. Set to 'null' to never \
+                       timeout")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("web3-signer-max-idle-connections")
+                .long("web3-signer-max-idle-connections")
+                .value_name("COUNT")
+                .help("Maximum number of idle connections to maintain per web3signer host. Default \
+                       is unlimited.")
+                .takes_value(true),
+        )
 }

validator_client/src/config.rs

@@ -14,6 +14,7 @@ use slog::{info, warn, Logger};
 use std::fs;
 use std::net::IpAddr;
 use std::path::PathBuf;
+use std::time::Duration;
 use types::{Address, GRAFFITI_BYTES_LEN};
 
 pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/";
@@ -81,6 +82,8 @@ pub struct Config {
     pub builder_boost_factor: Option<u64>,
     /// If true, Lighthouse will prefer builder proposals, if available.
     pub prefer_builder_proposals: bool,
+    pub web3_signer_keep_alive_timeout: Option<Duration>,
+    pub web3_signer_max_idle_connections: Option<usize>,
 }
 
 impl Default for Config {
@@ -124,6 +127,8 @@ impl Default for Config {
             produce_block_v3: false,
             builder_boost_factor: None,
             prefer_builder_proposals: false,
+            web3_signer_keep_alive_timeout: Some(Duration::from_secs(90)),
+            web3_signer_max_idle_connections: None,
         }
     }
 }
@@ -245,6 +250,22 @@ impl Config {
                 .collect::<Result<_, _>>()?;
         }
 
+        /*
+         * Web3 signer
+         */
+        if let Some(s) = parse_optional::<String>(cli_args, "web3-signer-keep-alive-timeout")? {
+            config.web3_signer_keep_alive_timeout = if s == "null" {
+                None
+            } else {
+                Some(Duration::from_millis(
+                    s.parse().map_err(|_| "invalid timeout value".to_string())?,
+                ))
+            }
+        }
+        if let Some(n) = parse_optional::<usize>(cli_args, "web3-signer-max-idle-connections")? {
+            config.web3_signer_max_idle_connections = Some(n);
+        }
+
         /*
          * Http API server
          */

validator_client/src/http_api/test_utils.rs

@@ -80,6 +80,7 @@ impl ApiTester {
         let initialized_validators = InitializedValidators::from_definitions(
             validator_defs,
             validator_dir.path().into(),
+            Default::default(),
             log.clone(),
         )
         .await

validator_client/src/http_api/tests.rs

@@ -68,6 +68,7 @@ impl ApiTester {
         let initialized_validators = InitializedValidators::from_definitions(
             validator_defs,
             validator_dir.path().into(),
+            Config::default(),
             log.clone(),
         )
         .await

validator_client/src/initialized_validators.rs

@@ -34,6 +34,7 @@ use validator_dir::Builder as ValidatorDirBuilder;
 
 use crate::key_cache;
 use crate::key_cache::KeyCache;
+use crate::Config;
 
 /// Default timeout for a request to a remote signer for a signature.
 ///
@@ -208,6 +209,7 @@ impl InitializedValidator {
         key_cache: &mut KeyCache,
         key_stores: &mut HashMap<PathBuf, Keystore>,
         web3_signer_client_map: &mut Option<HashMap<Web3SignerDefinition, Client>>,
+        config: &Config,
     ) -> Result<Self, Error> {
         if !def.enabled {
             return Err(Error::UnableToInitializeDisabledValidator);
@@ -311,6 +313,8 @@ impl InitializedValidator {
                                 web3_signer.client_identity_path.clone(),
                                 web3_signer.client_identity_password.clone(),
                                 request_timeout,
+                                config.web3_signer_keep_alive_timeout,
+                                config.web3_signer_max_idle_connections,
                             )?;
                             client_map.insert(web3_signer, client.clone());
                             client
@@ -325,6 +329,8 @@ impl InitializedValidator {
                         web3_signer.client_identity_path.clone(),
                         web3_signer.client_identity_password.clone(),
                         request_timeout,
+                        config.web3_signer_keep_alive_timeout,
+                        config.web3_signer_max_idle_connections,
                     )?;
                     new_web3_signer_client_map.insert(web3_signer, client.clone());
                     *web3_signer_client_map = Some(new_web3_signer_client_map);
@@ -393,8 +399,13 @@ fn build_web3_signer_client(
     client_identity_path: Option<PathBuf>,
     client_identity_password: Option<String>,
     request_timeout: Duration,
+    keep_alive_timeout: Option<Duration>,
+    max_idle_connections: Option<usize>,
 ) -> Result<Client, Error> {
-    let builder = Client::builder().timeout(request_timeout);
+    let builder = Client::builder()
+        .timeout(request_timeout)
+        .pool_idle_timeout(keep_alive_timeout)
+        .pool_max_idle_per_host(max_idle_connections.unwrap_or(usize::MAX));
 
     let builder = if let Some(path) = root_certificate_path {
         let certificate = load_pem_certificate(path)?;
@@ -475,20 +486,23 @@ pub struct InitializedValidators {
     web3_signer_client_map: Option<HashMap<Web3SignerDefinition, Client>>,
     /// For logging via `slog`.
     log: Logger,
+    config: Config,
 }
 
 impl InitializedValidators {
     /// Instantiates `Self`, initializing all validators in `definitions`.
     pub async fn from_definitions(
         definitions: ValidatorDefinitions,
         validators_dir: PathBuf,
+        config: Config,
         log: Logger,
     ) -> Result<Self, Error> {
         let mut this = Self {
             validators_dir,
             definitions,
             validators: HashMap::default(),
             web3_signer_client_map: None,
+            config,
             log,
         };
         this.update_validators().await?;
@@ -1234,6 +1248,7 @@ impl InitializedValidators {
                             &mut key_cache,
                             &mut key_stores,
                             &mut None,
+                            &self.config,
                         )
                         .await
                         {
@@ -1284,6 +1299,7 @@ impl InitializedValidators {
                             &mut key_cache,
                             &mut key_stores,
                             &mut self.web3_signer_client_map,
+                            &self.config,
                         )
                         .await
                         {

validator_client/src/lib.rs

@@ -192,6 +192,7 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
         let validators = InitializedValidators::from_definitions(
             validator_defs,
             config.validator_dir.clone(),
+            config.clone(),
             log.clone(),
         )
         .await