Implement Healthcheck status for keys loading

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 
 ## Upcoming version
+### Features Added
+- Enhanced Healthcheck endpoint reporting status of loading of signers keys [#738](https://github.com/ConsenSys/web3signer/pull/738)
+- Optional AWS endpoint overriding for bulk loading `--aws-endpoint-override`. Useful for local testing against localstack. [#730](https://github.com/ConsenSys/web3signer/issues/730)
+
 ### Bugs Fixed
 - Update of Azure libraries (transitive via signers library) and manual override to fix CVE-2023-1370
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/Signer.java

@@ -61,6 +61,7 @@ public class Signer extends FilecoinJsonRpcEndpoint {
   public static final ObjectMapper ETH_2_INTERFACE_OBJECT_MAPPER =
       SigningObjectMapperFactory.createObjectMapper().setSerializationInclusion(Include.NON_NULL);
   private static final String METRICS_ENDPOINT = "/metrics";
+  private static final String HEALTHCHECK_ENDPOINT = "/healthcheck";
 
   private final SignerConfiguration signerConfig;
   private final Web3SignerRunner runner;
@@ -185,4 +186,8 @@ public String getSlashingDbUrl() {
   public Response callReload() {
     return given().baseUri(getUrl()).post(RELOAD_ENDPOINT);
   }
+
+  public Response healthcheck() {
+    return given().baseUri(getUrl()).get(HEALTHCHECK_ENDPOINT);
+  }
 }

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsConfigFileImpl.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.web3signer.dsl.signer.runner;
 
+import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
@@ -422,6 +423,16 @@ private String awsBulkLoadingOptions(
               String.join(",", awsSecretsManagerParameters.getTagValuesFilter())));
     }
 
+    awsSecretsManagerParameters
+        .getEndpointOverride()
+        .ifPresent(
+            uri ->
+                yamlConfig.append(
+                    String.format(
+                        YAML_STRING_FMT,
+                        "eth2." + AWS_ENDPOINT_OVERRIDE_OPTION.substring(2),
+                        uri)));
+
     return yamlConfig.toString();
   }
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/dsl/signer/runner/CmdLineParamsDefaultImpl.java

@@ -12,6 +12,7 @@
  */
 package tech.pegasys.web3signer.dsl.signer.runner;
 
+import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_ENDPOINT_OVERRIDE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ACCESS_KEY_ID_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_AUTH_MODE_OPTION;
 import static tech.pegasys.web3signer.commandline.PicoCliAwsSecretsManagerParameters.AWS_SECRETS_ENABLED_OPTION;
@@ -255,6 +256,14 @@ private Collection<String> awsBulkLoadingOptions(
       params.add(awsSecretsManagerParameters.getRegion());
     }
 
+    awsSecretsManagerParameters
+        .getEndpointOverride()
+        .ifPresent(
+            uri -> {
+              params.add(AWS_ENDPOINT_OVERRIDE_OPTION);
+              params.add(uri.toString());
+            });
+
     if (!awsSecretsManagerParameters.getPrefixesFilter().isEmpty()) {
       params.add(AWS_SECRETS_PREFIXES_FILTER_OPTION);
       params.add(String.join(",", awsSecretsManagerParameters.getPrefixesFilter()));

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerAcceptanceTest.java

@@ -13,6 +13,7 @@
 package tech.pegasys.web3signer.tests.bulkloading;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
 import tech.pegasys.teku.bls.BLSKeyPair;
@@ -24,6 +25,7 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
@@ -69,13 +71,20 @@ public class AwsSecretsManagerAcceptanceTest extends AcceptanceTestBase {
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private AwsSecretsManagerUtil awsSecretsManagerUtil;
   private final List<BLSKeyPair> blsKeyPairs = new ArrayList<>();
 
   @BeforeAll
   void setupAwsResources() {
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     final SecureRandom secureRandom = new SecureRandom();
 
     for (int i = 0; i < 4; i++) {
@@ -88,7 +97,7 @@ void setupAwsResources() {
     }
   }
 
-  @ParameterizedTest
+  @ParameterizedTest(name = "{index} - Using config file: {0}")
   @ValueSource(booleans = {true, false})
   void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean useConfigFile) {
     final AwsSecretsManagerParameters awsSecretsManagerParameters =
@@ -100,6 +109,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("TagName0", "TagName1"))
             .withTagValuesFilter(List.of("TagValue0", "TagValue1", "TagValue2"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =
@@ -110,6 +120,13 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
 
     startSigner(configBuilder.build());
 
+    signer
+        .healthcheck()
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("status", equalTo("UP"));
+
     signer
         .callApiPublicKeys(KeyType.BLS)
         .then()
@@ -124,7 +141,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             hasSize(2));
   }
 
-  @ParameterizedTest
+  @ParameterizedTest(name = "{index} - Using config file: {0}")
   @ValueSource(booleans = {true, false})
   void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPublicApi(
       final boolean useConfigFile) {
@@ -134,6 +151,7 @@ void secretsAreLoadedFromAWSSecretsManagerWithEnvironmentAuthModeAndReportedByPu
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("TagName2", "TagName3"))
             .withTagValuesFilter(List.of("TagValue0", "TagValue2", "TagValue3"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerMultiValueAcceptanceTest.java

@@ -25,6 +25,7 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -70,6 +71,12 @@ public class AwsSecretsManagerMultiValueAcceptanceTest extends AcceptanceTestBas
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private AwsSecretsManagerUtil awsSecretsManagerUtil;
   private final List<BLSKeyPair> blsKeyPairList = new ArrayList<>(400);
 
@@ -80,7 +87,8 @@ void setupAwsResources() {
     }
 
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
 
     for (int i = 0; i < 4; i++) {
       String multilinePrivKeys =
@@ -103,6 +111,7 @@ void secretsAreLoadedFromAWSSecretsManagerAndReportedByPublicApi(final boolean u
             .withSecretAccessKey(RO_AWS_SECRET_ACCESS_KEY)
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
             .withTagNamesFilter(List.of("multivalue"))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AwsSecretsManagerPerformanceAcceptanceTest.java

@@ -23,6 +23,7 @@
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParametersBuilder;
 import tech.pegasys.web3signer.tests.AcceptanceTestBase;
 
+import java.net.URI;
 import java.security.SecureRandom;
 import java.time.Duration;
 import java.util.Collections;
@@ -79,6 +80,12 @@ public class AwsSecretsManagerPerformanceAcceptanceTest extends AcceptanceTestBa
   private static final String RO_AWS_SECRET_ACCESS_KEY = System.getenv("AWS_SECRET_ACCESS_KEY");
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
   private static final Integer NUMBER_OF_KEYS =
       Integer.parseInt(Optional.ofNullable(System.getenv("AWS_PERF_AT_KEYS_NUM")).orElse("1000"));
   private static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(10);
@@ -90,7 +97,8 @@ void setupAwsResources() {
     final StopWatch stopWatch = new StopWatch();
     stopWatch.start();
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     final SecureRandom secureRandom = new SecureRandom();
     LOG.info("Creating {} AWS keys in Secrets Manager in {}", NUMBER_OF_KEYS, AWS_REGION);
     blsKeyPairs =
@@ -119,6 +127,7 @@ void largeNumberOfKeysAreLoadedSuccessfully() {
             .withAccessKeyId(RO_AWS_ACCESS_KEY_ID)
             .withSecretAccessKey(RO_AWS_SECRET_ACCESS_KEY)
             .withPrefixesFilter(List.of(awsSecretsManagerUtil.getSecretsManagerPrefix()))
+            .withEndpointOverride(awsEndpointOverride)
             .build();
 
     final SignerConfigurationBuilder configBuilder =

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/bulkloading/AzureKeyVaultAcceptanceTest.java

@@ -13,6 +13,7 @@
 package tech.pegasys.web3signer.tests.bulkloading;
 
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 
 import tech.pegasys.web3signer.dsl.signer.SignerConfigurationBuilder;
@@ -58,6 +59,13 @@ void ensureSecretsInKeyVaultAreLoadedAndReportedViaPublicKeysApi() {
 
     final Response response = signer.callApiPublicKeys(KeyType.BLS);
     response.then().statusCode(200).contentType(ContentType.JSON).body("", contains(EXPECTED_KEY));
+
+    signer
+        .healthcheck()
+        .then()
+        .statusCode(200)
+        .contentType(ContentType.JSON)
+        .body("status", equalTo("UP"));
   }
 
   @Test
@@ -72,6 +80,13 @@ void invalidVaultParametersFailsToLoadKeys() {
 
     final Response response = signer.callApiPublicKeys(KeyType.BLS);
     response.then().statusCode(200).contentType(ContentType.JSON).body("", hasSize(0));
+
+    signer
+        .healthcheck()
+        .then()
+        .statusCode(503)
+        .contentType(ContentType.JSON)
+        .body("status", equalTo("DOWN"));
   }
 
   @Test

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/publickeys/AwsKeyIdentifiersAcceptanceTest.java

@@ -17,6 +17,7 @@
 
 import tech.pegasys.web3signer.AwsSecretsManagerUtil;
 
+import java.net.URI;
 import java.util.Collections;
 import java.util.Optional;
 
@@ -59,6 +60,12 @@ public class AwsKeyIdentifiersAcceptanceTest extends KeyIdentifiersAcceptanceTes
   private static final String AWS_REGION =
       Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
 
+  // can be pointed to localstack
+  private final Optional<URI> awsEndpointOverride =
+      System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+          ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+          : Optional.empty();
+
   private final String privateKey = privateKeys(BLS)[0]; // secret value
   private final String publicKey = BLS_PUBLIC_KEY_1;
 
@@ -67,7 +74,8 @@ public class AwsKeyIdentifiersAcceptanceTest extends KeyIdentifiersAcceptanceTes
   @BeforeAll
   void setup() {
     awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY);
+        new AwsSecretsManagerUtil(
+            AWS_REGION, RW_AWS_ACCESS_KEY_ID, RW_AWS_SECRET_ACCESS_KEY, awsEndpointOverride);
     awsSecretsManagerUtil.createSecret(publicKey, privateKey, Collections.emptyMap());
   }
 

acceptance-tests/src/test/java/tech/pegasys/web3signer/tests/signing/BlsSigningAcceptanceTest.java

@@ -35,6 +35,7 @@
 import tech.pegasys.web3signer.dsl.utils.MetadataFileHelpers;
 import tech.pegasys.web3signer.signing.KeyType;
 
+import java.net.URI;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Optional;
@@ -178,10 +179,16 @@ public void ableToSignUsingAws() throws JsonProcessingException {
     final String roAwsAccessKeyId = System.getenv("AWS_ACCESS_KEY_ID");
     final String roAwsSecretAccessKey = System.getenv("AWS_SECRET_ACCESS_KEY");
     final String region = Optional.ofNullable(System.getenv("AWS_REGION")).orElse("us-east-2");
+    // can be pointed to localstack
+    final Optional<URI> awsEndpointOverride =
+        System.getenv("AWS_ENDPOINT_OVERRIDE") != null
+            ? Optional.of(URI.create(System.getenv("AWS_ENDPOINT_OVERRIDE")))
+            : Optional.empty();
     final String publicKey = KEY_PAIR.getPublicKey().toString();
 
     final AwsSecretsManagerUtil awsSecretsManagerUtil =
-        new AwsSecretsManagerUtil(region, rwAwsAccessKeyId, rwAwsSecretAccessKey);
+        new AwsSecretsManagerUtil(
+            region, rwAwsAccessKeyId, rwAwsSecretAccessKey, awsEndpointOverride);
 
     awsSecretsManagerUtil.createSecret(publicKey, PRIVATE_KEY, Collections.emptyMap());
     final String fullyPrefixKeyName = awsSecretsManagerUtil.getSecretsManagerPrefix() + publicKey;

build.gradle

@@ -113,6 +113,7 @@ allprojects {
   targetCompatibility = 11
 
   repositories {
+    mavenLocal() //TODO: Remove me
     mavenCentral()
     maven { url "https://artifacts.consensys.net/public/maven/maven/" }
     maven { url "https://artifacts.consensys.net/public/teku/maven/" }

commandline/src/main/java/tech/pegasys/web3signer/commandline/PicoCliAwsSecretsManagerParameters.java

@@ -15,9 +15,11 @@
 import tech.pegasys.web3signer.signing.config.AwsAuthenticationMode;
 import tech.pegasys.web3signer.signing.config.AwsSecretsManagerParameters;
 
+import java.net.URI;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
 import picocli.CommandLine;
 import picocli.CommandLine.Option;
@@ -29,6 +31,7 @@ public class PicoCliAwsSecretsManagerParameters implements AwsSecretsManagerPara
   public static final String AWS_SECRETS_SECRET_ACCESS_KEY_OPTION =
       "--aws-secrets-secret-access-key";
   public static final String AWS_SECRETS_REGION_OPTION = "--aws-secrets-region";
+  public static final String AWS_ENDPOINT_OVERRIDE_OPTION = "--aws-endpoint-override";
   public static final String AWS_SECRETS_PREFIXES_FILTER_OPTION = "--aws-secrets-prefixes-filter";
   public static final String AWS_SECRETS_TAG_NAMES_FILTER_OPTION = "--aws-secrets-tag-names-filter";
   public static final String AWS_SECRETS_TAG_VALUES_FILTER_OPTION =
@@ -72,6 +75,12 @@ public class PicoCliAwsSecretsManagerParameters implements AwsSecretsManagerPara
       paramLabel = "<Region>")
   private String region;
 
+  @Option(
+      names = {AWS_ENDPOINT_OVERRIDE_OPTION},
+      description = "Override AWS endpoint.",
+      paramLabel = "<URI>")
+  private Optional<URI> endpointOverride;
+
   @Option(
       names = AWS_SECRETS_PREFIXES_FILTER_OPTION,
       description =
@@ -147,4 +156,9 @@ public Collection<String> getTagNamesFilter() {
   public Collection<String> getTagValuesFilter() {
     return tagValuesFilter;
   }
+
+  @Override
+  public Optional<URI> getEndpointOverride() {
+    return endpointOverride;
+  }
 }

core/src/main/java/tech/pegasys/web3signer/core/Eth1Runner.java

@@ -106,7 +106,8 @@ protected ArtifactSignerProvider createArtifactSignerProvider(
                     "yaml",
                     new YamlSignerParser(
                         List.of(ethSecpArtifactSignerFactory),
-                        YamlMapperFactory.createYamlMapper(config.getKeyStoreConfigFileMaxSize())));
+                        YamlMapperFactory.createYamlMapper(config.getKeyStoreConfigFileMaxSize())))
+                .getValues();
           }
         });
   }