Metrics Push Gateway Options

metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/MetricsConfiguration.java

@@ -22,13 +22,15 @@ public class MetricsConfiguration {
   private static final String DEFAULT_METRICS_HOST = "127.0.0.1";
   public static final int DEFAULT_METRICS_PORT = 9545;
 
-  public static final String MODE_PUSH_GATEWAY = "push";
-  public static final String MODE_SERVER_PULL = "pull";
+  private static final String DEFAULT_METRICS_PUSH_HOST = "127.0.0.1";
+  public static final int DEFAULT_METRICS_PUSH_PORT = 9001;
 
   private boolean enabled;
   private int port;
   private String host;
-  private String mode;
+  private boolean pushEnabled;
+  private int pushPort;
+  private String pushHost;
   private int pushInterval;
   private String prometheusJob;
   private Collection<String> hostsWhitelist = Collections.singletonList("localhost");
@@ -38,7 +40,9 @@ public static MetricsConfiguration createDefault() {
     metricsConfiguration.setEnabled(false);
     metricsConfiguration.setPort(DEFAULT_METRICS_PORT);
     metricsConfiguration.setHost(DEFAULT_METRICS_HOST);
-    metricsConfiguration.setMode(MODE_SERVER_PULL);
+    metricsConfiguration.setPushEnabled(false);
+    metricsConfiguration.setPushPort(DEFAULT_METRICS_PUSH_PORT);
+    metricsConfiguration.setPushHost(DEFAULT_METRICS_PUSH_HOST);
     metricsConfiguration.setPushInterval(15);
     metricsConfiguration.setPrometheusJob("pantheon-client");
 
@@ -71,12 +75,28 @@ public void setHost(final String host) {
     this.host = host;
   }
 
-  public String getMode() {
-    return mode;
+  public int getPushPort() {
+    return pushPort;
   }
 
-  public void setMode(final String mode) {
-    this.mode = mode;
+  public void setPushPort(final int pushPort) {
+    this.pushPort = pushPort;
+  }
+
+  public String getPushHost() {
+    return pushHost;
+  }
+
+  public void setPushHost(final String pushHost) {
+    this.pushHost = pushHost;
+  }
+
+  public boolean isPushEnabled() {
+    return pushEnabled;
+  }
+
+  public void setPushEnabled(final boolean pushEnabled) {
+    this.pushEnabled = pushEnabled;
   }
 
   public int getPushInterval() {
@@ -113,6 +133,18 @@ public String toString() {
         + ", host='"
         + host
         + '\''
+        + ", pushEnabled="
+        + pushEnabled
+        + ", pushPort="
+        + pushPort
+        + ", pushHost='"
+        + pushHost
+        + '\''
+        + ", pushInterval="
+        + pushInterval
+        + ", prometheusJob='"
+        + prometheusJob
+        + '\''
         + ", hostsWhitelist="
         + hostsWhitelist
         + '}';
@@ -125,13 +157,27 @@ public boolean equals(final Object o) {
     final MetricsConfiguration that = (MetricsConfiguration) o;
     return enabled == that.enabled
         && port == that.port
+        && pushEnabled == that.pushEnabled
+        && pushPort == that.pushPort
+        && pushInterval == that.pushInterval
         && Objects.equals(host, that.host)
+        && Objects.equals(pushHost, that.pushHost)
+        && Objects.equals(prometheusJob, that.prometheusJob)
         && com.google.common.base.Objects.equal(
             Lists.newArrayList(hostsWhitelist), Lists.newArrayList(that.hostsWhitelist));
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(enabled, port, host, hostsWhitelist);
+    return Objects.hash(
+        enabled,
+        port,
+        host,
+        pushEnabled,
+        pushPort,
+        pushHost,
+        pushInterval,
+        prometheusJob,
+        hostsWhitelist);
   }
 }

metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/MetricsHttpService.java

@@ -73,12 +73,8 @@ private void validateConfig(final MetricsConfiguration config) {
         "Invalid port configuration.");
     checkArgument(config.getHost() != null, "Required host is not configured.");
     checkArgument(
-        MetricsConfiguration.MODE_SERVER_PULL.equals(config.getMode()),
-        "Metrics Http Service cannot start up outside of '"
-            + MetricsConfiguration.MODE_SERVER_PULL
-            + "' mode, requested mode is '"
-            + config.getMode()
-            + "'.");
+        !(config.isEnabled() && config.isPushEnabled()),
+        "Metrics Http Service cannot run concurrent with push metrics.");
   }
 
   @Override

metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/MetricsPushGatewayService.java

@@ -45,24 +45,22 @@ class MetricsPushGatewayService implements MetricsService {
 
   private void validateConfig(final MetricsConfiguration config) {
     checkArgument(
-        config.getPort() == 0 || NetworkUtility.isValidPort(config.getPort()),
+        config.getPushPort() == 0 || NetworkUtility.isValidPort(config.getPushPort()),
         "Invalid port configuration.");
-    checkArgument(config.getHost() != null, "Required host is not configured.");
+    checkArgument(config.getPushHost() != null, "Required host is not configured.");
     checkArgument(
-        MetricsConfiguration.MODE_PUSH_GATEWAY.equals(config.getMode()),
-        "Metrics Push Gateway Service cannot start up outside of '"
-            + MetricsConfiguration.MODE_PUSH_GATEWAY
-            + "' mode.");
+        !(config.isEnabled() && config.isPushEnabled()),
+        "Metrics Push Gateway Service cannot run concurrent with the normal metrics.");
     checkArgument(
         metricsSystem instanceof PrometheusMetricsSystem,
         "Push Gateway requires a Prometheus Metrics System.");
   }
 
   @Override
   public CompletableFuture<?> start() {
-    LOG.info("Starting Metrics service on {}:{}", config.getHost(), config.getPort());
+    LOG.info("Starting Metrics service on {}:{}", config.getPushHost(), config.getPushPort());
 
-    pushGateway = new PushGateway(config.getHost() + ":" + config.getPort());
+    pushGateway = new PushGateway(config.getPushHost() + ":" + config.getPushPort());
 
     // Create the executor
     scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
@@ -81,9 +79,15 @@ public CompletableFuture<?> stop() {
       // Calling shutdown now cancels the pending push, which is desirable.
       scheduledExecutorService.shutdownNow();
       scheduledExecutorService.awaitTermination(30, TimeUnit.SECONDS);
-      pushGateway.delete(config.getPrometheusJob());
+      try {
+        pushGateway.delete(config.getPrometheusJob());
+      } catch (final Exception e) {
+        LOG.error("Could not clean up results on the  Prometheus Push Gateway.", e);
+        // Do not complete exceptionally, the gateway may be down and failures
+        // here cause the shutdown to loop.  Failure is acceptable.
+      }
       resultFuture.complete(null);
-    } catch (final IOException | InterruptedException e) {
+    } catch (final InterruptedException e) {
       LOG.error(e);
       resultFuture.completeExceptionally(e);
     }

metrics/src/main/java/tech/pegasys/pantheon/metrics/prometheus/MetricsService.java

@@ -25,13 +25,12 @@ static MetricsService create(
       final Vertx vertx,
       final MetricsConfiguration configuration,
       final MetricsSystem metricsSystem) {
-    switch (configuration.getMode()) {
-      case MetricsConfiguration.MODE_PUSH_GATEWAY:
-        return new MetricsPushGatewayService(configuration, metricsSystem);
-      case MetricsConfiguration.MODE_SERVER_PULL:
-        return new MetricsHttpService(vertx, configuration, metricsSystem);
-      default:
-        throw new RuntimeException("No metrics service for mode '" + configuration.getMode() + "'");
+    if (configuration.isEnabled()) {
+      return new MetricsPushGatewayService(configuration, metricsSystem);
+    } else if (configuration.isPushEnabled()) {
+      return new MetricsHttpService(vertx, configuration, metricsSystem);
+    } else {
+      throw new RuntimeException("No metrics service enabled.");
     }
   }
 

pantheon/src/main/java/tech/pegasys/pantheon/cli/PantheonCommand.java

@@ -22,6 +22,7 @@
 import static tech.pegasys.pantheon.ethereum.jsonrpc.websocket.WebSocketConfiguration.DEFAULT_WEBSOCKET_REFRESH_DELAY;
 import static tech.pegasys.pantheon.ethereum.p2p.peers.DefaultPeer.DEFAULT_PORT;
 import static tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PORT;
+import static tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration.DEFAULT_METRICS_PUSH_PORT;
 import static tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration.createDefault;
 
 import tech.pegasys.pantheon.PermissioningConfigurationBuilder;
@@ -351,13 +352,6 @@ private Long configureRefreshDelay(final Long refreshDelay) {
   )
   private final Boolean isMetricsEnabled = false;
 
-  @Option(
-    names = {"--metrics-mode"},
-    description =
-        "Mode for the metrics service to run in, 'push' or 'pull' (default: ${DEFAULT-VALUE})"
-  )
-  private String metricsMode = MetricsConfiguration.MODE_SERVER_PULL;
-
   @Option(
     names = {"--metrics-host"},
     paramLabel = MANDATORY_HOST_FORMAT_HELP,
@@ -375,6 +369,30 @@ private Long configureRefreshDelay(final Long refreshDelay) {
   )
   private final Integer metricsPort = DEFAULT_METRICS_PORT;
 
+  @Option(
+    names = {"--metrics-push-enabled"},
+    description =
+        "Set if the metrics push gateway integration should be started (default: ${DEFAULT-VALUE})"
+  )
+  private Boolean isMetricsPushEnabled = false;
+
+  @Option(
+    names = {"--metrics-push-host"},
+    paramLabel = MANDATORY_HOST_FORMAT_HELP,
+    description = "Host of the Prometheus Push Gateway for push mode (default: ${DEFAULT-VALUE})",
+    arity = "1"
+  )
+  private final HostSpecifier metricsPushHost =
+      HostSpecifier.fromValid(autoDiscoverDefaultIP().getHostAddress());
+
+  @Option(
+    names = {"--metrics-push-port"},
+    paramLabel = MANDATORY_PORT_FORMAT_HELP,
+    description = "Port of the Prometheus Push Gateway for push mode (default: ${DEFAULT-VALUE})",
+    arity = "1"
+  )
+  private final Integer metricsPushPort = DEFAULT_METRICS_PUSH_PORT;
+
   @Option(
     names = {"--metrics-push-interval"},
     paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
@@ -385,7 +403,7 @@ private Long configureRefreshDelay(final Long refreshDelay) {
   private final Integer metricsPushInterval = 15;
 
   @Option(
-    names = {"--metrics-prometheus-job"},
+    names = {"--metrics-push-prometheus-job"},
     description = "Job name to use when in push mode (default: ${DEFAULT-VALUE})",
     arity = "1"
   )
@@ -630,11 +648,20 @@ private WebSocketConfiguration webSocketConfiguration() {
   }
 
   MetricsConfiguration metricsConfiguration() {
+    if (isMetricsEnabled && isMetricsPushEnabled) {
+      throw new ParameterException(
+          new CommandLine(this),
+          "--metrics-enabled option and --metrics-push-enabled option can't be used at the same "
+              + "time.  Please refer to CLI reference for more details about this constraint.");
+    }
+
     final MetricsConfiguration metricsConfiguration = createDefault();
     metricsConfiguration.setEnabled(isMetricsEnabled);
-    metricsConfiguration.setMode(metricsMode);
     metricsConfiguration.setHost(metricsHost.toString());
     metricsConfiguration.setPort(metricsPort);
+    metricsConfiguration.setPushEnabled(isMetricsPushEnabled);
+    metricsConfiguration.setPushHost(metricsPushHost.toString());
+    metricsConfiguration.setPushPort(metricsPushPort);
     metricsConfiguration.setPushInterval(metricsPushInterval);
     metricsConfiguration.setPrometheusJob(metricsPrometheusJob);
     metricsConfiguration.setHostsWhitelist(hostsWhitelist);
@@ -752,8 +779,8 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) {
         // conflict error
         throw new ParameterException(
             new CommandLine(this),
-            "--network option and --genesis-file option can't be used at the same time."
-                + "Please refer to CLI reference for more details about this constraint.");
+            "--network option and --genesis-file option can't be used at the same time.  Please "
+                + "refer to CLI reference for more details about this constraint.");
       }
 
       builder.setGenesisConfig(genesisConfig());

pantheon/src/test/java/tech/pegasys/pantheon/cli/PantheonCommandTest.java

@@ -1248,13 +1248,29 @@ public void metricsHostAndPortOptionMustBeUsed() {
   }
 
   @Test
-  public void metricsModeOptionMustBeUsed() {
-    parseCommand("--metrics-mode", "pull");
+  public void metricsPushEnabledPropertyMustBeUsed() {
+    parseCommand("--metrics-push-enabled");
 
     verify(mockRunnerBuilder).metricsConfiguration(metricsConfigArgumentCaptor.capture());
     verify(mockRunnerBuilder).build();
 
-    assertThat(metricsConfigArgumentCaptor.getValue().getMode()).isEqualTo("pull");
+    assertThat(metricsConfigArgumentCaptor.getValue().isPushEnabled()).isTrue();
+
+    assertThat(commandOutput.toString()).isEmpty();
+    assertThat(commandErrorOutput.toString()).isEmpty();
+  }
+
+  @Test
+  public void metricsPushHostAndPushPortOptionMustBeUsed() {
+    final String host = "1.2.3.4";
+    final int port = 1234;
+    parseCommand("--metrics-push-host", host, "--metrics-push-port", String.valueOf(port));
+
+    verify(mockRunnerBuilder).metricsConfiguration(metricsConfigArgumentCaptor.capture());
+    verify(mockRunnerBuilder).build();
+
+    assertThat(metricsConfigArgumentCaptor.getValue().getPushHost()).isEqualTo(host);
+    assertThat(metricsConfigArgumentCaptor.getValue().getPushPort()).isEqualTo(port);
 
     assertThat(commandOutput.toString()).isEmpty();
     assertThat(commandErrorOutput.toString()).isEmpty();
@@ -1275,7 +1291,7 @@ public void metricsPushIntervalMustBeUsed() {
 
   @Test
   public void metricsPrometheusJobMustBeUsed() {
-    parseCommand("--metrics-prometheus-job", "pantheon-command-test");
+    parseCommand("--metrics-push-prometheus-job", "pantheon-command-test");
 
     verify(mockRunnerBuilder).metricsConfiguration(metricsConfigArgumentCaptor.capture());
     verify(mockRunnerBuilder).build();
@@ -1287,6 +1303,23 @@ public void metricsPrometheusJobMustBeUsed() {
     assertThat(commandErrorOutput.toString()).isEmpty();
   }
 
+  @Test
+  public void metricsAndMetricsPushMustNotBeUsedTogether() throws Exception {
+    assumeTrue(isFullInstantiation());
+
+    final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON);
+    final ArgumentCaptor<EthNetworkConfig> networkArg =
+        ArgumentCaptor.forClass(EthNetworkConfig.class);
+
+    parseCommand("--metrics-enabled", "--metrics-push-enabled");
+
+    verifyZeroInteractions(mockRunnerBuilder);
+
+    assertThat(commandOutput.toString()).isEmpty();
+    assertThat(commandErrorOutput.toString())
+        .startsWith("--metrics-enabled option and --metrics-push-enabled option can't be used");
+  }
+
   @Test
   public void pantheonDoesNotStartInMiningModeIfCoinbaseNotSet() {
     parseCommand("--miner-enabled");

pantheon/src/test/resources/everything_config.toml

@@ -51,11 +51,13 @@ rpc-ws-refresh-delay=500
 
 # Prometheus Metrics Endpoint
 metrics-enabled=false
-metrics-mode="pull"
 metrics-host="8.6.7.5"
 metrics-port=309
+metrics-push-enabled=false
+metrics-push-host="5.5.5.1"
+metrics-push-port=212
 metrics-push-interval=42
-metrics-prometheus-job="pantheon-everything"
+metrics-push-prometheus-job="pantheon-everything"
 
 # Mining
 miner-enabled=false