Add a configurable http metrics interface

README.md

@@ -242,6 +242,9 @@ https_cert_file = ""
 // The client key file to use for talking to HTTPS checks.
 https_key_file = ""
 
+// Client address to expose API endpoints. Required in order to expose /metrics endpoint for Prometheus. Example: "127.0.0.1:8080"
+client_address = ""
+
 // The method to use for pinging external nodes. Defaults to "udp" but can
 // also be set to "socket" to use ICMP (which requires root privileges).
 ping_type = "udp"

agent.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
+	"net/http"
 	"strings"
 	"sync"
 	"time"
@@ -13,6 +15,8 @@ import (
 	"github.com/hashicorp/consul-esm/version"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/lib"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const LeaderKey = "leader"
@@ -118,6 +122,11 @@ func (a *Agent) Run() error {
 
 	var wg sync.WaitGroup
 	wg.Add(1)
+	go func() {
+		a.runMetrics()
+		wg.Done()
+	}()
+	wg.Add(1)
 	go func() {
 		a.runRegister()
 		wg.Done()
@@ -191,6 +200,35 @@ func (a *Agent) register() error {
 	return nil
 }
 
+// runMetrics is a long-running goroutine that exposes an http metrics interface
+func (a *Agent) runMetrics() {
+	if a.config.Telemetry.PrometheusRetentionTime < 1 || a.config.ClientAddress == "" {
+		return
+	}
+
+	mux := http.NewServeMux()
+	srv := &http.Server{Addr: a.config.ClientAddress, Handler: mux}
+	handlerOptions := promhttp.HandlerOpts{
+		ErrorLog:      a.logger,
+		ErrorHandling: promhttp.ContinueOnError,
+	}
+
+	mux.Handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, handlerOptions))
+	go func() {
+		<-a.shutdownCh
+		deadline := time.Now().Add(5 * time.Second)
+		ctx, cancel := context.WithDeadline(context.Background(), deadline)
+		defer cancel()
+		if err := srv.Shutdown(ctx); err != nil {
+			a.logger.Printf("[ERR] Failed to shutdown metrics interface: %v", err)
+		}
+	}()
+
+	if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+		a.logger.Printf("[ERR] Failed to open metrics interface: %v", err)
+	}
+}
+
 // runRegister is a long-running goroutine that ensures this agent is registered
 // with Consul's service discovery. It will run until the shutdownCh is closed.
 func (a *Agent) runRegister() {

config.go

@@ -55,6 +55,8 @@ type Config struct {
 	HTTPSCertFile string
 	HTTPSKeyFile  string
 
+	ClientAddress string
+
 	PingType string
 
 	DisableCoordinateUpdates bool
@@ -178,6 +180,8 @@ type HumanConfig struct {
 	HTTPSCertFile flags.StringValue `mapstructure:"https_cert_file"`
 	HTTPSKeyFile  flags.StringValue `mapstructure:"https_key_file"`
 
+	ClientAddress flags.StringValue `mapstructure:"client_address"`
+
 	PingType flags.StringValue `mapstructure:"ping_type"`
 
 	DisableCoordinateUpdates flags.BoolValue `mapstructure:"disable_coordinate_updates"`
@@ -466,6 +470,7 @@ func MergeConfig(dst *Config, src *HumanConfig) error {
 	src.HTTPSCAPath.Merge(&dst.HTTPSCAPath)
 	src.HTTPSCertFile.Merge(&dst.HTTPSCertFile)
 	src.HTTPSKeyFile.Merge(&dst.HTTPSKeyFile)
+	src.ClientAddress.Merge(&dst.ClientAddress)
 	src.PingType.Merge(&dst.PingType)
 	src.DisableCoordinateUpdates.Merge(&dst.DisableCoordinateUpdates)
 	if len(src.Telemetry) == 1 {

config_test.go

@@ -37,6 +37,7 @@ https_ca_path = "CAPath/"
 https_cert_file = "server-cert.pem"
 https_key_file = "server-key.pem"
 disable_coordinate_updates = true
+client_address = "127.0.0.1:8080"
 ping_type = "socket"
 telemetry {
 	circonus_api_app = "circonus_api_app"
@@ -91,6 +92,7 @@ critical_threshold = 2
 		HTTPSCertFile:            "server-cert.pem",
 		HTTPSKeyFile:             "server-key.pem",
 		DisableCoordinateUpdates: true,
+		ClientAddress:            "127.0.0.1:8080",
 		PingType:                 PingTypeSocket,
 		Telemetry: lib.TelemetryConfig{
 			CirconusAPIApp:                     "circonus_api_app",

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/mitchellh/cli v1.0.0
 	github.com/mitchellh/hashstructure v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/prometheus/client_golang v1.4.0
 	github.com/sparrc/go-ping v0.0.0-20190613174326-4e5b6552494c
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect