Metrics for Hetzner API calls

Provide metrics for Hetzner API calls; helps identifying slowness, throttling causes, and errors bursts.

Signed-off-by: Maksim Paskal <[email protected]>

cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go

@@ -21,6 +21,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -32,7 +33,10 @@ import (
 )
 
 var (
-	version = "dev"
+	version    = "dev"
+	httpClient = &http.Client{
+		Transport: instrumentedRoundTripper(),
+	}
 )
 
 // hetznerManager handles Hetzner communication and data caching of
@@ -60,7 +64,11 @@ func newManager() (*hetznerManager, error) {
 		return nil, errors.New("`HCLOUD_CLOUD_INIT` is not specified")
 	}
 
-	client := hcloud.NewClient(hcloud.WithToken(token))
+	client := hcloud.NewClient(
+		hcloud.WithToken(token),
+		hcloud.WithHTTPClient(httpClient),
+	)
+
 	ctx := context.Background()
 	cloudInit, err := base64.StdEncoding.DecodeString(cloudInitBase64)
 	if err != nil {

cluster-autoscaler/cloudprovider/hetzner/hetzner_metrics.go

@@ -0,0 +1,104 @@
+package hetzner
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	k8smetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const subsystemIdentifier = "api"
+
+func instrumentedRoundTripper() http.RoundTripper {
+	inFlightRequestsGauge := k8smetrics.NewGauge(&k8smetrics.GaugeOpts{
+		Name: fmt.Sprintf("hcloud_%s_in_flight_requests", subsystemIdentifier),
+		Help: fmt.Sprintf("A gauge of in-flight requests to the hcloud %s.", subsystemIdentifier),
+	})
+
+	requestsPerEndpointCounter := k8smetrics.NewCounterVec(
+		&k8smetrics.CounterOpts{
+			Name: fmt.Sprintf("hcloud_%s_requests_total", subsystemIdentifier),
+			Help: fmt.Sprintf("A counter for requests to the hcloud %s per endpoint.", subsystemIdentifier),
+		},
+		[]string{"code", "method", "api_endpoint"},
+	)
+
+	requestLatencyHistogram := k8smetrics.NewHistogramVec(
+		&k8smetrics.HistogramOpts{
+			Name:    fmt.Sprintf("hcloud_%s_request_duration_seconds", subsystemIdentifier),
+			Help:    fmt.Sprintf("A histogram of request latencies to the hcloud %s .", subsystemIdentifier),
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method"},
+	)
+
+	legacyregistry.MustRegister(requestsPerEndpointCounter)
+	legacyregistry.MustRegister(requestLatencyHistogram)
+	legacyregistry.MustRegister(inFlightRequestsGauge)
+
+	return instrumentRoundTripperInFlight(inFlightRequestsGauge,
+		instrumentRoundTripperDuration(requestLatencyHistogram,
+			instrumentRoundTripperEndpoint(requestsPerEndpointCounter,
+				http.DefaultTransport,
+			),
+		),
+	)
+}
+
+type RoundTripperFunc func(req *http.Request) (*http.Response, error)
+
+// RoundTrip implements the RoundTripper interface.
+func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return rt(r)
+}
+
+func instrumentRoundTripperInFlight(gauge *k8smetrics.Gauge, next http.RoundTripper) RoundTripperFunc {
+	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+		gauge.Inc()
+		defer gauge.Dec()
+		return next.RoundTrip(r)
+	})
+}
+
+func instrumentRoundTripperDuration(obs *k8smetrics.HistogramVec, next http.RoundTripper) RoundTripperFunc {
+	return RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+		start := time.Now()
+		resp, err := next.RoundTrip(r)
+		if err == nil {
+			obs.WithLabelValues(strings.ToLower(resp.Request.Method)).Observe(time.Since(start).Seconds())
+		}
+		return resp, err
+	})
+}
+
+func instrumentRoundTripperEndpoint(counter *k8smetrics.CounterVec, next http.RoundTripper) promhttp.RoundTripperFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		resp, err := next.RoundTrip(r)
+		if err == nil {
+			statusCode := strconv.Itoa(resp.StatusCode)
+			counter.WithLabelValues(statusCode, strings.ToLower(resp.Request.Method), preparePathForLabel(resp.Request.URL.Path)).Inc()
+		}
+		return resp, err
+	}
+}
+
+func preparePathForLabel(path string) string {
+	path = strings.ToLower(path)
+
+	// replace all numbers and chars that are not a-z, / or _
+	reg := regexp.MustCompile("[^a-z/_]+")
+	path = reg.ReplaceAllString(path, "")
+
+	// replace all artifacts of number replacement (//)
+	path = strings.ReplaceAll(path, "//", "/")
+
+	// replace the /v/ that indicated the API version
+	return strings.Replace(path, "/v/", "/", 1)
+}