feat(metrics): add requests and errors counters

Add counters for the number of requests processed and the number of
errors encountered.

agent-inject/handler.go

@@ -5,6 +5,7 @@ package agent_inject
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -85,19 +86,22 @@ type Handler struct {
 // served via an HTTP server.
 func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 	h.Log.Info("Request received", "Method", r.Method, "URL", r.URL)
+	var procErr error
 
 	// Measure request processing duration and monitor request queue
 	requestQueue.Inc()
 	requestStart := time.Now()
 	defer func() {
 		requestProcessingTime.Observe(float64(time.Since(requestStart).Milliseconds()))
 		requestQueue.Dec()
+		incrementRequests(procErr)
 	}()
 
 	if ct := r.Header.Get("Content-Type"); ct != "application/json" {
 		msg := fmt.Sprintf("Invalid content-type: %q", ct)
 		http.Error(w, msg, http.StatusBadRequest)
 		h.Log.Warn("warning for request", "Warn", msg, "Code", http.StatusBadRequest)
+		procErr = errors.New(msg)
 		return
 	}
 
@@ -108,13 +112,15 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 			msg := fmt.Sprintf("error reading request body: %s", err)
 			http.Error(w, msg, http.StatusBadRequest)
 			h.Log.Error("error on request", "Error", msg, "Code", http.StatusBadRequest)
+			procErr = errors.New(msg)
 			return
 		}
 	}
 	if len(body) == 0 {
 		msg := "Empty request body"
 		http.Error(w, msg, http.StatusBadRequest)
 		h.Log.Error("warning for request", "Warn", msg, "Code", http.StatusBadRequest)
+		procErr = errors.New(strings.ToLower(msg))
 		return
 	}
 
@@ -136,6 +142,7 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		msg := fmt.Sprintf("error decoding admission request: %s", err)
 		http.Error(w, msg, http.StatusInternalServerError)
 		h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError)
+		procErr = errors.New(msg)
 		return
 	} else {
 		mutateResp = h.Mutate(admReq.Request)
@@ -156,12 +163,15 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, msg, http.StatusInternalServerError)
 		h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError)
 		incrementInjectionFailures(admReq.Request.Namespace)
+		procErr = errors.New(msg)
 		return
 	}
 
 	if _, err := w.Write(resp); err != nil {
-		h.Log.Error("error writing response", "Error", err)
+		msg := "error writing response"
+		h.Log.Error(msg, "Error", err)
 		incrementInjectionFailures(admReq.Request.Namespace)
+		procErr = errors.New(msg)
 		return
 	}
 
@@ -170,6 +180,8 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 	} else {
 		incrementInjectionFailures(admReq.Request.Namespace)
 	}
+
+	return
 }
 
 type MutateResponse struct {

agent-inject/metrics.go

@@ -18,6 +18,20 @@ const (
 )
 
 var (
+	requestsReceived = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "requests_received_total",
+		Help:      "Count of webhook requests received by the injector",
+	})
+
+	requestsErrored = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "requests_errored_total",
+		Help:      "Count of webhook requests that could not be processed by the injector",
+	})
+
 	requestQueue = prometheus.NewGauge(prometheus.GaugeOpts{
 		Namespace: metricsNamespace,
 		Subsystem: metricsSubsystem,
@@ -48,6 +62,13 @@ var (
 	}, []string{metricsLabelNamespace})
 )
 
+func incrementRequests(err error) {
+	requestsReceived.Inc()
+	if err != nil {
+		requestsErrored.Inc()
+	}
+}
+
 func incrementInjections(namespace string, res MutateResponse) {
 	// Injection type can be one of: init_and_sidecar (default); init_only; or sidecar_only
 	typeLabel := metricsLabelTypeBoth
@@ -69,6 +90,8 @@ func incrementInjectionFailures(namespace string) {
 
 func MustRegisterInjectorMetrics(registry prometheus.Registerer) {
 	registry.MustRegister(
+		requestsReceived,
+		requestsErrored,
 		requestQueue,
 		requestProcessingTime,
 		injectionsByNamespace,