feat: add agent injector telemetry

Add Prometheus metrics to monitor the Agent Injector's performance. New
metrics include a gauge of current requests being processed by the
webhook, a summary of request processing times, and a count of
successful and failed injections by Kubernetes namespace.

Successful injections are broken down by injection type. The `injection_type`
label can assume the value `init_only` for injections with only an
initContainer (no sidecar) and `sidecar` for all other cases (sidecar
only or sidecar + initContainer).

Fixes AG-005161.

agent-inject/handler.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault-k8s/agent-inject/agent"
@@ -25,6 +26,8 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+const warningInitOnlyInjection = "workload uses initContainer only; consider migrating to VSO"
+
 var (
 	admissionScheme = runtime.NewScheme()
 	deserializer    = func() runtime.Decoder {
@@ -85,6 +88,14 @@ type Handler struct {
 func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 	h.Log.Info("Request received", "Method", r.Method, "URL", r.URL)
 
+	// Measure request processing duration and monitor request queue
+	requestQueue.Inc()
+	requestStart := time.Now()
+	defer func() {
+		requestProcessingTime.Observe(float64(time.Since(requestStart).Milliseconds()))
+		requestQueue.Dec()
+	}()
+
 	if ct := r.Header.Get("Content-Type"); ct != "application/json" {
 		msg := fmt.Sprintf("Invalid content-type: %q", ct)
 		http.Error(w, msg, http.StatusBadRequest)
@@ -142,11 +153,20 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
 		msg := fmt.Sprintf("error marshalling admission response: %s", err)
 		http.Error(w, msg, http.StatusInternalServerError)
 		h.Log.Error("error on request", "Error", msg, "Code", http.StatusInternalServerError)
+		incrementInjectionFailures(admReq.Request.Namespace)
 		return
 	}
 
 	if _, err := w.Write(resp); err != nil {
 		h.Log.Error("error writing response", "Error", err)
+		incrementInjectionFailures(admReq.Request.Namespace)
+		return
+	}
+
+	if admResp.Response.Allowed {
+		incrementInjections(admReq.Request.Namespace, *admResp.Response)
+	} else {
+		incrementInjectionFailures(admReq.Request.Namespace)
 	}
 }
 
@@ -245,6 +265,13 @@ func (h *Handler) Mutate(req *admissionv1.AdmissionRequest) *admissionv1.Admissi
 		return admissionError(req.UID, err)
 	}
 
+	// Expose when workloads are injecting initContainers only. This is useful to identify
+	// workloads that are likely good candidates for migration to Vault Secrets Operator for
+	// greater scalability and performance.
+	if agentSidecar.PrePopulateOnly {
+		resp.Warnings = append(resp.Warnings, warningInitOnlyInjection)
+	}
+
 	resp.Patch = patch
 	patchType := admissionv1.PatchTypeJSONPatch
 	resp.PatchType = &patchType

agent-inject/metrics.go

@@ -0,0 +1,73 @@
+package agent_inject
+
+import (
+	"slices"
+
+	"github.com/prometheus/client_golang/prometheus"
+	admissionv1 "k8s.io/api/admission/v1"
+)
+
+const (
+	metricsNamespace         = "vault"
+	metricsSubsystem         = "agent_injector"
+	metricsLabelNamespace    = "namespace"
+	metricsLabelType         = "injection_type"
+	metricsLabelTypeDefault  = "sidecar"
+	metricsLabelTypeInitOnly = "init_only"
+)
+
+var (
+	requestQueue = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "request_queue_total",
+		Help:      "Total count of webhook requests in the queue",
+	})
+
+	requestProcessingTime = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "request_processing_duration_ms",
+		Help:      "Histogram of webhook request processing times in milliseconds",
+		Buckets:   []float64{5, 10, 25, 50, 75, 100, 250, 500, 1000, 2500, 5000, 7500, 10000},
+	})
+
+	injectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "injections_by_namespace_total",
+		Help:      "Total count of Agent Sidecar injections by namespace",
+	}, []string{metricsLabelNamespace, metricsLabelType})
+
+	failedInjectionsByNamespace = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: metricsNamespace,
+		Subsystem: metricsSubsystem,
+		Name:      "failed_injections_by_namespace_total",
+		Help:      "Total count of failed Agent Sidecar injections by namespace",
+	}, []string{metricsLabelNamespace})
+)
+
+func incrementInjections(namespace string, res admissionv1.AdmissionResponse) {
+	typeLabel := metricsLabelTypeDefault
+	if slices.Contains(res.Warnings, warningInitOnlyInjection) {
+		typeLabel = metricsLabelTypeInitOnly
+	}
+
+	injectionsByNamespace.With(prometheus.Labels{
+		metricsLabelNamespace: namespace,
+		metricsLabelType:      typeLabel,
+	}).Inc()
+}
+
+func incrementInjectionFailures(namespace string) {
+	failedInjectionsByNamespace.With(prometheus.Labels{metricsLabelNamespace: namespace}).Inc()
+}
+
+func MustRegisterInjectorMetrics(registry prometheus.Registerer) {
+	registry.MustRegister(
+		requestQueue,
+		requestProcessingTime,
+		injectionsByNamespace,
+		failedInjectionsByNamespace,
+	)
+}

subcommand/injector/command.go

@@ -27,6 +27,7 @@ import (
 	"github.com/hashicorp/vault-k8s/leader"
 	"github.com/hashicorp/vault-k8s/version"
 	"github.com/mitchellh/cli"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	adminv1 "k8s.io/api/admissionregistration/v1"
 	adminv1beta "k8s.io/api/admissionregistration/v1beta1"
@@ -231,6 +232,7 @@ func (c *Command) Run(args []string) int {
 
 	// Registering path to expose metrics
 	if c.flagTelemetryPath != "" {
+		agentInject.MustRegisterInjectorMetrics(prometheus.DefaultRegisterer)
 		c.UI.Info(fmt.Sprintf("Registering telemetry path on %q", c.flagTelemetryPath))
 		mux.Handle(c.flagTelemetryPath, promhttp.Handler())
 	}