[FEATURE] add metrics to zarf-agent

Makefile

@@ -140,6 +140,8 @@ build-examples: ## Build all of the example packages
 
 	@test -s ./build/zarf-package-yolo-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/yolo -o build -a $(ARCH) --confirm
 
+	@test -s ./build/zarf-package-scrape-zarf-agent-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/scraping-zarf-agent -o build -a $(ARCH) --confirm
+
 ## NOTE: Requires an existing cluster or the env var APPLIANCE_MODE=true
 .PHONY: test-e2e
 test-e2e: build-examples ## Run all of the core Zarf CLI E2E tests (builds any deps that aren't present)

examples/big-bang/README.md

@@ -42,3 +42,9 @@ To view the example in its entirety, select the `Edit this page` link below the
 :::
 
 <ExampleYAML example="big-bang/yolo" showLink={false} />
+
+## Big Bang Zarf Agent Metrics Support
+
+Using Big Bang's Prometheus instance, we can easily add a `ServiceMonitor` to scrape the Zarf Agent's metrics. Check out the [scraping-zarf-agent example](../scraping-zarf-agent/README.md) for more details. Make sure monitoring in enabled.
+
+:::

examples/scraping-zarf-agent/README.md

@@ -0,0 +1,76 @@
+import ExampleYAML from "@site/src/components/ExampleYAML";
+
+# Scrape Zarf Agent
+
+This example demonstrates how to scrape the Zarf Agent container image from the Prometheus Operator.
+
+## Prerequisites
+
+- A running K8s cluster.
+
+:::note
+
+The cluster does not need to have the Zarf init package installed or any other Zarf-related bootstrapping.
+
+:::
+
+## Instructions
+
+Initialize Zarf (interactively):
+
+```bash
+zarf init
+# Make these choices at the prompts
+# ? Do you want to download this init package? Yes
+# ? Deploy this Zarf package? Yes
+# ? Deploy the k3s component? No
+# ? Deploy the logging component? No
+# ? Deploy the git-server component? No
+```
+
+Create the package:
+
+```bash
+zarf package create --confirm
+```
+
+Deploy the package
+
+```bash
+# Run the following command to deploy the created package to the cluster
+zarf package deploy
+
+# Choose the yolo package from the list
+? Choose or type the package file [tab for suggestions]
+> zarf-package-scrape-zarf-agent-<ARCH>.tar.zst
+
+# Confirm the deployment
+? Deploy this Zarf package? (y/N) [y]
+```
+
+Wait for the `prometheus-k8s` StatefulSet's replicas to become ready:
+
+```bash
+zarf tools kubectl wait --for=jsonpath='{.status.availableReplicas}'=1  sts/prometheus-k8s -n monitoring --timeout=180s
+```
+
+Port-forward the Prometheus Operator's Prometheus instance:
+
+```bash
+zarf connect --name=prometheus-operated --namespace monitoring --remote-port 9090 --local-port=9090
+```
+
+Navigate to the [Prometheus UI targets](http://localhost:9090/targets) at http://localhost:9090/targets.
+
+Checkout metrics emitted by the Zarf Agent by querying against the `agent-hook` job. Click this [link](http://localhost:9090/graph?g0.expr=%7Bjob%3D%22agent-hook%22%7D&g0.tab=1&g0.stacked=0&g0.show_exemplars=0&g0.range_input=1h) to see the Zarf Agent metrics while port-forwarding.
+
+
+## `zarf.yaml` {#zarf.yaml}
+
+:::info
+
+To view the example in its entirety, select the `Edit this page` link below the article and select the parent folder.
+
+:::
+
+<ExampleYAML example="scraping-zarf-agent" showLink={false} />

examples/scraping-zarf-agent/manifests/prometheus-operator/clusterrole.yaml

@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: scrape-cr
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/status
+  - endpoints
+  - services
+  verbs:
+  - watch
+  - get
+  - list

examples/scraping-zarf-agent/manifests/prometheus-operator/clusterrolebinding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: scrape-cr-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: scrape-cr
+subjects:
+- kind: ServiceAccount
+  name: prometheus-operator
+  namespace: monitoring

examples/scraping-zarf-agent/manifests/prometheus-operator/prometheus.yaml

@@ -0,0 +1,17 @@
+kind: Prometheus
+apiVersion: monitoring.coreos.com/v1
+metadata:
+  name: k8s
+  namespace: monitoring
+spec:
+  serviceMonitorSelector: {}
+  serviceMonitorNamespaceSelector: {}
+  logLevel: debug
+  logFormat: json
+  replicas: 1
+  image: quay.io/prometheus/prometheus:v2.45.0
+  serviceAccountName: prometheus-operator
+  resources:
+    requests:
+      memory: 400Mi
+

examples/scraping-zarf-agent/manifests/prometheus-operator/servicemonitor-agent.yaml

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    artifact: monitoring-agent-hook
+  name: monitoring-agent-hook
+  namespace: monitoring
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    port: https
+    # targetPort: 443
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecureSkipVerify: true
+      # host name for the TLS handshake
+      serverName: agent-hook.zarf.svc
+  jobLabel: zarf-agent
+  namespaceSelector:
+    matchNames:
+    - zarf
+  selector:
+    matchLabels:
+      app: agent-hook

examples/scraping-zarf-agent/zarf.yaml

@@ -0,0 +1,38 @@
+kind: ZarfPackageConfig
+metadata:
+  name: scrape-zarf-agent
+  yolo: false
+  description: Scrape Zarf Agent with Prometheus Operator.
+
+components:
+  - name: prometheus-deps
+    required: true
+    manifests:
+      - name: prometheus-deps
+        namespace: monitoring
+        files:
+          - manifests/prometheus-deps/bundle.yaml
+    images:
+      - quay.io/prometheus-operator/prometheus-operator:v0.66.0
+    actions:
+      onDeploy:
+        after:
+          - wait:
+              cluster:
+                kind: deployment
+                name: prometheus-operator
+                namespace: monitoring
+                condition: available
+  - name: prometheus-operator
+    required: true
+    manifests:
+      - name: prometheus-operator
+        namespace: monitoring
+        files:
+        - manifests/prometheus-operator/prometheus.yaml
+        - manifests/prometheus-operator/servicemonitor-agent.yaml
+        - manifests/prometheus-operator/clusterrole.yaml
+        - manifests/prometheus-operator/clusterrolebinding.yaml
+    images:
+      - quay.io/prometheus/prometheus:v2.45.0
+      - quay.io/prometheus-operator/prometheus-config-reloader:v0.66.0

packages/zarf-agent/manifests/service.yaml

@@ -3,9 +3,12 @@ kind: Service
 metadata:
   name: agent-hook
   namespace: zarf
+  labels:
+    app: agent-hook
 spec:
   selector:
     app: agent-hook
   ports:
     - port: 443
       targetPort: 8443
+      name: https

src/internal/agent/http/server.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/defenseunicorns/zarf/src/internal/agent/hooks"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // NewAdmissionServer creates an http.Server for the mutating webhook admission handler.
@@ -26,6 +27,7 @@ func NewAdmissionServer(port string) *http.Server {
 	mux.Handle("/healthz", healthz())
 	mux.Handle("/mutate/pod", ah.Serve(podsMutation))
 	mux.Handle("/mutate/flux-gitrepository", ah.Serve(gitRepositoryMutation))
+	mux.Handle("/metrics", promhttp.Handler())
 
 	return &http.Server{
 		Addr:    fmt.Sprintf(":%s", port),
@@ -40,6 +42,7 @@ func NewProxyServer(port string) *http.Server {
 	mux := http.NewServeMux()
 	mux.Handle("/healthz", healthz())
 	mux.Handle("/", ProxyHandler())
+	mux.Handle("/metrics", promhttp.Handler())
 
 	return &http.Server{
 		Addr:    fmt.Sprintf(":%s", port),

src/internal/cluster/tunnel.go

@@ -44,6 +44,7 @@ const (
 	ZarfLogging  = "LOGGING"
 	ZarfGit      = "GIT"
 	ZarfInjector = "INJECTOR"
+	Prometheus   = "PROMETHEUS"
 
 	// See https://regex101.com/r/OWVfAO/1.
 	serviceURLPattern = `^(?P<name>[^\.]+)\.(?P<namespace>[^\.]+)\.svc\.cluster\.local$`
@@ -244,6 +245,10 @@ func (tunnel *Tunnel) Connect(target string, blocking bool) error {
 		tunnel.resourceName = "zarf-injector"
 		tunnel.remotePort = 5000
 
+	case Prometheus:
+		tunnel.resourceName = "prometheus-operator"
+		tunnel.remotePort = 8080
+
 	default:
 		if target != "" {
 			if err := tunnel.checkForZarfConnectLabel(target); err != nil {

src/test/e2e/26_simple_packages_test.go

@@ -13,6 +13,32 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestPrometheus(t *testing.T) {
+	t.Log("E2E: Prometheus")
+	e2e.SetupWithCluster(t)
+
+	path := fmt.Sprintf("build/zarf-package-scrape-zarf-agent-%s.tar.zst", e2e.Arch)
+
+	// Deploy Prometheus
+	stdOut, stdErr, err := e2e.Zarf("package", "deploy", path, "--confirm")
+	require.NoError(t, err, stdOut, stdErr)
+
+	// tunnel, err := cluster.NewTunnel("monitoring", "svc", "prometheus-operator", 8080, 8080)
+	tunnel, err := cluster.NewTunnel("monitoring", "svc", "", 8080, 0)
+	require.NoError(t, err)
+	err = tunnel.Connect("PROMETHEUS", false)
+	require.NoError(t, err)
+	defer tunnel.Close()
+
+	// Check that 'curl' returns something.
+	resp, err := http.Get(tunnel.HTTPEndpoint() + "/healthz")
+	require.NoError(t, err, resp)
+	require.Equal(t, 200, resp.StatusCode)
+
+	stdOut, stdErr, err = e2e.Zarf("package", "remove", "scrape-zarf-agent", "--confirm")
+	require.NoError(t, err, stdOut, stdErr)
+
+}
 func TestDosGames(t *testing.T) {
 	t.Log("E2E: Dos games")
 	e2e.SetupWithCluster(t)