From 1409e705c44b1f7fca32d9af219c8b8bc66e4d17 Mon Sep 17 00:00:00 2001 From: Michael Nairn Date: Wed, 4 Dec 2024 16:46:12 +0000 Subject: [PATCH] add metrics Signed-off-by: Michael Nairn --- config/metallb/kustomization.yaml | 2 ++ .../observability/k8s_prometheus_patch.yaml | 12 ++++++++ config/observability/kustomization.yaml | 16 ++++++++++ .../observability/thanos/kustomization.yaml | 10 +++++++ test/scale/README.md | 29 +++++++++++++++++-- test/scale/config.yaml | 15 ++++++++++ .../dns-operator/dns-operator-deployment.yaml | 4 +++ test/scale/dns-operator/kustomization.yaml | 2 +- test/scale/dns-operator/metrics-service.yaml | 13 +++++++++ test/scale/dns-operator/service-monitor.yaml | 14 +++++++++ test/scale/metrics.yaml | 2 ++ test/scale/metrics/.gitignore | 2 ++ 12 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 config/metallb/kustomization.yaml create mode 100644 config/observability/k8s_prometheus_patch.yaml create mode 100644 config/observability/kustomization.yaml create mode 100644 config/observability/thanos/kustomization.yaml create mode 100644 test/scale/dns-operator/metrics-service.yaml create mode 100644 test/scale/dns-operator/service-monitor.yaml create mode 100644 test/scale/metrics.yaml create mode 100644 test/scale/metrics/.gitignore diff --git a/config/metallb/kustomization.yaml b/config/metallb/kustomization.yaml new file mode 100644 index 0000000..0a5e085 --- /dev/null +++ b/config/metallb/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- github.com/metallb/metallb/config/native?ref=v0.13.7 diff --git a/config/observability/k8s_prometheus_patch.yaml b/config/observability/k8s_prometheus_patch.yaml new file mode 100644 index 0000000..3002fc6 --- /dev/null +++ b/config/observability/k8s_prometheus_patch.yaml @@ -0,0 +1,12 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: k8s + namespace: monitoring +spec: + remoteWrite: + - url: http://thanos-receive-router:19291/api/v1/receive + writeRelabelConfigs: + - action: replace + replacement: cluster1 + targetLabel: cluster_id diff --git a/config/observability/kustomization.yaml b/config/observability/kustomization.yaml new file mode 100644 index 0000000..f9b64ec --- /dev/null +++ b/config/observability/kustomization.yaml @@ -0,0 +1,16 @@ +resources: + - github.com/kuadrant/kuadrant-operator/config/observability?ref=main + - ./thanos + - github.com/kuadrant/kuadrant-operator/examples/dashboards?ref=main + - github.com/kuadrant/kuadrant-operator/examples/alerts?ref=main + +patches: + - target: + kind: ServiceMonitor + patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: ANY + - path: k8s_prometheus_patch.yaml diff --git a/config/observability/thanos/kustomization.yaml b/config/observability/thanos/kustomization.yaml new file mode 100644 index 0000000..ab52431 --- /dev/null +++ b/config/observability/thanos/kustomization.yaml @@ -0,0 +1,10 @@ +resources: + - github.com/kuadrant/kuadrant-operator/config/thanos?ref=main + +patches: + - patch: |- + $patch: delete + apiVersion: v1 + kind: Namespace + metadata: + name: monitoring diff --git a/test/scale/README.md b/test/scale/README.md index f5a0b9b..cb79a2a 100644 --- a/test/scale/README.md +++ b/test/scale/README.md @@ -1,5 +1,28 @@ +Create a kind cluster with prometheus/thanos installed and configured ```shell -make kube-burner -./bin/kube-burner init -c config.yaml -``` \ No newline at end of file +make local-setup +kubectl apply --server-side -k config/observability +kubectl apply --server-side -k config/observability # Run twice if it fails the first time +``` + +Forward port for prometheus +```shell +kubectl -n monitoring port-forward service/thanos-query 9090:9090 +``` + +Forward port for graphana (Optional) +```shell +kubectl -n monitoring port-forward service/grafana 3000:3000 +``` +Access dashboards http://127.0.0.1:3000 + +Tail all operator logs (Optional) +```shell +kubectl stern -l control-plane=dns-operator-controller-manager -A +``` + +Run default scale test(1 iteration using the inmemory provider) +```shell +PROMETHEUS_URL=http://127.0.0.1:9090 PROMETHEUS_TOKEN="" make test-scale +``` diff --git a/test/scale/config.yaml b/test/scale/config.yaml index 5397a6d..6d15eb1 100644 --- a/test/scale/config.yaml +++ b/test/scale/config.yaml @@ -1,3 +1,12 @@ +metricsEndpoints: + - endpoint: {{ .PROMETHEUS_URL }} + token: {{ .PROMETHEUS_TOKEN }} + metrics: + - ./metrics.yaml + indexer: + type: local + metricsDirectory: ./metrics + jobs: - name: scale-test-setup-dns-operator-cluster-roles jobIterations: 1 @@ -35,6 +44,12 @@ jobs: - objectTemplate: ./dns-operator/dns-operator-deployment.yaml kind: Deployment replicas: 1 + - objectTemplate: ./dns-operator/metrics-service.yaml + kind: Service + replicas: 1 + - objectTemplate: ./dns-operator/service-monitor.yaml + kind: ServiceMonitor + replicas: 1 - name: scale-test-setup jobType: create #default jobIterations: {{ .JOB_ITERATIONS }} diff --git a/test/scale/dns-operator/dns-operator-deployment.yaml b/test/scale/dns-operator/dns-operator-deployment.yaml index 14c2fc7..39eead6 100644 --- a/test/scale/dns-operator/dns-operator-deployment.yaml +++ b/test/scale/dns-operator/dns-operator-deployment.yaml @@ -19,6 +19,7 @@ spec: containers: - args: - --leader-elect + - --metrics-bind-address=:8080 - --provider=aws,google,inmemory,azure - --zap-log-level=debug command: @@ -34,6 +35,9 @@ spec: initialDelaySeconds: 15 periodSeconds: 20 name: manager + ports: + - containerPort: 8080 + name: metrics readinessProbe: httpGet: path: /readyz diff --git a/test/scale/dns-operator/kustomization.yaml b/test/scale/dns-operator/kustomization.yaml index 09a1672..626119b 100644 --- a/test/scale/dns-operator/kustomization.yaml +++ b/test/scale/dns-operator/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- ../../../config/manager +- ../../../config/default patches: - patch: |- diff --git a/test/scale/dns-operator/metrics-service.yaml b/test/scale/dns-operator/metrics-service.yaml new file mode 100644 index 0000000..e49af14 --- /dev/null +++ b/test/scale/dns-operator/metrics-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: dns-operator-controller-manager + name: dns-operator-controller-manager-metrics-service +spec: + ports: + - name: metrics + port: 8080 + targetPort: metrics + selector: + control-plane: dns-operator-controller-manager diff --git a/test/scale/dns-operator/service-monitor.yaml b/test/scale/dns-operator/service-monitor.yaml new file mode 100644 index 0000000..da8a552 --- /dev/null +++ b/test/scale/dns-operator/service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: dns-operator-metrics-monitor +spec: + endpoints: + - path: /metrics + port: metrics + scheme: http + selector: + matchLabels: + control-plane: dns-operator-controller-manager diff --git a/test/scale/metrics.yaml b/test/scale/metrics.yaml new file mode 100644 index 0000000..8a491f9 --- /dev/null +++ b/test/scale/metrics.yaml @@ -0,0 +1,2 @@ +- query: sum(rate(container_cpu_usage_seconds_total{container="",namespace=~"kuadrant-system|kuadrant-dns-operator-*|scale-test-.*"}[5m])) by(namespace) + metricName: namespaceCPU diff --git a/test/scale/metrics/.gitignore b/test/scale/metrics/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/test/scale/metrics/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore