From d73ee75fd96f7637ae4f0f16872ddc3f964acffb Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Tue, 19 Nov 2019 10:12:54 -0800 Subject: [PATCH 1/5] charts: import `stable/prometheus-adapter` 1.4.0 ``` $ helm fetch -d charts --untar stable/prometheus-adapter ``` See: https://github.com/helm/charts/tree/805af2adba1a1fd6ea57239f01f3828da6fb7e06/stable/prometheus-adapter --- charts/prometheus-adapter/.helmignore | 21 +++ charts/prometheus-adapter/Chart.yaml | 20 +++ charts/prometheus-adapter/OWNERS | 8 + charts/prometheus-adapter/README.md | 157 ++++++++++++++++++ .../prometheus-adapter/ci/default-values.yaml | 0 .../ci/external-rules-values.yaml | 9 + charts/prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 43 +++++ ...r-auth-delegator-cluster-role-binding.yaml | 19 +++ ...cs-apiserver-auth-reader-role-binding.yaml | 19 +++ .../custom-metrics-apiserver-deployment.yaml | 114 +++++++++++++ ...-resource-reader-cluster-role-binding.yaml | 19 +++ ...tom-metrics-apiserver-service-account.yaml | 11 ++ .../custom-metrics-apiserver-service.yaml | 21 +++ .../templates/custom-metrics-apiservice.yaml | 21 +++ .../custom-metrics-cluster-role.yaml | 16 ++ .../templates/custom-metrics-configmap.yaml | 93 +++++++++++ ...-metrics-resource-reader-cluster-role.yaml | 22 +++ .../external-metrics-apiservice.yaml | 23 +++ .../external-metrics-cluster-role.yaml | 20 +++ ...a-custom-metrics-cluster-role-binding.yaml | 19 +++ ...external-metrics-cluster-role-binding.yaml | 19 +++ .../resource-metrics-apiservice.yaml | 23 +++ ...resource-metrics-cluster-role-binding.yaml | 19 +++ .../resource-metrics-cluster-role.yaml | 22 +++ .../prometheus-adapter/templates/secret.yaml | 15 ++ charts/prometheus-adapter/values.yaml | 127 ++++++++++++++ 27 files changed, 909 insertions(+) create mode 100644 charts/prometheus-adapter/.helmignore create mode 100644 charts/prometheus-adapter/Chart.yaml create mode 100644 charts/prometheus-adapter/OWNERS create mode 100644 charts/prometheus-adapter/README.md create mode 100644 charts/prometheus-adapter/ci/default-values.yaml create mode 100644 charts/prometheus-adapter/ci/external-rules-values.yaml create mode 100644 charts/prometheus-adapter/templates/NOTES.txt create mode 100644 charts/prometheus-adapter/templates/_helpers.tpl create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-configmap.yaml create mode 100644 charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml create mode 100644 charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100644 charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100644 charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100644 charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100644 charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100644 charts/prometheus-adapter/templates/secret.yaml create mode 100644 charts/prometheus-adapter/values.yaml diff --git a/charts/prometheus-adapter/.helmignore b/charts/prometheus-adapter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/prometheus-adapter/Chart.yaml b/charts/prometheus-adapter/Chart.yaml new file mode 100644 index 0000000000..c7f22d6ed7 --- /dev/null +++ b/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +appVersion: v0.5.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/DirectXMan12/k8s-prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/DirectXMan12/k8s-prometheus-adapter +version: 1.4.0 diff --git a/charts/prometheus-adapter/OWNERS b/charts/prometheus-adapter/OWNERS new file mode 100644 index 0000000000..0c15fcf344 --- /dev/null +++ b/charts/prometheus-adapter/OWNERS @@ -0,0 +1,8 @@ +approvers: + - mattiasgees + - steven-sheehy + - hectorj2f +reviewers: + - mattiasgees + - steven-sheehy + - hectorj2f diff --git a/charts/prometheus-adapter/README.md b/charts/prometheus-adapter/README.md new file mode 100644 index 0000000000..4c43d2f63d --- /dev/null +++ b/charts/prometheus-adapter/README.md @@ -0,0 +1,157 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.11+ + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +$ helm install --name my-release stable/prometheus-adapter +``` + +This command deploys the prometheus adapter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +## Using the Chart + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md) for the proper format). + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +``` +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +``` +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +``` +rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod_name: + resource: pod + containerLabel: container_name + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```console +$ helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following table lists the configurable parameters of the Prometheus Adapter chart and their default values. + +| Parameter | Description | Default | +| ------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------------| +| `affinity` | Node affinity | `{}` | +| `image.repository` | Image repository | `directxman12/k8s-prometheus-adapter-amd64` | +| `image.tag` | Image tag | `v0.5.0` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets | `{}` | +| `logLevel` | Log level | `4` | +| `metricsRelistInterval` | Interval at which to re-list the set of all available metrics from Prometheus | `1m` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `podAnnotations` | Annotations to add to the pod | `{}` | +| `priorityClassName` | Pod priority | `` | +| `prometheus.url` | Url of where we can find the Prometheus service | `http://prometheus.default.svc` | +| `prometheus.port` | Port of where we can find the Prometheus service, zero to omit this option | `9090` | +| `rbac.create` | If true, create & use RBAC resources | `true` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `rules.default` | If `true`, enable a set of default rules in the configmap | `true` | +| `rules.custom` | A list of custom configmap rules | `[]` | +| `rules.existing` | The name of an existing configMap with rules. Overrides default, custom and external. | `` | +| `rules.external` | A list of custom rules for external metrics API | `[]` | +| `rules.resource` | `resourceRules` to set in configmap rules | `{}` | +| `service.annotations` | Annotations to add to the service | `{}` | +| `service.port` | Service port to expose | `443` | +| `service.type` | Type of service to create | `ClusterIP` | +| `serviceAccount.create` | If true, create & use Serviceaccount | `true` | +| `serviceAccount.name` | If not set and create is true, a name is generated using the fullname template | `` | +| `tls.enable` | If true, use the provided certificates. If false, generate self-signed certs | `false` | +| `tls.ca` | Public CA file that signed the APIService (ignored if tls.enable=false) | `` | +| `tls.key` | Private key of the APIService (ignored if tls.enable=false) | `` | +| `tls.certificate` | Public key of the APIService (ignored if tls.enable=false) | `` | +| `extraVolumeMounts` | Any extra volumes mounts | `[]` | +| `extraVolumes` | Any extra volumes | `[]` | +| `tolerations` | List of node taints to tolerate | `[]` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +$ helm install --name my-release \ + --set logLevel=1 \ + stable/prometheus-adapter +``` + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +$ helm install --name my-release -f values.yaml stable/prometheus-adapter +``` diff --git a/charts/prometheus-adapter/ci/default-values.yaml b/charts/prometheus-adapter/ci/default-values.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/charts/prometheus-adapter/ci/external-rules-values.yaml b/charts/prometheus-adapter/ci/external-rules-values.yaml new file mode 100644 index 0000000000..2dafb56211 --- /dev/null +++ b/charts/prometheus-adapter/ci/external-rules-values.yaml @@ -0,0 +1,9 @@ +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) diff --git a/charts/prometheus-adapter/templates/NOTES.txt b/charts/prometheus-adapter/templates/NOTES.txt new file mode 100644 index 0000000000..b7b9b99322 --- /dev/null +++ b/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/prometheus-adapter/templates/_helpers.tpl b/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100644 index 0000000000..5d0b993c52 --- /dev/null +++ b/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml new file mode 100644 index 0000000000..44761c4569 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml new file mode 100644 index 0000000000..79c0af39f6 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml new file mode 100644 index 0000000000..fa98872e6f --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/custom-metrics-configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - /adapter + - --secure-port=6443 + {{- if .Values.tls.enable }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url={{ .Values.prometheus.url }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + ports: + - containerPort: 6443 + name: https + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if .Values.tls.enable }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if .Values.tls.enable }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml new file mode 100644 index 0000000000..ec7e5e4762 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml new file mode 100644 index 0000000000..d73deacb78 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml new file mode 100644 index 0000000000..83dd70a91b --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + type: {{ .Values.service.type }} + diff --git a/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100644 index 0000000000..676402143c --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,21 @@ +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 diff --git a/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100644 index 0000000000..66a3d399a6 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml b/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml new file mode 100644 index 0000000000..5b035900ad --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml @@ -0,0 +1,93 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.yaml: | + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml b/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml new file mode 100644 index 0000000000..af10d0aaa8 --- /dev/null +++ b/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list +{{- end -}} diff --git a/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100644 index 0000000000..560b2ffcd8 --- /dev/null +++ b/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.external }} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100644 index 0000000000..4adbd65372 --- /dev/null +++ b/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml b/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..641b9d3a44 --- /dev/null +++ b/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml b/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..0776029af2 --- /dev/null +++ b/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100644 index 0000000000..f1a619912f --- /dev/null +++ b/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.resource}} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100644 index 0000000000..0534af11e6 --- /dev/null +++ b/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100644 index 0000000000..01a307d69b --- /dev/null +++ b/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/prometheus-adapter/templates/secret.yaml b/charts/prometheus-adapter/templates/secret.yaml new file mode 100644 index 0000000000..38e7cb624f --- /dev/null +++ b/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/prometheus-adapter/values.yaml b/charts/prometheus-adapter/values.yaml new file mode 100644 index 0000000000..4e13ad6c51 --- /dev/null +++ b/charts/prometheus-adapter/values.yaml @@ -0,0 +1,127 @@ +# Default values for k8s-prometheus-adapter.. +affinity: {} + +image: + repository: directxman12/k8s-prometheus-adapter-amd64 + tag: v0.5.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +nodeSelector: {} + +priorityClassName: "" + +# Url to access prometheus +prometheus: + url: http://prometheus.default.svc + port: 9090 + +replicas: 1 + +rbac: + # Specifies whether RBAC resources should be created + create: true + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +rules: + default: true + custom: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_custom_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + external: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_external_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + resource: {} +# cpu: +# containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) +# nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod_name: +# resource: pod +# containerLabel: container_name +# memory: +# containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) +# nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod_name: +# resource: pod +# containerLabel: container_name +# window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Annotations added to the pod +podAnnotations: {} From 049647f80ab2b96ca880016c14b257973da76773 Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Thu, 12 Dec 2019 21:34:02 +0100 Subject: [PATCH 2/5] ci: make multi-node bootstrap 'm1.large' An 'm1.medium' node is too small to run all services we now allocate to the bootstrap node. A proper set of infra nodes could remediate this over time... --- eve/workers/openstack-multiple-nodes/terraform/nodes.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eve/workers/openstack-multiple-nodes/terraform/nodes.tf b/eve/workers/openstack-multiple-nodes/terraform/nodes.tf index da4a5b1ce5..40fa90fd49 100644 --- a/eve/workers/openstack-multiple-nodes/terraform/nodes.tf +++ b/eve/workers/openstack-multiple-nodes/terraform/nodes.tf @@ -65,7 +65,7 @@ resource "openstack_compute_instance_v2" "bastion" { resource "openstack_compute_instance_v2" "bootstrap" { name = "${local.prefix}-bootstrap" image_name = var.openstack_image_name - flavor_name = var.openstack_flavour_name + flavor_name = "m1.large" key_pair = openstack_compute_keypair_v2.local_ssh_key.name scheduler_hints { From 68e70cde7870c89d5a5e575ba7e8907d48e64b10 Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Tue, 19 Nov 2019 11:56:08 -0800 Subject: [PATCH 3/5] monitoring: deploy `prometheus-adapter` This should enable the `metrics.k8s.io` API, and as such reinstate the `kubectl top` functionality and allow the `HorizontalPodAutoscaler` controller to work. The chart is rendered using ``` $ ./charts/render.py prometheus-adapter metalk8s-monitoring charts/prometheus-adapter.yaml charts/prometheus-adapter/ > salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls ``` Fixes: #2049 See: https://github.com/scality/metalk8s/issues/2049 --- CHANGELOG.md | 7 + buildchain/buildchain/constants.py | 17 +- buildchain/buildchain/image.py | 3 + buildchain/buildchain/salt_tree.py | 3 + buildchain/buildchain/versions.py | 5 + charts/prometheus-adapter.yaml | 47 ++ .../prometheus-adapter/deployed/chart.sls | 418 ++++++++++++++++++ .../prometheus-adapter/deployed/init.sls | 3 + salt/metalk8s/deployed.sls | 1 + 9 files changed, 496 insertions(+), 8 deletions(-) create mode 100644 charts/prometheus-adapter.yaml create mode 100644 salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls create mode 100644 salt/metalk8s/addons/prometheus-adapter/deployed/init.sls diff --git a/CHANGELOG.md b/CHANGELOG.md index 67df1f033f..38b23b6c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Release 2.4.2 (in development) +### Features added +- [#2049](https://github.com/scality/metalk8s/issues/2049) - Deploy +[prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter/) +to implement the `metrics.k8s.io` API, to support `kubectl top` and other +consumers of this API +(PR [#2057](https://github.com/scality/metalk8s/pull/2057)) + ## Release 2.4.1 ### Features added diff --git a/buildchain/buildchain/constants.py b/buildchain/buildchain/constants.py index c19ae771f1..ee25093696 100644 --- a/buildchain/buildchain/constants.py +++ b/buildchain/buildchain/constants.py @@ -17,14 +17,15 @@ CMD_WIDTH : int = 14 # URLs of the main container repositories. -CALICO_REPOSITORY : str = 'quay.io/calico' -COREOS_REPOSITORY : str = 'quay.io/coreos' -DOCKER_REPOSITORY : str = 'docker.io/library' -GOOGLE_REPOSITORY : str = 'k8s.gcr.io' -GRAFANA_REPOSITORY : str = 'docker.io/grafana' -INGRESS_REPOSITORY : str = 'quay.io/kubernetes-ingress-controller' -KIWIGRID_REPOSITORY : str = 'docker.io/kiwigrid' -PROMETHEUS_REPOSITORY : str = 'quay.io/prometheus' +CALICO_REPOSITORY : str = 'quay.io/calico' +COREOS_REPOSITORY : str = 'quay.io/coreos' +DOCKER_REPOSITORY : str = 'docker.io/library' +GOOGLE_REPOSITORY : str = 'k8s.gcr.io' +GRAFANA_REPOSITORY : str = 'docker.io/grafana' +INGRESS_REPOSITORY : str = 'quay.io/kubernetes-ingress-controller' +KIWIGRID_REPOSITORY : str = 'docker.io/kiwigrid' +PROMETHEUS_ADAPTER_REPOSITORY : str = 'docker.io/directxman12' +PROMETHEUS_REPOSITORY : str = 'quay.io/prometheus' # Paths {{{ diff --git a/buildchain/buildchain/image.py b/buildchain/buildchain/image.py index f76a772f51..a8d80342c8 100644 --- a/buildchain/buildchain/image.py +++ b/buildchain/buildchain/image.py @@ -191,6 +191,9 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage: constants.KIWIGRID_REPOSITORY: [ 'k8s-sidecar', ], + constants.PROMETHEUS_ADAPTER_REPOSITORY: [ + 'k8s-prometheus-adapter-amd64', + ], constants.PROMETHEUS_REPOSITORY: [ 'alertmanager', 'node-exporter', diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index fce9d147d6..8fe662ed9b 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -216,6 +216,9 @@ def _get_parts(self) -> Iterator[str]: renderer=targets.Renderer.JSON, ), + Path('salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls'), + Path('salt/metalk8s/addons/prometheus-adapter/deployed/init.sls'), + Path('salt/metalk8s/addons/prometheus-operator/deployed/chart.sls'), Path('salt/metalk8s/addons/prometheus-operator/deployed/cleanup.sls'), Path('salt/metalk8s/addons/prometheus-operator/deployed/dashboards.sls'), diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index b41134ddbb..4712915694 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -173,6 +173,11 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: version='v2.12.0', digest='sha256:cd93b8711bb92eb9c437d74217311519e0a93bc55779aa664325dc83cd13cb3', ), + Image( + name='k8s-prometheus-adapter-amd64', + version='v0.5.0', + digest='sha256:acf4c4dc6aaa51e3004eebdc410b509db45d1a6ebd255457c5745f0b4671678f', + ), Image( name='prometheus-config-reloader', version='v0.32.0', diff --git a/charts/prometheus-adapter.yaml b/charts/prometheus-adapter.yaml new file mode 100644 index 0000000000..53c9a11511 --- /dev/null +++ b/charts/prometheus-adapter.yaml @@ -0,0 +1,47 @@ +image: + repository: '{%- endraw -%}{{ build_image_name(\"k8s-prometheus-adapter-amd64\", False) }}{%- raw -%}' + +nodeSelector: + node-role.kubernetes.io/infra: '' + +prometheus: + url: http://prometheus-operator-prometheus + +replicas: 1 + +rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container_name + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + node: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container_name + window: 3m + + +tolerations: + - key: 'node-role.kubernetes.io/bootstrap' + operator: 'Exists' + effect: 'NoSchedule' + - key: 'node-role.kubernetes.io/infra' + operator: 'Exists' + effect: 'NoSchedule' diff --git a/salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls b/salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls new file mode 100644 index 0000000000..37f0a524ea --- /dev/null +++ b/salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls @@ -0,0 +1,418 @@ +#!jinja | metalk8s_kubernetes +{%- from "metalk8s/repo/macro.sls" import build_image_name with context %} + +{% raw %} + +apiVersion: v1 +data: + config.yaml: "rules:\n- seriesQuery: '{__name__=~\"^container_.*\",container_name!=\"\ + POD\",namespace!=\"\",pod_name!=\"\"}'\n seriesFilters: []\n resources:\n \ + \ overrides:\n namespace:\n resource: namespace\n pod_name:\n\ + \ resource: pod\n name:\n matches: ^container_(.*)_seconds_total$\n\ + \ as: \"\"\n metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!=\"\ + POD\"}[5m]))\n by (<<.GroupBy>>)\n- seriesQuery: '{__name__=~\"^container_.*\"\ + ,container_name!=\"POD\",namespace!=\"\",pod_name!=\"\"}'\n seriesFilters:\n\ + \ - isNot: ^container_.*_seconds_total$\n resources:\n overrides:\n \ + \ namespace:\n resource: namespace\n pod_name:\n resource:\ + \ pod\n name:\n matches: ^container_(.*)_total$\n as: \"\"\n metricsQuery:\ + \ sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!=\"POD\"}[5m]))\n \ + \ by (<<.GroupBy>>)\n- seriesQuery: '{__name__=~\"^container_.*\",container_name!=\"\ + POD\",namespace!=\"\",pod_name!=\"\"}'\n seriesFilters:\n - isNot: ^container_.*_total$\n\ + \ resources:\n overrides:\n namespace:\n resource: namespace\n\ + \ pod_name:\n resource: pod\n name:\n matches: ^container_(.*)$\n\ + \ as: \"\"\n metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!=\"\ + POD\"}) by (<<.GroupBy>>)\n- seriesQuery: '{namespace!=\"\",__name__!~\"^container_.*\"\ + }'\n seriesFilters:\n - isNot: .*_total$\n resources:\n template: <<.Resource>>\n\ + \ name:\n matches: \"\"\n as: \"\"\n metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>})\ + \ by (<<.GroupBy>>)\n- seriesQuery: '{namespace!=\"\",__name__!~\"^container_.*\"\ + }'\n seriesFilters:\n - isNot: .*_seconds_total\n resources:\n template:\ + \ <<.Resource>>\n name:\n matches: ^(.*)_total$\n as: \"\"\n metricsQuery:\ + \ sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)\n- seriesQuery:\ + \ '{namespace!=\"\",__name__!~\"^container_.*\"}'\n seriesFilters: []\n resources:\n\ + \ template: <<.Resource>>\n name:\n matches: ^(.*)_seconds_total$\n \ + \ as: \"\"\n metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by\ + \ (<<.GroupBy>>)\nresourceRules:\n cpu:\n containerLabel: container_name\n\ + \ containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m]))\n\ + \ by (<<.GroupBy>>)\n nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,\ + \ id='/'}[3m]))\n by (<<.GroupBy>>)\n resources:\n overrides:\n \ + \ namespace:\n resource: namespace\n node:\n resource:\ + \ node\n pod:\n resource: pod\n memory:\n containerLabel:\ + \ container_name\n containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>})\ + \ by (<<.GroupBy>>)\n nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'})\ + \ by\n (<<.GroupBy>>)\n resources:\n overrides:\n namespace:\n\ + \ resource: namespace\n node:\n resource: node\n \ + \ pod:\n resource: pod\n window: 3m\n \n" +kind: ConfigMap +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-server-resources + namespace: metalk8s-monitoring +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-resource-reader + namespace: metalk8s-monitoring +rules: +- apiGroups: + - '' + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-metrics + namespace: metalk8s-monitoring +rules: +- apiGroups: + - '' + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter:system:auth-delegator + namespace: metalk8s-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-resource-reader + namespace: metalk8s-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter-resource-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-hpa-controller + namespace: metalk8s-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter-server-resources +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-hpa-controller-metrics + namespace: metalk8s-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter-metrics +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter-auth-reader + namespace: metalk8s-monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: metalk8s-monitoring +--- +apiVersion: v1 +kind: Service +metadata: + annotations: {} + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter + namespace: metalk8s-monitoring +spec: + ports: + - port: 443 + protocol: TCP + targetPort: https + selector: + app: prometheus-adapter + release: prometheus-adapter + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter + namespace: metalk8s-monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-adapter + release: prometheus-adapter + template: + metadata: + annotations: + checksum/config: af5232ebf7c9aae4f20e8e0491bce5ee713153bc0ca342a4c4c275e44ef3d6e7 + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: prometheus-adapter + spec: + affinity: {} + containers: + - args: + - /adapter + - --secure-port=6443 + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url=http://prometheus-operator-prometheus:9090 + - --metrics-relist-interval=1m + - --v=4 + - --config=/etc/adapter/config.yaml + image: '{%- endraw -%}{{ build_image_name("k8s-prometheus-adapter-amd64", + False) }}{%- raw -%}:v0.5.0' + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + name: prometheus-adapter + ports: + - containerPort: 6443 + name: https + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + volumeMounts: + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + nodeSelector: + node-role.kubernetes.io/infra: '' + priorityClassName: null + serviceAccountName: prometheus-adapter + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/bootstrap + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/infra + operator: Exists + volumes: + - configMap: + name: prometheus-adapter + name: config + - emptyDir: {} + name: tmp +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: v1beta1.custom.metrics.k8s.io + namespace: metalk8s-monitoring +spec: + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: prometheus-adapter + namespace: metalk8s-monitoring + version: v1beta1 + versionPriority: 100 +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: prometheus-adapter + app.kubernetes.io/managed-by: salt + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: metalk8s + chart: prometheus-adapter-1.4.0 + heritage: metalk8s + release: prometheus-adapter + name: v1beta1.metrics.k8s.io + namespace: metalk8s-monitoring +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: prometheus-adapter + namespace: metalk8s-monitoring + version: v1beta1 + versionPriority: 100 + +{% endraw %} diff --git a/salt/metalk8s/addons/prometheus-adapter/deployed/init.sls b/salt/metalk8s/addons/prometheus-adapter/deployed/init.sls new file mode 100644 index 0000000000..439a7d1625 --- /dev/null +++ b/salt/metalk8s/addons/prometheus-adapter/deployed/init.sls @@ -0,0 +1,3 @@ +include: + - metalk8s.addons.prometheus-operator.deployed.namespace + - .chart diff --git a/salt/metalk8s/deployed.sls b/salt/metalk8s/deployed.sls index a06d4086f1..00baa7342e 100644 --- a/salt/metalk8s/deployed.sls +++ b/salt/metalk8s/deployed.sls @@ -10,3 +10,4 @@ include: - metalk8s.addons.volumes.deployed - metalk8s.addons.solutions.deployed - metalk8s.addons.ui.deployed + - metalk8s.addons.prometheus-adapter.deployed From ec577c8f70ce4fddcd75ac4e16f1a660a6d6d40f Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Tue, 26 Nov 2019 14:53:03 -0800 Subject: [PATCH 4/5] tests: re-instate `metrics.k8s.io` `APIService` test See: edf10efd0cc6e310346f11846e3e007e89ea81e6 See: https://github.com/scality/metalk8s/issues/1624 See: https://github.com/scality/metalk8s/pull/1630 --- tests/post/features/monitoring.feature | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/post/features/monitoring.feature b/tests/post/features/monitoring.feature index 18ba13cda6..4ba09c7a7f 100644 --- a/tests/post/features/monitoring.feature +++ b/tests/post/features/monitoring.feature @@ -24,3 +24,9 @@ Feature: Monitoring is up and running And job 'prometheus-operator-operator' in namespace 'metalk8s-monitoring' is 'up' And job 'prometheus-operator-prometheus' in namespace 'metalk8s-monitoring' is 'up' And job 'kube-state-metrics' in namespace 'metalk8s-monitoring' is 'up' + + Scenario: The metrics.k8s.io/v1beta1 API is available + Given the Kubernetes API is available + And we have 1 running pod labeled 'name=prometheus-adapter' in namespace 'metalk8s-monitoring' + And the 'v1beta1.metrics.k8s.io' APIService exists + Then the 'v1beta1.metrics.k8s.io' APIService is Available From 456cf72bfc10bd3d7d657e7a40ae5f3c64aafa44 Mon Sep 17 00:00:00 2001 From: Nicolas Trangez Date: Tue, 26 Nov 2019 15:35:47 -0800 Subject: [PATCH 5/5] tests: test `metrics.k8s.io/v1beta1` We assume that if this API is available and returns 'expected' results, the `kubectl top` command works (since it uses this API under the hood). See: 241c355c11afb039696e2280eae6ee6f0e3d158d See: https://github.com/scality/metalk8s/issues/2049 See: https://github.com/scality/metalk8s/pull/2057 --- tests/post/features/monitoring.feature | 8 ++++ tests/post/steps/test_monitoring.py | 66 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/tests/post/features/monitoring.feature b/tests/post/features/monitoring.feature index 4ba09c7a7f..5069d68322 100644 --- a/tests/post/features/monitoring.feature +++ b/tests/post/features/monitoring.feature @@ -30,3 +30,11 @@ Feature: Monitoring is up and running And we have 1 running pod labeled 'name=prometheus-adapter' in namespace 'metalk8s-monitoring' And the 'v1beta1.metrics.k8s.io' APIService exists Then the 'v1beta1.metrics.k8s.io' APIService is Available + + Scenario: Pod metrics can be retrieved using metrics.k8s.io/v1beta1 + Given the Kubernetes API is available + Then a pod with label 'component=kube-apiserver' in namespace 'kube-system' has metrics + + Scenario: Node metrics can be retrieved using metrics.k8s.io/v1beta1 + Given the Kubernetes API is available + Then a node with label 'node-role.kubernetes.io/bootstrap=' has metrics diff --git a/tests/post/steps/test_monitoring.py b/tests/post/steps/test_monitoring.py index f6204c33f9..044de8e3ae 100644 --- a/tests/post/steps/test_monitoring.py +++ b/tests/post/steps/test_monitoring.py @@ -27,6 +27,20 @@ def test_monitored_components(host): pass +@scenario( + '../features/monitoring.feature', + 'Pod metrics can be retrieved using metrics.k8s.io/v1beta1') +def test_pod_metrics(host): + pass + + +@scenario( + '../features/monitoring.feature', + 'Node metrics can be retrieved using metrics.k8s.io/v1beta1') +def test_node_metrics(host): + pass + + # }}} # Given {{{ @@ -108,6 +122,58 @@ def _check_object_exists(): utils.retry(_check_object_exists, times=20, wait=3) +@then(parsers.parse( + "a pod with label '{label}' in namespace '{namespace}' has metrics")) +def pod_has_metrics(label, namespace, k8s_apiclient): + def _pod_has_metrics(): + result = k8s_apiclient.call_api( + resource_path='/apis/metrics.k8s.io/v1beta1/' + 'namespaces/{namespace}/pods', + method='GET', + response_type=object, + path_params={ + 'namespace': namespace, + }, + query_params=[ + ('labelSelector', label), + ], + _return_http_data_only=True, + ) + + assert result['apiVersion'] == 'metrics.k8s.io/v1beta1' + assert result['kind'] == 'PodMetricsList' + assert result['items'] != [] + assert result['items'][0]['containers'] != [] + assert result['items'][0]['containers'][0]['usage']['cpu'] + assert result['items'][0]['containers'][0]['usage']['memory'] + + # Metrics are only available after a while (by design) + utils.retry(_pod_has_metrics, times=60, wait=3) + + +@then(parsers.parse("a node with label '{label}' has metrics")) +def node_has_metrics(label, k8s_apiclient): + def _node_has_metrics(): + result = k8s_apiclient.call_api( + resource_path='/apis/metrics.k8s.io/v1beta1/nodes', + method='GET', + response_type=object, + query_params=[ + ('labelSelector', label), + ], + _return_http_data_only=True, + ) + + assert result['apiVersion'] == 'metrics.k8s.io/v1beta1' + assert result['kind'] == 'NodeMetricsList' + assert result['items'] != [] + assert result['items'][0]['usage']['cpu'] + assert result['items'][0]['usage']['memory'] + + # Metrics are only available after a while (by design) + utils.retry(_node_has_metrics, times=60, wait=3) + + # }}} # Helpers {{{