Skip to content

Commit

Permalink
Merge branches 'w/2.5/improvement/GH-2049-metrics-api' and 'q/2057/2.…
Browse files Browse the repository at this point in the history
…4/improvement/GH-2049-metrics-api' into tmp/octopus/q/2.5
  • Loading branch information
bert-e committed Dec 17, 2019
3 parents 6fbd96c + 6f9d5c0 + 68858be commit 4b895f8
Show file tree
Hide file tree
Showing 39 changed files with 1,486 additions and 10 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ host on which they are generated. You can override this when using them using
`kubectl`s `-s`/`--server` argument to point to another address.

### Features added
- [#2049](https://github.com/scality/metalk8s/issues/2049) - Deploy
[prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter/)
to implement the `metrics.k8s.io` API, to support `kubectl top` and other
consumers of this API
(PR [#2057](https://github.com/scality/metalk8s/pull/2057))

- [#2103](https://github.com/scality/metalk8s/issues/2103) - Add a host-local
`nginx` on every node to provide highly-available and load-balanced access to
Expand Down
19 changes: 10 additions & 9 deletions buildchain/buildchain/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,16 @@
CMD_WIDTH : int = 14

# URLs of the main container repositories.
CALICO_REPOSITORY : str = 'quay.io/calico'
COREOS_REPOSITORY : str = 'quay.io/coreos'
DEX_REPOSITORY : str = 'quay.io/dexidp'
DOCKER_REPOSITORY : str = 'docker.io/library'
GOOGLE_REPOSITORY : str = 'k8s.gcr.io'
GRAFANA_REPOSITORY : str = 'docker.io/grafana'
INGRESS_REPOSITORY : str = 'quay.io/kubernetes-ingress-controller'
KIWIGRID_REPOSITORY : str = 'docker.io/kiwigrid'
PROMETHEUS_REPOSITORY : str = 'quay.io/prometheus'
CALICO_REPOSITORY : str = 'quay.io/calico'
COREOS_REPOSITORY : str = 'quay.io/coreos'
DEX_REPOSITORY : str = 'quay.io/dexidp'
DOCKER_REPOSITORY : str = 'docker.io/library'
GOOGLE_REPOSITORY : str = 'k8s.gcr.io'
GRAFANA_REPOSITORY : str = 'docker.io/grafana'
INGRESS_REPOSITORY : str = 'quay.io/kubernetes-ingress-controller'
KIWIGRID_REPOSITORY : str = 'docker.io/kiwigrid'
PROMETHEUS_ADAPTER_REPOSITORY : str = 'docker.io/directxman12'
PROMETHEUS_REPOSITORY : str = 'quay.io/prometheus'

# Paths {{{

Expand Down
3 changes: 3 additions & 0 deletions buildchain/buildchain/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage:
constants.KIWIGRID_REPOSITORY: [
'k8s-sidecar',
],
constants.PROMETHEUS_ADAPTER_REPOSITORY: [
'k8s-prometheus-adapter-amd64',
],
constants.PROMETHEUS_REPOSITORY: [
'alertmanager',
'node-exporter',
Expand Down
4 changes: 4 additions & 0 deletions buildchain/buildchain/salt_tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,10 @@ def _get_parts(self) -> Iterator[str]:
file_dep=[SCALITY_LOGO, SCALITY_FAVICON, LOGIN_STYLE],
renderer=targets.Renderer.SLS,
),

Path('salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls'),
Path('salt/metalk8s/addons/prometheus-adapter/deployed/init.sls'),

Path('salt/metalk8s/addons/prometheus-operator/deployed/chart.sls'),
Path('salt/metalk8s/addons/prometheus-operator/deployed/cleanup.sls'),
Path('salt/metalk8s/addons/prometheus-operator/deployed/dashboards.sls'),
Expand Down
5 changes: 5 additions & 0 deletions buildchain/buildchain/versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,11 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
version='v2.12.0',
digest='sha256:cd93b8711bb92eb9c437d74217311519e0a93bc55779aa664325dc83cd13cb3',
),
Image(
name='k8s-prometheus-adapter-amd64',
version='v0.5.0',
digest='sha256:acf4c4dc6aaa51e3004eebdc410b509db45d1a6ebd255457c5745f0b4671678f',
),
Image(
name='prometheus-config-reloader',
version='v0.32.0',
Expand Down
47 changes: 47 additions & 0 deletions charts/prometheus-adapter.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
image:
repository: '{%- endraw -%}{{ build_image_name(\"k8s-prometheus-adapter-amd64\", False) }}{%- raw -%}'

nodeSelector:
node-role.kubernetes.io/infra: ''

prometheus:
url: http://prometheus-operator-prometheus

replicas: 1

rules:
resource:
cpu:
containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>)
nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>)
resources:
overrides:
node:
resource: node
namespace:
resource: namespace
pod:
resource: pod
containerLabel: container_name
memory:
containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
resources:
overrides:
node:
resource: node
namespace:
resource: namespace
pod:
resource: pod
containerLabel: container_name
window: 3m


tolerations:
- key: 'node-role.kubernetes.io/bootstrap'
operator: 'Exists'
effect: 'NoSchedule'
- key: 'node-role.kubernetes.io/infra'
operator: 'Exists'
effect: 'NoSchedule'
21 changes: 21 additions & 0 deletions charts/prometheus-adapter/.helmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
20 changes: 20 additions & 0 deletions charts/prometheus-adapter/Chart.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
apiVersion: v1
appVersion: v0.5.0
description: A Helm chart for k8s prometheus adapter
home: https://github.com/DirectXMan12/k8s-prometheus-adapter
keywords:
- hpa
- metrics
- prometheus
- adapter
maintainers:
- email: [email protected]
name: mattiasgees
- name: steven-sheehy
- email: [email protected]
name: hectorj2f
name: prometheus-adapter
sources:
- https://github.com/kubernetes/charts
- https://github.com/DirectXMan12/k8s-prometheus-adapter
version: 1.4.0
8 changes: 8 additions & 0 deletions charts/prometheus-adapter/OWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
approvers:
- mattiasgees
- steven-sheehy
- hectorj2f
reviewers:
- mattiasgees
- steven-sheehy
- hectorj2f
157 changes: 157 additions & 0 deletions charts/prometheus-adapter/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
# Prometheus Adapter

Installs the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics.

## Prerequisites

Kubernetes 1.11+

## Installing the Chart

To install the chart with the release name `my-release`:

```console
$ helm install --name my-release stable/prometheus-adapter
```

This command deploys the prometheus adapter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.

## Using the Chart

To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`.

Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md) for the proper format).

Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics).

The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis):

### Custom Metrics

Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`:

```
rules:
custom:
- seriesQuery: '{__name__=~"^some_metric_count$"}'
resources:
template: <<.Resource>>
name:
matches: ""
as: "my_custom_metric"
metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
```

### External Metrics

Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`:

```
rules:
external:
- seriesQuery: '{__name__=~"^some_metric_count$"}'
resources:
template: <<.Resource>>
name:
matches: ""
as: "my_external_metric"
metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
```

### Resource Metrics

Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`:

```
rules:
resource:
cpu:
containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>)
nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>)
resources:
overrides:
instance:
resource: node
namespace:
resource: namespace
pod_name:
resource: pod
containerLabel: container_name
memory:
containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
resources:
overrides:
instance:
resource: node
namespace:
resource: namespace
pod_name:
resource: pod
containerLabel: container_name
window: 3m
```

**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster.

## Uninstalling the Chart

To uninstall/delete the `my-release` deployment:

```console
$ helm delete my-release
```

The command removes all the Kubernetes components associated with the chart and deletes the release.

## Configuration

The following table lists the configurable parameters of the Prometheus Adapter chart and their default values.

| Parameter | Description | Default |
| ------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------------|
| `affinity` | Node affinity | `{}` |
| `image.repository` | Image repository | `directxman12/k8s-prometheus-adapter-amd64` |
| `image.tag` | Image tag | `v0.5.0` |
| `image.pullPolicy` | Image pull policy | `IfNotPresent` |
| `image.pullSecrets` | Image pull secrets | `{}` |
| `logLevel` | Log level | `4` |
| `metricsRelistInterval` | Interval at which to re-list the set of all available metrics from Prometheus | `1m` |
| `nodeSelector` | Node labels for pod assignment | `{}` |
| `podAnnotations` | Annotations to add to the pod | `{}` |
| `priorityClassName` | Pod priority | `` |
| `prometheus.url` | Url of where we can find the Prometheus service | `http://prometheus.default.svc` |
| `prometheus.port` | Port of where we can find the Prometheus service, zero to omit this option | `9090` |
| `rbac.create` | If true, create & use RBAC resources | `true` |
| `resources` | CPU/Memory resource requests/limits | `{}` |
| `rules.default` | If `true`, enable a set of default rules in the configmap | `true` |
| `rules.custom` | A list of custom configmap rules | `[]` |
| `rules.existing` | The name of an existing configMap with rules. Overrides default, custom and external. | `` |
| `rules.external` | A list of custom rules for external metrics API | `[]` |
| `rules.resource` | `resourceRules` to set in configmap rules | `{}` |
| `service.annotations` | Annotations to add to the service | `{}` |
| `service.port` | Service port to expose | `443` |
| `service.type` | Type of service to create | `ClusterIP` |
| `serviceAccount.create` | If true, create & use Serviceaccount | `true` |
| `serviceAccount.name` | If not set and create is true, a name is generated using the fullname template | `` |
| `tls.enable` | If true, use the provided certificates. If false, generate self-signed certs | `false` |
| `tls.ca` | Public CA file that signed the APIService (ignored if tls.enable=false) | `` |
| `tls.key` | Private key of the APIService (ignored if tls.enable=false) | `` |
| `tls.certificate` | Public key of the APIService (ignored if tls.enable=false) | `` |
| `extraVolumeMounts` | Any extra volumes mounts | `[]` |
| `extraVolumes` | Any extra volumes | `[]` |
| `tolerations` | List of node taints to tolerate | `[]` |

Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,

```console
$ helm install --name my-release \
--set logLevel=1 \
stable/prometheus-adapter
```

Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,

```console
$ helm install --name my-release -f values.yaml stable/prometheus-adapter
```
Empty file.
9 changes: 9 additions & 0 deletions charts/prometheus-adapter/ci/external-rules-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
rules:
external:
- seriesQuery: '{__name__=~"^some_metric_count$"}'
resources:
template: <<.Resource>>
name:
matches: ""
as: "my_custom_metric"
metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
9 changes: 9 additions & 0 deletions charts/prometheus-adapter/templates/NOTES.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed.
In a few minutes you should be able to list metrics using the following command(s):
{{ if .Values.rules.resource }}
kubectl get --raw /apis/metrics.k8s.io/v1beta1
{{- end }}
kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1
{{ if .Values.rules.external }}
kubectl get --raw /apis/external.metrics.k8s.io/v1beta1
{{- end }}
43 changes: 43 additions & 0 deletions charts/prometheus-adapter/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "k8s-prometheus-adapter.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "k8s-prometheus-adapter.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "k8s-prometheus-adapter.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}

{{/*
Create the name of the service account to use
*/}}
{{- define "k8s-prometheus-adapter.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
Loading

0 comments on commit 4b895f8

Please sign in to comment.