From c3032252b70a33b1807deed281069b3244607556 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Tue, 25 Jan 2022 17:04:36 +0100 Subject: [PATCH 1/5] Step 2 of consolidating Prometheus Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/deletions.yaml | 10 +++++----- cluster/manifests/prometheus/statefulset.yaml | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index 2ce5565b05..cf0e5f9d81 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -58,11 +58,11 @@ pre_apply: # kind: DaemonSet # propagation_policy: Orphan # step 2 (prometheus consolidation) -# - labels: -# application: prometheus -# namespace: kube-system -# kind: StatefulSet -# propagation_policy: Orphan +- labels: + application: prometheus + namespace: kube-system + kind: StatefulSet + propagation_policy: Orphan # everything defined under here will be deleted after applying the manifests post_apply: diff --git a/cluster/manifests/prometheus/statefulset.yaml b/cluster/manifests/prometheus/statefulset.yaml index 34dc4f4e1d..88593aa2fa 100644 --- a/cluster/manifests/prometheus/statefulset.yaml +++ b/cluster/manifests/prometheus/statefulset.yaml @@ -4,8 +4,7 @@ metadata: annotations: pdb-controller.zalando.org/non-ready-ttl: "5m" labels: - application: prometheus - # application: kubernetes # step 2 + application: kubernetes component: prometheus version: v2.32.1 {{- if ne .ConfigItems.prometheus_csi_ebs "true" }} @@ -19,8 +18,7 @@ spec: podManagementPolicy: Parallel selector: matchLabels: - application: prometheus - # statefulset: prometheus # step 2 + statefulset: prometheus serviceName: prometheus template: metadata: From 07957f1c5ccb9e8ac8134c71af14ab4dd08e6362 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 26 Jan 2022 12:17:09 +0100 Subject: [PATCH 2/5] skipper/routesrv: Delete routesrv deployment when not enabled Signed-off-by: Alexander Yastrebov --- cluster/config-defaults.yaml | 2 +- cluster/manifests/deletions.yaml | 6 ++++++ cluster/manifests/skipper/routesrv-deployment.yaml | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index e07f61f999..c3dca59a4d 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -114,7 +114,7 @@ skipper_routesrv_replicas: 3 skipper_routesrv_cpu: "1000m" skipper_routesrv_memory: "1Gi" {{else}} -skipper_routesrv_enabled: "true" +skipper_routesrv_enabled: "false" skipper_routesrv_replicas: 3 skipper_routesrv_cpu: "100m" skipper_routesrv_memory: "1Gi" diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index cf0e5f9d81..2375aee6dc 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -129,3 +129,9 @@ post_apply: - name: cluster-admin-okta kind: ClusterRoleBinding {{- end }} + +{{ if ne .Cluster.ConfigItems.skipper_routesrv_enabled "true" }} +- name: skipper-ingress-routesrv + namespace: kube-system + kind: Deployment +{{ end }} diff --git a/cluster/manifests/skipper/routesrv-deployment.yaml b/cluster/manifests/skipper/routesrv-deployment.yaml index 3bbe647cc1..d07291be4a 100644 --- a/cluster/manifests/skipper/routesrv-deployment.yaml +++ b/cluster/manifests/skipper/routesrv-deployment.yaml @@ -1,3 +1,4 @@ +{{ if ne .ConfigItems.skipper_routesrv_enabled "true" }} apiVersion: apps/v1 kind: Deployment metadata: @@ -108,3 +109,4 @@ spec: name: skipper-default-filters optional: true {{ end }} +{{ end }} From a665be2b95a5886fd2ebc64099ff2a14adabfa77 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 26 Jan 2022 15:09:09 +0100 Subject: [PATCH 3/5] admission-controller: update version to support setting replicas from the HPA Signed-off-by: Alexander Yastrebov --- cluster/node-pools/master-default/userdata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/node-pools/master-default/userdata.yaml b/cluster/node-pools/master-default/userdata.yaml index 8a89ae3ce8..ca8f37e86a 100644 --- a/cluster/node-pools/master-default/userdata.yaml +++ b/cluster/node-pools/master-default/userdata.yaml @@ -200,7 +200,7 @@ write_files: requests: cpu: 100m memory: 200Mi - - image: registry.opensource.zalan.do/teapot/admission-controller:master-138 + - image: registry.opensource.zalan.do/teapot/admission-controller:master-139 name: admission-controller lifecycle: preStop: From f4624eb992682a67c34d0c32394018ecc0661134 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Wed, 26 Jan 2022 15:42:38 +0100 Subject: [PATCH 4/5] Change cluster-id tag prefix for egress stack Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/kube-static-egress-controller/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/kube-static-egress-controller/deployment.yaml b/cluster/manifests/kube-static-egress-controller/deployment.yaml index 446dea83f9..02503a06df 100644 --- a/cluster/manifests/kube-static-egress-controller/deployment.yaml +++ b/cluster/manifests/kube-static-egress-controller/deployment.yaml @@ -44,8 +44,8 @@ spec: {{ end }} - "--stack-termination-protection" - "--cluster-id={{ .ID }}" + - "--cluster-id-tag-prefix=zalando.org/cluster/" - "--additional-stack-tags=InfrastructureComponent=true" - - "--additional-stack-tags=zalando.org/cluster/{{ .Cluster.ID }}=owned" env: - name: AWS_REGION value: {{ .Cluster.Region }} From d4f18a1575eb1fb5e85757adc6607fb43ef0b1b1 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 26 Jan 2022 21:53:21 +0100 Subject: [PATCH 5/5] ingress/routesrv: Makes skipper_routesrv_enabled a three state switch Signed-off-by: Alexander Yastrebov --- cluster/config-defaults.yaml | 6 ++++++ cluster/manifests/deletions.yaml | 2 +- cluster/manifests/skipper/deployment.yaml | 12 ++++++------ cluster/manifests/skipper/routesrv-deployment.yaml | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index c3dca59a4d..3e74ee0cd8 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -107,7 +107,13 @@ skipper_redis_write_timeout: "25ms" skipper_cluster_ratelimit_max_group_shards: 1 +# # skipper routesrv settings +# +# skipper_routesrv_enabled is a three state switch: +# - "false" - routesrv deployment is removed, skipper uses own k8s dataclient +# - "pre" - routesrv is deployed, skipper uses own k8s dataclient +# - "exec" - routesrv is deployed, skipper uses routesrv {{if eq .Cluster.Environment "production"}} skipper_routesrv_enabled: "false" skipper_routesrv_replicas: 3 diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml index 2375aee6dc..025bb7c7ab 100644 --- a/cluster/manifests/deletions.yaml +++ b/cluster/manifests/deletions.yaml @@ -130,7 +130,7 @@ post_apply: kind: ClusterRoleBinding {{- end }} -{{ if ne .Cluster.ConfigItems.skipper_routesrv_enabled "true" }} +{{ if eq .Cluster.ConfigItems.skipper_routesrv_enabled "false" }} - name: skipper-ingress-routesrv namespace: kube-system kind: Deployment diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 3921c75d6d..148698b3ad 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -91,7 +91,7 @@ spec: args: - "run.sh" - "skipper" -{{ if eq .ConfigItems.skipper_routesrv_enabled "true" }} +{{ if eq .ConfigItems.skipper_routesrv_enabled "exec" }} - "-routes-urls=http://skipper-ingress-routesrv.kube-system.svc.cluster.local/routes" - "-normalize-host" {{ else }} @@ -103,11 +103,11 @@ spec: - "-address=:9999" - "-wait-first-route-load" - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}" -{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_skipper_eastwest "true")}} +{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest "true")}} - "-enable-kubernetes-east-west" - "-kubernetes-east-west-domain=.ingress.cluster.local" {{ end }} -{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_skipper_eastwest_range "true")}} +{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest_range "true")}} - "-kubernetes-east-west-range-domains=ingress.cluster.local" - "-kubernetes-east-west-range-predicates=ClientIP(\"10.2.0.0/16\", \"{{ .Values.vpc_ipv4_cidr }}\")" {{ end }} @@ -135,7 +135,7 @@ spec: - "-api-usage-monitoring-client-keys=https://identity.zalando.com/managed-id,sub" - "-api-usage-monitoring-default-client-tracking-pattern=services[.].*" {{ end }} -{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true")}} +{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true")}} - "-default-filters-dir=/etc/config/default-filters" {{ end }} - "-max-audit-body=0" @@ -275,7 +275,7 @@ spec: - name: routes mountPath: /etc/routes {{ end }} -{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true")}} +{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true")}} - name: filters mountPath: /etc/config/default-filters {{ end }} @@ -295,7 +295,7 @@ spec: configMap: name: sandbox-tokeninfo-bridge-conf {{ end }} -{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true") }} +{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true") }} - name: filters configMap: name: skipper-default-filters diff --git a/cluster/manifests/skipper/routesrv-deployment.yaml b/cluster/manifests/skipper/routesrv-deployment.yaml index d07291be4a..b8e92e2139 100644 --- a/cluster/manifests/skipper/routesrv-deployment.yaml +++ b/cluster/manifests/skipper/routesrv-deployment.yaml @@ -1,4 +1,4 @@ -{{ if ne .ConfigItems.skipper_routesrv_enabled "true" }} +{{ if ne .ConfigItems.skipper_routesrv_enabled "false" }} apiVersion: apps/v1 kind: Deployment metadata: