alpha to beta

cluster/config-defaults.yaml

@@ -107,14 +107,20 @@ skipper_redis_write_timeout: "25ms"
 
 skipper_cluster_ratelimit_max_group_shards: 1
 
+#
 # skipper routesrv settings
+#
+# skipper_routesrv_enabled is a three state switch:
+# - "false" - routesrv deployment is removed, skipper uses own k8s dataclient
+# - "pre" - routesrv is deployed, skipper uses own k8s dataclient
+# - "exec" - routesrv is deployed, skipper uses routesrv
 {{if eq .Cluster.Environment "production"}}
 skipper_routesrv_enabled: "false"
 skipper_routesrv_replicas: 3
 skipper_routesrv_cpu: "1000m"
 skipper_routesrv_memory: "1Gi"
 {{else}}
-skipper_routesrv_enabled: "true"
+skipper_routesrv_enabled: "false"
 skipper_routesrv_replicas: 3
 skipper_routesrv_cpu: "100m"
 skipper_routesrv_memory: "1Gi"

cluster/manifests/deletions.yaml

@@ -58,11 +58,11 @@ pre_apply:
 #   kind: DaemonSet
 #   propagation_policy: Orphan
 # step 2 (prometheus consolidation)
-# - labels:
-#     application: prometheus
-#   namespace: kube-system
-#   kind: StatefulSet
-#   propagation_policy: Orphan
+- labels:
+    application: prometheus
+  namespace: kube-system
+  kind: StatefulSet
+  propagation_policy: Orphan
 
 # everything defined under here will be deleted after applying the manifests
 post_apply:
@@ -129,3 +129,9 @@ post_apply:
 - name: cluster-admin-okta
   kind: ClusterRoleBinding
 {{- end }}
+
+{{ if eq .Cluster.ConfigItems.skipper_routesrv_enabled "false" }}
+- name: skipper-ingress-routesrv
+  namespace: kube-system
+  kind: Deployment
+{{ end }}

cluster/manifests/kube-static-egress-controller/deployment.yaml

@@ -44,8 +44,8 @@ spec:
 {{ end }}
         - "--stack-termination-protection"
         - "--cluster-id={{ .ID }}"
+        - "--cluster-id-tag-prefix=zalando.org/cluster/"
         - "--additional-stack-tags=InfrastructureComponent=true"
-        - "--additional-stack-tags=zalando.org/cluster/{{ .Cluster.ID }}=owned"
         env:
         - name: AWS_REGION
           value: {{ .Cluster.Region }}

cluster/manifests/prometheus/statefulset.yaml

@@ -4,8 +4,7 @@ metadata:
   annotations:
     pdb-controller.zalando.org/non-ready-ttl: "5m"
   labels:
-    application: prometheus
-    # application: kubernetes # step 2
+    application: kubernetes
     component: prometheus
     version: v2.32.1
 {{- if ne .ConfigItems.prometheus_csi_ebs "true" }}
@@ -19,8 +18,7 @@ spec:
   podManagementPolicy: Parallel
   selector:
     matchLabels:
-      application: prometheus
-      # statefulset: prometheus # step 2
+      statefulset: prometheus
   serviceName: prometheus
   template:
     metadata:

cluster/manifests/skipper/deployment.yaml

@@ -91,7 +91,7 @@ spec:
         args:
           - "run.sh"
           - "skipper"
-{{ if eq .ConfigItems.skipper_routesrv_enabled "true" }}
+{{ if eq .ConfigItems.skipper_routesrv_enabled "exec" }}
           - "-routes-urls=http://skipper-ingress-routesrv.kube-system.svc.cluster.local/routes"
           - "-normalize-host"
 {{ else }}
@@ -103,11 +103,11 @@ spec:
           - "-address=:9999"
           - "-wait-first-route-load"
           - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}"
-{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_skipper_eastwest "true")}}
+{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest "true")}}
           - "-enable-kubernetes-east-west"
           - "-kubernetes-east-west-domain=.ingress.cluster.local"
 {{ end }}
-{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_skipper_eastwest_range "true")}}
+{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest_range "true")}}
           - "-kubernetes-east-west-range-domains=ingress.cluster.local"
           - "-kubernetes-east-west-range-predicates=ClientIP(\"10.2.0.0/16\", \"{{ .Values.vpc_ipv4_cidr }}\")"
 {{ end }}
@@ -135,7 +135,7 @@ spec:
           - "-api-usage-monitoring-client-keys=https://identity.zalando.com/managed-id,sub"
           - "-api-usage-monitoring-default-client-tracking-pattern=services[.].*"
 {{ end }}
-{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true")}}
+{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true")}}
           - "-default-filters-dir=/etc/config/default-filters"
 {{ end }}
           - "-max-audit-body=0"
@@ -275,7 +275,7 @@ spec:
           - name: routes
             mountPath: /etc/routes
 {{ end }}
-{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true")}}
+{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true")}}
           - name: filters
             mountPath: /etc/config/default-filters
 {{ end }}
@@ -295,7 +295,7 @@ spec:
           configMap:
             name: sandbox-tokeninfo-bridge-conf
 {{ end }}
-{{ if and (ne .ConfigItems.skipper_routesrv_enabled "true") (eq .ConfigItems.enable_apimonitoring "true") }}
+{{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_apimonitoring "true") }}
         - name: filters
           configMap:
             name: skipper-default-filters

cluster/manifests/skipper/routesrv-deployment.yaml

@@ -1,3 +1,4 @@
+{{ if ne .ConfigItems.skipper_routesrv_enabled "false" }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -108,3 +109,4 @@ spec:
             name: skipper-default-filters
             optional: true
 {{ end }}
+{{ end }}

cluster/node-pools/master-default/userdata.yaml

@@ -200,7 +200,7 @@ write_files:
             requests:
               cpu: 100m
               memory: 200Mi
-        - image: registry.opensource.zalan.do/teapot/admission-controller:master-138
+        - image: registry.opensource.zalan.do/teapot/admission-controller:master-139
           name: admission-controller
           lifecycle:
             preStop: