Merge pull request #11491 from olemarkus/cas-update-manifest

Update CAS manifest
kubernetes · May 21, 2021 · 8a6ec14 · 8a6ec14
2 parents 7aaa448 + 2fb1861
commit 8a6ec14
Show file tree

Hide file tree

Showing 2 changed files with 209 additions and 58 deletions.
diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
@@ -1,5 +1,19 @@
 {{ with .ClusterAutoscaler }}
-# Sourced from https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-release-1.20/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml
+# Sourced from https://github.com/kubernetes/autoscaler/
+---
+# Source: cluster-autoscaler/templates/pdb.yaml
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+  name: cluster-autoscaler
+spec:
+  selector:
+    matchLabels:
+      k8s-app: "cluster-autoscaler"
+  maxUnavailable: 1
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -18,52 +32,134 @@ metadata:
     k8s-addon: cluster-autoscaler.addons.k8s.io
     k8s-app: cluster-autoscaler
 rules:
-  - apiGroups: [""]
-    resources: ["events", "endpoints"]
-    verbs: ["create", "patch"]
-  - apiGroups: [""]
-    resources: ["pods/eviction"]
-    verbs: ["create"]
-  - apiGroups: [""]
-    resources: ["pods/status"]
-    verbs: ["update"]
-  - apiGroups: [""]
-    resources: ["endpoints"]
-    resourceNames: ["cluster-autoscaler"]
-    verbs: ["get", "update"]
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["watch", "list", "get", "update"]
-  - apiGroups: [""]
-    resources:
-      - "pods"
-      - "services"
-      - "replicationcontrollers"
-      - "persistentvolumeclaims"
-      - "persistentvolumes"
-    verbs: ["watch", "list", "get"]
-  - apiGroups: ["extensions"]
-    resources: ["replicasets", "daemonsets"]
-    verbs: ["watch", "list", "get"]
-  - apiGroups: ["policy"]
-    resources: ["poddisruptionbudgets"]
-    verbs: ["watch", "list"]
-  - apiGroups: ["apps"]
-    resources: ["statefulsets", "replicasets", "daemonsets"]
-    verbs: ["watch", "list", "get"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses", "csinodes"]
-    verbs: ["watch", "list", "get"]
-  - apiGroups: ["batch", "extensions"]
-    resources: ["jobs"]
-    verbs: ["get", "list", "watch", "patch"]
-  - apiGroups: ["coordination.k8s.io"]
-    resources: ["leases"]
-    verbs: ["create"]
-  - apiGroups: ["coordination.k8s.io"]
-    resourceNames: ["cluster-autoscaler"]
-    resources: ["leases"]
-    verbs: ["get", "update"]
+  - apiGroups:
+      - ""
+    resources:
+      - events
+      - endpoints
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+    - ""
+    resources:
+    - pods/eviction
+    verbs:
+    - create
+  - apiGroups:
+      - ""
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    resourceNames:
+      - cluster-autoscaler
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+    - watch
+    - list
+    - get
+    - update
+  - apiGroups:
+    - ""
+    resources:
+      - pods
+      - services
+      - replicationcontrollers
+      - persistentvolumeclaims
+      - persistentvolumes
+    verbs:
+      - watch
+      - list
+      - get
+  - apiGroups:
+    - batch
+    resources:
+      - jobs
+      - cronjobs
+    verbs:
+      - watch
+      - list
+      - get
+  - apiGroups:
+    - batch
+    - extensions
+    resources:
+    - jobs
+    verbs:
+    - get
+    - list
+    - patch
+    - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - replicasets
+      - daemonsets
+    verbs:
+      - watch
+      - list
+      - get
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - watch
+      - list
+  - apiGroups:
+    - apps
+    resources:
+    - daemonsets
+    - replicasets
+    - statefulsets
+    verbs:
+    - watch
+    - list
+    - get
+  - apiGroups:
+    - storage.k8s.io
+    resources:
+    - storageclasses
+    - csidrivers
+    - csinodes
+    - csistoragecapacities
+    verbs:
+    - watch
+    - list
+    - get
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+    - coordination.k8s.io
+    resources:
+    - leases
+    verbs:
+    - create
+  - apiGroups:
+    - coordination.k8s.io
+    resourceNames:
+    - cluster-autoscaler
+    resources:
+    - leases
+    verbs:
+    - get
+    - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
@@ -74,14 +170,22 @@ metadata:
     k8s-addon: cluster-autoscaler.addons.k8s.io
     k8s-app: cluster-autoscaler
 rules:
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["create","list","watch"]
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
-    verbs: ["delete", "get", "update", "watch"]
-
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    resourceNames:
+      - cluster-autoscaler-status
+    verbs:
+      - delete
+      - get
+      - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -124,9 +228,9 @@ metadata:
   name: cluster-autoscaler
   namespace: kube-system
   labels:
-    app: cluster-autoscaler
+    k8s-app: cluster-autoscaler
 spec:
-  replicas: 1
+  replicas: {{ ControlPlaneControllerReplicas }}
   selector:
     matchLabels:
       app: cluster-autoscaler
@@ -138,6 +242,26 @@ spec:
         prometheus.io/port: "8085"
         prometheus.io/scrape: "true"
     spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - cluster-autoscaler
+            topologyKey: kubernetes.com/hostname
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100 
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                  - cluster-autoscaler
+              topologyKey: topology.kubernetes.io/zone
       priorityClassName: system-cluster-critical
       serviceAccountName: cluster-autoscaler
       tolerations:
@@ -172,11 +296,12 @@ spec:
           ports:
           - containerPort: 8085
             protocol: TCP
+            name: http
           livenessProbe:
             failureThreshold: 3
             httpGet:
               path: /health-check
-              port: 8085
+              port: http
               scheme: HTTP
             periodSeconds: 10
             successThreshold: 1

diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go
@@ -97,6 +97,9 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 
 	dest["GetInstanceGroup"] = tf.GetInstanceGroup
 	dest["GetNodeInstanceGroups"] = tf.GetNodeInstanceGroups
+	dest["HasHighlyAvailableControlPlane"] = tf.HasHighlyAvailableControlPlane
+	dest["ControlPlaneControllerReplicas"] = tf.ControlPlaneControllerReplicas
+
 	dest["CloudTags"] = tf.CloudTagsForInstanceGroup
 	dest["KubeDNS"] = func() *kops.KubeDNSConfig {
 		return cluster.Spec.KubeDNS
@@ -272,6 +275,29 @@ func (tf *TemplateFunctions) GetInstanceGroup(name string) (*kops.InstanceGroup,
 	return ig, nil
 }
 
+// ControlPlaneControllerReplicas returns the amount of replicas for a controllers that should run in the cluster
+// If the cluster has a highly available control plane, this function will return 2, if it has 1 control plane node, it will return 1
+func (tf *TemplateFunctions) ControlPlaneControllerReplicas() int {
+	if tf.HasHighlyAvailableControlPlane() {
+		return 2
+	}
+	return 1
+}
+
+// HasHighlyAvailableControlPlane returns true of the cluster has more than one control plane node. False otherwise.
+func (tf *TemplateFunctions) HasHighlyAvailableControlPlane() bool {
+	cp := 0
+	for _, ig := range tf.InstanceGroups {
+		if ig.Spec.Role == kops.InstanceGroupRoleMaster {
+			cp++
+			if cp > 1 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // CloudControllerConfigArgv returns the args to external cloud controller
 func (tf *TemplateFunctions) CloudControllerConfigArgv() ([]string, error) {
 	cluster := tf.Cluster