Merge pull request #7783 from justinsb/move_kops_controller_to_daemonset

Move kops-controller to daemonset
kubernetes · Oct 15, 2019 · 8ae7ed6 · 8ae7ed6
2 parents 39cbb2b + ee04368
commit 8ae7ed6
Show file tree

Hide file tree

Showing 7 changed files with 263 additions and 22 deletions.
diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go
@@ -55,10 +55,6 @@ func main() {
 	// Disable metrics by default (avoid port conflicts, also risky because we are host network)
 	metricsAddress := ":0"
 	//flag.StringVar(&metricsAddr, "metrics-addr", metricsAddress, "The address the metric endpoint binds to.")
-	// Enable leader election
-	enableLeaderElection := false
-	//flag.BoolVar(&enableLeaderElection, "enable-leader-election", enableLeaderElection,
-	//	"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 
 	configPath := "/etc/kubernetes/kops-controller/config.yaml"
 	flag.StringVar(&configPath, "conf", configPath, "Location of yaml configuration file")
@@ -93,7 +89,8 @@ func main() {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddress,
-		LeaderElection:     enableLeaderElection,
+		LeaderElection:     true,
+		LeaderElectionID:   "kops-controller-leader",
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template
@@ -11,6 +11,9 @@ data:
 
 ---
 
+# Deployment of size 0, to move from Deployment to DaemonSet
+# TODO: Remove in beta? (it's only been on master branch)
+
 kind: Deployment
 apiVersion: apps/v1
 metadata:
@@ -21,7 +24,7 @@ metadata:
     k8s-app: kops-controller
     version: v1.15.0-alpha.1
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       k8s-app: kops-controller
@@ -31,12 +34,43 @@ spec:
         k8s-addon: kops-controller.addons.k8s.io
         k8s-app: kops-controller
         version: v1.15.0-alpha.1
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      serviceAccountName: default
+      containers:
+      - name: sleep
+        image: k8s.gcr.io/pause-amd64:3.0
+        command: [ "/pause" ]
+
+---
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kops-controller
+  namespace: kube-system
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.15.0-alpha.1
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.15.0-alpha.1
+    spec:
+      priorityClassName: system-node-critical
       tolerations:
       - key: "node-role.kubernetes.io/master"
-        effect: NoSchedule
+        operator: Exists
       nodeSelector:
         node-role.kubernetes.io/master: ""
       dnsPolicy: Default  # Don't use cluster DNS (we are likely running before kube-dns)
@@ -124,3 +158,53 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  - kops-controller-leader
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+  - delete
+# Workaround for https://github.com/kubernetes/kubernetes/issues/80295
+# We can't restrict creation of objects by name
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
diff --git a/.../cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/.../cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml
@@ -21,14 +21,41 @@ metadata:
   name: kops-controller
   namespace: kube-system
 spec:
-  replicas: 1
+  replicas: 0
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.15.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /pause
+        image: gcr.io/google_containers/pause-amd64:3.0
+        name: sleep
+      serviceAccountName: default
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.15.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
   selector:
     matchLabels:
       k8s-app: kops-controller
   template:
     metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
       labels:
         k8s-addon: kops-controller.addons.k8s.io
         k8s-app: kops-controller
@@ -52,14 +79,19 @@ spec:
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      priorityClassName: system-node-critical
       serviceAccount: kops-controller
       tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
+      - key: node-role.kubernetes.io/master
+        operator: Exists
       volumes:
       - configMap:
           name: kops-controller
         name: kops-controller-config
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
 
 ---
 
@@ -106,3 +138,51 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kops-controller-leader
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
@@ -7,7 +7,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592
+    manifestHash: 24cf09054ddfdcb490b878b04ff321026daa10c7
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io

diff --git a/.../cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/.../cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml
@@ -21,14 +21,41 @@ metadata:
   name: kops-controller
   namespace: kube-system
 spec:
-  replicas: 1
+  replicas: 0
+  selector:
+    matchLabels:
+      k8s-app: kops-controller
+  template:
+    metadata:
+      labels:
+        k8s-addon: kops-controller.addons.k8s.io
+        k8s-app: kops-controller
+        version: v1.15.0-alpha.1
+    spec:
+      containers:
+      - command:
+        - /pause
+        image: gcr.io/google_containers/pause-amd64:3.0
+        name: sleep
+      serviceAccountName: default
+
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+    k8s-app: kops-controller
+    version: v1.15.0-alpha.1
+  name: kops-controller
+  namespace: kube-system
+spec:
   selector:
     matchLabels:
       k8s-app: kops-controller
   template:
     metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
       labels:
         k8s-addon: kops-controller.addons.k8s.io
         k8s-app: kops-controller
@@ -52,14 +79,19 @@ spec:
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      priorityClassName: system-node-critical
       serviceAccount: kops-controller
       tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/master
+      - key: node-role.kubernetes.io/master
+        operator: Exists
       volumes:
       - configMap:
           name: kops-controller
         name: kops-controller-config
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
 
 ---
 
@@ -106,3 +138,51 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - kops-controller-leader
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kops-controller
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:serviceaccount:kube-system:kops-controller
diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml
@@ -7,7 +7,7 @@ spec:
   - id: k8s-1.16
     kubernetesVersion: '>=1.16.0-alpha.0'
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: 2e308b2c4d21ed023e2418068669afd4a5eb0592
+    manifestHash: 24cf09054ddfdcb490b878b04ff321026daa10c7
     name: kops-controller.addons.k8s.io
     selector:
       k8s-addon: kops-controller.addons.k8s.io