Merge pull request #10945 from olemarkus/exclude-cp-nodes-from-lbs

Exclude CP nodes from load balancers
kubernetes · Mar 3, 2021 · fcefcb5 · fcefcb5
2 parents 83e1424 + 32fce0d
commit fcefcb5
Show file tree

Hide file tree

Showing 121 changed files with 1,730 additions and 1,054 deletions.
diff --git a/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml b/...servergroup/multizone-setup-3-masters-3-nodes-without-bastion-auto-zone-distribution.yaml
@@ -71,6 +71,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-1-cluster
@@ -151,6 +152,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-2-cluster
@@ -231,6 +233,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-3-cluster

diff --git a/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml b/.../servergroup/multizone-setup-3-masters-3-nodes-without-bastion-with-API-loadbalancer.yaml
@@ -72,6 +72,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
 Name: master-a-1-cluster
@@ -140,6 +141,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
 Name: master-b-1-cluster
@@ -208,6 +210,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c
 Name: master-c-1-cluster

diff --git a/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml b/...l/openstackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-bastion.yaml
@@ -79,6 +79,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
 Name: master-a-1-cluster
@@ -159,6 +160,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
 Name: master-b-1-cluster
@@ -239,6 +241,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c
 Name: master-c-1-cluster

diff --git a/...ackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml b/...ackmodel/tests/servergroup/multizone-setup-3-masters-3-nodes-without-external-router.yaml
@@ -31,6 +31,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-a
 Name: master-a-1-cluster
@@ -105,6 +106,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-b
 Name: master-b-1-cluster
@@ -179,6 +181,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master-c
 Name: master-c-1-cluster

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion-2.yaml
@@ -90,6 +90,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-1-cluster

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node-one-bastion.yaml
@@ -116,6 +116,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-1-cluster

diff --git a/...model/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml b/...model/tests/servergroup/one-master-one-node-without-bastion-no-public-ip-association.yaml
@@ -23,6 +23,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-1-cluster

diff --git a/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml b/pkg/model/openstackmodel/tests/servergroup/one-master-one-node.yaml
@@ -43,6 +43,7 @@ Metadata:
   k8s.io_cluster-autoscaler_node-template_label_kubernetes.io_role: master
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_control-plane: ""
   k8s.io_cluster-autoscaler_node-template_label_node-role.kubernetes.io_master: ""
+  k8s.io_cluster-autoscaler_node-template_label_node.kubernetes.io_exclude-from-external-load-balancers: ""
   k8s.io_role_master: "1"
   kops.k8s.io_instancegroup: master
 Name: master-1-cluster

diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt
@@ -208,6 +208,7 @@ KubeletConfig:
     labelname: labelvalue
     node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
   taints:
   - key1=value1:NoSchedule
   - key2=value2:NoExecute

diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt
@@ -225,6 +225,7 @@ KubeletConfig:
     labelname: labelvalue
     node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
   taints:
   - key1=value1:NoSchedule
   - key2=value2:NoExecute

diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt
@@ -225,6 +225,7 @@ KubeletConfig:
     labelname: labelvalue
     node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
   taints:
   - key1=value1:NoSchedule
   - key2=value2:NoExecute

diff --git a/pkg/nodelabels/builder.go b/pkg/nodelabels/builder.go
@@ -84,5 +84,6 @@ func BuildMandatoryControlPlaneLabels() map[string]string {
 	nodeLabels[RoleLabelControlPlane20] = ""
 	nodeLabels[RoleLabelName15] = RoleMasterLabelValue15
 	nodeLabels["kops.k8s.io/kops-controller-pki"] = ""
+	nodeLabels["node.kubernetes.io/exclude-from-external-load-balancers"] = ""
 	return nodeLabels
 }
diff --git a/pkg/nodelabels/builder_test.go b/pkg/nodelabels/builder_test.go
@@ -61,14 +61,15 @@ func TestBuildNodeLabels(t *testing.T) {
 				},
 			},
 			expected: map[string]string{
-				RoleLabelMaster16:                 "",
-				RoleLabelControlPlane20:           "",
-				RoleLabelName15:                   RoleMasterLabelValue15,
-				"kops.k8s.io/kops-controller-pki": "",
-				"controlPlane1":                   "controlPlane1",
-				"controlPlane2":                   "controlPlane2",
-				"node1":                           "override1",
-				"node3":                           "override3",
+				RoleLabelMaster16:       "",
+				RoleLabelControlPlane20: "",
+				RoleLabelName15:         RoleMasterLabelValue15,
+				"node.kubernetes.io/exclude-from-external-load-balancers": "",
+				"kops.k8s.io/kops-controller-pki":                         "",
+				"controlPlane1":                                           "controlPlane1",
+				"controlPlane2":                                           "controlPlane2",
+				"node1":                                                   "override1",
+				"node3":                                                   "override3",
 			},
 		},
 		{

diff --git a/.../data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data b/.../data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data
@@ -305,6 +305,7 @@ KubeletConfig:
     kubernetes.io/role: master
     node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
   registerSchedulable: false

diff --git a/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf b/tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf
@@ -200,6 +200,11 @@ resource "aws_autoscaling_group" "master-us-test-1a-masters-bastionuserdata-exam
     propagate_at_launch = true
     value               = ""
   }
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers"
+    propagate_at_launch = true
+    value               = ""
+  }
   tag {
     key                 = "k8s.io/role/master"
     propagate_at_launch = true
@@ -547,41 +552,44 @@ resource "aws_launch_template" "master-us-test-1a-masters-bastionuserdata-exampl
   tag_specifications {
     resource_type = "instance"
     tags = {
-      "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
-      "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"       = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
-      "k8s.io/role/master"                                                                  = "1"
-      "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
-      "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
+      "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/bastionuserdata.example.com"                                                     = "owned"
     }
   }
   tag_specifications {
     resource_type = "volume"
     tags = {
-      "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
-      "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
-      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"       = ""
-      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
-      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
-      "k8s.io/role/master"                                                                  = "1"
-      "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
-      "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
+      "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
+      "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+      "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+      "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+      "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+      "k8s.io/role/master"                                                                                    = "1"
+      "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+      "kubernetes.io/cluster/bastionuserdata.example.com"                                                     = "owned"
     }
   }
   tags = {
-    "KubernetesCluster"                                                                   = "bastionuserdata.example.com"
-    "Name"                                                                                = "master-us-test-1a.masters.bastionuserdata.example.com"
-    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"       = ""
-    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                    = "master"
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = ""
-    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"        = ""
-    "k8s.io/role/master"                                                                  = "1"
-    "kops.k8s.io/instancegroup"                                                           = "master-us-test-1a"
-    "kubernetes.io/cluster/bastionuserdata.example.com"                                   = "owned"
+    "KubernetesCluster"                                                                                     = "bastionuserdata.example.com"
+    "Name"                                                                                                  = "master-us-test-1a.masters.bastionuserdata.example.com"
+    "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki"                         = ""
+    "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role"                                      = "master"
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane"                   = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master"                          = ""
+    "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = ""
+    "k8s.io/role/master"                                                                                    = "1"
+    "kops.k8s.io/instancegroup"                                                                             = "master-us-test-1a"
+    "kubernetes.io/cluster/bastionuserdata.example.com"                                                     = "owned"
   }
   user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data")
 }

diff --git a/tests/integration/update_cluster/complex/cloudformation.json b/tests/integration/update_cluster/complex/cloudformation.json
@@ -63,6 +63,11 @@
             "Value": "",
             "PropagateAtLaunch": true
           },
+          {
+            "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
+            "Value": "",
+            "PropagateAtLaunch": true
+          },
           {
             "Key": "k8s.io/role/master",
             "Value": "1",
@@ -338,6 +343,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
                   "Value": ""
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
+                  "Value": ""
+                },
                 {
                   "Key": "k8s.io/role/master",
                   "Value": "1"
@@ -387,6 +396,10 @@
                   "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master",
                   "Value": ""
                 },
+                {
+                  "Key": "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers",
+                  "Value": ""
+                },
                 {
                   "Key": "k8s.io/role/master",
                   "Value": "1"

diff --git a/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/complex/cloudformation.json.extracted.yaml
@@ -321,6 +321,7 @@ Resources.AWSEC2LaunchTemplatemasterustest1amasterscomplexexamplecom.Properties.
       kubernetes.io/role: master
       node-role.kubernetes.io/control-plane: ""
       node-role.kubernetes.io/master: ""
+      node.kubernetes.io/exclude-from-external-load-balancers: ""
     nonMasqueradeCIDR: 100.64.0.0/10
     podManifestPath: /etc/kubernetes/manifests
     registerSchedulable: false

diff --git a/.../complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data b/.../complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data
@@ -320,6 +320,7 @@ KubeletConfig:
     kubernetes.io/role: master
     node-role.kubernetes.io/control-plane: ""
     node-role.kubernetes.io/master: ""
+    node.kubernetes.io/exclude-from-external-load-balancers: ""
   nonMasqueradeCIDR: 100.64.0.0/10
   podManifestPath: /etc/kubernetes/manifests
   registerSchedulable: false