Add support for scaling out the control plane with dedicated apiserve…

…r nodes Ensure apiserver role can only be used on AWS (because of firewalling) Apply api-server label to CP as well Consolidate node not ready validation message Guard apiserver nodes with a feature flag Rename Apiserver role to APIServer Add an integration test for apiserver nodes Rename Apiserver role to APIServer Enumerate all roles in rolling update docs Apply suggestions from code review Co-authored-by: Steven E. Harris <[email protected]>
kubernetes · Mar 20, 2021 · 20bd724 · 20bd724
1 parent bf2105b
commit 20bd724
Show file tree

Hide file tree

Showing 37 changed files with 2,732 additions and 73 deletions.
diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go
@@ -328,6 +328,17 @@ func TestDockerCustom(t *testing.T) {
 	newIntegrationTest("docker.example.com", "docker-custom").runTestCloudformation(t)
 }
 
+// TestAPIServerNodes runs a simple configuration with dedicated apiserver nodes
+func TestAPIServerNodes(t *testing.T) {
+	featureflag.ParseFlags("+APIServerNodes")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-APIServerNodes")
+	}
+	defer unsetFeatureFlags()
+
+	newIntegrationTest("minimal.example.com", "apiservernodes").runTestCloudformation(t)
+}
+
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
 	ctx := context.Background()
 

diff --git a/cmd/kops/rollingupdatecluster.go b/cmd/kops/rollingupdatecluster.go
@@ -177,6 +177,11 @@ func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		Example: rollingupdateExample,
 	}
 
+	allRoles := make([]string, 0, len(kopsapi.AllInstanceGroupRoles))
+	for _, r := range kopsapi.AllInstanceGroupRoles {
+		allRoles = append(allRoles, string(r))
+	}
+
 	cmd.Flags().BoolVarP(&options.Yes, "yes", "y", options.Yes, "Perform rolling update immediately, without --yes rolling-update executes a dry-run")
 	cmd.Flags().BoolVar(&options.Force, "force", options.Force, "Force rolling update, even if no changes")
 	cmd.Flags().BoolVar(&options.CloudOnly, "cloudonly", options.CloudOnly, "Perform rolling update without confirming progress with k8s")
@@ -189,7 +194,7 @@ func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.PostDrainDelay, "post-drain-delay", options.PostDrainDelay, "Time to wait after draining each node")
 	cmd.Flags().BoolVarP(&options.Interactive, "interactive", "i", options.Interactive, "Prompt to continue after each instance is updated")
 	cmd.Flags().StringSliceVar(&options.InstanceGroups, "instance-group", options.InstanceGroups, "List of instance groups to update (defaults to all if not specified)")
-	cmd.Flags().StringSliceVar(&options.InstanceGroupRoles, "instance-group-roles", options.InstanceGroupRoles, "If specified, only instance groups of the specified role will be updated (e.g. Master,Node,Bastion)")
+	cmd.Flags().StringSliceVar(&options.InstanceGroupRoles, "instance-group-roles", options.InstanceGroupRoles, "If specified, only instance groups of the specified role will be updated ("+strings.Join(allRoles, ",")+")")
 
 	cmd.Flags().BoolVar(&options.FailOnDrainError, "fail-on-drain-error", true, "The rolling-update will fail if draining a node fails.")
 	cmd.Flags().BoolVar(&options.FailOnValidate, "fail-on-validate-error", true, "The rolling-update will fail if the cluster fails to validate.")
@@ -302,11 +307,14 @@ func RunRollingUpdateCluster(ctx context.Context, f *util.Factory, out io.Writer
 	if len(options.InstanceGroupRoles) != 0 {
 		var filtered []*kopsapi.InstanceGroup
 
-		for _, ig := range instanceGroups {
-			for _, role := range options.InstanceGroupRoles {
-				if ig.Spec.Role == kopsapi.InstanceGroupRole(strings.Title(strings.ToLower(role))) {
+		for _, role := range options.InstanceGroupRoles {
+			s, f := kopsapi.ParseInstanceGroupRole(role, true)
+			if !f {
+				return fmt.Errorf("invalid instance group role %q", role)
+			}
+			for _, ig := range instanceGroups {
+				if ig.Spec.Role == s {
 					filtered = append(filtered, ig)
-					continue
 				}
 			}
 		}

diff --git a/docs/cli/kops_create_instancegroup.md b/docs/cli/kops_create_instancegroup.md
diff --git a/docs/cli/kops_rolling-update_cluster.md b/docs/cli/kops_rolling-update_cluster.md
diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go
@@ -57,6 +57,9 @@ type NodeupModelContext struct {
 	// IsMaster is true if the InstanceGroup has a role of master (populated by Init)
 	IsMaster bool
 
+	// HasAPIServer is true if the InstanceGroup has a role of master or apiserver (pupulated by Init)
+	HasAPIServer bool
+
 	kubernetesVersion semver.Version
 	bootstrapCerts    map[string]*nodetasks.BootstrapCert
 }
@@ -70,10 +73,15 @@ func (c *NodeupModelContext) Init() error {
 	c.kubernetesVersion = *k8sVersion
 	c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{}
 
-	if c.NodeupConfig.InstanceGroupRole == kops.InstanceGroupRoleMaster {
+	role := c.NodeupConfig.InstanceGroupRole
+
+	if role == kops.InstanceGroupRoleMaster {
 		c.IsMaster = true
 	}
 
+	if role == kops.InstanceGroupRoleMaster || role == kops.InstanceGroupRoleAPIServer {
+		c.HasAPIServer = true
+	}
 	return nil
 }
 

diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go
@@ -32,17 +32,26 @@ var _ fi.ModelBuilder = &EtcdManagerTLSBuilder{}
 
 // Build is responsible for TLS configuration for etcd-manager
 func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
-	if !b.IsMaster || !b.UseEtcdManager() {
+	if !b.HasAPIServer || !b.UseEtcdManager() {
 		return nil
 	}
 
+	// We also dynamically generate the client keypair for apiserver
+	if err := b.buildKubeAPIServerKeypair(ctx); err != nil {
+		return err
+	}
+
 	for _, k := range []string{"main", "events"} {
 		d := "/etc/kubernetes/pki/etcd-manager-" + k
 
 		keys := make(map[string]string)
-		keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
-		keys["etcd-peers-ca"] = "etcd-peers-ca-" + k
-		// Because API server can only have a single client-cert, we need to share a client CA
+
+		// Only nodes running etcd need the peers CA
+		if b.IsMaster {
+			keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
+			keys["etcd-peers-ca"] = "etcd-peers-ca-" + k
+		}
+		// Because API server can only have a single client certificate for etcd, we need to share a client CA
 		keys["etcd-clients-ca"] = "etcd-clients-ca"
 
 		for fileName, keystoreName := range keys {
@@ -63,10 +72,6 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
 		}
 	}
 
-	// We also dynamically generate the client keypair for apiserver
-	if err := b.buildKubeAPIServerKeypair(ctx); err != nil {
-		return err
-	}
 	return nil
 }
 

diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go
@@ -52,7 +52,7 @@ var _ fi.ModelBuilder = &KubeAPIServerBuilder{}
 
 // Build is responsible for generating the configuration for the kube-apiserver
 func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.HasAPIServer {
 		return nil
 	}
 
@@ -316,19 +316,29 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 		}
 	}
 
+	var mainEtcdCluster, eventsEtcdCluster string
+	if b.IsMaster {
+		mainEtcdCluster = "https://127.0.0.1:4001"
+		eventsEtcdCluster = "https://127.0.0.1:4002"
+	} else {
+		host := b.Cluster.ObjectMeta.Name
+		mainEtcdCluster = "https://main.etcd." + host + ":4001"
+		eventsEtcdCluster = "https://events.etcd." + host + ":4002"
+	}
+
 	if b.UseEtcdManager() && b.UseEtcdTLS() {
 		basedir := "/etc/kubernetes/pki/kube-apiserver"
 		kubeAPIServer.EtcdCAFile = filepath.Join(basedir, "etcd-ca.crt")
 		kubeAPIServer.EtcdCertFile = filepath.Join(basedir, "etcd-client.crt")
 		kubeAPIServer.EtcdKeyFile = filepath.Join(basedir, "etcd-client.key")
-		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
-		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+		kubeAPIServer.EtcdServers = []string{mainEtcdCluster}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#" + eventsEtcdCluster}
 	} else if b.UseEtcdTLS() {
 		kubeAPIServer.EtcdCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 		kubeAPIServer.EtcdCertFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client.pem")
 		kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem")
-		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
-		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+		kubeAPIServer.EtcdServers = []string{mainEtcdCluster}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#" + eventsEtcdCluster}
 	}
 
 	// @check if we are using secure kubelet client certificates

diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
@@ -421,6 +421,7 @@ func (b *KubeletBuilder) addContainerizedMounter(c *fi.ModelBuilderContext) erro
 // buildKubeletConfigSpec returns the kubeletconfig for the specified instanceGroup
 func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, error) {
 	isMaster := b.IsMaster
+	isAPIServer := b.InstanceGroup.Spec.Role == kops.InstanceGroupRoleAPIServer
 
 	// Merge KubeletConfig for NodeLabels
 	c := b.NodeupConfig.KubeletConfig
@@ -490,6 +491,10 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro
 			// (Even though the value is empty, we still expect <Key>=<Value>:<Effect>)
 			c.Taints = append(c.Taints, nodelabels.RoleLabelMaster16+"=:"+string(v1.TaintEffectNoSchedule))
 		}
+		if len(c.Taints) == 0 && isAPIServer {
+			// (Even though the value is empty, we still expect <Key>=<Value>:<Effect>)
+			c.Taints = append(c.Taints, nodelabels.RoleLabelAPIServer16+"=:"+string(v1.TaintEffectNoSchedule))
+		}
 
 		// Enable scheduling since it can be controlled via taints.
 		c.RegisterSchedulable = fi.Bool(true)

diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go
@@ -69,8 +69,8 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
-	// if we are not a master we can stop here
-	if !b.IsMaster {
+	// If we do not run the Kubernetes API server we can stop here.
+	if !b.HasAPIServer {
 		return nil
 	}
 

diff --git a/pkg/apis/kops/instancegroup.go b/pkg/apis/kops/instancegroup.go
@@ -18,7 +18,6 @@ package kops
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/klog/v2"
 )
 
 const (
@@ -59,12 +58,15 @@ const (
 	InstanceGroupRoleNode InstanceGroupRole = "Node"
 	// InstanceGroupRoleBastion is a bastion role
 	InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
+	// InstanceGroupRoleAPIServer is an API server role
+	InstanceGroupRoleAPIServer InstanceGroupRole = "APIServer"
 )
 
 // AllInstanceGroupRoles is a slice of all valid InstanceGroupRole values
 var AllInstanceGroupRoles = []InstanceGroupRole{
-	InstanceGroupRoleNode,
 	InstanceGroupRoleMaster,
+	InstanceGroupRoleAPIServer,
+	InstanceGroupRoleNode,
 	InstanceGroupRoleBastion,
 }
 
@@ -286,27 +288,32 @@ func (g *InstanceGroup) IsMaster() bool {
 	switch g.Spec.Role {
 	case InstanceGroupRoleMaster:
 		return true
-	case InstanceGroupRoleNode:
-		return false
-	case InstanceGroupRoleBastion:
+	default:
 		return false
+	}
+}
+
+// IsAPIServerOnly checks if instanceGroup runs only the API Server
+func (g *InstanceGroup) IsAPIServerOnly() bool {
+	switch g.Spec.Role {
+	case InstanceGroupRoleAPIServer:
+		return true
 	default:
-		klog.Fatalf("Role not set in group %v", g)
 		return false
 	}
 }
 
+// hasAPIServer checks if instanceGroup runs an API Server
+func (g *InstanceGroup) HasAPIServer() bool {
+	return g.IsMaster() || g.IsAPIServerOnly()
+}
+
 // IsBastion checks if instanceGroup is a bastion
 func (g *InstanceGroup) IsBastion() bool {
 	switch g.Spec.Role {
-	case InstanceGroupRoleMaster:
-		return false
-	case InstanceGroupRoleNode:
-		return false
 	case InstanceGroupRoleBastion:
 		return true
 	default:
-		klog.Fatalf("Role not set in group %v", g)
 		return false
 	}
 }

diff --git a/pkg/apis/kops/parse.go b/pkg/apis/kops/parse.go
@@ -23,7 +23,10 @@ import (
 	"k8s.io/kops/upup/pkg/fi/utils"
 )
 
-// ParseInstanceGroupRole converts a string to an InstanceGroupRole
+// ParseInstanceGroupRole converts a string to an InstanceGroupRole.
+//
+// If lenient is set to true, the function will match pluralised words too.
+// It will return the instance group role and true if a match was found.
 func ParseInstanceGroupRole(input string, lenient bool) (InstanceGroupRole, bool) {
 	findRole := strings.ToLower(input)
 	if lenient {

diff --git a/pkg/apis/kops/util/labels.go b/pkg/apis/kops/util/labels.go
@@ -25,9 +25,15 @@ func GetNodeRole(node *v1.Node) string {
 	if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
 		return "master"
 	}
+	if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+		return "control-plane"
+	}
 	if _, ok := node.Labels["node-role.kubernetes.io/node"]; ok {
 		return "node"
 	}
+	if _, ok := node.Labels["node-role.kubernetes.io/api-server"]; ok {
+		return "apiserver"
+	}
 	// Older label
 	return node.Labels["kubernetes.io/role"]
 }
diff --git a/pkg/apis/kops/validation/instancegroup.go b/pkg/apis/kops/validation/instancegroup.go
@@ -47,6 +47,7 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud) field.ErrorLis
 		}
 	case kops.InstanceGroupRoleNode:
 	case kops.InstanceGroupRoleBastion:
+	case kops.InstanceGroupRoleAPIServer:
 	default:
 		var supported []string
 		for _, role := range kops.AllInstanceGroupRoles {
@@ -186,6 +187,10 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 		allErrs = append(allErrs, ValidateMasterInstanceGroup(g, cluster)...)
 	}
 
+	if g.Spec.Role == kops.InstanceGroupRoleAPIServer && kops.CloudProviderID(cluster.Spec.CloudProvider) != kops.CloudProviderAWS {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Apiserver role only supported on AWS"))
+	}
+
 	// Check that instance groups are defined in subnets that are defined in the cluster
 	{
 		clusterSubnets := make(map[string]*kops.ClusterSubnetSpec)