Merge pull request kubernetes#10722 from olemarkus/apiserver-nodes

Apiserver nodes

cmd/kops/integration_test.go

@@ -328,6 +328,17 @@ func TestDockerCustom(t *testing.T) {
 	newIntegrationTest("docker.example.com", "docker-custom").runTestCloudformation(t)
 }
 
+// TestAPIServerNodes runs a simple configuration with dedicated apiserver nodes
+func TestAPIServerNodes(t *testing.T) {
+	featureflag.ParseFlags("+APIServerNodes")
+	unsetFeatureFlags := func() {
+		featureflag.ParseFlags("-APIServerNodes")
+	}
+	defer unsetFeatureFlags()
+
+	newIntegrationTest("minimal.example.com", "apiservernodes").runTestCloudformation(t)
+}
+
 func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) {
 	ctx := context.Background()
 

cmd/kops/rollingupdatecluster.go

@@ -177,6 +177,11 @@ func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		Example: rollingupdateExample,
 	}
 
+	allRoles := make([]string, 0, len(kopsapi.AllInstanceGroupRoles))
+	for _, r := range kopsapi.AllInstanceGroupRoles {
+		allRoles = append(allRoles, string(r))
+	}
+
 	cmd.Flags().BoolVarP(&options.Yes, "yes", "y", options.Yes, "Perform rolling update immediately, without --yes rolling-update executes a dry-run")
 	cmd.Flags().BoolVar(&options.Force, "force", options.Force, "Force rolling update, even if no changes")
 	cmd.Flags().BoolVar(&options.CloudOnly, "cloudonly", options.CloudOnly, "Perform rolling update without confirming progress with k8s")
@@ -189,7 +194,7 @@ func NewCmdRollingUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.PostDrainDelay, "post-drain-delay", options.PostDrainDelay, "Time to wait after draining each node")
 	cmd.Flags().BoolVarP(&options.Interactive, "interactive", "i", options.Interactive, "Prompt to continue after each instance is updated")
 	cmd.Flags().StringSliceVar(&options.InstanceGroups, "instance-group", options.InstanceGroups, "List of instance groups to update (defaults to all if not specified)")
-	cmd.Flags().StringSliceVar(&options.InstanceGroupRoles, "instance-group-roles", options.InstanceGroupRoles, "If specified, only instance groups of the specified role will be updated (e.g. Master,Node,Bastion)")
+	cmd.Flags().StringSliceVar(&options.InstanceGroupRoles, "instance-group-roles", options.InstanceGroupRoles, "If specified, only instance groups of the specified role will be updated ("+strings.Join(allRoles, ",")+")")
 
 	cmd.Flags().BoolVar(&options.FailOnDrainError, "fail-on-drain-error", true, "The rolling-update will fail if draining a node fails.")
 	cmd.Flags().BoolVar(&options.FailOnValidate, "fail-on-validate-error", true, "The rolling-update will fail if the cluster fails to validate.")
@@ -302,11 +307,14 @@ func RunRollingUpdateCluster(ctx context.Context, f *util.Factory, out io.Writer
 	if len(options.InstanceGroupRoles) != 0 {
 		var filtered []*kopsapi.InstanceGroup
 
-		for _, ig := range instanceGroups {
-			for _, role := range options.InstanceGroupRoles {
-				if ig.Spec.Role == kopsapi.InstanceGroupRole(strings.Title(strings.ToLower(role))) {
+		for _, role := range options.InstanceGroupRoles {
+			s, f := kopsapi.ParseInstanceGroupRole(role, true)
+			if !f {
+				return fmt.Errorf("invalid instance group role %q", role)
+			}
+			for _, ig := range instanceGroups {
+				if ig.Spec.Role == s {
 					filtered = append(filtered, ig)
-					continue
 				}
 			}
 		}

nodeup/pkg/model/context.go

@@ -57,6 +57,9 @@ type NodeupModelContext struct {
 	// IsMaster is true if the InstanceGroup has a role of master (populated by Init)
 	IsMaster bool
 
+	// HasAPIServer is true if the InstanceGroup has a role of master or apiserver (pupulated by Init)
+	HasAPIServer bool
+
 	kubernetesVersion semver.Version
 	bootstrapCerts    map[string]*nodetasks.BootstrapCert
 }
@@ -70,10 +73,15 @@ func (c *NodeupModelContext) Init() error {
 	c.kubernetesVersion = *k8sVersion
 	c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{}
 
-	if c.NodeupConfig.InstanceGroupRole == kops.InstanceGroupRoleMaster {
+	role := c.NodeupConfig.InstanceGroupRole
+
+	if role == kops.InstanceGroupRoleMaster {
 		c.IsMaster = true
 	}
 
+	if role == kops.InstanceGroupRoleMaster || role == kops.InstanceGroupRoleAPIServer {
+		c.HasAPIServer = true
+	}
 	return nil
 }
 

nodeup/pkg/model/etcd_manager_tls.go

@@ -32,17 +32,26 @@ var _ fi.ModelBuilder = &EtcdManagerTLSBuilder{}
 
 // Build is responsible for TLS configuration for etcd-manager
 func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
-	if !b.IsMaster || !b.UseEtcdManager() {
+	if !b.HasAPIServer || !b.UseEtcdManager() {
 		return nil
 	}
 
+	// We also dynamically generate the client keypair for apiserver
+	if err := b.buildKubeAPIServerKeypair(ctx); err != nil {
+		return err
+	}
+
 	for _, k := range []string{"main", "events"} {
 		d := "/etc/kubernetes/pki/etcd-manager-" + k
 
 		keys := make(map[string]string)
-		keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
-		keys["etcd-peers-ca"] = "etcd-peers-ca-" + k
-		// Because API server can only have a single client-cert, we need to share a client CA
+
+		// Only nodes running etcd need the peers CA
+		if b.IsMaster {
+			keys["etcd-manager-ca"] = "etcd-manager-ca-" + k
+			keys["etcd-peers-ca"] = "etcd-peers-ca-" + k
+		}
+		// Because API server can only have a single client certificate for etcd, we need to share a client CA
 		keys["etcd-clients-ca"] = "etcd-clients-ca"
 
 		for fileName, keystoreName := range keys {
@@ -63,10 +72,6 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error {
 		}
 	}
 
-	// We also dynamically generate the client keypair for apiserver
-	if err := b.buildKubeAPIServerKeypair(ctx); err != nil {
-		return err
-	}
 	return nil
 }
 

nodeup/pkg/model/kube_apiserver.go

@@ -52,7 +52,7 @@ var _ fi.ModelBuilder = &KubeAPIServerBuilder{}
 
 // Build is responsible for generating the configuration for the kube-apiserver
 func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.HasAPIServer {
 		return nil
 	}
 
@@ -316,19 +316,29 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 		}
 	}
 
+	var mainEtcdCluster, eventsEtcdCluster string
+	if b.IsMaster {
+		mainEtcdCluster = "https://127.0.0.1:4001"
+		eventsEtcdCluster = "https://127.0.0.1:4002"
+	} else {
+		host := b.Cluster.ObjectMeta.Name
+		mainEtcdCluster = "https://main.etcd." + host + ":4001"
+		eventsEtcdCluster = "https://events.etcd." + host + ":4002"
+	}
+
 	if b.UseEtcdManager() && b.UseEtcdTLS() {
 		basedir := "/etc/kubernetes/pki/kube-apiserver"
 		kubeAPIServer.EtcdCAFile = filepath.Join(basedir, "etcd-ca.crt")
 		kubeAPIServer.EtcdCertFile = filepath.Join(basedir, "etcd-client.crt")
 		kubeAPIServer.EtcdKeyFile = filepath.Join(basedir, "etcd-client.key")
-		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
-		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+		kubeAPIServer.EtcdServers = []string{mainEtcdCluster}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#" + eventsEtcdCluster}
 	} else if b.UseEtcdTLS() {
 		kubeAPIServer.EtcdCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 		kubeAPIServer.EtcdCertFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client.pem")
 		kubeAPIServer.EtcdKeyFile = filepath.Join(b.PathSrvKubernetes(), "etcd-client-key.pem")
-		kubeAPIServer.EtcdServers = []string{"https://127.0.0.1:4001"}
-		kubeAPIServer.EtcdServersOverrides = []string{"/events#https://127.0.0.1:4002"}
+		kubeAPIServer.EtcdServers = []string{mainEtcdCluster}
+		kubeAPIServer.EtcdServersOverrides = []string{"/events#" + eventsEtcdCluster}
 	}
 
 	// @check if we are using secure kubelet client certificates

nodeup/pkg/model/kubelet.go

@@ -421,6 +421,7 @@ func (b *KubeletBuilder) addContainerizedMounter(c *fi.ModelBuilderContext) erro
 // buildKubeletConfigSpec returns the kubeletconfig for the specified instanceGroup
 func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, error) {
 	isMaster := b.IsMaster
+	isAPIServer := b.InstanceGroup.Spec.Role == kops.InstanceGroupRoleAPIServer
 
 	// Merge KubeletConfig for NodeLabels
 	c := b.NodeupConfig.KubeletConfig
@@ -490,6 +491,10 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro
 			// (Even though the value is empty, we still expect <Key>=<Value>:<Effect>)
 			c.Taints = append(c.Taints, nodelabels.RoleLabelMaster16+"=:"+string(v1.TaintEffectNoSchedule))
 		}
+		if len(c.Taints) == 0 && isAPIServer {
+			// (Even though the value is empty, we still expect <Key>=<Value>:<Effect>)
+			c.Taints = append(c.Taints, nodelabels.RoleLabelAPIServer16+"=:"+string(v1.TaintEffectNoSchedule))
+		}
 
 		// Enable scheduling since it can be controlled via taints.
 		c.RegisterSchedulable = fi.Bool(true)

nodeup/pkg/model/secrets.go

@@ -69,8 +69,8 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 	}
 
-	// if we are not a master we can stop here
-	if !b.IsMaster {
+	// If we do not run the Kubernetes API server we can stop here.
+	if !b.HasAPIServer {
 		return nil
 	}
 

pkg/apis/kops/instancegroup.go

@@ -18,7 +18,6 @@ package kops
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/klog/v2"
 )
 
 const (
@@ -59,12 +58,15 @@ const (
 	InstanceGroupRoleNode InstanceGroupRole = "Node"
 	// InstanceGroupRoleBastion is a bastion role
 	InstanceGroupRoleBastion InstanceGroupRole = "Bastion"
+	// InstanceGroupRoleAPIServer is an API server role
+	InstanceGroupRoleAPIServer InstanceGroupRole = "APIServer"
 )
 
 // AllInstanceGroupRoles is a slice of all valid InstanceGroupRole values
 var AllInstanceGroupRoles = []InstanceGroupRole{
-	InstanceGroupRoleNode,
 	InstanceGroupRoleMaster,
+	InstanceGroupRoleAPIServer,
+	InstanceGroupRoleNode,
 	InstanceGroupRoleBastion,
 }
 
@@ -286,27 +288,32 @@ func (g *InstanceGroup) IsMaster() bool {
 	switch g.Spec.Role {
 	case InstanceGroupRoleMaster:
 		return true
-	case InstanceGroupRoleNode:
-		return false
-	case InstanceGroupRoleBastion:
+	default:
 		return false
+	}
+}
+
+// IsAPIServerOnly checks if instanceGroup runs only the API Server
+func (g *InstanceGroup) IsAPIServerOnly() bool {
+	switch g.Spec.Role {
+	case InstanceGroupRoleAPIServer:
+		return true
 	default:
-		klog.Fatalf("Role not set in group %v", g)
 		return false
 	}
 }
 
+// hasAPIServer checks if instanceGroup runs an API Server
+func (g *InstanceGroup) HasAPIServer() bool {
+	return g.IsMaster() || g.IsAPIServerOnly()
+}
+
 // IsBastion checks if instanceGroup is a bastion
 func (g *InstanceGroup) IsBastion() bool {
 	switch g.Spec.Role {
-	case InstanceGroupRoleMaster:
-		return false
-	case InstanceGroupRoleNode:
-		return false
 	case InstanceGroupRoleBastion:
 		return true
 	default:
-		klog.Fatalf("Role not set in group %v", g)
 		return false
 	}
 }

pkg/apis/kops/parse.go

@@ -23,7 +23,10 @@ import (
 	"k8s.io/kops/upup/pkg/fi/utils"
 )
 
-// ParseInstanceGroupRole converts a string to an InstanceGroupRole
+// ParseInstanceGroupRole converts a string to an InstanceGroupRole.
+//
+// If lenient is set to true, the function will match pluralised words too.
+// It will return the instance group role and true if a match was found.
 func ParseInstanceGroupRole(input string, lenient bool) (InstanceGroupRole, bool) {
 	findRole := strings.ToLower(input)
 	if lenient {

pkg/apis/kops/util/labels.go

@@ -25,9 +25,15 @@ func GetNodeRole(node *v1.Node) string {
 	if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
 		return "master"
 	}
+	if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+		return "control-plane"
+	}
 	if _, ok := node.Labels["node-role.kubernetes.io/node"]; ok {
 		return "node"
 	}
+	if _, ok := node.Labels["node-role.kubernetes.io/api-server"]; ok {
+		return "apiserver"
+	}
 	// Older label
 	return node.Labels["kubernetes.io/role"]
 }

pkg/apis/kops/validation/instancegroup.go

@@ -47,6 +47,7 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud) field.ErrorLis
 		}
 	case kops.InstanceGroupRoleNode:
 	case kops.InstanceGroupRoleBastion:
+	case kops.InstanceGroupRoleAPIServer:
 	default:
 		var supported []string
 		for _, role := range kops.AllInstanceGroupRoles {
@@ -186,6 +187,10 @@ func CrossValidateInstanceGroup(g *kops.InstanceGroup, cluster *kops.Cluster, cl
 		allErrs = append(allErrs, ValidateMasterInstanceGroup(g, cluster)...)
 	}
 
+	if g.Spec.Role == kops.InstanceGroupRoleAPIServer && kops.CloudProviderID(cluster.Spec.CloudProvider) != kops.CloudProviderAWS {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "role"), "Apiserver role only supported on AWS"))
+	}
+
 	// Check that instance groups are defined in subnets that are defined in the cluster
 	{
 		clusterSubnets := make(map[string]*kops.ClusterSubnetSpec)