Kubelet API Certificate

A while back options to permit secure kube-apiserver to kubelet api was #2831 using the server.cert and server.key as testing grouns. This PR formalizes the options and generates a client certificate on their behalf (note, the server{.cert,key} can no longer be used post 1.7 as the certificate usage is checked i.e. it's not using a client cert). The users now only need to add anonymousAuth: false to enable secure api to kubelet. I'd like to make this default to all new builds i'm not sure where to place it.

- updated the security.md to reflect the changes
- issue a new client kubelet-api certificate used to secure authorize comms between api and kubelet
- fixed any formatting issues i came across on the journey

docs/security.md

@@ -34,13 +34,25 @@ This stores the [config.json](https://docs.docker.com/engine/reference/commandli
 All Pods running on your cluster have access to underlying instance IAM role.
 Currently permission scope is quite broad. See [iam_roles.md](iam_roles.md) for details and ways to mitigate that.
 
-
 ## Kubernetes API
 
 (this section is a work in progress)
 
 Kubernetes has a number of authentication mechanisms:
 
+## Kubelet API
+
+By default AnonymousAuth on the kubelet is off and so communication between kube-apiserver and kubelet api is not authenticated. In order to switch on authentication;
+
+```YAML
+# In the cluster spec
+spec:
+  kubelet:
+    anonymousAuth: false
+```
+
+**Note** on a existing cluster with 'anonymousAuth' unset you would need to first roll out the masters and then update the pools. 
+
 ### API Bearer Token
 
 The API bearer token is a secret named 'admin'.

nodeup/pkg/model/context.go

@@ -183,3 +183,30 @@ func (c *NodeupModelContext) UsesCNI() bool {
 	}
 	return true
 }
+
+// UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are be
+// in one of three section, master specific kubelet, cluster wide kubelet or the InstanceGroup. Though arguably is
+// doesn't make much sense to unset this on a per InstanceGroup level, but hey :)
+func (c *NodeupModelContext) UseSecureKubelet() bool {
+	cluster := &c.Cluster.Spec // just to shorten the typing
+	group := &c.InstanceGroup.Spec
+
+	// @check if we have anything specific to master kubelet
+	if c.IsMaster {
+		if cluster.MasterKubelet != nil && cluster.MasterKubelet.AnonymousAuth != nil && *cluster.MasterKubelet.AnonymousAuth == true {
+			return true
+		}
+	}
+
+	// @check the default settings for master and kubelet
+	if cluster.Kubelet != nil && cluster.Kubelet.AnonymousAuth != nil && *cluster.Kubelet.AnonymousAuth == false {
+		return true
+	}
+
+	// @check on the InstanceGroup itself
+	if group.Kubelet != nil && group.Kubelet.AnonymousAuth != nil && *group.Kubelet.AnonymousAuth == false {
+		return true
+	}
+
+	return false
+}

nodeup/pkg/model/convenience.go

@@ -16,7 +16,13 @@ limitations under the License.
 
 package model
 
-import "k8s.io/kops/upup/pkg/fi"
+import (
+	"fmt"
+	"path/filepath"
+
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+)
 
 // s is a helper that builds a *string from a string value
 func s(v string) *string {
@@ -27,3 +33,55 @@ func s(v string) *string {
 func i64(v int64) *int64 {
 	return fi.Int64(v)
 }
+
+// buildCertificateRequest retrieves the certificate from a keystore
+func buildCertificateRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error {
+	cert, err := b.KeyStore.Cert(name)
+	if err != nil {
+		return err
+	}
+
+	serialized, err := cert.AsString()
+	if err != nil {
+		return err
+	}
+
+	location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s.pem", name))
+	if path != "" {
+		location = path
+	}
+
+	c.AddTask(&nodetasks.File{
+		Path:     location,
+		Contents: fi.NewStringResource(serialized),
+		Type:     nodetasks.FileType_File,
+	})
+
+	return nil
+}
+
+// buildPrivateKeyRequest retrieves a private key from the store
+func buildPrivateKeyRequest(c *fi.ModelBuilderContext, b *NodeupModelContext, name, path string) error {
+	k, err := b.KeyStore.PrivateKey(name)
+	if err != nil {
+		return err
+	}
+
+	serialized, err := k.AsString()
+	if err != nil {
+		return err
+	}
+
+	location := filepath.Join(b.PathSrvKubernetes(), fmt.Sprintf("%s-key.pem", name))
+	if path != "" {
+		location = path
+	}
+
+	c.AddTask(&nodetasks.File{
+		Path:     location,
+		Contents: fi.NewStringResource(serialized),
+		Type:     nodetasks.FileType_File,
+	})
+
+	return nil
+}

nodeup/pkg/model/kubeapiserver.go

@@ -41,6 +41,7 @@ type KubeAPIServerBuilder struct {
 
 var _ fi.ModelBuilder = &KubeAPIServerBuilder{}
 
+// Build is responsible for generating the configuration for the kube-apiserver
 func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 	if !b.IsMaster {
 		return nil
@@ -70,6 +71,19 @@ func (b *KubeAPIServerBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
+	// @check if we are using secure client certificates for kubelet and grab the certificates
+	{
+		if b.UseSecureKubelet() {
+			name := "kubelet-api"
+			if err := buildCertificateRequest(c, b.NodeupModelContext, name, ""); err != nil {
+				return err
+			}
+			if err := buildPrivateKeyRequest(c, b.NodeupModelContext, name, ""); err != nil {
+				return err
+			}
+		}
+	}
+
 	// Touch log file, so that docker doesn't create a directory instead
 	{
 		t := &nodetasks.File{
@@ -127,22 +141,30 @@ func (b *KubeAPIServerBuilder) writeAuthenticationConfig(c *fi.ModelBuilderConte
 			Type:     nodetasks.FileType_File,
 		}
 		c.AddTask(t)
+
 		return nil
-	} else {
-		return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication)
 	}
+
+	return fmt.Errorf("Unrecognized authentication config %v", b.Cluster.Spec.Authentication)
 }
 
+// buildPod is responsible for generating the kube-apiserver pod and thus manifest file
 func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	kubeAPIServer := b.Cluster.Spec.KubeAPIServer
-
 	kubeAPIServer.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
 	kubeAPIServer.TLSCertFile = filepath.Join(b.PathSrvKubernetes(), "server.cert")
 	kubeAPIServer.TLSPrivateKeyFile = filepath.Join(b.PathSrvKubernetes(), "server.key")
-
 	kubeAPIServer.BasicAuthFile = filepath.Join(b.PathSrvKubernetes(), "basic_auth.csv")
 	kubeAPIServer.TokenAuthFile = filepath.Join(b.PathSrvKubernetes(), "known_tokens.csv")
 
+	// @check if we are using secure kubelet client certificates
+	if b.UseSecureKubelet() {
+		// @note we are making assumption we are using the one's created by the pki model, not custom defined ones
+		kubeAPIServer.KubeletClientCertificate = filepath.Join(b.PathSrvKubernetes(), "kubelet-api.pem")
+		kubeAPIServer.KubeletClientKey = filepath.Join(b.PathSrvKubernetes(), "kubelet-api-key.pem")
+	}
+
+	// build the kube-apiserver flags for the service
 	flags, err := flagbuilder.BuildFlags(b.Cluster.Spec.KubeAPIServer)
 	if err != nil {
 		return nil, fmt.Errorf("error building kube-apiserver flags: %v", err)
@@ -203,6 +225,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 			InitialDelaySeconds: 15,
 			TimeoutSeconds:      15,
 		},
+
 		Ports: []v1.ContainerPort{
 			{
 				Name:          "https",
@@ -219,7 +242,6 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 
 	for _, path := range b.SSLHostPaths() {
 		name := strings.Replace(path, "/", "", -1)
-
 		addHostPathMapping(pod, container, name, path)
 	}
 

nodeup/pkg/model/kubelet.go

@@ -18,8 +18,8 @@ package model
 
 import (
 	"fmt"
-	"github.com/blang/semver"
-	"github.com/golang/glog"
+	"path/filepath"
+
 	"k8s.io/client-go/pkg/api/v1"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
@@ -29,6 +29,9 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
 )
 
 // KubeletBuilder install kubelet
@@ -38,6 +41,7 @@ type KubeletBuilder struct {
 
 var _ fi.ModelBuilder = &KubeletBuilder{}
 
+// Build is responsible for building the kubelet configuration
 func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 	kubeletConfig, err := b.buildKubeletConfig()
 	if err != nil {
@@ -78,7 +82,6 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 	// Add kubeconfig
 	{
 		// TODO: Change kubeconfig to be https
-
 		kubeconfig, err := b.buildPKIKubeconfig("kubelet")
 		if err != nil {
 			return err
@@ -265,6 +268,12 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro
 		utils.JsonMergeStruct(c, b.Cluster.Spec.Kubelet)
 	}
 
+	// @check if we are using secure kubelet <-> api settings
+	if b.UseSecureKubelet() {
+		// @TODO these filenames need to be a constant somewhere
+		c.ClientCAFile = filepath.Join(b.PathSrvKubernetes(), "ca.crt")
+	}
+
 	if b.InstanceGroup.Spec.Kubelet != nil {
 		utils.JsonMergeStruct(c, b.InstanceGroup.Spec.Kubelet)
 	}

pkg/model/pki.go

@@ -30,6 +30,7 @@ type PKIModelBuilder struct {
 
 var _ fi.ModelBuilder = &PKIModelBuilder{}
 
+// Build is responsible for generating the various pki assets
 func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	{
 		// Keypair used by the kubelet
@@ -43,6 +44,18 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 		c.AddTask(t)
 	}
 
+	{
+		// Generate a kubelet client certificate for api to speak securely to kubelets. This change was first
+		// introduced in https://github.com/kubernetes/kops/pull/2831 where server.cert/key were used. With kubernetes >= 1.7
+		// the certificate usage is being checked (obviously the above was server not client certificate) and so now fails
+		c.AddTask(&fitasks.Keypair{
+			Name:      fi.String("kubelet-api"),
+			Lifecycle: b.Lifecycle,
+			Subject:   "cn=kubelet-api",
+			Type:      "client",
+		})
+	}
+
 	{
 		// Secret used by the kubelet
 		// TODO: Can this be removed... at least from 1.6 on?