[wip] Kubelet TLS bootstrapping

core/controlplane/config/config.go

@@ -55,6 +55,9 @@ func NewDefaultCluster() *Cluster {
 		ClusterAutoscalerSupport{
 			Enabled: false,
 		},
+		ClusterTLSBootstrap{
+			Enabled: false,
+		},
 		EphemeralImageStorage{
 			Enabled:    false,
 			Disk:       "xvdb",
@@ -456,6 +459,7 @@ type Experimental struct {
 	AwsEnvironment           AwsEnvironment           `yaml:"awsEnvironment"`
 	AwsNodeLabels            AwsNodeLabels            `yaml:"awsNodeLabels"`
 	ClusterAutoscalerSupport ClusterAutoscalerSupport `yaml:"clusterAutoscalerSupport"`
+	ClusterTLSBootstrap      ClusterTLSBootstrap      `yaml:"clusterTLSBootstrap"`
 	EphemeralImageStorage    EphemeralImageStorage    `yaml:"ephemeralImageStorage"`
 	Kube2IamSupport          Kube2IamSupport          `yaml:"kube2IamSupport,omitempty"`
 	LoadBalancer             LoadBalancer             `yaml:"loadBalancer"`
@@ -503,6 +507,10 @@ type ClusterAutoscalerSupport struct {
 	Enabled bool `yaml:"enabled"`
 }
 
+type ClusterTLSBootstrap struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type EphemeralImageStorage struct {
 	Enabled    bool   `yaml:"enabled"`
 	Disk       string `yaml:"disk"`
@@ -908,6 +916,10 @@ func (c Cluster) valid() error {
 		fmt.Println(`WARNING: instance types "t2.nano" and "t2.micro" are not recommended. See https://github.com/coreos/kube-aws/issues/258 for more information`)
 	}
 
+	if c.Experimental.ClusterTLSBootstrap.Enabled && !c.Experimental.Plugins.Rbac.Enabled {
+		fmt.Println(`WARNING: enabling cluster-level TLS bootstrapping without RBAC is not recommended. See https://kubernetes.io/docs/admin/kubelet-tls-bootstrapping/ for more information`)
+	}
+
 	return nil
 }
 

core/controlplane/config/config_test.go

@@ -751,6 +751,69 @@ nodeDrainer:
 
 }
 
+func TestClusterTLSBootstrapConfig(t *testing.T) {
+
+	validConfigs := []struct {
+		conf                string
+		clusterTLSBootstrap ClusterTLSBootstrap
+	}{
+		{
+			conf: `
+`,
+			clusterTLSBootstrap: ClusterTLSBootstrap{
+				Enabled: false,
+			},
+		},
+		{
+			conf: `
+experimental:
+  clusterTLSBootstrap:
+    enabled: false
+`,
+			clusterTLSBootstrap: ClusterTLSBootstrap{
+				Enabled: false,
+			},
+		},
+		{
+			conf: `
+experimental:
+  clusterTLSBootstrap:
+    enabled: true
+`,
+			clusterTLSBootstrap: ClusterTLSBootstrap{
+				Enabled: true,
+			},
+		},
+		{
+			conf: `
+# Settings for an experimental feature must be under the "experimental" field. Ignored.
+clusterTLSBootstrap:
+  enabled: true
+`,
+			clusterTLSBootstrap: ClusterTLSBootstrap{
+				Enabled: false,
+			},
+		},
+	}
+
+	for _, conf := range validConfigs {
+		confBody := singleAzConfigYaml + conf.conf
+		c, err := ClusterFromBytes([]byte(confBody))
+		if err != nil {
+			t.Errorf("failed to parse config %s: %v", confBody, err)
+			continue
+		}
+		if !reflect.DeepEqual(c.Experimental.ClusterTLSBootstrap, conf.clusterTLSBootstrap) {
+			t.Errorf(
+				"parsed cluster TLS bootstrap settings %+v does not match config: %s",
+				c.Experimental.ClusterTLSBootstrap,
+				confBody,
+			)
+		}
+	}
+
+}
+
 func TestRktConfig(t *testing.T) {
 	validChannels := []string{
 		"alpha",

core/controlplane/config/templates/cloud-config-controller

@@ -487,6 +487,14 @@ write_files:
           "http://127.0.0.1:8080/apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings"
       done
 
+      {{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+      post_yaml "@${mfdir}/cluster-roles/csr.yaml" \
+      "http://127.0.0.1:8080/apis/rbac.authorization.k8s.io/v1alpha1/clusterroles" |true
+
+      post_yaml "@${mfdir}/cluster-role-bindings/csr.yaml" \
+      "http://127.0.0.1:8080/apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings" |true
+      {{ end }}
+
       {{ end }}
 
   - path: /etc/kubernetes/cni/docker_opts_cni.env
@@ -862,6 +870,37 @@ write_files:
           kind: ClusterRole
           name: cluster-admin
           apiGroup: rbac.authorization.k8s.io
+
+{{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+  - path: /srv/kubernetes/rbac/cluster-roles/csr.yaml
+    content: |
+        kind: ClusterRole
+        apiVersion: rbac.authorization.k8s.io/v1alpha1
+        metadata:
+          name: certificate-signing-requests
+        rules:
+          - apiGroups:
+              - '*'
+            resources:
+              - certificatesigningrequests
+            verbs:
+              - '*'
+
+  - path: /srv/kubernetes/rbac/cluster-role-bindings/csr.yaml
+    content: |
+        kind: ClusterRoleBinding
+        apiVersion: rbac.authorization.k8s.io/v1alpha1
+        metadata:
+          name: kubelet-bootstrap-csr
+        subjects:
+          - kind: Group
+            namespace: '*'
+            name: system:kubelet-bootstrap
+        roleRef:
+          kind: ClusterRole
+          name: certificate-signing-requests
+          apiGroup: rbac.authorization.k8s.io
+{{ end }}
 {{ end }}
 
   - path: /etc/kubernetes/manifests/kube-proxy.yaml
@@ -949,6 +988,9 @@ write_files:
           - --client-ca-file=/etc/kubernetes/ssl/ca.pem
           - --service-account-key-file=/etc/kubernetes/ssl/apiserver-key.pem
           - --runtime-config=extensions/v1beta1/networkpolicies=true,batch/v2alpha1{{if .Experimental.Plugins.Rbac.Enabled}},rbac.authorization.k8s.io/v1alpha1=true{{ end }}{{if .Experimental.Admission.PodSecurityPolicy.Enabled}},extensions/v1beta1/podsecuritypolicy=true{{ end }}
+          {{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+          - --token-auth-file=/etc/kubernetes/ssl/csr-bootstrap-token.csv
+          {{ end }}
           - --cloud-provider=aws
           livenessProbe:
             httpGet:
@@ -1008,6 +1050,11 @@ write_files:
           - --master=http://127.0.0.1:8080
           - --leader-elect=true
           - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
+          {{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+          - --insecure-experimental-approve-all-kubelet-csrs-for-group=system:kubelet-bootstrap
+          - --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem
+          - --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem
+          {{ end }}
           - --root-ca-file=/etc/kubernetes/ssl/ca.pem
           - --cloud-provider=aws
           resources:
@@ -1425,11 +1472,24 @@ write_files:
           - port: 80
             targetPort: 9090
 
+{{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+  # TODO: remove hard-coded token
+  - path: /etc/kubernetes/ssl/csr-bootstrap-token.csv
+    content: |
+      WNY9xavOd1SrFmffhBBXgFX1vTw3lgRh,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
+{{ end }}
+
 {{ if .ManageCertificates }}
   - path: /etc/kubernetes/ssl/ca.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
     encoding: gzip+base64
     content: {{.TLSConfig.CACert}}
 
+{{ if .Experimental.ClusterTLSBootstrap.Enabled }}
+  - path: /etc/kubernetes/ssl/ca-key.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
+    encoding: gzip+base64
+    content: {{.TLSConfig.CAKey}}
+{{ end }}
+
   - path: /etc/kubernetes/ssl/apiserver.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
     encoding: gzip+base64
     content: {{.TLSConfig.APIServerCert}}

core/controlplane/config/templates/cloud-config-worker

@@ -146,7 +146,8 @@ coreos:
         --cluster_dns={{.DNSServiceIP}} \
         --cluster_domain=cluster.local \
         --cloud-provider=aws \
-        --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \
+        {{if .ClusterTLSBootstrap.Enabled}}--experimental-bootstrap-kubeconfig=/etc/kubernetes/worker-bootstrap-kubeconfig.yaml
+        {{end}}--kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml \
         --tls-cert-file=/etc/kubernetes/ssl/worker.pem \
         --tls-private-key-file=/etc/kubernetes/ssl/worker-key.pem
         Restart=always
@@ -522,13 +523,15 @@ write_files:
     encoding: gzip+base64
     content: {{.TLSConfig.EtcdClientKey}}
 
+{{ if not .ClusterTLSBootstrap.Enabled }}
   - path: /etc/kubernetes/ssl/worker.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
     encoding: gzip+base64
     content: {{.TLSConfig.WorkerCert}}
 
   - path: /etc/kubernetes/ssl/worker-key.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
     encoding: gzip+base64
     content: {{.TLSConfig.WorkerKey}}
+{{ end }}
 
   - path: /etc/kubernetes/ssl/ca.pem{{if .TLSAssetsEncryptionEnabled}}.enc{{end}}
     encoding: gzip+base64
@@ -704,6 +707,28 @@ write_files:
                 path: /var/run/dbus
               name: dbus
 
+{{ if .ClusterTLSBootstrap.Enabled }}
+  - path: /etc/kubernetes/worker-bootstrap-kubeconfig.yaml
+    content: |
+        apiVersion: v1
+        kind: Config
+        clusters:
+        - name: local
+          cluster:
+            certificate-authority: /etc/kubernetes/ssl/ca.pem
+            server: https://{{.ExternalDNSName}}:443
+        users:
+        - name: kubelet-bootstrap
+          user:
+            # TODO: generate a dedicated token, send it encrypted to the CloudFormation template, and decrypt it on provisioning
+            token: WNY9xavOd1SrFmffhBBXgFX1vTw3lgRh
+        contexts:
+        - context:
+            cluster: local
+            user: kubelet-bootstrap
+          name: kubelet-bootstrap-context
+        current-context: kubelet-bootstrap-context
+{{ else }}
   - path: /etc/kubernetes/worker-kubeconfig.yaml
     content: |
         apiVersion: v1
@@ -723,6 +748,7 @@ write_files:
             user: kubelet
           name: kubelet-context
         current-context: kubelet-context
+{{ end }}
 
 {{ if not .UseCalico }}
   - path: /etc/kubernetes/cni/net.d/10-flannel.conf

core/controlplane/config/templates/cluster.yaml

@@ -749,6 +749,10 @@ worker:
 #   # Will provision controller nodes with IAM permissions to run cluster-autoscaler
 #   clusterAutoscalerSupport:
 #     enable: true
+#   # If enabled, instructs the controller manager to automatically issue TLS certificates to worker nodes via certificate signing requests (csr) made to the API server using the bootstrap token.
+#   # If enabled, also consider enabling the rbac plugin to limit requests using the bootstrap token to only be able to make requests related to certificate provisioning.
+#   clusterTLSBootstrap:
+#     enable: true
 #   # This option has not yet been tested with rkt as container runtime
 #   ephemeralImageStorage:
 #     enabled: true

core/controlplane/config/tls_config.go

@@ -191,7 +191,7 @@ func ReadRawTLSAssets(dirname string) (*RawTLSAssets, error) {
 		name      string
 		cert, key *[]byte
 	}{
-		{"ca", &r.CACert, nil},
+		{"ca", &r.CACert, &r.CAKey},
 		{"apiserver", &r.APIServerCert, &r.APIServerKey},
 		{"worker", &r.WorkerCert, &r.WorkerKey},
 		{"admin", &r.AdminCert, &r.AdminKey},
@@ -225,7 +225,7 @@ func ReadEncryptedTLSAssets(dirname string) (*EncryptedTLSAssets, error) {
 		name      string
 		cert, key *[]byte
 	}{
-		{"ca", &r.CACert, nil},
+		{"ca", &r.CACert, &r.CAKey},
 		{"apiserver", &r.APIServerCert, &r.APIServerKey},
 		{"worker", &r.WorkerCert, &r.WorkerKey},
 		{"admin", &r.AdminCert, &r.AdminKey},
@@ -306,6 +306,7 @@ func (r *RawTLSAssets) Encrypt(kMSKeyARN string, kmsSvc EncryptService) (*Encryp
 	}
 	encryptedAssets := EncryptedTLSAssets{TLSAssets{
 		CACert:         encrypt(r.CACert),
+		CAKey:          encrypt(r.CAKey),
 		APIServerCert:  encrypt(r.APIServerCert),
 		APIServerKey:   encrypt(r.APIServerKey),
 		WorkerCert:     encrypt(r.WorkerCert),
@@ -367,6 +368,7 @@ func (r *TLSAssets) Compact() (*CompactTLSAssets, error) {
 	}
 	compactAssets := CompactTLSAssets{
 		CACert:         compact(r.CACert),
+		CAKey:          compact(r.CAKey),
 		APIServerCert:  compact(r.APIServerCert),
 		APIServerKey:   compact(r.APIServerKey),
 		WorkerCert:     compact(r.WorkerCert),

test/integration/maincluster_test.go

@@ -95,6 +95,9 @@ func TestMainClusterConfig(t *testing.T) {
 			ClusterAutoscalerSupport: controlplane_config.ClusterAutoscalerSupport{
 				Enabled: false,
 			},
+			ClusterTLSBootstrap: controlplane_config.ClusterTLSBootstrap{
+				Enabled: false,
+			},
 			EphemeralImageStorage: controlplane_config.EphemeralImageStorage{
 				Enabled:    false,
 				Disk:       "xvdb",
@@ -871,6 +874,8 @@ experimental:
     enabled: true
   clusterAutoscalerSupport:
     enabled: true
+  clusterTLSBootstrap:
+    enabled: true
   ephemeralImageStorage:
     enabled: true
   kube2IamSupport:
@@ -936,6 +941,9 @@ worker:
 						ClusterAutoscalerSupport: controlplane_config.ClusterAutoscalerSupport{
 							Enabled: true,
 						},
+						ClusterTLSBootstrap: controlplane_config.ClusterTLSBootstrap{
+							Enabled: true,
+						},
 						EphemeralImageStorage: controlplane_config.EphemeralImageStorage{
 							Enabled:    true,
 							Disk:       "xvdb",
@@ -1007,6 +1015,8 @@ worker:
       enabled: true
     clusterAutoscalerSupport:
       enabled: true
+    clusterTLSBootstrap:
+      enabled: true
     ephemeralImageStorage:
       enabled: true
     kube2IamSupport:
@@ -1049,6 +1059,9 @@ worker:
 						ClusterAutoscalerSupport: controlplane_config.ClusterAutoscalerSupport{
 							Enabled: true,
 						},
+						ClusterTLSBootstrap: controlplane_config.ClusterTLSBootstrap{
+							Enabled: true,
+						},
 						EphemeralImageStorage: controlplane_config.EphemeralImageStorage{
 							Enabled:    true,
 							Disk:       "xvdb",