--requestheader-client-ca-file for aggregated API servers (#1406)

* --requestheader-client-ca-file scaffolding * added genCertForAggregatedAPIServers() boilerplate * generate proxy CA and keypair by default * added proxy certs to master as files * test level-up * eliminating old implementation * added new proxy cert params * added proxy vars * no longer throwing away private key * added TODO scaffolding for generating certs on master * pseudo code for master cert generation * todo clarification * first draft of cert generation script * removed in-code cert generation foo * cleanup * missing comma * cert script mods * clean up certs on disk * add AnableAggregatedAPIs config guard * improvements to cert generation script * enforcing k8s 1.7 only for enableAggregatedAPIs * generate certs in /tmp * added add’l requestheader apiserver options and required rbac * add retry logic
Azure · Sep 20, 2017 · 1cab24a · 1cab24a
1 parent 57e58b6
commit 1cab24a
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 11 deletions.
diff --git a/parts/kubernetesmaster-kube-apiserver.yaml b/parts/kubernetesmaster-kube-apiserver.yaml
@@ -28,13 +28,20 @@ spec:
         - "--tls-cert-file=/etc/kubernetes/certs/apiserver.crt"
         - "--tls-private-key-file=/etc/kubernetes/certs/apiserver.key"
         - "--client-ca-file=/etc/kubernetes/certs/ca.crt"
+        - "--requestheader-client-ca-file=/etc/kubernetes/certs/proxy-ca.crt"
+        - "--proxy-client-cert-file=/etc/kubernetes/certs/proxy.crt"
+        - "--proxy-client-key-file=/etc/kubernetes/certs/proxy.key"
         - "--service-account-key-file=/etc/kubernetes/certs/apiserver.key"
         - "--oidc-client-id="
         - "--oidc-issuer-url="
         - "--oidc-username-claim=oid"
         - "--storage-backend=etcd2"
         - "--v=4"
         - "<kubernetesEnableRbac>"
+        - "--requestheader-allowed-names="
+        - "--requestheader-extra-headers-prefix=X-Remote-Extra-"
+        - "--requestheader-group-headers=X-Remote-Group"
+        - "--requestheader-username-headers=X-Remote-User"
       volumeMounts:
         - name: "etc-kubernetes"
           mountPath: "/etc/kubernetes"

diff --git a/parts/kubernetesmastercustomdata.yml b/parts/kubernetesmastercustomdata.yml
@@ -55,6 +55,15 @@ write_files:
   content: |
     {{WrapAsVariable "clientCertificate"}}
 
+{{if .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}}
+- path: "/etc/kubernetes/generate-proxy-certs.sh"
+  permissions: "0744"
+  encoding: "gzip"
+  owner: "root"
+  content: !!binary |
+    {{WrapAsVariable "generateProxyCertsScript"}}
+{{end}}
+
 - path: "/var/lib/kubelet/kubeconfig"
   permissions: "0644"
   owner: "root"
@@ -273,6 +282,16 @@ write_files:
     sed -i "s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml"
 {{end}}
 
+{{if not .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}}
+    sed -i "/requestheader-client-ca-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/proxy-client-cert-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/proxy-client-key-file/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/requestheader-allowed-names/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/requestheader-extra-headers-prefix/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/requestheader-group-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+    sed -i "/requestheader-username-headers/d" "/etc/kubernetes/manifests/kube-apiserver.yaml"
+{{end}}
+
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
   encoding: gzip
@@ -338,5 +357,8 @@ runcmd:
 - mkdir -p /etc/kubernetes/manifests
 - usermod -aG docker {{WrapAsVariable "username"}}
 - /usr/lib/apt/apt.systemd.daily
+{{if .OrchestratorProfile.KubernetesConfig.EnableAggregatedAPIs}}
+- sudo bash /etc/kubernetes/generate-proxy-certs.sh
+{{end}}
 - apt-mark unhold walinuxagent
 - touch /opt/azure/containers/runcmd.complete
diff --git a/parts/kubernetesmastergenerateproxycertscript.sh b/parts/kubernetesmastergenerateproxycertscript.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+PROXY_CA_KEY="${PROXY_CA_KEY:=/tmp/proxy-client-ca.key}"
+PROXY_CRT="${PROXY_CRT:=/tmp/proxy-client-ca.crt}"
+PROXY_CLIENT_KEY="${PROXY_CLIENT_KEY:=/tmp/proxy-client.key}"
+PROXY_CLIENT_CSR="${PROXY_CLIENT_CSR:=/tmp/proxy-client.csr}"
+PROXY_CLIENT_CRT="${PROXY_CLIENT_CRT:=/tmp/proxy-client.crt}"
+ETCD_REQUESTHEADER_CLIENT_CA="${ETCD_REQUESTHEADER_CLIENT_CA:=/proxycerts/requestheader-client-ca-file}"
+ETCD_PROXY_CERT="${ETCD_PROXY_CERT:=/proxycerts/proxy-client-cert-file}"
+ETCD_PROXY_KEY="${ETCD_PROXY_KEY:=/proxycerts/proxy-client-key-file}"
+K8S_PROXY_CA_CRT_FILEPATH="${K8S_PROXY_CA_CRT_FILEPATH:=/etc/kubernetes/certs/proxy-ca.crt}"
+K8S_PROXY_KEY_FILEPATH="${K8S_PROXY_KEY_FILEPATH:=/etc/kubernetes/certs/proxy.key}"
+K8S_PROXY_CRT_FILEPATH="${K8S_PROXY_CRT_FILEPATH:=/etc/kubernetes/certs/proxy.crt}"
+
+# generate root CA
+openssl genrsa -out $PROXY_CA_KEY 2048
+openssl req -new -x509 -days 1826 -key $PROXY_CA_KEY -out $PROXY_CRT -subj '/CN=proxyClientCA'
+# generate new cert
+openssl genrsa -out $PROXY_CLIENT_KEY 2048
+openssl req -new -key $PROXY_CLIENT_KEY -out $PROXY_CLIENT_CSR -subj '/CN=aggregator/O=system:masters'
+openssl x509 -req -days 730 -in $PROXY_CLIENT_CSR -CA $PROXY_CRT -CAkey $PROXY_CA_KEY -set_serial 02 -out $PROXY_CLIENT_CRT
+
+retrycmd_if_failure() { for i in 1 2 3 4 5 6 7 8 9 10; do $@; [ $? -eq 0  ] && break || sleep 30; done ; }
+
+write_certs_to_disk() {
+    etcdctl get $ETCD_REQUESTHEADER_CLIENT_CA > $K8S_PROXY_CA_CRT_FILEPATH
+    etcdctl get $ETCD_PROXY_KEY > $K8S_PROXY_KEY_FILEPATH
+    etcdctl get $ETCD_PROXY_CERT > $K8S_PROXY_CRT_FILEPATH
+    # Remove whitespace padding at beginning of 1st line
+    sed -i '1s/\s//' $K8S_PROXY_CA_CRT_FILEPATH $K8S_PROXY_CRT_FILEPATH $K8S_PROXY_KEY_FILEPATH
+    chmod 600 $K8S_PROXY_KEY_FILEPATH
+}
+
+write_certs_to_disk_with_retry() {
+    for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+        write_certs_to_disk
+        [ $? -eq 0  ] && break || sleep 5
+    done
+}
+
+# block until all etcd is ready
+retrycmd_if_failure etcdctl cluster-health
+# Make etcd keys, adding a leading whitespace because etcd won't accept a val that begins with a '-' (hyphen)!
+if etcdctl mk $ETCD_REQUESTHEADER_CLIENT_CA " $(cat ${PROXY_CRT})"; then
+    etcdctl mk $ETCD_PROXY_KEY " $(cat ${PROXY_CLIENT_KEY})"
+    etcdctl mk $ETCD_PROXY_CERT " $(cat ${PROXY_CLIENT_CRT})"
+    sleep 5
+    write_certs_to_disk_with_retry
+# If the etcdtl mk command failed, that means the key already exists
+else
+    sleep 5
+    write_certs_to_disk_with_retry
+fi
diff --git a/parts/kubernetesmastervars.t b/parts/kubernetesmastervars.t
@@ -116,6 +116,7 @@
   {{end}}
 {{end}}
     "provisionScript": "{{GetKubernetesB64Provision}}",
+    "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}",
     "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}",
 {{if IsVNETIntegrated}}
     "allocateNodeCidrs": false,
@@ -267,4 +268,4 @@
     "agentWindowsVersion": "[parameters('agentWindowsVersion')]",
     "singleQuote": "'",
     "windowsCustomScriptSuffix": " $inputFile = '%SYSTEMDRIVE%\\AzureData\\CustomData.bin' ; $outputFile = '%SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.ps1' ; Copy-Item $inputFile $outputFile ; Invoke-Expression('{0} {1}' -f $outputFile, $arguments) ; "
-{{end}}
+{{end}}
diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go
@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	kubernetesMasterCustomDataYaml      = "kubernetesmastercustomdata.yml"
-	kubernetesMasterCustomScript        = "kubernetesmastercustomscript.sh"
-	kubernetesAgentCustomDataYaml       = "kubernetesagentcustomdata.yml"
-	kubeConfigJSON                      = "kubeconfig.json"
-	kubernetesWindowsAgentCustomDataPS1 = "kuberneteswindowssetup.ps1"
+	kubernetesMasterCustomDataYaml           = "kubernetesmastercustomdata.yml"
+	kubernetesMasterCustomScript             = "kubernetesmastercustomscript.sh"
+	kubernetesMasterGenerateProxyCertsScript = "kubernetesmastergenerateproxycertscript.sh"
+	kubernetesAgentCustomDataYaml            = "kubernetesagentcustomdata.yml"
+	kubeConfigJSON                           = "kubeconfig.json"
+	kubernetesWindowsAgentCustomDataPS1      = "kuberneteswindowssetup.ps1"
 )
 
 const (
@@ -101,13 +102,15 @@ var kubernetesManifestYamls = map[string]string{
 }
 
 var kubernetesAritfacts = map[string]string{
-	"MASTER_PROVISION_B64_GZIP_STR": kubernetesMasterCustomScript,
-	"KUBELET_SERVICE_B64_GZIP_STR":  kubernetesKubeletService,
+	"MASTER_PROVISION_B64_GZIP_STR":            kubernetesMasterCustomScript,
+	"MASTER_GENERATE_PROXY_CERTS_B64_GZIP_STR": kubernetesMasterGenerateProxyCertsScript,
+	"KUBELET_SERVICE_B64_GZIP_STR":             kubernetesKubeletService,
 }
 
 var kubernetesAritfacts15 = map[string]string{
-	"MASTER_PROVISION_B64_GZIP_STR": kubernetesMasterCustomScript,
-	"KUBELET_SERVICE_B64_GZIP_STR":  "kuberneteskubelet1.5.service",
+	"MASTER_PROVISION_B64_GZIP_STR":            kubernetesMasterCustomScript,
+	"MASTER_GENERATE_PROXY_CERTS_B64_GZIP_STR": kubernetesMasterGenerateProxyCertsScript,
+	"KUBELET_SERVICE_B64_GZIP_STR":             "kuberneteskubelet1.5.service",
 }
 
 var kubernetesAddonYamls = map[string]string{
@@ -942,6 +945,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 		"GetKubernetesB64Provision": func() string {
 			return getBase64CustomScript(kubernetesMasterCustomScript)
 		},
+		"GetKubernetesB64GenerateProxyCerts": func() string {
+			return getBase64CustomScript(kubernetesMasterGenerateProxyCertsScript)
+		},
 		"GetKubernetesMasterPreprovisionYaml": func() string {
 			str := ""
 			if cs.Properties.MasterProfile.PreprovisionExtension != nil {

diff --git a/pkg/acsengine/engine_test.go b/pkg/acsengine/engine_test.go
@@ -291,7 +291,10 @@ func TestTemplateOutputPresence(t *testing.T) {
 	}
 
 	var template TestARMTemplate
-	json.Unmarshal([]byte(armTemplate), &template)
+	err = json.Unmarshal([]byte(armTemplate), &template)
+	if err != nil {
+		t.Fatalf("couldn't unmarshall ARM template: %#v\n", err)
+	}
 
 	tt := []struct {
 		key   string

diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go
@@ -673,6 +673,7 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage
 	vlabs.UseInstanceMetadata = api.UseInstanceMetadata
 	vlabs.EnableRbac = api.EnableRbac
+	vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs
 	vlabs.GCHighThreshold = api.GCHighThreshold
 	vlabs.GCLowThreshold = api.GCLowThreshold
 }

diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go
@@ -641,6 +641,7 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage
 	api.UseInstanceMetadata = vlabs.UseInstanceMetadata
 	api.EnableRbac = vlabs.EnableRbac
+	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs
 	api.GCHighThreshold = vlabs.GCHighThreshold
 	api.GCLowThreshold = vlabs.GCLowThreshold
 }

diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -172,6 +172,7 @@ type KubernetesConfig struct {
 	CustomHyperkubeImage             string  `json:"customHyperkubeImage,omitempty"`
 	UseInstanceMetadata              bool    `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool    `json:"enableRbac,omitempty"`
+	EnableAggregatedAPIs             bool    `json:"enableAggregatedAPIs,omitempty"`
 	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
 	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
 }

diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go
@@ -200,6 +200,7 @@ type KubernetesConfig struct {
 	CustomHyperkubeImage             string  `json:"customHyperkubeImage,omitempty"`
 	UseInstanceMetadata              bool    `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool    `json:"enableRbac,omitempty"`
+	EnableAggregatedAPIs             bool    `json:"enableAggregatedAPIs,omitempty"`
 	GCHighThreshold                  int     `json:"gchighthreshold,omitempty"`
 	GCLowThreshold                   int     `json:"gclowthreshold,omitempty"`
 }

diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go
@@ -57,6 +57,16 @@ func (o *OrchestratorProfile) Validate() error {
 			if err != nil {
 				return err
 			}
+			if o.KubernetesConfig.EnableAggregatedAPIs {
+				if o.OrchestratorRelease != common.KubernetesRelease1Dot7 {
+					return fmt.Errorf("enableAggregatedAPIs is only available in Kubernetes version %s; unable to validate for Kubernetes version %s",
+						common.KubernetesRelease1Dot7, o.OrchestratorRelease)
+				}
+
+				if !o.KubernetesConfig.EnableRbac {
+					return fmt.Errorf("enableAggregatedAPIs requires the enableRbac feature as a prerequisite")
+				}
+			}
 		}
 
 	default: