Keyvault etcd certs

parts/k8s/kubernetesmastervars.t

@@ -5,13 +5,46 @@
     "apiServerCertificate": "[parameters('apiServerCertificate')]",
 {{ if not IsHostedMaster }}
     "apiServerPrivateKey": "[parameters('apiServerPrivateKey')]",
-{{end}}
     "etcdServerCertificate": "[parameters('etcdServerCertificate')]",
     "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]",
     "etcdClientPrivateKey": "[parameters('etcdClientPrivateKey')]",
     "etcdClientCertificate": "[parameters('etcdClientCertificate')]",
-    "etcdPeerPrivateKeys": "[parameters('etcdPeerPrivateKeys')]",
-    "etcdPeerCertificates": "[parameters('etcdPeerCertificates')]",
+    {{if eq .MasterProfile.Count 1}}
+    "etcdPeerPrivateKeys": [
+        "[parameters('etcdPeerPrivateKey0')]"
+    ],
+    "etcdPeerCertificates": [
+        "[parameters('etcdPeerCertificate0')]"
+    ],
+    {{end}}
+    {{if eq .MasterProfile.Count 3}}
+    "etcdPeerPrivateKeys": [
+        "[parameters('etcdPeerPrivateKey0')]",
+        "[parameters('etcdPeerPrivateKey1')]",
+        "[parameters('etcdPeerPrivateKey2')]"
+    ],
+    "etcdPeerCertificates": [
+        "[parameters('etcdPeerCertificate0')]",
+        "[parameters('etcdPeerCertificate1')]",
+        "[parameters('etcdPeerCertificate2')]"
+    ],
+    {{end}}
+    {{if eq .MasterProfile.Count 5}}
+    "etcdPeerPrivateKeys": [
+        "[parameters('etcdPeerPrivateKey0')]",
+        "[parameters('etcdPeerPrivateKey1')]",
+        "[parameters('etcdPeerPrivateKey2')]",
+        "[parameters('etcdPeerPrivateKey3')]",
+        "[parameters('etcdPeerPrivateKey4')]"
+    ],
+    "etcdPeerCertificates": [
+        "[parameters('etcdPeerCertificate0')]",
+        "[parameters('etcdPeerCertificate1')]",
+        "[parameters('etcdPeerCertificate2')]",
+        "[parameters('etcdPeerCertificate3')]",
+        "[parameters('etcdPeerCertificate4')]"
+    ],
+    {{end}}
     "etcdPeerCertFilepath":[
         "/etc/kubernetes/certs/etcdpeer0.crt",
         "/etc/kubernetes/certs/etcdpeer1.crt",
@@ -31,6 +64,7 @@
     "etcdClientKeyFilepath": "/etc/kubernetes/certs/etcdclient.key",
     "etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt",
     "etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key",
+{{end}}
     "caCertificate": "[parameters('caCertificate')]",
     "caPrivateKey": "[parameters('caPrivateKey')]",
     "clientCertificate": "[parameters('clientCertificate')]",

parts/k8s/kubernetesparams.t

@@ -14,18 +14,14 @@
       "type": "string"
     },
 {{end}}
-    "apiServerCertificate": {
+{{if IsHostedMaster}}
+    "kubernetesEndpoint": {
       "metadata": {
-        "description": "The base 64 server certificate used on the master"
+        "description": "The Kubernetes API endpoint https://<kubernetesEndpoint>:443"
       },
       "type": "string"
     },
-    "apiServerPrivateKey": {
-      "metadata": {
-        "description": "The base 64 server private key used on the master."
-      },
-      "type": "securestring"
-    },
+{{else}}
     "etcdServerCertificate": {
       "metadata": {
         "description": "The base 64 server certificate used on the master"
@@ -50,17 +46,82 @@
       }, 
       "type": "securestring"
     },
-    "etcdPeerCertificates": {
+    "etcdPeerCertificate0": {
       "metadata": {
         "description": "The base 64 server certificates used on the master"
       }, 
-      "type": "array"
-    }, 
-    "etcdPeerPrivateKeys": {
+      "type": "string"
+    },
+    "etcdPeerPrivateKey0": {
       "metadata": {
         "description": "The base 64 server private keys used on the master."
       }, 
-      "type": "array"
+      "type": "securestring"
+    },
+    {{if ge .MasterProfile.Count 3}}
+      "etcdPeerCertificate1": {
+        "metadata": {
+          "description": "The base 64 server certificates used on the master"
+        }, 
+        "type": "string"
+      }, 
+      "etcdPeerCertificate2": {
+        "metadata": {
+          "description": "The base 64 server certificates used on the master"
+        }, 
+        "type": "string"
+      },
+      "etcdPeerPrivateKey1": {
+        "metadata": {
+          "description": "The base 64 server private keys used on the master."
+        }, 
+        "type": "securestring"
+      },
+      "etcdPeerPrivateKey2": {
+        "metadata": {
+          "description": "The base 64 server private keys used on the master."
+        }, 
+        "type": "securestring"
+      },
+      {{if ge .MasterProfile.Count 5}}
+        "etcdPeerCertificate3": {
+          "metadata": {
+            "description": "The base 64 server certificates used on the master"
+          }, 
+          "type": "string"
+        }, 
+        "etcdPeerCertificate4": {
+          "metadata": {
+            "description": "The base 64 server certificates used on the master"
+          }, 
+          "type": "string"
+        },
+        "etcdPeerPrivateKey3": {
+          "metadata": {
+            "description": "The base 64 server private keys used on the master."
+          }, 
+          "type": "securestring"
+        },
+        "etcdPeerPrivateKey4": {
+          "metadata": {
+            "description": "The base 64 server private keys used on the master."
+          }, 
+          "type": "securestring"
+        },
+      {{end}}
+    {{end}}
+{{end}}
+    "apiServerCertificate": {
+      "metadata": {
+        "description": "The base 64 server certificate used on the master"
+      },
+      "type": "string"
+    },
+    "apiServerPrivateKey": {
+      "metadata": {
+        "description": "The base 64 server private key used on the master."
+      },
+      "type": "securestring"
     },
     "caCertificate": {
       "metadata": {
@@ -75,14 +136,6 @@
       },
       "type": "securestring"
     },
-{{if IsHostedMaster}}
-    "kubernetesEndpoint": {
-      "metadata": {
-        "description": "The Kubernetes API endpoint https://<kubernetesEndpoint>:443"
-      },
-      "type": "string"
-    },
-{{end}}
     "clientCertificate": {
       "metadata": {
         "description": "The base 64 client certificate used to communicate with the master"

pkg/acsengine/engine.go

@@ -114,6 +114,12 @@ var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentReso
  - kubeConfigCertificate
  - kubeConfigPrivateKey
  - servicePrincipalClientSecret
+ - etcdClientCertificate
+ - etcdClientPrivateKey
+ - etcdServerCertificate
+ - etcdServerPrivateKey
+ - etcdPeerCertificates
+ - etcdPeerPrivateKeys
 
  To refer to a keyvault secret, the value of the parameter in the api model file should be formatted as:
 
@@ -502,12 +508,18 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
 			addSecret(parametersMap, "clientPrivateKey", properties.CertificateProfile.ClientPrivateKey, true)
 			addSecret(parametersMap, "kubeConfigCertificate", properties.CertificateProfile.KubeConfigCertificate, true)
 			addSecret(parametersMap, "kubeConfigPrivateKey", properties.CertificateProfile.KubeConfigPrivateKey, true)
-			addSecret(parametersMap, "etcdServerCertificate", properties.CertificateProfile.EtcdServerCertificate, true)
-			addSecret(parametersMap, "etcdServerPrivateKey", properties.CertificateProfile.EtcdServerPrivateKey, true)
-			addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true)
-			addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true)
-			addArraySecret(parametersMap, "etcdPeerCertificates", properties.CertificateProfile.EtcdPeerCertificates, true)
-			addArraySecret(parametersMap, "etcdPeerPrivateKeys", properties.CertificateProfile.EtcdPeerPrivateKeys, true)
+			if properties.MasterProfile != nil {
+				addSecret(parametersMap, "etcdServerCertificate", properties.CertificateProfile.EtcdServerCertificate, true)
+				addSecret(parametersMap, "etcdServerPrivateKey", properties.CertificateProfile.EtcdServerPrivateKey, true)
+				addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true)
+				addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true)
+				for i, pc := range properties.CertificateProfile.EtcdPeerCertificates {
+					addSecret(parametersMap, "etcdPeerCertificate"+strconv.Itoa(i), pc, true)
+				}
+				for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys {
+					addSecret(parametersMap, "etcdPeerPrivateKey"+strconv.Itoa(i), pk, true)
+				}
+			}
 		}
 
 		if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" {
@@ -773,24 +785,6 @@ func addSecret(m paramsMap, k string, v interface{}, encode bool) {
 	addKeyvaultReference(m, k, parts[1], parts[2], parts[4])
 }
 
-func addArraySecret(m paramsMap, k string, v interface{}, encode bool) {
-	arr, ok := v.([]string)
-	if !ok {
-		addValue(m, k, v)
-		return
-	}
-	values := make([]string, len(arr))
-	for i := 0; i < len(arr); i++ {
-		if encode {
-			values[i] = base64.StdEncoding.EncodeToString([]byte(arr[i]))
-		} else {
-			values[i] = arr[i]
-		}
-
-	}
-	addValue(m, k, values)
-}
-
 // getStorageAccountType returns the support managed disk storage tier for a give VM size
 func getStorageAccountType(sizeName string) (string, error) {
 	spl := strings.Split(sizeName, "_")