From 3e6ae6fd0dcb91a18eccd539c7e077127f463d5d Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 24 Jan 2018 17:22:28 -0800 Subject: [PATCH 1/7] Use single values for etcdpeer key params --- parts/k8s/kubernetesmastervars.t | 16 +++++++-- parts/k8s/kubernetesparams.t | 62 +++++++++++++++++++++++++++++--- pkg/acsengine/engine.go | 25 +++++++++---- 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index ea008c0376..025177f228 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -10,8 +10,20 @@ "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]", "etcdClientPrivateKey": "[parameters('etcdClientPrivateKey')]", "etcdClientCertificate": "[parameters('etcdClientCertificate')]", - "etcdPeerPrivateKeys": "[parameters('etcdPeerPrivateKeys')]", - "etcdPeerCertificates": "[parameters('etcdPeerCertificates')]", + "etcdPeerPrivateKeys": [ + "[parameters('etcdPeerPrivateKey0')]", + "[parameters('etcdPeerPrivateKey1')]", + "[parameters('etcdPeerPrivateKey2')]", + "[parameters('etcdPeerPrivateKey3')]", + "[parameters('etcdPeerPrivateKey4')]" + ], + "etcdPeerCertificates": [ + "[parameters('etcdPeerCertificate0')]", + "[parameters('etcdPeerCertificate1')]", + "[parameters('etcdPeerCertificate2')]", + "[parameters('etcdPeerCertificate3')]", + "[parameters('etcdPeerCertificate4')]" + ], "etcdPeerCertFilepath":[ "/etc/kubernetes/certs/etcdpeer0.crt", "/etc/kubernetes/certs/etcdpeer1.crt", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 280ba7e904..2e332ec9c0 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -50,18 +50,70 @@ }, "type": "securestring" }, - "etcdPeerCertificates": { + "etcdPeerCertificate0": { "metadata": { "description": "The base 64 server certificates used on the master" }, - "type": "array" - }, - "etcdPeerPrivateKeys": { + "type": "string" + }, + "etcdPeerPrivateKey0": { "metadata": { "description": "The base 64 server private keys used on the master." }, - "type": "array" + "type": "securestring" }, + {{if eq .MasterProfile.Count 3}} + "etcdPeerCertificate1": { + "metadata": { + "description": "The base 64 server certificates used on the master" + }, + "type": "string" + }, + "etcdPeerCertificate2": { + "metadata": { + "description": "The base 64 server certificates used on the master" + }, + "type": "string" + }, + "etcdPeerPrivateKey1": { + "metadata": { + "description": "The base 64 server private keys used on the master." + }, + "type": "securestring" + }, + "etcdPeerPrivateKey2": { + "metadata": { + "description": "The base 64 server private keys used on the master." + }, + "type": "securestring" + }, + {{if eq .MasterProfile.Count 5}} + "etcdPeerCertificate3": { + "metadata": { + "description": "The base 64 server certificates used on the master" + }, + "type": "string" + }, + "etcdPeerCertificate4": { + "metadata": { + "description": "The base 64 server certificates used on the master" + }, + "type": "string" + }, + "etcdPeerPrivateKey3": { + "metadata": { + "description": "The base 64 server private keys used on the master." + }, + "type": "securestring" + }, + "etcdPeerPrivateKey4": { + "metadata": { + "description": "The base 64 server private keys used on the master." + }, + "type": "securestring" + }, + {{end}} + {{end}} "caCertificate": { "metadata": { "description": "The base 64 certificate authority certificate" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 76a0650880..0b14fc445e 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -506,8 +506,14 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addSecret(parametersMap, "etcdServerPrivateKey", properties.CertificateProfile.EtcdServerPrivateKey, true) addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true) addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true) - addArraySecret(parametersMap, "etcdPeerCertificates", properties.CertificateProfile.EtcdPeerCertificates, true) - addArraySecret(parametersMap, "etcdPeerPrivateKeys", properties.CertificateProfile.EtcdPeerPrivateKeys, true) + for i, pc := range properties.CertificateProfile.EtcdPeerCertificates { + addSecret(parametersMap, "etcdPeerCertificates"+strconv.Itoa(i), pc, true) + } + for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys { + addSecret(parametersMap, "etcdPeerPrivateKeys"+strconv.Itoa(i), pk, true) + } + //addArraySecret(parametersMap, "etcdPeerCertificates", properties.CertificateProfile.EtcdPeerCertificates, true) + //addArraySecret(parametersMap, "etcdPeerPrivateKeys", properties.CertificateProfile.EtcdPeerPrivateKeys, true) } if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" { @@ -781,12 +787,17 @@ func addArraySecret(m paramsMap, k string, v interface{}, encode bool) { } values := make([]string, len(arr)) for i := 0; i < len(arr); i++ { - if encode { - values[i] = base64.StdEncoding.EncodeToString([]byte(arr[i])) - } else { - values[i] = arr[i] + str := arr[i] + parts := keyvaultSecretPathRe.FindStringSubmatch(str) + if parts == nil || len(parts) != 5 { + if encode { + values[i] = base64.StdEncoding.EncodeToString([]byte(str)) + } else { + values[i] = str + } + continue } - + addKeyvaultReference(m, k, parts[1], parts[2], parts[4]) } addValue(m, k, values) } From 09839c0d76608c28e973eacc766d49c2428c06f5 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 24 Jan 2018 17:55:49 -0800 Subject: [PATCH 2/7] fixed param logic and added logic to vars --- parts/k8s/kubernetesmastervars.t | 22 ++++++++++++++++++++++ parts/k8s/kubernetesparams.t | 4 ++-- pkg/acsengine/engine.go | 4 ++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 025177f228..6ed0781e17 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -10,6 +10,27 @@ "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]", "etcdClientPrivateKey": "[parameters('etcdClientPrivateKey')]", "etcdClientCertificate": "[parameters('etcdClientCertificate')]", +{{if eq .MasterProfile.Count 1}} + "etcdPeerPrivateKeys": [ + "[parameters('etcdPeerPrivateKey0')]" + ], + "etcdPeerCertificates": [ + "[parameters('etcdPeerCertificate0')]" + ], +{{end}} +{{if eq .MasterProfile.Count 3}} + "etcdPeerPrivateKeys": [ + "[parameters('etcdPeerPrivateKey0')]", + "[parameters('etcdPeerPrivateKey1')]", + "[parameters('etcdPeerPrivateKey2')]" + ], + "etcdPeerCertificates": [ + "[parameters('etcdPeerCertificate0')]", + "[parameters('etcdPeerCertificate1')]", + "[parameters('etcdPeerCertificate2')]" + ], +{{end}} +{{if eq .MasterProfile.Count 5}} "etcdPeerPrivateKeys": [ "[parameters('etcdPeerPrivateKey0')]", "[parameters('etcdPeerPrivateKey1')]", @@ -24,6 +45,7 @@ "[parameters('etcdPeerCertificate3')]", "[parameters('etcdPeerCertificate4')]" ], +{{end}} "etcdPeerCertFilepath":[ "/etc/kubernetes/certs/etcdpeer0.crt", "/etc/kubernetes/certs/etcdpeer1.crt", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 2e332ec9c0..4b9e9d11ac 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -62,7 +62,7 @@ }, "type": "securestring" }, - {{if eq .MasterProfile.Count 3}} + {{if ge .MasterProfile.Count 3}} "etcdPeerCertificate1": { "metadata": { "description": "The base 64 server certificates used on the master" @@ -87,7 +87,7 @@ }, "type": "securestring" }, - {{if eq .MasterProfile.Count 5}} + {{if ge .MasterProfile.Count 5}} "etcdPeerCertificate3": { "metadata": { "description": "The base 64 server certificates used on the master" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 0b14fc445e..7946553cc1 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -507,10 +507,10 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true) addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true) for i, pc := range properties.CertificateProfile.EtcdPeerCertificates { - addSecret(parametersMap, "etcdPeerCertificates"+strconv.Itoa(i), pc, true) + addSecret(parametersMap, "etcdPeerCertificate"+strconv.Itoa(i), pc, true) } for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys { - addSecret(parametersMap, "etcdPeerPrivateKeys"+strconv.Itoa(i), pk, true) + addSecret(parametersMap, "etcdPeerPrivateKey"+strconv.Itoa(i), pk, true) } //addArraySecret(parametersMap, "etcdPeerCertificates", properties.CertificateProfile.EtcdPeerCertificates, true) //addArraySecret(parametersMap, "etcdPeerPrivateKeys", properties.CertificateProfile.EtcdPeerPrivateKeys, true) From 0213de06dc23a1124510ac6eb39f5682160463e0 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Thu, 25 Jan 2018 10:17:59 -0800 Subject: [PATCH 3/7] remove unused code --- pkg/acsengine/engine.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 7946553cc1..9402391cf1 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -512,8 +512,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys { addSecret(parametersMap, "etcdPeerPrivateKey"+strconv.Itoa(i), pk, true) } - //addArraySecret(parametersMap, "etcdPeerCertificates", properties.CertificateProfile.EtcdPeerCertificates, true) - //addArraySecret(parametersMap, "etcdPeerPrivateKeys", properties.CertificateProfile.EtcdPeerPrivateKeys, true) } if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" { @@ -779,29 +777,6 @@ func addSecret(m paramsMap, k string, v interface{}, encode bool) { addKeyvaultReference(m, k, parts[1], parts[2], parts[4]) } -func addArraySecret(m paramsMap, k string, v interface{}, encode bool) { - arr, ok := v.([]string) - if !ok { - addValue(m, k, v) - return - } - values := make([]string, len(arr)) - for i := 0; i < len(arr); i++ { - str := arr[i] - parts := keyvaultSecretPathRe.FindStringSubmatch(str) - if parts == nil || len(parts) != 5 { - if encode { - values[i] = base64.StdEncoding.EncodeToString([]byte(str)) - } else { - values[i] = str - } - continue - } - addKeyvaultReference(m, k, parts[1], parts[2], parts[4]) - } - addValue(m, k, values) -} - // getStorageAccountType returns the support managed disk storage tier for a give VM size func getStorageAccountType(sizeName string) (string, error) { spl := strings.Split(sizeName, "_") From 14c0492f7c31ff48114725c9b793d0b6797d7aed Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 7 Feb 2018 11:40:53 -0800 Subject: [PATCH 4/7] only add master certs/keys to params and vars if master is not hosted --- parts/k8s/kubernetesmastervars.t | 14 +++++++------- parts/k8s/kubernetesparams.t | 17 +++++++++-------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 6ed0781e17..d45d853ecc 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -5,20 +5,19 @@ "apiServerCertificate": "[parameters('apiServerCertificate')]", {{ if not IsHostedMaster }} "apiServerPrivateKey": "[parameters('apiServerPrivateKey')]", -{{end}} "etcdServerCertificate": "[parameters('etcdServerCertificate')]", "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]", "etcdClientPrivateKey": "[parameters('etcdClientPrivateKey')]", "etcdClientCertificate": "[parameters('etcdClientCertificate')]", -{{if eq .MasterProfile.Count 1}} + {{if eq .MasterProfile.Count 1}} "etcdPeerPrivateKeys": [ "[parameters('etcdPeerPrivateKey0')]" ], "etcdPeerCertificates": [ "[parameters('etcdPeerCertificate0')]" ], -{{end}} -{{if eq .MasterProfile.Count 3}} + {{end}} + {{if eq .MasterProfile.Count 3}} "etcdPeerPrivateKeys": [ "[parameters('etcdPeerPrivateKey0')]", "[parameters('etcdPeerPrivateKey1')]", @@ -29,8 +28,8 @@ "[parameters('etcdPeerCertificate1')]", "[parameters('etcdPeerCertificate2')]" ], -{{end}} -{{if eq .MasterProfile.Count 5}} + {{end}} + {{if eq .MasterProfile.Count 5}} "etcdPeerPrivateKeys": [ "[parameters('etcdPeerPrivateKey0')]", "[parameters('etcdPeerPrivateKey1')]", @@ -45,7 +44,7 @@ "[parameters('etcdPeerCertificate3')]", "[parameters('etcdPeerCertificate4')]" ], -{{end}} + {{end}} "etcdPeerCertFilepath":[ "/etc/kubernetes/certs/etcdpeer0.crt", "/etc/kubernetes/certs/etcdpeer1.crt", @@ -65,6 +64,7 @@ "etcdClientKeyFilepath": "/etc/kubernetes/certs/etcdclient.key", "etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt", "etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key", +{{end}} "caCertificate": "[parameters('caCertificate')]", "caPrivateKey": "[parameters('caPrivateKey')]", "clientCertificate": "[parameters('clientCertificate')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 4b9e9d11ac..7e53232741 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -14,6 +14,14 @@ "type": "string" }, {{end}} +{{if IsHostedMaster}} + "kubernetesEndpoint": { + "metadata": { + "description": "The Kubernetes API endpoint https://:443" + }, + "type": "string" + }, +{{else}} "apiServerCertificate": { "metadata": { "description": "The base 64 server certificate used on the master" @@ -114,6 +122,7 @@ }, {{end}} {{end}} +{{end}} "caCertificate": { "metadata": { "description": "The base 64 certificate authority certificate" @@ -127,14 +136,6 @@ }, "type": "securestring" }, -{{if IsHostedMaster}} - "kubernetesEndpoint": { - "metadata": { - "description": "The Kubernetes API endpoint https://:443" - }, - "type": "string" - }, -{{end}} "clientCertificate": { "metadata": { "description": "The base 64 client certificate used to communicate with the master" From 6804fdaa2e6c61ca9e00e1cdda03b0450a54f248 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 7 Feb 2018 16:08:38 -0800 Subject: [PATCH 5/7] move apiserver cert --- parts/k8s/kubernetesparams.t | 12 ++++++------ pkg/acsengine/engine.go | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 7e53232741..ccd283ca58 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -22,12 +22,6 @@ "type": "string" }, {{else}} - "apiServerCertificate": { - "metadata": { - "description": "The base 64 server certificate used on the master" - }, - "type": "string" - }, "apiServerPrivateKey": { "metadata": { "description": "The base 64 server private key used on the master." @@ -123,6 +117,12 @@ {{end}} {{end}} {{end}} + "apiServerCertificate": { + "metadata": { + "description": "The base 64 server certificate used on the master" + }, + "type": "string" + }, "caCertificate": { "metadata": { "description": "The base 64 certificate authority certificate" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index de112e0d65..0a32a53ed8 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -114,6 +114,12 @@ var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentReso - kubeConfigCertificate - kubeConfigPrivateKey - servicePrincipalClientSecret + - etcdClientCertificate + - etcdClientPrivateKey + - etcdServerCertificate + - etcdServerPrivateKey + - etcdPeerCertificates + - etcdPeerPrivateKeys To refer to a keyvault secret, the value of the parameter in the api model file should be formatted as: From e50568c9d3b9138175a66fbe0ac47090302a1a0d Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 7 Feb 2018 16:15:51 -0800 Subject: [PATCH 6/7] add master profile != nil check --- pkg/acsengine/engine.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 0a32a53ed8..9668803c67 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -508,15 +508,17 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addSecret(parametersMap, "clientPrivateKey", properties.CertificateProfile.ClientPrivateKey, true) addSecret(parametersMap, "kubeConfigCertificate", properties.CertificateProfile.KubeConfigCertificate, true) addSecret(parametersMap, "kubeConfigPrivateKey", properties.CertificateProfile.KubeConfigPrivateKey, true) - addSecret(parametersMap, "etcdServerCertificate", properties.CertificateProfile.EtcdServerCertificate, true) - addSecret(parametersMap, "etcdServerPrivateKey", properties.CertificateProfile.EtcdServerPrivateKey, true) - addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true) - addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true) - for i, pc := range properties.CertificateProfile.EtcdPeerCertificates { - addSecret(parametersMap, "etcdPeerCertificate"+strconv.Itoa(i), pc, true) - } - for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys { - addSecret(parametersMap, "etcdPeerPrivateKey"+strconv.Itoa(i), pk, true) + if properties.MasterProfile != nil { + addSecret(parametersMap, "etcdServerCertificate", properties.CertificateProfile.EtcdServerCertificate, true) + addSecret(parametersMap, "etcdServerPrivateKey", properties.CertificateProfile.EtcdServerPrivateKey, true) + addSecret(parametersMap, "etcdClientCertificate", properties.CertificateProfile.EtcdClientCertificate, true) + addSecret(parametersMap, "etcdClientPrivateKey", properties.CertificateProfile.EtcdClientPrivateKey, true) + for i, pc := range properties.CertificateProfile.EtcdPeerCertificates { + addSecret(parametersMap, "etcdPeerCertificate"+strconv.Itoa(i), pc, true) + } + for i, pk := range properties.CertificateProfile.EtcdPeerPrivateKeys { + addSecret(parametersMap, "etcdPeerPrivateKey"+strconv.Itoa(i), pk, true) + } } } From a130be934ca53973c284dc6a63458df9d4fa2230 Mon Sep 17 00:00:00 2001 From: CecileRobertMichon Date: Wed, 7 Feb 2018 16:30:20 -0800 Subject: [PATCH 7/7] undo move api server key --- parts/k8s/kubernetesparams.t | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index ccd283ca58..ae7255acc1 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -22,12 +22,6 @@ "type": "string" }, {{else}} - "apiServerPrivateKey": { - "metadata": { - "description": "The base 64 server private key used on the master." - }, - "type": "securestring" - }, "etcdServerCertificate": { "metadata": { "description": "The base 64 server certificate used on the master" @@ -123,6 +117,12 @@ }, "type": "string" }, + "apiServerPrivateKey": { + "metadata": { + "description": "The base 64 server private key used on the master." + }, + "type": "securestring" + }, "caCertificate": { "metadata": { "description": "The base 64 certificate authority certificate"