From 976cf6ceca9985fd1706c5988cf21d548482e76e Mon Sep 17 00:00:00 2001 From: Wenjun Wu Date: Wed, 7 Mar 2018 18:12:21 -0800 Subject: [PATCH] Squashed commit of the following: commit 203efbfd76e1cda15b32cf84d917984ed09ff2a0 Author: Jiangtian Li Date: Fri Jan 19 09:07:14 2018 -0800 Extend windows os drive size when customized OSDiskSizeGB is used (#2097) commit 88ec2fb54f913a7dc41588ca901e9f709c6bb0d4 Author: Robbie Zhang Date: Thu Jan 11 13:49:44 2018 -0800 Update the kube-dns addon commit 217ad8d47324c8cb1834b84c85b6088b49ca7a5f Merge: 530bedbd d8856c8b Author: Wenjun Wu Date: Mon Jan 8 16:22:56 2018 -0800 Merge remote-tracking branch 'origin/migration' into migration commit d8856c8ba57c88155e849c9a9b6e586b5cf82f69 Author: Robbie Zhang Date: Fri Jan 5 15:39:28 2018 -0800 Remove the Allow SSH and RDP Rules from NSG commit 530bedbd1203c5577c3e50e15b072c26aebc5b7e Merge: f3389a6b 50709347 Author: Wenjun Wu Date: Fri Jan 5 15:38:54 2018 -0800 Merge tag 'v0.9.4' into migration commit f3389a6baf654fd5025efc6f51d1cd9345b8b27e Author: Wenjun Wu Date: Fri Dec 15 11:11:13 2017 -0800 remove agent customscript and service file (#13) * remove agent specific custom script and service file. * remove cloud provider from windows start ps1 commit c2eda57d3ef4123e8c7173e209f9aedc2b344c22 Merge: 8ef4f2b9 004145cb Author: Wenjun Wu Date: Tue Dec 12 18:05:13 2017 -0800 Merge commit '004145cba163' into migration commit 004145cba1637f15134f9e01a4052c64d56f87ec Author: Wenjun Wu Date: Tue Dec 12 18:03:36 2017 -0800 fix merge error: azure storage classes yaml commit 8ef4f2b9a2a2d2f6e9ccf1d76fcc65ffb83f6c2b Merge: adbc1cf6 bd006fc9 Author: Wenjun Wu Date: Mon Nov 27 18:24:06 2017 -0800 Merge tag 'v0.9.3' into migration commit adbc1cf6beb1d3bb3ed695b12d707ea23f61971b Merge: f8da501a 79572455 Author: Wenjun Wu Date: Wed Oct 25 14:36:24 2017 -0700 Merge tag 'v0.8.0' into migration commit f8da501a6a16c3cf8269212eff4dd2221108fb19 Author: Robbie Zhang Date: Fri Sep 1 16:38:00 2017 -0700 Disable Windows Update commit ac838689d05bacd26013ab80360f1d31f7cf4249 Author: Robbie Zhang Date: Fri Sep 1 16:37:36 2017 -0700 Use kubelet v1.6.6.1 for Windows agent commit 5424f1470445897c412fbc1fc2e098dcadef090c Author: Robbie Zhang Date: Fri Sep 1 16:36:47 2017 -0700 Set master AvailabilitySet FaultDomainCount and UpdateDomainCount to 1 commit 5b1fbb0c9fb8f5333583a88a399330a2c5e689f4 Author: Robbie Zhang Date: Tue Aug 15 12:23:41 2017 -0700 Enable StorageAccount Encryption and Enforce HTTPS commit 12fd01dbae3487b380fe5b1fee37a3c5e27f8df8 Author: Harry He Date: Fri Jul 7 10:16:03 2017 -0700 Remove Resource Requests from kube-proxy (#5) Previously kube-proxy requested 100m CPU. It prevented containers requesting 1 CPU from being deployed onto nodes with 1 CPU, because there is only 900m CPU left. This change remove resource requests from kube-proxy. commit 52416399282f57d09d6d5e21ea87f0ef01083891 Author: Robbie Zhang Date: Fri Jul 7 14:23:32 2017 -0700 Set the default CloudProvider backoff values commit 549a4c20f346dfef11c73f3f5fcd6918d727f77e Merge: 0506730a 8a47cbd8 Author: Robbie Zhang Date: Fri Jul 7 16:14:12 2017 -0700 Merge with v0.3.0 commit 0506730a8a11c29e3fd18030eb97266e65b0188d Author: Robbie Zhang Date: Fri Jul 7 13:01:18 2017 -0700 Disable Automatic Windows Update commit 8eb8afe355b2819a2bea0471a81780804c6b454d Merge: 639e36a7 fb09cdf0 Author: Robbie Zhang Date: Fri Jul 7 12:07:03 2017 -0700 Merge from upstream release v0.2.0 commit 639e36a79893f4aa9013aea653e5b0e2f64d6bde Author: Robbie Zhang Date: Mon Jul 3 11:05:10 2017 -0700 Remove azure.json from Windows Agent commit c9d0704cd9ec06e3f029cd916a5c13af521d72dd Merge: bae0a8b3 579e8b83 Author: Robbie Zhang Date: Mon Jun 19 10:13:37 2017 -0700 Merge tag 'v0.1.2' into migration commit bae0a8b3c0d4e77542b645819b14cb7d792e89fb Author: Raghu Shantha [MSFT] Date: Thu Jun 15 11:36:03 2017 -0700 Enable Firewall on Node, Add Windows Firewall rules for required ports (#2) * Enable Firewall on Node, Add Windows Firewall rules for required ports * Added comments for firewall rules * Allow all traffic; lockdown kubectl Node ports to Master only * Remove & and single quote in comment section resource group deployment parser does not like these chars in the comment section commit af24ad6f2d9b07e563c4a2f00579fc0daaed34f6 Author: Robbie Zhang Date: Tue Jun 6 18:20:40 2017 -0700 Enable RBAC on APIServer commit e648d3d5077e44e785bcc949e926f3dc5900bba9 Merge: 380bc587 cc95f47e Author: Robbie Zhang Date: Wed May 24 11:01:11 2017 -0700 Merge branch 'master' into migration commit 380bc587b34672430f576453496c565f8229851f Author: Robbie Zhang Date: Mon May 15 11:39:43 2017 -0700 Fix: add the size map for F1 commit e64b44653c397d058f21c96a8c4717efc6dd7c74 Merge: 87c56c3b 253dd41d Author: Wenjun Wu Date: Sun May 14 15:47:20 2017 -0700 Merge branch 'master' into migration commit 87c56c3b4b3b51eac22f5906a71f7ea036729f38 Author: Robbie Zhang Date: Fri Apr 14 12:55:21 2017 -0700 Private Commit for Azure Console Shell Remove SPN secrets from agent node Remove the Kube Dashboard and Heapster Addons Add agentpool label on the agent nodes Use static IP address for system and agentpool1 --- parts/defaultpolicy.json | 2 + parts/kubernetesagentcustomdata.yml | 3 + parts/kubernetesagentresourcesvmas.t | 42 +++++- parts/kubernetesagentvars.t | 2 +- parts/kuberneteskubelet.service | 6 +- parts/kubernetesmaster-kube-apiserver.yaml | 2 + ...netesmasteraddons-heapster-deployment.yaml | 140 ------------------ ...netesmasteraddons-kube-dns-deployment.yaml | 6 + ...etesmasteraddons-kube-proxy-daemonset.yaml | 3 - parts/kubernetesmastercustomdata.yml | 26 ++-- parts/kubernetesmastercustomscript.sh | 31 ++-- parts/kubernetesmasterresources.t | 51 +++---- parts/kubernetesmastervars.t | 8 +- parts/kuberneteswinagentresourcesvmas.t | 41 ++++- parts/kuberneteswindowssetup.ps1 | 63 +++----- pkg/acsengine/azureconst.go | 2 +- pkg/acsengine/engine.go | 5 +- 17 files changed, 172 insertions(+), 261 deletions(-) create mode 100644 parts/defaultpolicy.json delete mode 100644 parts/kubernetesmasteraddons-heapster-deployment.yaml diff --git a/parts/defaultpolicy.json b/parts/defaultpolicy.json new file mode 100644 index 0000000000..fd9b8b245f --- /dev/null +++ b/parts/defaultpolicy.json @@ -0,0 +1,2 @@ +{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"client", "namespace": "*", "resource": "*", "apiGroup": "*"}} +{"apiVersion": "abac.authorization.kubernetes.io/v1beta1", "kind": "Policy", "spec": {"user":"system:serviceaccount:kube-system:default", "namespace": "*", "resource": "*", "apiGroup": "*"}} \ No newline at end of file diff --git a/parts/kubernetesagentcustomdata.yml b/parts/kubernetesagentcustomdata.yml index ab2a2dee18..febf0520cb 100644 --- a/parts/kubernetesagentcustomdata.yml +++ b/parts/kubernetesagentcustomdata.yml @@ -109,6 +109,9 @@ write_files: KUBE_CTRL_MGR_ROUTE_RECONCILIATION_PERIOD={{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}} KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}} KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}} + CLOUD_PROVIDER= + CLOUD_CONFIG= + AZURE_CONTAINER_REGISTRY_CONFIG= {{if IsKubernetesVersionGe "1.6.0"}} KUBELET_NON_MASQUERADE_CIDR=--non-masquerade-cidr={{WrapAsVariable "kubernetesNonMasqueradeCidr"}} KUBELET_FEATURE_GATES=--feature-gates=Accelerators=true diff --git a/parts/kubernetesagentresourcesvmas.t b/parts/kubernetesagentresourcesvmas.t index 15ca0bf30b..15780104b8 100644 --- a/parts/kubernetesagentresourcesvmas.t +++ b/parts/kubernetesagentresourcesvmas.t @@ -27,7 +27,15 @@ {{if eq $seq 1}} "primary": true, {{end}} + {{if eq $.Name "system"}} + "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(50, int(variables('masterFirstAddrOctet4')))))]", + "privateIPAllocationMethod": "Static", + {{else if eq $.Name "agentpool1"}} + "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(100, int(variables('masterFirstAddrOctet4')))))]", + "privateIPAllocationMethod": "Static", + {{else}} "privateIPAllocationMethod": "Dynamic", + {{end}} "subnet": { "id": "[variables('{{$.Name}}VnetSubnetID')]" } @@ -69,10 +77,25 @@ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], {{end}} + "kind": "Storage", "location": "[variables('location')]", "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", "properties": { - "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, + "sku": { + "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -88,10 +111,25 @@ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], {{end}} + "kind": "Storage", "location": "[variables('location')]", "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", "properties": { - "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, + "sku": { + "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, diff --git a/parts/kubernetesagentvars.t b/parts/kubernetesagentvars.t index 923bef1c62..e7f9481b59 100644 --- a/parts/kubernetesagentvars.t +++ b/parts/kubernetesagentvars.t @@ -19,4 +19,4 @@ {{else}} "{{.Name}}VnetSubnetID": "[variables('vnetSubnetID')]", "{{.Name}}SubnetName": "[variables('subnetName')]", -{{end}} +{{end}} \ No newline at end of file diff --git a/parts/kuberneteskubelet.service b/parts/kuberneteskubelet.service index d55dbda43f..5ea6bfaadc 100644 --- a/parts/kuberneteskubelet.service +++ b/parts/kuberneteskubelet.service @@ -44,9 +44,9 @@ ExecStart=/usr/bin/docker run \ --cluster-dns=${KUBELET_CLUSTER_DNS} \ --cluster-domain=cluster.local \ --node-labels="${KUBELET_NODE_LABELS}" \ - --cloud-provider=azure \ - --cloud-config=/etc/kubernetes/azure.json \ - --azure-container-registry-config=/etc/kubernetes/azure.json \ + --cloud-provider=${CLOUD_PROVIDER} \ + --cloud-config=${CLOUD_CONFIG} \ + --azure-container-registry-config=${AZURE_CONTAINER_REGISTRY_CONFIG} \ --network-plugin=${KUBELET_NETWORK_PLUGIN} \ --max-pods=${KUBELET_MAX_PODS} \ --node-status-update-frequency=${KUBELET_NODE_STATUS_UPDATE_FREQUENCY} \ diff --git a/parts/kubernetesmaster-kube-apiserver.yaml b/parts/kubernetesmaster-kube-apiserver.yaml index 77cc56e551..caec976570 100644 --- a/parts/kubernetesmaster-kube-apiserver.yaml +++ b/parts/kubernetesmaster-kube-apiserver.yaml @@ -36,6 +36,8 @@ spec: - "--oidc-issuer-url=" - "--oidc-username-claim=oid" - "--storage-backend=" + - "--authorization-mode=ABAC,RBAC" + - "--authorization-policy-file=/etc/kubernetes/manifests/defaultpolicy.json" - "--v=4" - "" - "--requestheader-allowed-names=" diff --git a/parts/kubernetesmasteraddons-heapster-deployment.yaml b/parts/kubernetesmasteraddons-heapster-deployment.yaml deleted file mode 100644 index 1255b3d052..0000000000 --- a/parts/kubernetesmasteraddons-heapster-deployment.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: heapster - namespace: kube-system - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: system:heapster-with-nanny - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists -rules: -- apiGroups: - - extensions - - apps - resources: - - deployments - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - events - - namespaces - - nodes - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: system:heapster-with-nanny - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:heapster-with-nanny -subjects: -- kind: ServiceAccount - name: heapster - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - kubernetes.io/cluster-service: "true" - kubernetes.io/name: Heapster - name: heapster - namespace: kube-system -spec: - ports: - - port: 80 - targetPort: 8082 - selector: - k8s-app: heapster ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - labels: - k8s-app: heapster - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: EnsureExists - namespace: kube-system - name: heapster -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: heapster - template: - metadata: - labels: - k8s-app: heapster - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - spec: - tolerations: - - key: CriticalAddonsOnly - operator: Exists - serviceAccountName: heapster - containers: - - image: - command: - - "/heapster" - - "--source=kubernetes.summary_api:\"\"" - name: heapster - resources: - requests: - cpu: 80m - memory: 140Mi - limits: - cpu: 80m - memory: 140Mi - - image: - command: - - "/pod_nanny" - - "--cpu=80m" - - "--extra-cpu=0.5m" - - "--memory=140Mi" - - "--extra-memory=4Mi" - - "--threshold=5" - - "--deployment=heapster" - - "--container=heapster" - - "--poll-period=300000" - - "--estimator=exponential" - name: heapster-nanny - resources: - requests: - cpu: 50m - memory: 90Mi - limits: - cpu: 50m - memory: 90Mi - env: - - valueFrom: - fieldRef: - fieldPath: metadata.name - name: MY_POD_NAME - - valueFrom: - fieldRef: - fieldPath: metadata.namespace - name: MY_POD_NAMESPACE - nodeSelector: - beta.kubernetes.io/os: linux diff --git a/parts/kubernetesmasteraddons-kube-dns-deployment.yaml b/parts/kubernetesmasteraddons-kube-dns-deployment.yaml index 4d05fc38ac..fa5ca691a5 100644 --- a/parts/kubernetesmasteraddons-kube-dns-deployment.yaml +++ b/parts/kubernetesmasteraddons-kube-dns-deployment.yaml @@ -43,6 +43,11 @@ spec: matchLabels: k8s-app: kube-dns version: v20 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate template: metadata: annotations: @@ -168,3 +173,4 @@ spec: serviceAccountName: kube-dns nodeSelector: beta.kubernetes.io/os: linux + agentpool: system diff --git a/parts/kubernetesmasteraddons-kube-proxy-daemonset.yaml b/parts/kubernetesmasteraddons-kube-proxy-daemonset.yaml index 1a2adfc837..b2cfc9ffb7 100644 --- a/parts/kubernetesmasteraddons-kube-proxy-daemonset.yaml +++ b/parts/kubernetesmasteraddons-kube-proxy-daemonset.yaml @@ -28,9 +28,6 @@ spec: - "--feature-gates=ExperimentalCriticalPodAnnotation=true" image: "" name: kube-proxy - resources: - requests: - cpu: 100m securityContext: privileged: true volumeMounts: diff --git a/parts/kubernetesmastercustomdata.yml b/parts/kubernetesmastercustomdata.yml index 6ed1199565..654435b049 100644 --- a/parts/kubernetesmastercustomdata.yml +++ b/parts/kubernetesmastercustomdata.yml @@ -79,6 +79,13 @@ write_files: name: localclustercontext current-context: localclustercontext +- path: /etc/kubernetes/manifests/defaultpolicy.json + permissions: "0644" + encoding: gzip + owner: "root" + content: !!binary | + API_SERVER_POLICY_B64_GZIP_STR + - path: /etc/kubernetes/manifests/kube-apiserver.yaml permissions: "0644" encoding: gzip @@ -122,21 +129,7 @@ write_files: MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR {{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}} -- path: /etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml - permissions: "0644" - encoding: gzip - owner: "root" - content: !!binary | - MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR {{end}} - -- path: /etc/kubernetes/addons/kube-heapster-deployment.yaml - permissions: "0644" - encoding: gzip - owner: "root" - content: !!binary | - MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR - - path: /etc/kubernetes/addons/azure-storage-classes.yaml permissions: "0644" encoding: gzip @@ -203,6 +196,9 @@ write_files: KUBE_CTRL_MGR_ROUTE_RECONCILIATION_PERIOD={{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}} KUBELET_IMAGE_GC_HIGH_THRESHOLD={{WrapAsVariable "gchighthreshold"}} KUBELET_IMAGE_GC_LOW_THRESHOLD={{WrapAsVariable "gclowthreshold"}} + CLOUD_PROVIDER=azure + CLOUD_CONFIG=/etc/kubernetes/azure.json + AZURE_CONTAINER_REGISTRY_CONFIG=/etc/kubernetes/azure.json {{if IsKubernetesVersionGe "1.6.0"}} {{if HasLinuxAgents}} KUBELET_NON_MASQUERADE_CIDR=--non-masquerade-cidr={{WrapAsVariable "kubernetesNonMasqueradeCidr"}} @@ -266,9 +262,7 @@ write_files: sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" sed -i "s||{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s||{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s||{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s||{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml" {{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}} - sed -i "s||{{WrapAsVariable "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesDashboardCPURequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesDashboardMemoryRequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesDashboardCPULimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" diff --git a/parts/kubernetesmastercustomscript.sh b/parts/kubernetesmastercustomscript.sh index 97d9f96214..392ff3230c 100644 --- a/parts/kubernetesmastercustomscript.sh +++ b/parts/kubernetesmastercustomscript.sh @@ -41,11 +41,7 @@ echo `date`,`hostname`, startscript>>/opt/m # A delay to start the kubernetes processes is necessary # if a reboot is required. Otherwise, the agents will encounter issue: # https://github.com/kubernetes/kubernetes/issues/41185 -if [ -f /var/run/reboot-required ]; then - REBOOTREQUIRED=true -else - REBOOTREQUIRED=false -fi +REBOOTREQUIRED=false # If APISERVER_PRIVATE_KEY is empty, then we are not on the master if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then @@ -85,11 +81,15 @@ chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" chown root:root "${APISERVER_PUBLIC_KEY_PATH}" echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" -AZURE_JSON_PATH="/etc/kubernetes/azure.json" -touch "${AZURE_JSON_PATH}" -chmod 0600 "${AZURE_JSON_PATH}" -chown root:root "${AZURE_JSON_PATH}" -cat << EOF > "${AZURE_JSON_PATH}" +# If APISERVER_PRIVATE_KEY is empty, then we are not on the master +if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then + echo "APISERVER_PRIVATE_KEY is non-empty, assuming master node, configure azure json." + + AZURE_JSON_PATH="/etc/kubernetes/azure.json" + touch "${AZURE_JSON_PATH}" + chmod 0600 "${AZURE_JSON_PATH}" + chown root:root "${AZURE_JSON_PATH}" + cat << EOF > "${AZURE_JSON_PATH}" { "cloud":"${TARGET_ENVIRONMENT}", "tenantId": "${TENANT_ID}", @@ -116,6 +116,9 @@ cat << EOF > "${AZURE_JSON_PATH}" "useInstanceMetadata": ${USE_INSTANCE_METADATA} } EOF +else + echo "APISERVER_PRIVATE_KEY is empty, assuming worker node, skip azure json." +fi ########################################################### # END OF SECRET DATA @@ -418,6 +421,8 @@ if [[ ! -z "${APISERVER_PRIVATE_KEY}" ]]; then ensureEtcdDataDir ensureEtcd ensureApiserver + + /usr/local/bin/kubectl create clusterrolebinding superuser --clusterrole=cluster-admin --user=client fi # mitigation for bug https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1676635 @@ -428,6 +433,12 @@ sed -i "13i\echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/h echo "Install complete successfully" apt-mark unhold walinuxagent +if [ -f /var/run/reboot-required ]; then + REBOOTREQUIRED=true +else + REBOOTREQUIRED=false +fi + if $REBOOTREQUIRED; then # wait 1 minute to restart node, so that the custom script extension can complete echo 'reboot required, rebooting node in 1 minute' diff --git a/parts/kubernetesmasterresources.t b/parts/kubernetesmasterresources.t index e3f4754929..ef4f09fcc2 100644 --- a/parts/kubernetesmasterresources.t +++ b/parts/kubernetesmasterresources.t @@ -5,8 +5,8 @@ "name": "[variables('masterAvailabilitySet')]", "properties": { - "platformFaultDomainCount": "2", - "platformUpdateDomainCount": "3", + "platformFaultDomainCount": "1", + "platformUpdateDomainCount": "1", "managed" : "true" }, "type": "Microsoft.Compute/availabilitySets" @@ -24,10 +24,25 @@ "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], + "kind": "Storage", "location": "[variables('location')]", "name": "[variables('masterStorageAccountName')]", "properties": { - "accountType": "[variables('vmSizesMap')[variables('masterVMSize')].storageAccountType]" + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, + "sku": { + "name": "[variables('vmSizesMap')[variables('masterVMSize')].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -77,36 +92,6 @@ "name": "[variables('nsgName')]", "properties": { "securityRules": [ -{{if .HasWindows}} - { - "name": "allow_rdp", - "properties": { - "access": "Allow", - "description": "Allow RDP traffic to master", - "destinationAddressPrefix": "*", - "destinationPortRange": "3389-3389", - "direction": "Inbound", - "priority": 102, - "protocol": "Tcp", - "sourceAddressPrefix": "*", - "sourcePortRange": "*" - } - }, -{{end}} - { - "name": "allow_ssh", - "properties": { - "access": "Allow", - "description": "Allow SSH traffic to master", - "destinationAddressPrefix": "*", - "destinationPortRange": "22-22", - "direction": "Inbound", - "priority": 101, - "protocol": "Tcp", - "sourceAddressPrefix": "*", - "sourcePortRange": "*" - } - }, { "name": "allow_kube_tls", "properties": { diff --git a/parts/kubernetesmastervars.t b/parts/kubernetesmastervars.t index 613e403fed..474d85c551 100644 --- a/parts/kubernetesmastervars.t +++ b/parts/kubernetesmastervars.t @@ -108,7 +108,7 @@ "sshKeyPath": "[concat('/home/',variables('username'),'/.ssh/authorized_keys')]", {{if .HasStorageAccountDisks}} - "apiVersionStorage": "2015-06-15", + "apiVersionStorage": "2016-12-01", "maxVMsPerStorageAccount": 20, "maxStorageAccountsPerAgent": "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]", "dataStorageAccountPrefixSeed": 97, @@ -131,10 +131,10 @@ {{end}} "provisionScript": "{{GetKubernetesB64Provision}}", "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", - "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('servicePrincipalClientSecret'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' MAX_PODS=',variables('maxPods'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'))]", + "provisionScriptParametersCommon": "[concat('KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' NETWORK_POLICY=',variables('networkPolicy'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' MAX_PODS=',variables('maxPods'))]", {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ADMINUSER=',variables('username'))]", + "provisionScriptParametersMaster": "[concat('TENANT_ID=',variables('tenantID'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('servicePrincipalClientSecret'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ADMINUSER=',variables('username'))]", {{end}} "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}", "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}", @@ -189,7 +189,7 @@ "nsgName": "[concat(variables('agentNamePrefix'), 'nsg')]", {{end}} "nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]", - "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',variables('nameSuffix'))]", + "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',variables('nameSuffix'))]", {{if not IsHostedMaster }} "masterPublicIPAddressName": "[concat(variables('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', variables('nameSuffix'))]", "masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]", diff --git a/parts/kuberneteswinagentresourcesvmas.t b/parts/kuberneteswinagentresourcesvmas.t index 6ec43361d3..d9c06727be 100644 --- a/parts/kuberneteswinagentresourcesvmas.t +++ b/parts/kuberneteswinagentresourcesvmas.t @@ -69,10 +69,25 @@ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], {{end}} + "kind": "Storage", "location": "[variables('location')]", "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", "properties": { - "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, + "sku": { + "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -88,10 +103,25 @@ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], {{end}} + "kind": "Storage", "location": "[variables('location')]", "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", "properties": { - "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, + "sku": { + "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -156,7 +186,10 @@ "computername": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", {{GetKubernetesWindowsAgentCustomData .}} "adminUsername": "[variables('windowsAdminUsername')]", - "adminPassword": "[variables('windowsAdminPassword')]" + "adminPassword": "[variables('windowsAdminPassword')]", + "windowsConfiguration": { + "enableAutomaticUpdates": false + } }, "storageProfile": { {{GetDataDisks .}} @@ -245,7 +278,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AzureHostname ',variables('{{.Name}}VMNamePrefix'),copyIndex(variables('{{.Name}}Offset')),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" + "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',variables('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',variables('clientPrivateKey'),' -AzureHostname ',variables('{{.Name}}VMNamePrefix'),copyIndex(variables('{{.Name}}Offset')),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" } } } \ No newline at end of file diff --git a/parts/kuberneteswindowssetup.ps1 b/parts/kuberneteswindowssetup.ps1 index 4f14f28739..0d8226e660 100644 --- a/parts/kuberneteswindowssetup.ps1 +++ b/parts/kuberneteswindowssetup.ps1 @@ -29,15 +29,7 @@ param( [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $AzureHostname, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientSecret + $AzureHostname ) $global:CACertificate = "{{WrapAsVariable "caCertificate"}}" @@ -84,6 +76,17 @@ function Set-TelemetrySetting() Set-ItemProperty -Path "HKLM:\Software\Microsoft\Windows\CurrentVersion\Policies\DataCollection" -Name "CommercialId" -Value $global:WindowsTelemetryGUID -Force } +function Resize-OSDrive() +{ + $osDrive = ((Get-WmiObject Win32_OperatingSystem).SystemDrive).TrimEnd(":") + $size = (Get-Partition -DriveLetter $osDrive).Size + $maxSize = (Get-PartitionSupportedSize -DriveLetter $osDrive).SizeMax + if ($size -lt $maxSize) + { + Resize-Partition -DriveLetter $osDrive -Size $maxSize + } +} + function Get-KubeBinaries() { @@ -92,32 +95,6 @@ Get-KubeBinaries() Expand-Archive -path $zipfile -DestinationPath C:\ } -function -Write-AzureConfig() -{ - $azureConfigFile = $global:KubeDir + "\azure.json" - - $azureConfig = @" -{ - "tenantId": "$global:TenantId", - "subscriptionId": "$global:SubscriptionId", - "aadClientId": "$AADClientId", - "aadClientSecret": "$AADClientSecret", - "resourceGroup": "$global:ResourceGroup", - "location": "$Location", - "subnetName": "$global:SubnetName", - "securityGroupName": "$global:SecurityGroupName", - "vnetName": "$global:VNetName", - "routeTableName": "$global:RouteTableName", - "primaryAvailabilitySetName": "$global:PrimaryAvailabilitySetName", - "useManagedIdentityExtension": $global:UseManagedIdentityExtension, - "useInstanceMetadata": $global:UseInstanceMetadata -} -"@ - - $azureConfig | Out-File -encoding ASCII -filepath "$azureConfigFile" -} - function Write-KubeConfig() { @@ -158,9 +135,9 @@ New-InfraContainer() function Write-KubernetesStartFiles($podCIDR) { - $KubeletArgList = @("--hostname-override=`$global:AzureHostname","--pod-infra-container-image=kubletwin/pause","--resolv-conf=""""""""","--kubeconfig=c:\k\config","--cloud-provider=azure","--cloud-config=c:\k\azure.json") + $KubeletArgList = @("--hostname-override=`$global:AzureHostname","--pod-infra-container-image=kubletwin/pause","--resolv-conf=""""""""","--kubeconfig=c:\k\config") $KubeletCommandLine = @" -c:\k\kubelet.exe --hostname-override=`$global:AzureHostname --pod-infra-container-image=kubletwin/pause --resolv-conf="" --allow-privileged=true --enable-debugging-handlers --cluster-dns=`$global:KubeDnsServiceIp --cluster-domain=cluster.local --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge --v=2 --azure-container-registry-config=c:\k\azure.json --runtime-request-timeout=10m --cloud-provider=azure --cloud-config=c:\k\azure.json +c:\k\kubelet.exe --hostname-override=`$global:AzureHostname --pod-infra-container-image=kubletwin/pause --resolv-conf="" --allow-privileged=true --enable-debugging-handlers --cluster-dns=`$global:KubeDnsServiceIp --cluster-domain=cluster.local --kubeconfig=c:\k\config --hairpin-mode=promiscuous-bridge --v=2 --runtime-request-timeout=10m "@ if ($global:KubeBinariesVersion -lt "1.8.0") @@ -394,17 +371,21 @@ try # to the windows machine, and run the script manually to watch # the output. if ($true) { + Write-Log "Disable automatic Windows update" + reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update" /v AUOptions /t REG_DWORD /d 1 /f + Write-Log "Provisioning $global:DockerServiceName... with IP $MasterIP" + net start Docker Write-Log "apply telemetry data setting" Set-TelemetrySetting + Write-Log "resize os drive if possible" + Resize-OSDrive + Write-Log "download kubelet binaries and unzip" Get-KubeBinaries - Write-Log "Write azure config" - Write-AzureConfig - Write-Log "Write kube config" Write-KubeConfig @@ -425,7 +406,7 @@ try else { # keep for debugging purposes - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AzureHostname $AzureHostname -AADClientId $AADClientId -AADClientSecret $AADClientSecret" + Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AzureHostname $AzureHostname" } } catch diff --git a/pkg/acsengine/azureconst.go b/pkg/acsengine/azureconst.go index 936ba46fa5..895bdf3c69 100644 --- a/pkg/acsengine/azureconst.go +++ b/pkg/acsengine/azureconst.go @@ -1732,4 +1732,4 @@ func GetClassicSizeMap() string { } } ` -} +} \ No newline at end of file diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 29c991995a..62bd4e2c68 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -33,6 +33,7 @@ const ( kubernetesAgentCustomDataYaml = "kubernetesagentcustomdata.yml" kubeConfigJSON = "kubeconfig.json" kubernetesWindowsAgentCustomDataPS1 = "kuberneteswindowssetup.ps1" + kubePolicyJSON = "defaultpolicy.json" ) const ( @@ -106,6 +107,7 @@ var kubernetesAritfacts = map[string]string{ "MASTER_PROVISION_B64_GZIP_STR": kubernetesMasterCustomScript, "MASTER_GENERATE_PROXY_CERTS_B64_GZIP_STR": kubernetesMasterGenerateProxyCertsScript, "KUBELET_SERVICE_B64_GZIP_STR": kubernetesKubeletService, + "API_SERVER_POLICY_B64_GZIP_STR": kubePolicyJSON, } var kubernetesAritfacts15 = map[string]string{ @@ -115,12 +117,9 @@ var kubernetesAritfacts15 = map[string]string{ } var kubernetesAddonYamls = map[string]string{ - "MASTER_ADDON_HEAPSTER_DEPLOYMENT_B64_GZIP_STR": "kubernetesmasteraddons-heapster-deployment.yaml", "MASTER_ADDON_KUBE_DNS_DEPLOYMENT_B64_GZIP_STR": "kubernetesmasteraddons-kube-dns-deployment.yaml", "MASTER_ADDON_KUBE_PROXY_DAEMONSET_B64_GZIP_STR": "kubernetesmasteraddons-kube-proxy-daemonset.yaml", - "MASTER_ADDON_KUBERNETES_DASHBOARD_DEPLOYMENT_B64_GZIP_STR": "kubernetesmasteraddons-kubernetes-dashboard-deployment.yaml", "MASTER_ADDON_AZURE_STORAGE_CLASSES_B64_GZIP_STR": "kubernetesmasteraddons-azure-storage-classes.yaml", - "MASTER_ADDON_TILLER_DEPLOYMENT_B64_GZIP_STR": "kubernetesmasteraddons-tiller-deployment.yaml", } var kubernetesAddonYamls15 = map[string]string{