From 99ff435667382215ae0172c35f8c7e0b73b931e1 Mon Sep 17 00:00:00 2001 From: Tariq Ibrahim Date: Thu, 1 Nov 2018 08:45:51 -0700 Subject: [PATCH] Merging kubernetesmastervarsvmss into kubernetesmastervars (#4116) --- parts/k8s/kubernetesbase.t | 6 +- parts/k8s/kubernetesmastercustomdata.yml | 45 +- parts/k8s/kubernetesmastercustomdatavmss.yml | 577 ------------------- parts/k8s/kubernetesmasterresourcesvmss.t | 2 +- parts/k8s/kubernetesmastervars.t | 30 +- parts/k8s/kubernetesmastervarsvmss.t | 275 --------- pkg/acsengine/const.go | 4 +- pkg/acsengine/engine.go | 2 +- pkg/acsengine/template_generator.go | 4 - 9 files changed, 75 insertions(+), 870 deletions(-) diff --git a/parts/k8s/kubernetesbase.t b/parts/k8s/kubernetesbase.t index 33e47d0cd8..0ccff6b622 100644 --- a/parts/k8s/kubernetesbase.t +++ b/parts/k8s/kubernetesbase.t @@ -49,11 +49,7 @@ "{{.Name}}AccountName": "[concat(variables('storageAccountBaseName'), 'agnt{{$index}}')]", {{end}} {{end}} - {{if IsMasterVirtualMachineScaleSets}} - {{template "k8s/kubernetesmastervarsvmss.t" .}} - {{else}} - {{template "k8s/kubernetesmastervars.t" .}} - {{end}} + {{template "k8s/kubernetesmastervars.t" .}} }, "resources": [ {{if UserAssignedIDEnabled}} diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 4c913dddcc..d4c39a33a9 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -182,7 +182,11 @@ write_files: - name: localcluster cluster: certificate-authority: /etc/kubernetes/certs/ca.crt + {{if IsMasterVirtualMachineScaleSets}} + server: + {{else}} server: {{WrapAsVerbatim "concat('https://', variables('masterPrivateIpAddrs')[copyIndex(variables('masterOffset'))], ':443')"}} + {{end}} users: - name: client user: @@ -266,6 +270,10 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER content: | #!/bin/bash set -e + {{if IsMasterVirtualMachineScaleSets}} + PRIVATE_IP=$(hostname -I | cut -d" " -f1) + ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} + {{end}} {{if gt .MasterProfile.Count 1}} # Redirect ILB (4443) traffic to port 443 (ELB) in the prerouting chain iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443 @@ -389,7 +397,11 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/flannel-daemonset.yaml" {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "cilium"}} + {{if IsMasterVirtualMachineScaleSets}} + sed -i "s||"https://$PRIVATE_IP:$ETCD_CLIENT_PORT"|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" + {{else}} sed -i "s||{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" + {{end}} sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/ca.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/etcdclient.key)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/etcdclient.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" @@ -474,12 +486,43 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER content: | #!/bin/bash set -x + {{if IsMasterVirtualMachineScaleSets}} + MASTER_VM_NAME=$(hostname) + MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//") + MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}} + MASTER_INDEX=$(hostname | tail -c 2) + PRIVATE_IP=$(hostname -I | cut -d" " -f1) + PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3) + MASTER_COUNT={{WrapAsVariable "masterCount"}} + IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}} + echo $IPADDRESS_COUNT + ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}} + ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} + MASTER_URLS="" + index=0 + while [ $index -lt $MASTER_COUNT ] + do + echo $index + offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4` + echo $offset + MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT," + index=`expr $index + 1` + done + MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//") + echo $MASTER_URLS + sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:$ETCD_CLIENT_PORT" /etc/environment + sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment + sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment + sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment + sudo sed -i "s||https://$PRIVATE_IP:443|g" "/var/lib/kubelet/kubeconfig" + /bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.crt --peer-key-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.key --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd + {{else}} sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:2379" /etc/environment sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster {{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd - + {{end}} {{if .MasterProfile.IsCoreOS}} - path: "/opt/azure/containers/provision-setup.sh" permissions: "0755" diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index 5e44ea0736..e69de29bb2 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -1,577 +0,0 @@ -#cloud-config - -{{if not .MasterProfile.IsCoreOS}} -packages: - - jq - - traceroute -{{end}} - -write_files: -- path: "/opt/azure/containers/provision_source.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "provisionSource"}} - -- path: "/opt/azure/containers/provision.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "provisionScript"}} - -- path: "/opt/azure/containers/provision_installs.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "provisionInstalls"}} - -- path: "/opt/azure/containers/provision_configs.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "provisionConfigs"}} - -- path: "/etc/ssh/sshd_config" - permissions: "0644" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "sshdConfig"}} - -- path: "/etc/systemd/system.conf" - permissions: "0644" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "systemConf"}} - -- path: "/usr/local/bin/health-monitor.sh" - permissions: "0544" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "healthMonitorScript"}} - -- path: "/etc/systemd/system/kubelet-monitor.timer" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=a timer that delays kubelet-monitor from starting too soon after boot - [Timer] - OnBootSec=30min - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet-monitor.service" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=a script that checks kubelet health and restarts if needed - After=kubelet.service - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - ExecStart=/usr/local/bin/health-monitor.sh kubelet - -- path: "/etc/systemd/system/docker-monitor.timer" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=a timer that delays docker-monitor from starting too soon after boot - [Timer] - OnBootSec=30min - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/docker-monitor.service" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=a script that checks docker health and restarts if needed - After=docker.service - [Service] - Restart=always - RestartSec=10 - RemainAfterExit=yes - ExecStart=/usr/local/bin/health-monitor.sh container-runtime - -{{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} - {{if not .MasterProfile.IsCoreOS}} -- path: "/etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf" - permissions: "0644" - owner: "root" - content: | - [Service] - MountFlags=shared - {{end}} - -- path: "/etc/systemd/system/docker.service.d/exec_start.conf" - permissions: "0644" - owner: "root" - content: | - [Service] - ExecStart= - {{if .MasterProfile.IsCoreOS}} - ExecStart=/usr/bin/env PATH=${TORCX_BINDIR}:${PATH} ${TORCX_BINDIR}/dockerd --host=fd:// --containerd=/var/run/docker/libcontainerd/docker-containerd.sock --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} $DOCKER_SELINUX $DOCKER_OPTS $DOCKER_CGROUPS $DOCKER_OPT_BIP $DOCKER_OPT_MTU $DOCKER_OPT_IPMASQ - {{else}} - ExecStart=/usr/bin/dockerd -H fd:// --storage-driver=overlay2 --bip={{WrapAsParameter "dockerBridgeCidr"}} - {{end}} - -- path: "/etc/docker/daemon.json" - permissions: "0644" - owner: "root" - content: | - { - "live-restore": true, - "log-driver": "json-file", - "log-opts": { - "max-size": "50m", - "max-file": "5" - } - } -{{end}} - -- path: "/etc/kubernetes/certs/ca.crt" - permissions: "0644" - encoding: "base64" - owner: "root" - content: | - {{WrapAsParameter "caCertificate"}} - -- path: "/etc/kubernetes/certs/client.crt" - permissions: "0644" - encoding: "base64" - owner: "root" - content: | - {{WrapAsParameter "clientCertificate"}} - -{{if EnableAggregatedAPIs}} -- path: "/etc/kubernetes/generate-proxy-certs.sh" - permissions: "0744" - encoding: "gzip" - owner: "root" - content: !!binary | - {{WrapAsVariable "generateProxyCertsScript"}} -{{end}} - -{{if HasCustomSearchDomain}} -- path: "/opt/azure/containers/setup-custom-search-domains.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "customSearchDomainsScript"}} -{{end}} - -- path: "/var/lib/kubelet/kubeconfig" - permissions: "0644" - owner: "root" - content: | - apiVersion: v1 - kind: Config - clusters: - - name: localcluster - cluster: - certificate-authority: /etc/kubernetes/certs/ca.crt - server: - users: - - name: client - user: - client-certificate: /etc/kubernetes/certs/client.crt - client-key: /etc/kubernetes/certs/client.key - contexts: - - context: - cluster: localcluster - user: client - name: localclustercontext - current-context: localclustercontext - -{{if EnableDataEncryptionAtRest}} -- path: "/etc/kubernetes/encryption-config.yaml" - permissions: "0600" - owner: "root" - content: | - kind: EncryptionConfig - apiVersion: v1 - resources: - - resources: - - secrets - providers: - - aescbc: - keys: - - name: key1 - secret: - - identity: {} -{{end}} - -{{if EnableEncryptionWithExternalKms}} -- path: "/etc/kubernetes/encryption-config.yaml" - permissions: "0444" - owner: "root" - content: | - kind: EncryptionConfig - apiVersion: v1 - resources: - - resources: - - secrets - providers: - - kms: - name: azurekmsprovider - endpoint: unix:///opt/azurekms.socket - cachesize: 0 - - identity: {} -{{end}} - -MASTER_MANIFESTS_CONFIG_PLACEHOLDER - -MASTER_ADDONS_CONFIG_PLACEHOLDER - -MASTER_CUSTOM_FILES_PLACEHOLDER - -- path: "/etc/default/kubelet" - permissions: "0644" - owner: "root" - content: | -{{if IsKubernetesVersionLt "1.8.0"}} - KUBELET_OPTS=--require-kubeconfig -{{else}} - KUBELET_OPTS= -{{end}} - KUBELET_CONFIG={{GetKubeletConfigKeyVals .MasterProfile.KubernetesConfig}} - KUBELET_IMAGE={{WrapAsParameter "kubernetesHyperkubeSpec"}} - KUBELET_NODE_LABELS={{GetMasterKubernetesLabels "',variables('labelResourceGroup'),'"}} -{{if IsKubernetesVersionGe "1.6.0"}} - {{if HasLinuxAgents}} - KUBELET_REGISTER_NODE=--register-node=true - KUBELET_REGISTER_WITH_TAINTS=--register-with-taints=node-role.kubernetes.io/master=true:NoSchedule - {{end}} -{{else}} - KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}} -{{end}} - -MASTER_ARTIFACTS_CONFIG_PLACEHOLDER - -- path: "/opt/azure/containers/kubelet.sh" - permissions: "0755" - owner: "root" - content: | - #!/bin/bash - set -e - PRIVATE_IP=$(hostname -I | cut -d" " -f1) - ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} -{{if gt .MasterProfile.Count 1}} - # Redirect ILB (4443) traffic to port 443 (ELB) in the prerouting chain - iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443 -{{end}} - -{{if IsAzureCNI}} - sed -i "s||{{WrapAsParameter "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml" -{{end}} - sed -i "s||{{WrapAsParameter "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" - sed -i "s||{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "s||{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" - sed -i "s||{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" - sed -i "s||{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s||{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml" -{{if NeedsKubeDNSWithExecHealthz}} - sed -i "s||{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s||{{WrapAsParameter "kubernetesExecHealthzSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s||{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s||{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml" -{{else if IsKubernetesVersionGe "1.12.0"}} - sed -i "s||{{WrapAsParameter "kubernetesCoreDNSSpec"}}|g; s||{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s||{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/coredns.yaml" -{{else}} - sed -i "s||{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s||{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s||{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml" -{{end}} - sed -i "s||{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s||{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s||{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml" - -{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesDashboardCPURequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesDashboardMemoryRequests"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesDashboardCPULimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesDashboardMemoryLimit"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsTillerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesTillerSpec"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesTillerCPURequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesTillerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesTillerCPULimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesTillerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesTillerMaxHistory"}}|g" "/etc/kubernetes/addons/kube-tiller-deployment.yaml" -{{end}} - -{{if AdminGroupID }} - sed -i "s||{{WrapAsParameter "aadAdminGroupId"}}|g" "/etc/kubernetes/addons/aad-default-admin-group-rbac.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorSpec"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorNodeName"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorOS"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorTaint"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorRegion"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorCPURequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorMemoryRequests"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorCPULimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesACIConnectorMemoryLimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerAzureCloud"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" -{{end}} - -{{if eq .OrchestratorProfile.KubernetesConfig.LoadBalancerSku "Standard"}} - sed -i "s||{{WrapAsParameter "kuberneteselbsvcname"}}|g" "/etc/kubernetes/addons/elb-svc.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsBlobfuseFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesBlobfuseFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/blobfuse-flexvolume-installer.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsSMBFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameterObject "flexVolumeDriverConfig" "kubernetesSMBFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/smb-flexvolume-installer.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsKeyVaultFlexVolumeEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPURequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryRequests"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerCPULimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml" - sed -i "s||{{WrapAsParameter "kubernetesKeyVaultFlexVolumeInstallerMemoryLimit"}}|g" "/etc/kubernetes/addons/keyvault-flexvolume-installer.yaml" -{{end}} - -{{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesReschedulerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesReschedulerCPULimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" - sed -i "s||{{WrapAsParameter "kubernetesReschedulerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" -{{end}} - -{{if .OrchestratorProfile.IsMetricsServerEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesMetricsServerSpec"}}|g" "/etc/kubernetes/addons/kube-metrics-server-deployment.yaml" -{{end}} - -{{if IsNVIDIADevicePluginEnabled}} - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginSpec"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml" - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginCPURequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml" - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryRequests"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml" - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginCPULimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml" - sed -i "s||{{WrapAsParameter "kubernetesNVIDIADevicePluginMemoryLimit"}}|g" "/etc/kubernetes/addons/nvidia-device-plugin.yaml" -{{end}} - -{{if EnableDataEncryptionAtRest }} - sed -i "s||\"{{WrapAsParameter "etcdEncryptionKey"}}\"|g" "/etc/kubernetes/encryption-config.yaml" -{{end}} - -{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} - sed -i "s||{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/calico-daemonset.yaml" -{{end}} -{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPlugin "flannel"}} - sed -i "s||{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/flannel-daemonset.yaml" -{{end}} -{{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "cilium"}} - sed -i "s||"https://$PRIVATE_IP:$ETCD_CLIENT_PORT"|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" - sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/ca.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" - sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/etcdclient.key)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" - sed -i "s||$(base64 -w 0 /etc/kubernetes/certs/etcdclient.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml" -{{end}} -{{if UseCloudControllerManager }} - sed -i "s||{{WrapAsParameter "kubernetesCcmImageSpec"}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml" - sed -i "s||{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig}}|g" "/etc/kubernetes/manifests/cloud-controller-manager.yaml" -{{end}} - sed -i "s||{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.ControllerManagerConfig}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml" - sed -i "s||{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.APIServerConfig}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" - sed -i "s||{{GetK8sRuntimeConfigKeyVals .OrchestratorProfile.KubernetesConfig.SchedulerConfig}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml" - sed -i "s||{{WrapAsVariable "kubernetesAPIServerIP"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" -{{if not EnablePodSecurityPolicy}} - sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service" -{{end}} -{{if EnableEncryptionWithExternalKms}} - sed -i "s|# Required|Requires=kms.service|g" "/etc/systemd/system/kubelet.service" -{{end}} -{{if HasCustomSearchDomain}} - sed -i "s||{{WrapAsParameter "searchDomainName"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh" - sed -i "s||{{WrapAsParameter "searchDomainRealmUser"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh" - sed -i "s||{{WrapAsParameter "searchDomainRealmPassword"}}|g" "/opt/azure/containers/setup-custom-search-domains.sh" -{{end}} -{{if .OrchestratorProfile.KubernetesConfig.IsContainerMonitoringEnabled}} - sed -i "s||{{WrapAsParameter "omsAgentVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "omsAgentDockerProviderVersion"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "omsAgentImage"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "omsAgentWorkspaceGuid"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "omsAgentWorkspaceKey"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "kubernetesOMSAgentCPURequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "kubernetesOMSAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "kubernetesOMSAgentCPULimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" - sed -i "s||{{WrapAsParameter "kubernetesOMSAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/omsagent-daemonset.yaml" -{{end}} - - sed -i "s||{{WrapAsParameter "kubernetesNonMasqueradeCidr"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPURequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryRequests"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentCPULimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||{{WrapAsParameter "kubernetesIPMasqAgentMemoryLimit"}}|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" -{{if IsAzureCNI}} - sed -i "s||168.63.129.16/32|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||true|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" -{{else}} - sed -i "\||d" "/etc/kubernetes/addons/ip-masq-agent.yaml" - sed -i "s||false|g" "/etc/kubernetes/addons/ip-masq-agent.yaml" -{{end}} - -- path: "/opt/azure/containers/mountetcd.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - {{WrapAsVariable "mountetcdScript"}} - -- path: "/etc/systemd/system/etcd.service" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=etcd - highly-available key value store - Documentation=https://github.com/coreos/etcd - Documentation=man:etcd - After=network.target - Wants=network-online.target - [Service] - Environment=DAEMON_ARGS= - Environment=ETCD_NAME=%H - Environment=ETCD_DATA_DIR= - EnvironmentFile=-/etc/default/%p - Type=notify - User=etcd - PermissionsStartOnly=true - ExecStart=/usr/bin/etcd $DAEMON_ARGS - Restart=always - [Install] - WantedBy=multi-user.target - -- path: "/opt/azure/containers/setup-etcd.sh" - permissions: "0744" - owner: "root" - content: | - #!/bin/bash - set -x - MASTER_VM_NAME=$(hostname) - MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//") - MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}} - MASTER_INDEX=$(hostname | tail -c 2) - PRIVATE_IP=$(hostname -I | cut -d" " -f1) - PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3) - MASTER_COUNT={{WrapAsVariable "masterCount"}} - IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}} - echo $IPADDRESS_COUNT - ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}} - ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} - MASTER_URLS="" - index=0 - while [ $index -lt $MASTER_COUNT ] - do - echo $index - offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4` - echo $offset - MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT," - index=`expr $index + 1` - done - MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//") - echo $MASTER_URLS - sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:$ETCD_CLIENT_PORT" /etc/environment - sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment - sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment - sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment - sudo sed -i "s||https://$PRIVATE_IP:443|g" "/var/lib/kubelet/kubeconfig" - /bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.crt --peer-key-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.key --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd - -{{if .MasterProfile.IsCoreOS}} -- path: "/opt/azure/containers/provision-setup.sh" - permissions: "0755" - owner: "root" - content: | - #!/bin/bash - source /opt/azure/containers/provision_source.sh - set -x - MASTER_VM_NAME=$(hostname) - MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//") - MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}} - MASTER_INDEX=$(hostname | tail -c 2) - PRIVATE_IP=$(hostname -I | cut -d" " -f1) - PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3) - MASTER_COUNT={{WrapAsVariable "masterCount"}} - IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}} - echo $IPADDRESS_COUNT - ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}} - ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}} - MASTER_URLS="" - index=0 - while [ $index -lt $MASTER_COUNT ] - do - echo $index - offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4` - echo $offset - MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT," - index=`expr $index + 1` - done - MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//") - echo $MASTER_URLS - - /bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk"" --initial-cluster-state "new" | tee -a /etc/default/etcd - /opt/azure/containers/mountetcd.sh - sudo /bin/chown -R etcd:etcd /var/lib/etcddisk - systemctl stop etcd-member - sudo /bin/sed -i s/Restart=on-failure/Restart=always/g /lib/systemd/system/etcd-member.service - systemctl daemon-reload - systemctl restart etcd-member - retrycmd_if_failure 5 5 10 curl --retry 5 --retry-delay 10 --retry-max-time 10 --max-time 60 http://127.0.0.1:2379/v2/machines - mkdir -p /etc/kubernetes/manifests - - {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} - usermod -aG docker {{WrapAsParameter "linuxAdminUsername"}} - {{end}} - - {{if EnableAggregatedAPIs}} - sudo bash /etc/kubernetes/generate-proxy-certs.sh - {{end}} - - touch /opt/azure/containers/runcmd.complete - -coreos: - units: - - name: start-provision-setup.service - command: "start" - content: | - [Unit] - Description=Start provision setup service - - [Service] - ExecStart=/opt/azure/containers/provision-setup.sh -{{else}} -runcmd: -- set -x -- timeout 10 apt-mark hold walinuxagent{{GetKubernetesMasterPreprovisionYaml}} -- timeout 10 apt-mark unhold walinuxagent -{{end}} diff --git a/parts/k8s/kubernetesmasterresourcesvmss.t b/parts/k8s/kubernetesmasterresourcesvmss.t index b6aa240519..edaccf2cb4 100644 --- a/parts/k8s/kubernetesmasterresourcesvmss.t +++ b/parts/k8s/kubernetesmasterresourcesvmss.t @@ -360,7 +360,7 @@ "osProfile": { "adminUsername": "[parameters('linuxAdminUsername')]", "computerNamePrefix": "[concat(variables('masterVMNamePrefix'), 'vmss')]", - {{GetKubernetesMasterCustomDataVMSS .}} + {{GetKubernetesMasterCustomData .}} "linuxConfiguration": { "disablePasswordAuthentication": "true", "ssh": { diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 0ea3120777..2b2eb1d355 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -82,7 +82,12 @@ "masterFqdnPrefix": "[tolower(parameters('masterEndpointDNSNamePrefix'))]", {{if not IsHostedMaster}} "masterCount": {{.MasterProfile.Count}}, + {{if IsMasterVirtualMachineScaleSets}} + "masterOffset": "", + "masterIpAddressCount": {{.MasterProfile.IPAddressCount}}, + {{ else }} "masterOffset": "[parameters('masterOffset')]", + {{ end }} {{end}} "apiVersionCompute": "2018-06-01", "apiVersionStorage": "2018-07-01", @@ -138,8 +143,12 @@ {{if not IsOpenShift}} "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", {{if not IsHostedMaster}} + {{if IsMasterVirtualMachineScaleSets}} + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", + {{else}} "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} + {{end}} {{end}} "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}", "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}", @@ -167,15 +176,26 @@ {{end}} {{else}} {{if .MasterProfile.IsCustomVNET}} + {{if IsMasterVirtualMachineScaleSets}} + "vnetSubnetID": "[parameters('agentVnetSubnetID')]", + "vnetSubnetIDMaster": "[parameters('masterVnetSubnetID')]", + {{else}} "vnetSubnetID": "[parameters('masterVnetSubnetID')]", + {{end}} "subnetName": "[split(parameters('masterVnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]", "virtualNetworkName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]", "virtualNetworkResourceGroupName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]", {{else}} + {{if IsMasterVirtualMachineScaleSets}} + "subnetName": "subnetmaster", + "vnetSubnetID": "[concat(variables('vnetID'),'/subnets/subnetagent')]", + "vnetSubnetIDMaster": "[concat(variables('vnetID'),'/subnets/subnetmaster')]", + {{else}} "subnetName": "[concat(parameters('orchestratorName'), '-subnet')]", - "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", "vnetSubnetID": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", "virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]", + {{end}} + "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", "virtualNetworkResourceGroupName": "''", {{end}} {{end}} @@ -236,6 +256,11 @@ "masterFirstAddrOctets": "[split(parameters('firstConsecutiveStaticIP'),'.')]", "masterFirstAddrOctet4": "[variables('masterFirstAddrOctets')[3]]", "masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]", + "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, + "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, + {{if IsMasterVirtualMachineScaleSets}} + "masterVMNamePrefix": "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]", + {{else}} "masterVMNamePrefix": "{{GetMasterVMPrefix}}", "masterVMNames": [ "[concat(variables('masterVMNamePrefix'), '0')]", @@ -251,8 +276,6 @@ "[concat(variables('masterFirstAddrPrefix'), add(3, int(variables('masterFirstAddrOctet4'))))]", "[concat(variables('masterFirstAddrPrefix'), add(4, int(variables('masterFirstAddrOctet4'))))]" ], - "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, - "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, "masterEtcdPeerURLs":[ "[concat('https://', variables('masterPrivateIpAddrs')[0], ':', variables('masterEtcdServerPort'))]", "[concat('https://', variables('masterPrivateIpAddrs')[1], ':', variables('masterEtcdServerPort'))]", @@ -272,6 +295,7 @@ "[concat(variables('masterVMNames')[0], '=', variables('masterEtcdPeerURLs')[0], ',', variables('masterVMNames')[1], '=', variables('masterEtcdPeerURLs')[1], ',', variables('masterVMNames')[2], '=', variables('masterEtcdPeerURLs')[2])]", "[concat(variables('masterVMNames')[0], '=', variables('masterEtcdPeerURLs')[0], ',', variables('masterVMNames')[1], '=', variables('masterEtcdPeerURLs')[1], ',', variables('masterVMNames')[2], '=', variables('masterEtcdPeerURLs')[2], ',', variables('masterVMNames')[3], '=', variables('masterEtcdPeerURLs')[3], ',', variables('masterVMNames')[4], '=', variables('masterEtcdPeerURLs')[4])]" ], + {{end}} {{end}} "subscriptionId": "[subscription().subscriptionId]", "contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", diff --git a/parts/k8s/kubernetesmastervarsvmss.t b/parts/k8s/kubernetesmastervarsvmss.t index 9fe3d097c7..e69de29bb2 100644 --- a/parts/k8s/kubernetesmastervarsvmss.t +++ b/parts/k8s/kubernetesmastervarsvmss.t @@ -1,275 +0,0 @@ - "maxVMsPerPool": 100, -{{ if not IsHostedMaster }} - {{if eq .MasterProfile.Count 1}} - "etcdPeerPrivateKeys": [ - "[parameters('etcdPeerPrivateKey0')]" - ], - "etcdPeerCertificates": [ - "[parameters('etcdPeerCertificate0')]" - ], - {{end}} - {{if eq .MasterProfile.Count 3}} - "etcdPeerPrivateKeys": [ - "[parameters('etcdPeerPrivateKey0')]", - "[parameters('etcdPeerPrivateKey1')]", - "[parameters('etcdPeerPrivateKey2')]" - ], - "etcdPeerCertificates": [ - "[parameters('etcdPeerCertificate0')]", - "[parameters('etcdPeerCertificate1')]", - "[parameters('etcdPeerCertificate2')]" - ], - {{end}} - {{if eq .MasterProfile.Count 5}} - "etcdPeerPrivateKeys": [ - "[parameters('etcdPeerPrivateKey0')]", - "[parameters('etcdPeerPrivateKey1')]", - "[parameters('etcdPeerPrivateKey2')]", - "[parameters('etcdPeerPrivateKey3')]", - "[parameters('etcdPeerPrivateKey4')]" - ], - "etcdPeerCertificates": [ - "[parameters('etcdPeerCertificate0')]", - "[parameters('etcdPeerCertificate1')]", - "[parameters('etcdPeerCertificate2')]", - "[parameters('etcdPeerCertificate3')]", - "[parameters('etcdPeerCertificate4')]" - ], - {{end}} - "etcdPeerCertFilepath":[ - "/etc/kubernetes/certs/etcdpeer0.crt", - "/etc/kubernetes/certs/etcdpeer1.crt", - "/etc/kubernetes/certs/etcdpeer2.crt", - "/etc/kubernetes/certs/etcdpeer3.crt", - "/etc/kubernetes/certs/etcdpeer4.crt" - ], - "etcdPeerKeyFilepath":[ - "/etc/kubernetes/certs/etcdpeer0.key", - "/etc/kubernetes/certs/etcdpeer1.key", - "/etc/kubernetes/certs/etcdpeer2.key", - "/etc/kubernetes/certs/etcdpeer3.key", - "/etc/kubernetes/certs/etcdpeer4.key" - ], - "etcdCaFilepath": "/etc/kubernetes/certs/ca.crt", - "etcdClientCertFilepath": "/etc/kubernetes/certs/etcdclient.crt", - "etcdClientKeyFilepath": "/etc/kubernetes/certs/etcdclient.key", - "etcdServerCertFilepath": "/etc/kubernetes/certs/etcdserver.crt", - "etcdServerKeyFilepath": "/etc/kubernetes/certs/etcdserver.key", -{{end}} - "useManagedIdentityExtension": "{{ UseManagedIdentity }}", - "userAssignedID": "{{UserAssignedID}}", - "userAssignedClientID": "{{UserAssignedClientID}}", - "userAssignedIDReference": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities/', variables('userAssignedID'))]", - "useInstanceMetadata": "{{ UseInstanceMetadata }}", - "loadBalancerSku": "{{ LoadBalancerSku }}", - "excludeMasterFromStandardLB": "{{ ExcludeMasterFromStandardLB }}", -{{ if UseManagedIdentity }} - "servicePrincipalClientId": "msi", - "servicePrincipalClientSecret": "msi", -{{ else }} - "servicePrincipalClientId": "[parameters('servicePrincipalClientId')]", - "servicePrincipalClientSecret": "[parameters('servicePrincipalClientSecret')]", -{{ end }} - "masterFqdnPrefix": "[tolower(parameters('masterEndpointDNSNamePrefix'))]", -{{if not IsHostedMaster}} - "masterCount": {{.MasterProfile.Count}}, - "masterOffset": "", - "masterIpAddressCount": {{.MasterProfile.IPAddressCount}}, -{{end}} - "apiVersionCompute": "2018-06-01", - "apiVersionStorage": "2018-07-01", - "apiVersionKeyVault": "2018-02-14", - "apiVersionNetwork": "2018-08-01", - "apiVersionManagedIdentity": "2015-08-31-preview", - "apiVersionAuthorization": "2018-09-01-preview", - "locations": [ - "[resourceGroup().location]", - "[parameters('location')]" - ], - "location": "[variables('locations')[mod(add(2,length(parameters('location'))),add(1,length(parameters('location'))))]]", - "resourceGroup": "[resourceGroup().name]", - "truncatedResourceGroup": "[take(replace(replace(resourceGroup().name, '(', '-'), ')', '-'), 63)]", - "labelResourceGroup": "[if(or(or(endsWith(variables('truncatedResourceGroup'), '-'), endsWith(variables('truncatedResourceGroup'), '_')), endsWith(variables('truncatedResourceGroup'), '.')), concat(take(variables('truncatedResourceGroup'), 62), 'z'), variables('truncatedResourceGroup'))]", -{{if IsHostedMaster}} - "routeTableName": "[concat(variables('agentNamePrefix'), 'routetable')]", -{{else}} - "routeTableName": "[concat(variables('masterVMNamePrefix'),'routetable')]", -{{end}} - "routeTableID": "[resourceId('Microsoft.Network/routeTables', variables('routeTableName'))]", - "sshNatPorts": [22,2201,2202,2203,2204], - "sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]", - -{{if .HasStorageAccountDisks}} - "maxVMsPerStorageAccount": 20, - "maxStorageAccountsPerAgent": "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]", - "dataStorageAccountPrefixSeed": 97, - "storageAccountPrefixes": [ "0", "6", "c", "i", "o", "u", "1", "7", "d", "j", "p", "v", "2", "8", "e", "k", "q", "w", "3", "9", "f", "l", "r", "x", "4", "a", "g", "m", "s", "y", "5", "b", "h", "n", "t", "z" ], - "storageAccountPrefixesCount": "[length(variables('storageAccountPrefixes'))]", - "vmsPerStorageAccount": 20, - "storageAccountBaseName": "[uniqueString(concat(variables('masterFqdnPrefix'),variables('location')))]", - {{GetSizeMap}}, -{{else}} - "storageAccountPrefixes": [], - "storageAccountBaseName": "", -{{end}} -{{if not IsHostedMaster}} - {{if .MasterProfile.IsStorageAccount}} - "masterStorageAccountName": "[concat(variables('storageAccountBaseName'), 'mstr0')]", - {{end}} -{{end}} - "provisionScript": "{{GetKubernetesB64Provision}}", - "provisionSource": "{{GetKubernetesB64ProvisionSource}}", - "healthMonitorScript": "{{GetKubernetesB64HealthMonitorScript}}", - "provisionInstalls": "{{GetKubernetesB64Installs}}", - "provisionConfigs": "{{GetKubernetesB64Configs}}", - "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", - "customSearchDomainsScript": "{{GetKubernetesB64CustomSearchDomainsScript}}", - "sshdConfig": "{{GetB64sshdConfig}}", - "systemConf": "{{GetB64systemConf}}", - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'))]", - {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", - {{end}} - "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}", - "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}", - -{{if IsAzureCNI}} - "allocateNodeCidrs": false, -{{else}} - "allocateNodeCidrs": true, -{{end}} - "subnetNameResourceSegmentIndex": 10, - "vnetNameResourceSegmentIndex": 8, - "vnetResourceGroupNameResourceSegmentIndex": 4, -{{if IsHostedMaster}} - {{if IsCustomVNET}} - "vnetSubnetID": "[parameters('{{ (index .AgentPoolProfiles 0).Name }}VnetSubnetID')]", - "subnetName": "[split(variables('vnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]", - "virtualNetworkName": "[split(variables('vnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]", - "virtualNetworkResourceGroupName": "[split(variables('vnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]", - {{else}} - "subnetName": "[concat(parameters('orchestratorName'), '-subnet')]", - "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", - "vnetSubnetID": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", - "virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]", - "virtualNetworkResourceGroupName": "", - {{end}} -{{else}} - {{if .MasterProfile.IsCustomVNET}} - "vnetSubnetID": "[parameters('agentVnetSubnetID')]", - "vnetSubnetIDMaster": "[parameters('masterVnetSubnetID')]", - "subnetName": "[split(parameters('masterVnetSubnetID'), '/')[variables('subnetNameResourceSegmentIndex')]]", - "virtualNetworkName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetNameResourceSegmentIndex')]]", - "virtualNetworkResourceGroupName": "[split(parameters('masterVnetSubnetID'), '/')[variables('vnetResourceGroupNameResourceSegmentIndex')]]", - {{else}} - "subnetName": "subnetmaster", - "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", - "vnetSubnetID": "[concat(variables('vnetID'),'/subnets/subnetagent')]", - "vnetSubnetIDMaster": "[concat(variables('vnetID'),'/subnets/subnetmaster')]", - "virtualNetworkName": "[concat(parameters('orchestratorName'), '-vnet-', parameters('nameSuffix'))]", - "virtualNetworkResourceGroupName": "''", - {{end}} -{{end}} -{{if IsHostedMaster }} - "nsgName": "[concat(variables('agentNamePrefix'), 'nsg')]", -{{else}} - "nsgName": "[concat(variables('masterVMNamePrefix'), 'nsg')]", -{{end}} - "nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]", -{{if AnyAgentUsesVirtualMachineScaleSets}} - "primaryScaleSetName": "[concat(parameters('orchestratorName'), '-{{ (index .AgentPoolProfiles 0).Name }}-',parameters('nameSuffix'), '-vmss')]", - "primaryAvailabilitySetName": "", - "vmType": "vmss", -{{else}} - "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',parameters('nameSuffix'))]", - "primaryScaleSetName": "", - "vmType": "standard", -{{end}} -{{if IsHostedMaster }} - "kubernetesAPIServerIP": "[parameters('kubernetesEndpoint')]", - "agentNamePrefix": "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]", -{{else}} - {{if IsPrivateCluster}} - "kubeconfigServer": "[concat('https://', variables('kubernetesAPIServerIP'), ':443')]", - {{if ProvisionJumpbox}} - "jumpboxOSDiskName": "[concat(parameters('jumpboxVMName'), '-osdisk')]", - "jumpboxPublicIpAddressName": "[concat(parameters('jumpboxVMName'), '-ip')]", - "jumpboxNetworkInterfaceName": "[concat(parameters('jumpboxVMName'), '-nic')]", - "jumpboxNetworkSecurityGroupName": "[concat(parameters('jumpboxVMName'), '-nsg')]", - "kubeconfig": "{{GetKubeConfig}}", - {{if not JumpboxIsManagedDisks}} - "jumpboxStorageAccountName": "[concat(variables('storageAccountBaseName'), 'jb')]", - {{end}} - {{if not .HasStorageAccountDisks}} - {{GetSizeMap}}, - {{end}} - {{end}} - {{else}} - "masterPublicIPAddressName": "[concat(parameters('orchestratorName'), '-master-ip-', variables('masterFqdnPrefix'), '-', parameters('nameSuffix'))]", - "masterLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterLbName'))]", - "masterLbIPConfigID": "[concat(variables('masterLbID'),'/frontendIPConfigurations/', variables('masterLbIPConfigName'))]", - "masterLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-lbFrontEnd-', parameters('nameSuffix'))]", - "masterLbName": "[concat(parameters('orchestratorName'), '-master-lb-', parameters('nameSuffix'))]", - "kubeconfigServer": "[concat('https://', variables('masterFqdnPrefix'), '.', variables('location'), '.', parameters('fqdnEndpointSuffix'))]", - {{end}} - {{if gt .MasterProfile.Count 1}} - "masterInternalLbName": "[concat(parameters('orchestratorName'), '-master-internal-lb-', parameters('nameSuffix'))]", - "masterInternalLbID": "[resourceId('Microsoft.Network/loadBalancers',variables('masterInternalLbName'))]", - "masterInternalLbIPConfigName": "[concat(parameters('orchestratorName'), '-master-internal-lbFrontEnd-', parameters('nameSuffix'))]", - "masterInternalLbIPConfigID": "[concat(variables('masterInternalLbID'),'/frontendIPConfigurations/', variables('masterInternalLbIPConfigName'))]", - "masterInternalLbIPOffset": {{GetDefaultInternalLbStaticIPOffset}}, - "kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]", - {{else}} - "kubernetesAPIServerIP": "[parameters('firstConsecutiveStaticIP')]", - {{end}} - "masterLbBackendPoolName": "[concat(parameters('orchestratorName'), '-master-pool-', parameters('nameSuffix'))]", - "masterFirstAddrComment": "these MasterFirstAddrComment are used to place multiple masters consecutively in the address space", - "masterFirstAddrOctets": "[split(parameters('firstConsecutiveStaticIP'),'.')]", - "masterFirstAddrOctet4": "[variables('masterFirstAddrOctets')[3]]", - "masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]", - "masterVMNamePrefix": "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]", - "masterVMNames": [ - "[concat(variables('masterVMNamePrefix'), 'vmss000000')]", - "[concat(variables('masterVMNamePrefix'), 'vmss000001')]", - "[concat(variables('masterVMNamePrefix'), 'vmss000002')]", - "[concat(variables('masterVMNamePrefix'), 'vmss000003')]", - "[concat(variables('masterVMNamePrefix'), 'vmss000004')]" - ], - "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, - "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, -{{end}} - "subscriptionId": "[subscription().subscriptionId]", - "contributorRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]", - "readerRoleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'acdd72a7-3385-48ef-bd42-f606fba81ae7')]", - "scope": "[resourceGroup().id]", - "tenantId": "[subscription().tenantId]", - "singleQuote": "'" -{{if .LinuxProfile.HasSecrets}} - , "linuxProfileSecrets" : - [ - {{range $vIndex, $vault := .LinuxProfile.Secrets}} - {{if $vIndex}} , {{end}} - { - "sourceVault":{ - "id":"[parameters('linuxKeyVaultID{{$vIndex}}')]" - }, - "vaultCertificates":[ - {{range $cIndex, $cert := $vault.VaultCertificates}} - {{if $cIndex}} , {{end}} - { - "certificateUrl" :"[parameters('linuxKeyVaultID{{$vIndex}}CertificateURL{{$cIndex}}')]" - } - {{end}} - ] - } - {{end}} - ] -{{end}} -{{if .HasWindows}} - ,"windowsCustomScriptSuffix": " $inputFile = '%SYSTEMDRIVE%\\AzureData\\CustomData.bin' ; $outputFile = '%SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.ps1' ; Copy-Item $inputFile $outputFile ; Invoke-Expression('{0} {1}' -f $outputFile, $arguments) ; " -{{end}} -{{if EnableEncryptionWithExternalKms}} - ,"clusterKeyVaultName": "[take(concat('kv', tolower(uniqueString(concat(variables('masterFqdnPrefix'),variables('location'),parameters('nameSuffix'))))), 22)]" -{{else}} - ,"clusterKeyVaultName": "" -{{end}} \ No newline at end of file diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index bff34ab682..710b99ecff 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -108,7 +108,6 @@ const ( const ( kubernetesMasterCustomDataYaml = "k8s/kubernetesmastercustomdata.yml" - kubernetesMasterCustomDataVMSSYaml = "k8s/kubernetesmastercustomdatavmss.yml" kubernetesCustomScript = "k8s/kubernetescustomscript.sh" kubernetesProvisionSourceScript = "k8s/kubernetesprovisionsource.sh" kubernetesHealthMonitorScript = "k8s/health-monitor.sh" @@ -183,8 +182,7 @@ const ( kubernetesAgentVars = "k8s/kubernetesagentvars.t" kubernetesMasterResourcesVMAS = "k8s/kubernetesmasterresources.t" kubernetesMasterResourcesVMSS = "k8s/kubernetesmasterresourcesvmss.t" - kubernetesMasterVarsVMAS = "k8s/kubernetesmastervars.t" - kubernetesMasterVarsVMSS = "k8s/kubernetesmastervarsvmss.t" + kubernetesMasterVars = "k8s/kubernetesmastervars.t" kubernetesParams = "k8s/kubernetesparams.t" kubernetesWinAgentVars = "k8s/kuberneteswinagentresourcesvmas.t" kubernetesWinAgentVarsVMSS = "k8s/kuberneteswinagentresourcesvmss.t" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index bbb9b9c2a2..1448523107 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -26,7 +26,7 @@ import ( var commonTemplateFiles = []string{agentOutputs, agentParams, masterOutputs, iaasOutputs, masterParams, windowsParams} var dcosTemplateFiles = []string{dcosBaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcosMasterResources, dcosMasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS} var dcos2TemplateFiles = []string{dcos2BaseFile, dcosAgentResourcesVMAS, dcosAgentResourcesVMSS, dcosAgentVars, dcos2MasterResources, dcos2BootstrapResources, dcos2MasterVars, dcosParams, dcosWindowsAgentResourcesVMAS, dcosWindowsAgentResourcesVMSS, dcos2BootstrapVars, dcos2BootstrapParams} -var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResourcesVMAS, kubernetesMasterResourcesVMSS, kubernetesMasterVarsVMAS, kubernetesMasterVarsVMSS, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS} +var kubernetesTemplateFiles = []string{kubernetesBaseFile, kubernetesAgentResourcesVMAS, kubernetesAgentResourcesVMSS, kubernetesAgentVars, kubernetesMasterResourcesVMAS, kubernetesMasterResourcesVMSS, kubernetesMasterVars, kubernetesParams, kubernetesWinAgentVars, kubernetesWinAgentVarsVMSS} var swarmTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS} var swarmModeTemplateFiles = []string{swarmBaseFile, swarmParams, swarmAgentResourcesVMAS, swarmAgentVars, swarmAgentResourcesVMSS, swarmBaseFile, swarmMasterResources, swarmMasterVars, swarmWinAgentResourcesVMAS, swarmWinAgentResourcesVMSS} var openshiftTemplateFiles = append( diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 72a4ab3f81..9f5b6f6ebd 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -559,10 +559,6 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat str := t.getMasterCustomData(cs, kubernetesMasterCustomDataYaml, profile) return str }, - "GetKubernetesMasterCustomDataVMSS": func(profile *api.Properties) string { - str := t.getMasterCustomData(cs, kubernetesMasterCustomDataVMSSYaml, profile) - return str - }, "GetKubernetesAgentCustomData": func(profile *api.AgentPoolProfile) string { str, e := t.getSingleLineForTemplate(kubernetesAgentCustomDataYaml, cs, profile)