Merging kubernetesmastervarsvmss into kubernetesmastervars (#4116)

parts/k8s/kubernetesbase.t

@@ -49,11 +49,7 @@
           "{{.Name}}AccountName": "[concat(variables('storageAccountBaseName'), 'agnt{{$index}}')]",
         {{end}}
     {{end}}
-    {{if IsMasterVirtualMachineScaleSets}}
-      {{template "k8s/kubernetesmastervarsvmss.t" .}}
-    {{else}}
-      {{template "k8s/kubernetesmastervars.t" .}}
-    {{end}}
+    {{template "k8s/kubernetesmastervars.t" .}}
   },
   "resources": [
     {{if UserAssignedIDEnabled}}

parts/k8s/kubernetesmastercustomdata.yml

@@ -182,7 +182,11 @@ write_files:
     - name: localcluster
       cluster:
         certificate-authority: /etc/kubernetes/certs/ca.crt
+      {{if IsMasterVirtualMachineScaleSets}}
+        server: <SERVERIP>
+      {{else}}
         server: {{WrapAsVerbatim "concat('https://', variables('masterPrivateIpAddrs')[copyIndex(variables('masterOffset'))], ':443')"}}
+      {{end}}
     users:
     - name: client
       user:
@@ -266,6 +270,10 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
   content: |
     #!/bin/bash
     set -e
+  {{if IsMasterVirtualMachineScaleSets}}
+    PRIVATE_IP=$(hostname -I | cut -d" " -f1)
+    ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
+  {{end}}
 {{if gt .MasterProfile.Count 1}}
     # Redirect ILB (4443) traffic to port 443 (ELB) in the prerouting chain
     iptables -t nat -A PREROUTING -p tcp --dport 4443 -j REDIRECT --to-port 443
@@ -389,7 +397,11 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/flannel-daemonset.yaml"
 {{end}}
 {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "cilium"}}
+  {{if IsMasterVirtualMachineScaleSets}}
+    sed -i "s|<ETCD_URL>|"https://$PRIVATE_IP:$ETCD_CLIENT_PORT"|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
+  {{else}}
     sed -i "s|<ETCD_URL>|{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
+  {{end}}
     sed -i "s|<ETCD_CA>|$(base64 -w 0 /etc/kubernetes/certs/ca.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
     sed -i "s|<ETCD_CLIENT_KEY>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.key)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
     sed -i "s|<ETCD_CLIENT_CERT>|$(base64 -w 0 /etc/kubernetes/certs/etcdclient.crt)|g" "/etc/kubernetes/addons/cilium-daemonset.yaml"
@@ -474,12 +486,43 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
   content: |
     #!/bin/bash
     set -x
+  {{if IsMasterVirtualMachineScaleSets}}
+    MASTER_VM_NAME=$(hostname)
+    MASTER_VM_NAME_BASE=$(hostname | sed "s/.$//")
+    MASTER_FIRSTADDR_OCTET4={{WrapAsVariable "masterFirstAddrOctet4"}}
+    MASTER_INDEX=$(hostname | tail -c 2)
+    PRIVATE_IP=$(hostname -I | cut -d" " -f1)
+    PRIVATE_IP_BASE=$(hostname -I | cut -d" " -f1 | cut -d. -f1-3)
+    MASTER_COUNT={{WrapAsVariable "masterCount"}}
+    IPADDRESS_COUNT={{WrapAsVariable "masterIpAddressCount"}}
+    echo $IPADDRESS_COUNT
+    ETCD_SERVER_PORT={{WrapAsVariable "masterEtcdServerPort"}}
+    ETCD_CLIENT_PORT={{WrapAsVariable "masterEtcdClientPort"}}
+    MASTER_URLS=""
+    index=0
+    while [ $index -lt $MASTER_COUNT ]
+    do
+        echo $index
+        offset=`expr $index \\* $IPADDRESS_COUNT + $MASTER_FIRSTADDR_OCTET4`
+        echo $offset
+        MASTER_URLS="$MASTER_URLS$MASTER_VM_NAME_BASE$index=https://$PRIVATE_IP_BASE.$offset:$ETCD_SERVER_PORT,"
+        index=`expr $index + 1`
+    done
+    MASTER_URLS=$(echo $MASTER_URLS | sed "s/.$//")
+    echo $MASTER_URLS
+    sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:$ETCD_CLIENT_PORT" /etc/environment
+    sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment
+    sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment
+    sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment
+    sudo sed -i "s|<SERVERIP>|https://$PRIVATE_IP:443|g" "/var/lib/kubelet/kubeconfig"
+    /bin/echo DAEMON_ARGS=--name $MASTER_VM_NAME --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.crt --peer-key-file=/etc/kubernetes/certs/etcdpeer$MASTER_INDEX.key --initial-advertise-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --listen-peer-urls "https://$PRIVATE_IP:$ETCD_SERVER_PORT" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT" --listen-client-urls "https://$PRIVATE_IP:$ETCD_CLIENT_PORT,https://127.0.0.1:$ETCD_CLIENT_PORT" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster $MASTER_URLS --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd
+  {{else}}
     sudo sed -i "1iETCDCTL_ENDPOINTS=https://127.0.0.1:2379" /etc/environment
     sudo sed -i "1iETCDCTL_CA_FILE={{WrapAsVariable "etcdCaFilepath"}}" /etc/environment
     sudo sed -i "1iETCDCTL_KEY_FILE={{WrapAsVariable "etcdClientKeyFilepath"}}" /etc/environment
     sudo sed -i "1iETCDCTL_CERT_FILE={{WrapAsVariable "etcdClientCertFilepath"}}" /etc/environment
     /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster {{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd
-
+  {{end}}
 {{if .MasterProfile.IsCoreOS}}
 - path: "/opt/azure/containers/provision-setup.sh"
   permissions: "0755"