Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

fix: check network to k8s api for vmss #2938

Merged
merged 7 commits into from
Mar 25, 2020
Merged

fix: check network to k8s api for vmss #2938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2e4c602
Check network to k8s api for vmss
bowang-666 Mar 19, 2020
6c236f0
Update error name
bowang-666 Mar 19, 2020
c1dd180
Move check to end
bowang-666 Mar 21, 2020
024eb0e
Move check to cse_main
bowang-666 Mar 24, 2020
aaf027c
use static nc command
bowang-666 Mar 24, 2020
30d04d6
Add generated file
bowang-666 Mar 24, 2020
9e8e6b6
Fix unit tests
bowang-666 Mar 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion parts/k8s/cloud-init/artifacts/cse_helpers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ ERR_MS_PROD_DEB_PKG_ADD_FAIL=43
ERR_SYSTEMD_INSTALL_FAIL=48
ERR_MODPROBE_FAIL=49
ERR_OUTBOUND_CONN_FAIL=50
ERR_K8S_API_CONN_FAIL=51
ERR_K8S_API_SERVER_CONN_FAIL=51
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60
ERR_KATA_APT_KEY_TIMEOUT=61
ERR_KATA_INSTALL_TIMEOUT=62
Expand Down
4 changes: 4 additions & 0 deletions parts/k8s/cloud-init/artifacts/cse_main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
fi
{{end}}

{{- if IsHostedMaster }}
retrycmd_if_failure 50 1 3 nc -vz ${API_SERVER_IP} 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL
{{end}}

if $REBOOTREQUIRED; then
echo 'reboot required, rebooting node in 1 minute'
/bin/bash -c "shutdown -r 1 &"
Expand Down
10 changes: 5 additions & 5 deletions pkg/engine/armresources_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func TestGenerateARMResourcesWithVMSSAgentPool(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -556,7 +556,7 @@ func TestGenerateARMResourcesWithVMSSAgentPool(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -856,7 +856,7 @@ func TestGenerateARMResourceWithVMASAgents(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: &map[string]interface{}{},
ProtectedSettings: &map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`},
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`},
},
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), copyIndex(variables('agentpool1Offset')),'/cse', '-agent-', copyIndex(variables('agentpool1Offset')))]"),
Type: to.StringPtr("Microsoft.Compute/virtualMachines/extensions"),
Expand Down Expand Up @@ -1354,7 +1354,7 @@ func TestGenerateARMResourcesWithVMSSAgentPoolAndSLB(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -1979,7 +1979,7 @@ func TestGenerateARMResourcesWithVMSSAgentPoolAndSLB(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-computeAksLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down
49 changes: 48 additions & 1 deletion pkg/engine/armvariables_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,30 @@ func TestK8sVars(t *testing.T) {
if err != nil {
t.Fatal(err)
}
expectedMap["cloudInitFiles"] = map[string]interface{}{
"provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs),
"provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs),
"provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs),
"provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs),
"provisionCIS": getBase64EncodedGzippedCustomScript(kubernetesCISScript, cs),
"healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs),
"customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs),
"generateProxyCertsScript": getBase64EncodedGzippedCustomScript(kubernetesMasterGenerateProxyCertsScript, cs),
"mountEtcdScript": getBase64EncodedGzippedCustomScript(kubernetesMountEtcd, cs),
"kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs),
"kmsSystemdService": getBase64EncodedGzippedCustomScript(kmsSystemdService, cs),
"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs),
"dockerMonitorSystemdTimer": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdTimer, cs),
"dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
"labelNodesScript": getBase64EncodedGzippedCustomScript(labelNodesScript, cs),
"labelNodesSystemdService": getBase64EncodedGzippedCustomScript(labelNodesSystemdService, cs),
"aptPreferences": getBase64EncodedGzippedCustomScript(aptPreferences, cs),
"dockerClearMountPropagationFlags": getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, cs),
"auditdRules": getBase64EncodedGzippedCustomScript(auditdRules, cs),
"etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService, cs),
"dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs),
"dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs),
}
expectedMap["agentNamePrefix"] = "[concat(parameters('orchestratorName'), '-agentpool-', parameters('nameSuffix'), '-')]"
expectedMap["agentpool1AccountName"] = "[concat(variables('storageAccountBaseName'), 'agnt0')]"
expectedMap["agentpool1StorageAccountOffset"] = "[mul(variables('maxStorageAccountsPerAgent'),variables('agentpool1Index'))]"
Expand Down Expand Up @@ -449,7 +473,30 @@ func TestK8sVars(t *testing.T) {
if err != nil {
t.Fatal(err)
}

expectedMap["cloudInitFiles"] = map[string]interface{}{
"provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, cs),
"provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, cs),
"provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, cs),
"provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, cs),
"provisionCIS": getBase64EncodedGzippedCustomScript(kubernetesCISScript, cs),
"healthMonitorScript": getBase64EncodedGzippedCustomScript(kubernetesHealthMonitorScript, cs),
"customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, cs),
"generateProxyCertsScript": getBase64EncodedGzippedCustomScript(kubernetesMasterGenerateProxyCertsScript, cs),
"mountEtcdScript": getBase64EncodedGzippedCustomScript(kubernetesMountEtcd, cs),
"kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, cs),
"kmsSystemdService": getBase64EncodedGzippedCustomScript(kmsSystemdService, cs),
"kubeletMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesKubeletMonitorSystemdService, cs),
"dockerMonitorSystemdTimer": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdTimer, cs),
"dockerMonitorSystemdService": getBase64EncodedGzippedCustomScript(kubernetesDockerMonitorSystemdService, cs),
"labelNodesScript": getBase64EncodedGzippedCustomScript(labelNodesScript, cs),
"labelNodesSystemdService": getBase64EncodedGzippedCustomScript(labelNodesSystemdService, cs),
"aptPreferences": getBase64EncodedGzippedCustomScript(aptPreferences, cs),
"dockerClearMountPropagationFlags": getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, cs),
"auditdRules": getBase64EncodedGzippedCustomScript(auditdRules, cs),
"etcdSystemdService": getBase64EncodedGzippedCustomScript(etcdSystemdService, cs),
"dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, cs),
"dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, cs),
}
expectedMap["useManagedIdentityExtension"] = "true"
expectedMap["provisionScriptParametersCommon"] = "[concat('" + cs.GetProvisionScriptParametersCommon(api.ProvisionScriptParametersInput{Location: common.WrapAsARMVariable("location"), ResourceGroup: common.WrapAsARMVariable("resourceGroup"), TenantID: common.WrapAsARMVariable("tenantID"), SubscriptionID: common.WrapAsARMVariable("subscriptionId"), ClientID: common.WrapAsARMVariable("servicePrincipalClientId"), ClientSecret: common.WrapAsARMVariable("singleQuote") + common.WrapAsARMVariable("servicePrincipalClientSecret") + common.WrapAsARMVariable("singleQuote"), APIServerCertificate: common.WrapAsParameter("apiServerCertificate"), KubeletPrivateKey: common.WrapAsParameter("clientPrivateKey"), ClusterKeyVaultName: common.WrapAsARMVariable("clusterKeyVaultName")}) + "')]"

Expand Down
6 changes: 5 additions & 1 deletion pkg/engine/templates_generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion pkg/engine/virtualmachinescalesets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -755,7 +755,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
auditDEnabled := strconv.FormatBool(to.Bool(profile.AuditDEnabled))
isVHD := strconv.FormatBool(profile.IsVHDDistro())

commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' API_SERVER_IP=',variables('kubernetesAPIServerIP'),' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
vmssCSE = compute.VirtualMachineScaleSetExtension{
Name: to.StringPtr("vmssCSE"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Expand Down
Loading