Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

fix: check network to k8s api for vmss #2938

Merged
merged 7 commits into from
Mar 25, 2020
Merged

fix: check network to k8s api for vmss #2938

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2e4c602
Check network to k8s api for vmss
bowang-666 Mar 19, 2020
6c236f0
Update error name
bowang-666 Mar 19, 2020
c1dd180
Move check to end
bowang-666 Mar 21, 2020
024eb0e
Move check to cse_main
bowang-666 Mar 24, 2020
aaf027c
use static nc command
bowang-666 Mar 24, 2020
30d04d6
Add generated file
bowang-666 Mar 24, 2020
9e8e6b6
Fix unit tests
bowang-666 Mar 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion parts/k8s/cloud-init/artifacts/cse_helpers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ ERR_MS_PROD_DEB_PKG_ADD_FAIL=43
ERR_SYSTEMD_INSTALL_FAIL=48
ERR_MODPROBE_FAIL=49
ERR_OUTBOUND_CONN_FAIL=50
ERR_K8S_API_CONN_FAIL=51
ERR_K8S_API_SERVER_CONN_FAIL=51
ERR_KATA_KEY_DOWNLOAD_TIMEOUT=60
ERR_KATA_APT_KEY_TIMEOUT=61
ERR_KATA_INSTALL_TIMEOUT=62
Expand Down
2 changes: 1 addition & 1 deletion pkg/engine/templates_generated.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion pkg/engine/virtualmachinescalesets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -707,6 +707,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
var vmssExtensions []compute.VirtualMachineScaleSetExtension

outBoundCmd := ""
outBoundK8sCmd := ""
registry := ""
ncBinary := "nc"
if profile.IsCoreOS() {
Expand All @@ -721,6 +722,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
registry = `mcr.microsoft.com 443`
}
outBoundCmd = `retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz ` + registry + ` || exit $ERR_OUTBOUND_CONN_FAIL;`
outBoundK8sCmd = `;ERR_K8S_API_SERVER_CONN_FAIL=51; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL;`
}

var vmssCSE compute.VirtualMachineScaleSetExtension
Expand Down Expand Up @@ -755,7 +757,7 @@ func CreateAgentVMSS(cs *api.ContainerService, profile *api.AgentPoolProfile) Vi
auditDEnabled := strconv.FormatBool(to.Bool(profile.AuditDEnabled))
isVHD := strconv.FormatBool(profile.IsVHDDistro())

commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"%s')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground, outBoundK8sCmd)
vmssCSE = compute.VirtualMachineScaleSetExtension{
Name: to.StringPtr("vmssCSE"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Expand Down
4 changes: 2 additions & 2 deletions pkg/engine/virtualmachinescalesets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,7 @@ func TestCreateAgentVMSSHostedMasterProfile(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz mcr.microsoft.com 443 || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz mcr.microsoft.com 443 || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1";ERR_K8S_API_SERVER_CONN_FAIL=51; retrycmd_if_failure 50 1 3 nc -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL;')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-AKSLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down Expand Up @@ -736,7 +736,7 @@ func TestCreateAgentVMSSHostedMasterProfile(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: map[string]interface{}{},
ProtectedSettings: map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz gcr.azk8s.cn 443 || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`}}}, {
"commandToExecute": `[concat('echo $(date),$(hostname); retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz gcr.azk8s.cn 443 || exit $ERR_OUTBOUND_CONN_FAIL; for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1";ERR_K8S_API_SERVER_CONN_FAIL=51; retrycmd_if_failure 50 1 3 nc -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL;')]`}}}, {
Name: to.StringPtr("[concat(variables('agentpool1VMNamePrefix'), '-AKSLinuxBilling')]"),
VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{
Publisher: to.StringPtr("Microsoft.AKS"),
Expand Down
6 changes: 4 additions & 2 deletions pkg/engine/vmextensions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ func createAgentVMASCustomScriptExtension(cs *api.ContainerService, profile *api
location := "[variables('location')]"
name := fmt.Sprintf("[concat(variables('%[1]sVMNamePrefix'), copyIndex(variables('%[1]sOffset')),'/cse', '-agent-', copyIndex(variables('%[1]sOffset')))]", profile.Name)
outBoundCmd := ""
outBoundK8sCmd := ""
registry := ""
ncBinary := "nc"
if profile.IsCoreOS() {
Expand All @@ -129,7 +130,8 @@ func createAgentVMASCustomScriptExtension(cs *api.ContainerService, profile *api
} else {
registry = `mcr.microsoft.com 443`
}
outBoundCmd = `retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; ERR_K8S_API_CONN_FAIL=51; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz ` + registry + `; if [ $? -ne 0 ];then exit $ERR_OUTBOUND_CONN_FAIL;fi; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_CONN_FAIL;`
outBoundCmd = `retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz ` + registry + ` || exit $ERR_OUTBOUND_CONN_FAIL;`
outBoundK8sCmd = `;ERR_K8S_API_SERVER_CONN_FAIL=51; retrycmd_if_failure 50 1 3 ` + ncBinary + ` -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL;`
}

runInBackground := ""
Expand Down Expand Up @@ -164,7 +166,7 @@ func createAgentVMASCustomScriptExtension(cs *api.ContainerService, profile *api
vmExtension.Publisher = to.StringPtr("Microsoft.Azure.Extensions")
vmExtension.VirtualMachineExtensionProperties.Type = to.StringPtr("CustomScript")
vmExtension.TypeHandlerVersion = to.StringPtr("2.0")
commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground)
commandExec := fmt.Sprintf("[concat('echo $(date),$(hostname); %s for i in $(seq 1 1200); do grep -Fq \"EOF\" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),%s,' IS_VHD=%s GPU_NODE=%s SGX_NODE=%s AUDITD_ENABLED=%s /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1%s\"%s')]", outBoundCmd, generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled), isVHD, nVidiaEnabled, sgxEnabled, auditDEnabled, runInBackground, outBoundK8sCmd)
vmExtension.ProtectedSettings = &map[string]interface{}{
"commandToExecute": commandExec,
}
Expand Down
5 changes: 3 additions & 2 deletions pkg/engine/vmextensions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,8 @@ func TestCreateAgentVMASCustomScriptExtension(t *testing.T) {
var userAssignedIDEnabled = false

// Test CSE with an outboundCmd
expectedOutboundCmd := "retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; ERR_K8S_API_CONN_FAIL=51; retrycmd_if_failure 50 1 3 nc -vz mcr.microsoft.com 443; if [ $? -ne 0 ];then exit $ERR_OUTBOUND_CONN_FAIL;fi; retrycmd_if_failure 50 1 3 nc -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_CONN_FAIL;"
expectedOutboundCmd := "retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done }; ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz mcr.microsoft.com 443 || exit $ERR_OUTBOUND_CONN_FAIL;"
expectedOutboundK8sCmd := ";ERR_K8S_API_SERVER_CONN_FAIL=51; retrycmd_if_failure 50 1 3 nc -vz variables('kubernetesAPIServerIP') 443 || exit $ERR_K8S_API_SERVER_CONN_FAIL;"
expectedCSE := VirtualMachineExtensionARM{
ARMResource: ARMResource{
APIVersion: "[variables('apiVersionCompute')]",
Expand All @@ -378,7 +379,7 @@ func TestCreateAgentVMASCustomScriptExtension(t *testing.T) {
AutoUpgradeMinorVersion: to.BoolPtr(true),
Settings: &map[string]interface{}{},
ProtectedSettings: &map[string]interface{}{
"commandToExecute": `[concat('echo $(date),$(hostname); ` + expectedOutboundCmd + ` for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"')]`,
"commandToExecute": `[concat('echo $(date),$(hostname); ` + expectedOutboundCmd + ` for i in $(seq 1 1200); do grep -Fq "EOF" /opt/azure/containers/provision.sh && break; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),` + generateUserAssignedIdentityClientIDParameter(userAssignedIDEnabled) + `,' IS_VHD=true GPU_NODE=false SGX_NODE=false AUDITD_ENABLED=false /usr/bin/nohup /bin/bash -c "/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1"` + expectedOutboundK8sCmd + `')]`,
},
},
Type: to.StringPtr("Microsoft.Compute/virtualMachines/extensions"),
Expand Down