Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: set master distro to aks-ubuntu-18.04 for master branch jobs #178

Merged
merged 2 commits into from
Jul 7, 2020
Merged

chore: set master distro to aks-ubuntu-18.04 for master branch jobs #178

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ef4952
chore: set master distro to aks-ubuntu-18.04 for master branch jobs
Jul 6, 2020
1747b18
Address PR comments
Jul 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 3 additions & 10 deletions extensions/master_extension/v1/win-e2e-master-extension.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#!/bin/bash

echo "start master extension" >> /tmp/master_extension.log

KUBECONFIG="$(find /home/*/.kube/config)"
KUBECTL="kubectl --kubeconfig=${KUBECONFIG}"

Expand Down Expand Up @@ -30,13 +32,4 @@ master_node=$(${KUBECTL} get nodes | grep master | awk '{print $1}')
${KUBECTL} taint nodes "$master_node" node-role.kubernetes.io/master=:NoSchedule
${KUBECTL} label nodes "$master_node" node-role.kubernetes.io/master=NoSchedule

# Prepull images
${KUBECTL} create -f https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull.yaml
# Wait 15 minutes for the test images to be pulled onto the nodes.
sleep 15m
# Check the status of the pods.
${KUBECTL} get pods -o wide >> /tmp/master_extension.log
# Delete the pods anyway since pre-pulling is best-effort
${KUBECTL} delete -f https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull.yaml
# Wait a few more minutes for the pod to be cleaned up.
sleep 3m
echo "finish master extension" >> /tmp/master_extension.log
1 change: 1 addition & 0 deletions extensions/master_extension_prepull/v1/supported-orchestrators.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
["Kubernetes"]
39 changes: 39 additions & 0 deletions extensions/master_extension_prepull/v1/template-link.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET), 'win-e2e-master-extension')]",
"type": "Microsoft.Resources/deployments",
"apiVersion": "[variables('apiVersionDeployments')]",
"dependsOn": [
"[concat('Microsoft.Compute/virtualMachines/', EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET), '/extensions/cse', '-EXTENSION_TARGET_VM_TYPE-', copyIndex(EXTENSION_LOOP_OFFSET))]"
],
"copy": {
"count": "EXTENSION_LOOP_COUNT",
"name": "wine2emasterextensionloop"
},
"properties": {
"mode": "Incremental",
"templateLink": {
"uri": "EXTENSION_URL_REPLACEextensions/master_extension_prepull/v1/template.json",
"contentVersion": "1.0.0.0"
},
"parameters": {
"artifactsLocation": {
"value": "EXTENSION_URL_REPLACE"
},
"apiVersionDeployments": {
"value": "[variables('apiVersionDeployments')]"
},
"targetVMName": {
"value": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET))]"
},
"targetVMType": {
"value": "EXTENSION_TARGET_VM_TYPE"
},
"extensionParameters": {
"value": "EXTENSION_PARAMETERS_REPLACE"
},
"vmIndex":{
"value": "[copyIndex(EXTENSION_LOOP_OFFSET)]"
}
}
}
}
75 changes: 75 additions & 0 deletions extensions/master_extension_prepull/v1/template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"artifactsLocation": {
"type": "string",
"minLength": 1,
"metadata": {
"description": "Artifacts Location - URL"
}
},
"apiVersionDeployments": {
"type": "string",
"minLength": 1,
"metadata": {
"description": "Deployments API Version"
}
},
"targetVMName":{
"type": "string",
"minLength": 1,
"metadata": {
"description": "Name of the vm to run the "
}
},
"targetVMType":{
"type": "string",
"minLength": 1,
"metadata": {
"description": "Type of the vm to run the extension: master or agent "
}
},
"extensionParameters": {
"type": "securestring",
"minLength": 0,
"metadata": {
"description": "Custom Parameter for Extension - for hello-world, this is empty"
}
},
"vmIndex": {
"type": "int",
"metadata": {
"description": "index in the pool of the current agent, used so that we can get the extension name right"
}
}
},
"variables": {
"singleQuote": "'",
"initScriptUrl": "[concat(parameters('artifactsLocation'), 'extensions/master_extension_prepull/v1/win-e2e-master-extension.sh')]"
},
"resources": [
{
"apiVersion": "[parameters('apiVersionDeployments')]",
"dependsOn": [],
"location": "[resourceGroup().location]",
"type": "Microsoft.Compute/virtualMachines/extensions",
"name": "[concat(parameters('targetVMName'),'/cse', '-', parameters('targetVMType'), '-', parameters('vmIndex'))]",
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.0",
"autoUpgradeMinorVersion": true,
"settings": {
"fileUris": [
"[variables('initScriptUrl')]"
]
},
"protectedSettings": {
"commandToExecute": "[concat('/bin/bash -c \"/bin/bash ./win-e2e-master-extension.sh ', variables('singleQuote'), parameters('extensionParameters'), variables('singleQuote'), ' >> /var/log/azure/win-e2e-master-extension.log 2>&1\"')]"
}
}
}
],
"outputs": { }
}
46 changes: 46 additions & 0 deletions extensions/master_extension_prepull/v1/win-e2e-master-extension.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

echo "start master extension" >> /tmp/master_extension.log

KUBECONFIG="$(find /home/*/.kube/config)"
KUBECTL="kubectl --kubeconfig=${KUBECONFIG}"

# Wait for coredns pod to be ready
while true; do
COREDNS_READY=$(${KUBECTL} get pods -l k8s-app=kube-dns -n kube-system -o custom-columns=STATUS:status.containerStatuses[0].ready --no-headers)
if [ "${COREDNS_READY}" = "true" ]; then
echo "$(date -R) - coredns is ready" >> /tmp/master_extension.log
${KUBECTL} get pods --all-namespaces >> /tmp/master_extension.log
break
fi
sleep 2
done
chewong marked this conversation as resolved.
Show resolved Hide resolved

# Wait for all pods in kube-system namespace to be ready before applying taints
while true; do
NOTREADY_PODS=$(${KUBECTL} get pods -n kube-system -o custom-columns=STATUS:status.containerStatuses[0].ready --no-headers | grep -v "true")
if [ -z "${NOTREADY_PODS}" ]; then
echo "$(date -R) - All kube-system pods are ready" >> /tmp/master_extension.log
${KUBECTL} get pods --all-namespaces >> /tmp/master_extension.log
break
fi
sleep 2
done

master_node=$(${KUBECTL} get nodes | grep master | awk '{print $1}')

${KUBECTL} taint nodes "$master_node" node-role.kubernetes.io/master=:NoSchedule
${KUBECTL} label nodes "$master_node" node-role.kubernetes.io/master=NoSchedule

# Prepull images
${KUBECTL} create -f https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull.yaml
# Wait 15 minutes for the test images to be pulled onto the nodes.
sleep 15m
# Check the status of the pods.
${KUBECTL} get pods -o wide >> /tmp/master_extension.log
# Delete the pods anyway since pre-pulling is best-effort
${KUBECTL} delete -f https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/gce/prepull.yaml
# Wait a few more minutes for the pod to be cleaned up.
sleep 3m

echo "finish master extension" >> /tmp/master_extension.log
13 changes: 8 additions & 5 deletions hack/update-job-templates.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@ set -o errexit
set -o nounset
set -o pipefail

repo_root=$(dirname "${BASH_SOURCE[0]}")/..
lts2019="2019-Datacenter-Core-with-Containers-smalldisk"
sac1903="Datacenter-Core-1903-with-Containers-smalldisk"
sac1909="Datacenter-Core-1909-with-Containers-smalldisk"

sac2004="Datacenter-Core-2004-with-Containers-smalldisk"

function setVersion() {
local windowsSku=$1
templateFiles=($(grep -rnwl '../job-templates' -e "$windowsSku"))
templateFiles=($(grep -rnwl "${repo_root}/job-templates" -e "$windowsSku"))
version=$(az vm image show --urn MicrosoftWindowsServer:WindowsServer:$windowsSku:latest | jq -r .name)

#can't do inline replacements with jq (https://github.com/stedolan/jq/issues/105)
tmp=$(mktemp)
tmp=$(mktemp)
for template in "${templateFiles[@]}"; do
jq --arg version "$version" '.properties.windowsProfile.imageVersion = $version' $template > "$tmp" && mv "$tmp" $template
echo "Updated file '$template' '$windowsSku' with '$version'"
Expand All @@ -23,4 +25,5 @@ function setVersion() {

setVersion $lts2019
setVersion $sac1903
setVersion $sac1909
setVersion $sac1909
setVersion $sac2004
2 changes: 1 addition & 1 deletion job-templates/kubernetes_1903_master.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"distro": "ubuntu",
"distro": "aks-ubuntu-18.04",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is possible to set the VHD imageVersion explicitly? This way we know the version that is deployed is consistent across tests?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAIK aks-engine API model doesn't allow users to override the Linux image version

"extensions": [
{
"name": "master_extension"
Expand Down
2 changes: 1 addition & 1 deletion job-templates/kubernetes_1909_master.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"distro": "ubuntu",
"distro": "aks-ubuntu-18.04",
"extensions": [
{
"name": "master_extension"
Expand Down
4 changes: 2 additions & 2 deletions job-templates/kubernetes_2004_master.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"distro": "ubuntu",
"distro": "aks-ubuntu-18.04",
"extensions": [
{
"name": "master_extension"
Expand All @@ -45,7 +45,7 @@
"sshEnabled": true,
"windowsPublisher": "MicrosoftWindowsServer",
"windowsOffer": "WindowsServer",
"windowsSku": "datacenter-core-2004-with-containers-smalldisk",
"windowsSku": "Datacenter-Core-2004-with-Containers-smalldisk",
"imageVersion": "19041.329.2006042019"
},
"linuxProfile": {
Expand Down
5 changes: 2 additions & 3 deletions job-templates/kubernetes_containerd.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"distro": "ubuntu",
"distro": "aks-ubuntu-18.04",
"extensions": [
{
"name": "master_extension"
Expand All @@ -48,7 +48,6 @@
"windowsProfile": {
"adminUsername": "azureuser",
"adminPassword": "replacepassword1234$",
"csiProxyURL": "https://k8scsi.blob.core.windows.net/csi-proxy/v0.1.0/binaries/csi-proxy.tar.gz",
"enableCSIProxy": true,
"sshEnabled": true
},
Expand Down Expand Up @@ -82,4 +81,4 @@
}
]
}
}
}
2 changes: 1 addition & 1 deletion job-templates/kubernetes_containerd_hyperv.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3",
"distro": "ubuntu",
"distro": "aks-ubuntu-18.04",
"extensions": [
{
"name": "master_extension"
Expand Down
86 changes: 0 additions & 86 deletions job-templates/kubernetes_master.json
View file
Open in desktop

This file was deleted.

Loading