From 79f8a72bdd437072c3bf8b2d1a6c140c9c537f48 Mon Sep 17 00:00:00 2001 From: chshou Date: Mon, 26 Mar 2018 15:47:21 -0700 Subject: [PATCH 1/4] fix merge errors and deployment succeeded --- parts/k8s/kubernetesmastercustomscript.sh | 7 ++++-- pkg/acsengine/defaults-kubelet.go | 28 +++++++++++------------ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 8ada7e1167..eda0be4c9a 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -134,6 +134,11 @@ touch "${APISERVER_PUBLIC_KEY_PATH}" chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" chown root:root "${APISERVER_PUBLIC_KEY_PATH}" +set +x +echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" +echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" +set -x + if [[ ! -z "${MASTER_NODE}" ]]; then echo "MASTER_NODE is non-empty, master node, configure azure json." @@ -143,8 +148,6 @@ if [[ ! -z "${MASTER_NODE}" ]]; then chown root:root "${AZURE_JSON_PATH}" set +x - echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" - echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" cat << EOF > "${AZURE_JSON_PATH}" { "cloud":"${TARGET_ENVIRONMENT}", diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go index 22abe02606..983278ace3 100644 --- a/pkg/acsengine/defaults-kubelet.go +++ b/pkg/acsengine/defaults-kubelet.go @@ -21,7 +21,6 @@ func setKubeletConfig(cs *api.ContainerService) { "--cgroups-per-qos": "true", "--enforce-node-allocatable": "pods", "--kubeconfig": "/var/lib/kubelet/kubeconfig", - "--azure-container-registry-config": "/etc/kubernetes/azure.json", "--keep-terminated-pod-volumes": "false", } @@ -32,19 +31,20 @@ func setKubeletConfig(cs *api.ContainerService) { // Default Kubelet config defaultKubeletConfig := map[string]string{ - "--cluster-domain": "cluster.local", - "--network-plugin": "cni", - "--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"], - "--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods), - "--eviction-hard": DefaultKubernetesHardEvictionThreshold, - "--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"], - "--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), - "--non-masquerade-cidr": DefaultNonMasqueradeCidr, - "--cloud-provider": "azure", - "--cloud-config": "/etc/kubernetes/azure.json", - "--event-qps": DefaultKubeletEventQPS, - "--cadvisor-port": DefaultKubeletCadvisorPort, + "--cluster-domain": "cluster.local", + "--network-plugin": "cni", + "--pod-infra-container-image": cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[o.OrchestratorVersion]["pause"], + "--max-pods": strconv.Itoa(DefaultKubernetesKubeletMaxPods), + "--eviction-hard": DefaultKubernetesHardEvictionThreshold, + "--node-status-update-frequency": KubeConfigs[o.OrchestratorVersion]["nodestatusfreq"], + "--image-gc-high-threshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "--image-gc-low-threshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "--non-masquerade-cidr": DefaultNonMasqueradeCidr, + "--cloud-provider": "azure", + "--cloud-config": "/etc/kubernetes/azure.json", + "--azure-container-registry-config": "/etc/kubernetes/azure.json", + "--event-qps": DefaultKubeletEventQPS, + "--cadvisor-port": DefaultKubeletCadvisorPort, } // If no user-configurable kubelet config values exists, use the defaults From 0cf97423a366960627ec0f95050c12be190bd804 Mon Sep 17 00:00:00 2001 From: chshou Date: Mon, 26 Mar 2018 16:43:11 -0700 Subject: [PATCH 2/4] added example --- examples/kubernetes-aci.json | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 examples/kubernetes-aci.json diff --git a/examples/kubernetes-aci.json b/examples/kubernetes-aci.json new file mode 100644 index 0000000000..40f9762741 --- /dev/null +++ b/examples/kubernetes-aci.json @@ -0,0 +1,91 @@ +{ + "apiVersion": "vlabs", + "plan": {}, + "properties": { + "provisioningState": "", + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorVersion": "1.8.2", + "kubernetesConfig": { + "networkPolicy": "none", + "kubeletConfig": { + "--cloud-provider": "", + "--cloud-config": "", + "--azure-container-registry-config": "" + }, + "addons": [ + { + "name": "tiller", + "enabled" : false + }, + { + "name": "kubernetes-dashboard", + "enabled" : false + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "caas-test-eastus-linux-03", + "vmSize": "Standard_D2_v2", + "firstConsecutiveStaticIP": "10.240.255.5" + }, + "agentPoolProfiles": [ + { + "name": "system", + "count": 2, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux" + }, + { + "name": "agentpool1", + "count": 2, + "vmSize": "Standard_F2", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux" + }, + { + "name": "agentpool2", + "count": 3, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux", + "osDiskSizeGB": 50 + }, + { + "name": "agentpool3", + "count": 3, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux", + "osDiskSizeGB": 50 + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA3fOxIwM0O3dXfiQgcdvqvlTJyDj+bIfnAn54G2KFXDr+cvlpmVSCdy31TwNQFZR1Ts14qK1t+Jc96FX0oy4QQlDQAZDvh2rjOoMvBdR6vIG4VGLBOMJB6TV6jNyTFI8yc77OCfjvw7NTXhLyCazp8Hyh9Yb14wskhQTO/qt2PqZ1/b5z0I/5ZZbFVLdEBJudFkyceO1B2TGYSUBqyMiO+4DT8mFFE+Cf4NcrsXcNEUw3jqKmUYsKlvIf1jKJNNKp2zHaw67ByZm/kOKZVQucBXBzXwWvbbjrrGzIuXVgBrEyeLNAH5oWl3VyrL+6vKz2mN3wGuTrNcLefIezwTc/tQ==" + } + ] + } + }, + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "value-in-key-vault" + }, + "servicePrincipalProfile": { + "clientId": "597ec5f8-9e25-425b-be42-deaf90dc5af5", + "secret": "value-in-key-vault" + }, + "certificateProfile": {}, + "nameSuffix": "a03eus0l" + } +} From f2bfe5823599a714462792b68f0f2108833880d9 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 26 Mar 2018 14:45:30 -0700 Subject: [PATCH 3/4] move sed to kubelet.sh, remove unnecessary exit 0 (#2520) * move sed to kubelet.sh, remove unnecessary exit 0 * circleci bump * circleci bump --- parts/k8s/kubernetesagentcustomdata.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index df70ef3c88..02c0d2dfbe 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -129,8 +129,9 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER # SNAT outbound traffic from pods to destinations outside of VNET. iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE {{end}} - - exit 0 +{{if not EnablePodSecurityPolicy}} + sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service" +{{end}} - path: "/opt/azure/containers/provision.sh" permissions: "0744" @@ -168,9 +169,6 @@ coreos: [Service] ExecStart=/opt/azure/containers/provision-setup.sh {{else}} -{{if not EnablePodSecurityPolicy}} - sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service" -{{end}} runcmd: - echo `date`,`hostname`, startruncmd>>/opt/m # the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run From d155d0bd22773233e854049d2bd6370e503194b4 Mon Sep 17 00:00:00 2001 From: chshou Date: Mon, 26 Mar 2018 17:39:16 -0700 Subject: [PATCH 4/4] sanitize example --- examples/kubernetes-aci.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/kubernetes-aci.json b/examples/kubernetes-aci.json index 40f9762741..ee975bcb46 100644 --- a/examples/kubernetes-aci.json +++ b/examples/kubernetes-aci.json @@ -72,20 +72,19 @@ "ssh": { "publicKeys": [ { - "keyData": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEA3fOxIwM0O3dXfiQgcdvqvlTJyDj+bIfnAn54G2KFXDr+cvlpmVSCdy31TwNQFZR1Ts14qK1t+Jc96FX0oy4QQlDQAZDvh2rjOoMvBdR6vIG4VGLBOMJB6TV6jNyTFI8yc77OCfjvw7NTXhLyCazp8Hyh9Yb14wskhQTO/qt2PqZ1/b5z0I/5ZZbFVLdEBJudFkyceO1B2TGYSUBqyMiO+4DT8mFFE+Cf4NcrsXcNEUw3jqKmUYsKlvIf1jKJNNKp2zHaw67ByZm/kOKZVQucBXBzXwWvbbjrrGzIuXVgBrEyeLNAH5oWl3VyrL+6vKz2mN3wGuTrNcLefIezwTc/tQ==" + "keyData": "" } ] } }, "windowsProfile": { - "adminUsername": "azureuser", - "adminPassword": "value-in-key-vault" + "adminUsername": "", + "adminPassword": "" }, "servicePrincipalProfile": { - "clientId": "597ec5f8-9e25-425b-be42-deaf90dc5af5", - "secret": "value-in-key-vault" + "clientId": "", + "secret": "" }, - "certificateProfile": {}, - "nameSuffix": "a03eus0l" + "certificateProfile": {} } }