From 2faac8b4a61233fdd6e526ed5c1f646f4de82551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Fri, 30 Mar 2018 17:22:58 -0700 Subject: [PATCH 01/17] initial commit --- parts/k8s/kubernetesagentresourcesvmss.t | 2 +- parts/k8s/kubernetescustomscript.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index 189d4afc99..c03062e69e 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -154,4 +154,4 @@ } }, "type": "Microsoft.Compute/virtualMachineScaleSets" - } \ No newline at end of file + } diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index c5e09c61f6..48ca340df2 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -188,7 +188,7 @@ function ensureFilepath() { echo "Timeout waiting for $1" exit 6 fi - + } function setKubeletOpts () { From 13487e3dd78f35f980ffff757ec822cf6714e7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Wed, 4 Apr 2018 14:34:31 -0700 Subject: [PATCH 02/17] added parameters for vmType and primaryScaleSetName --- parts/k8s/kubernetesmastervars.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index be39cf51e3..55b7e39398 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -228,7 +228,7 @@ "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", {{if not IsOpenShift}} {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',variables('kubernetesClusterAutoscalerEnabled'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", {{if EnableEncryptionWithExternalKms}} {{ if not UseManagedIdentity}} "servicePrincipalObjectId": "[parameters('servicePrincipalObjectId')]", From 16789452aa7fb1e565d55dff108e46355bf3e35e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Thu, 5 Apr 2018 15:13:26 -0700 Subject: [PATCH 03/17] initial autoscaler addon commit --- examples/addons/cluster-autoscaler/README.md | 99 ++++++++ .../kubernetes-cluster-autoscaler.json | 61 +++++ ...raddons-cluster-autoscaler-deployment.yaml | 218 ++++++++++++++++++ parts/k8s/kubernetescustomscript.sh | 15 ++ parts/k8s/kubernetesmastercustomdata.yml | 4 + parts/k8s/kubernetesmastervars.t | 1 + parts/k8s/kubernetesparams.t | 7 + pkg/acsengine/addons.go | 5 + pkg/acsengine/const.go | 4 +- pkg/acsengine/defaults.go | 29 +++ pkg/acsengine/engine.go | 28 +++ pkg/acsengine/k8s_versions.go | 180 ++++++++------- pkg/acsengine/types.go | 1 + pkg/api/const.go | 4 + pkg/api/types.go | 11 + pkg/api/types_test.go | 30 +++ test/e2e/kubernetes/kubernetes_test.go | 17 ++ 17 files changed, 624 insertions(+), 90 deletions(-) create mode 100644 examples/addons/cluster-autoscaler/README.md create mode 100644 examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json create mode 100644 parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml diff --git a/examples/addons/cluster-autoscaler/README.md b/examples/addons/cluster-autoscaler/README.md new file mode 100644 index 0000000000..dd9b3f5657 --- /dev/null +++ b/examples/addons/cluster-autoscaler/README.md @@ -0,0 +1,99 @@ +# Cluster Autoscaler Add-on + +This is the Cluster Autoscaler add-on. Add this add-on to your json file as shown below to automatically enable cluster autoscaler in your new Kubernetes cluster. + +``` +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.10", + "kubernetesConfig": { + "addons": [ + { + "name": "cluster-autoscaler", + "enabled" : true, + "config": { + "clientId": "", + "clientKey": "", + "tenantId": "", + "subscriptionId": "", + "resourceGroup": "", + "region": "eastus" + }, + "containers": [ + { + "name": "cluster-autoscaler", + "image": "gcr.io/google_containers/cluster-autoscaler:1.2", + "cpuRequests": "50m", + "memoryRequests": "150Mi", + "cpuLimits": "50m", + "memoryLimits": "150Mi" + } + ] + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 3, + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } + } +``` + +You should see cluster autoscaler as running after running: + +``` +$ kubectl get pods -n kube-system +``` + +Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler for examples. + +# Configuration + +| Name | Required | Description | Default Value | +| -------------- | -------- | ----------------------------------------------- | ---------------------------------- | +| clientId | yes | your client id | | +| clientKey | yes | your client key | | +| tenantId | yes | your tenant id | | +| resourceGroup | yes | your resource group | | +| region | no | Azure region | "westus" | +| nodeName | no | node name | "aci-connector" | +| os | no | operating system (Linux/Windows) | "Linux" | +| taint | no | apply taint to node, making scheduling explicit | "azure.com/aci" | +| name | no | container name | "aci-connector" | +| image | no | image | "microsoft/virtual-kubelet:latest" | +| cpuRequests | no | cpu requests for the container | "50m" | +| memoryRequests | no | memory requests for the container | "150Mi" | +| cpuLimits | no | cpu limits for the container | "50m" | +| memoryLimits | no | memory limits for the container | "150Mi" | + +# Supported Orchestrators + +Kubernetes diff --git a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json new file mode 100644 index 0000000000..f9686e92ec --- /dev/null +++ b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json @@ -0,0 +1,61 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "kubernetesConfig": { + "addons": [ + { + "name": "aci-connector", + "enabled" : true, + "config": { + "clientId": "", + "clientKey": "", + "tenantId": "", + "subscriptionId": "", + "resourceGroup": "", + "region": "eastus" + }, + "containers": [ + { + "name": "aci-connector", + "image": "microsoft/aci-connector-k8s:latest", + "cpuRequests": "50m", + "memoryRequests": "150Mi", + "cpuLimits": "50m", + "memoryLimits": "150Mi" + } + ] + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 3, + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "AvailabilitySet" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } + } diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml new file mode 100644 index 0000000000..55d031247d --- /dev/null +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -0,0 +1,218 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["events","endpoints"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get","update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch","list","get","update"] +- apiGroups: [""] + resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] + verbs: ["watch","list","get"] +- apiGroups: ["extensions"] + resources: ["replicasets","daemonsets"] + verbs: ["watch","list","get"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch","list"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["watch","list","get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status"] + verbs: ["delete","get","update"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: v1 +data: + ClientID: + ClientSecret: + ResourceGroup: + SubscriptionID: + TenantID: + VMType: + Deployment: +kind: Secret +metadata: + name: cluster-autoscaler-azure + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + containers: + - image: + imagePullPolicy: Always + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=3 + - --logtostderr=true + - --cloud-provider=azure + - --skip-nodes-with-local-storage=false + {{}} + - --nodes=:: + {{}} + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: cluster-autoscaler-azure + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: cluster-autoscaler-azure + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: cluster-autoscaler-azure + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: cluster-autoscaler-azure + +{{}} + - name: ARM_DEPLOYMENT + valueFrom: + secretKeyRef: + key: Deployment + name: cluster-autoscaler-azure +{{}} + + volumeMounts: + - mountPath: /etc/ssl/certs/ca-certificates.crt + name: ssl-certs + readOnly: true + +{{}} + - mountPath: /var/lib/azure/ + name: deploy-parameters + readOnly: true + dnsPolicy: ClusterFirst +{{}} + + restartPolicy: Always + volumes: + - hostPath: + path: /etc/ssl/certs/ca-certificates.crt + type: "" + name: ssl-certs + +{{}} + - name: deploy-parameters + secret: + secretName: cluster-autoscaler-azure-deploy-parameters + items: + - key: deploy-parameters + path: azuredeploy.parameters.json +{{}} \ No newline at end of file diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 48ca340df2..c5ed0186da 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -240,6 +240,21 @@ function configNetworkPlugin() { fi } +function configAddons() { + if [[ "${CLUSTER_AUTOSCALER_ADDON}" = true ]]; then + configClusterAutoscalerAddon +} + +function configClusterAutoscalerAddon() { + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + #TODO: this is for standard only, change this when we have VMSS PR (#) merged + sed -i "s||c3RhbmRhcmQ=|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" +} + function installClearContainersRuntime() { # Add Clear Containers repository key echo "Adding Clear Containers repository key..." diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 9265ed0f63..1ee1771e14 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -251,6 +251,10 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesACIConnectorMemoryLimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" {{end}} +{{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" +{{end}} + {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} sed -i "s||{{WrapAsVariable "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 55b7e39398..b0485c9dc5 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -109,6 +109,7 @@ "kubernetesACIConnectorMemoryRequests": "[parameters('kubernetesACIConnectorMemoryRequests')]", "kubernetesACIConnectorCPULimit": "[parameters('kubernetesACIConnectorCPULimit')]", "kubernetesACIConnectorMemoryLimit": "[parameters('kubernetesACIConnectorMemoryLimit')]", + "kubernetesClusterAutoscalerSpec": "[parameters('kubernetesClusterAutoscalerSpec')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 35b8dad041..6f0dd010bd 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -444,6 +444,13 @@ }, "type": "string" }, + "kubernetesClusterAutoscalerSpec": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerSpec"}} + "metadata": { + "description": "The container spec for the cluster autoscaler." + }, + "type": "string" + }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index 7494ca242e..814315774a 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -57,6 +57,11 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet "aci-connector-deployment.yaml", profile.OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled(), }, + { + "kubernetesmasteraddons-cluster-autoscaler-deployment.yaml", + "cluster-autoscaler-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled(), + }, { "kubernetesmasteraddons-kube-rescheduler-deployment.yaml", "kube-rescheduler-deployment.yaml", diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fe33c1cdc4..6f4005f286 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -104,8 +104,8 @@ const ( DefaultACIConnectorAddonName = "aci-connector" // DefaultDashboardAddonName is the name of the kubernetes-dashboard addon deployment DefaultDashboardAddonName = "kubernetes-dashboard" - // DefaultACIConnectorImage defines the ACI Connector deployment version on Kubernetes Clusters - DefaultACIConnectorImage = "virtual-kubelet:latest" + // DefaultClusterAutoscalerAddonName is the name of the autoscaler addon deployment + DefaultClusterAutoscalerAddonName = "cluster-autoscaler" // DefaultKubernetesDNSServiceIP specifies the IP address that kube-dns // listens on by default. must by in the default Service CIDR range. DefaultKubernetesDNSServiceIP = "10.0.0.10" diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 9f30cea5b4..89e0be55b2 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -34,6 +34,7 @@ var ( KubernetesImageBase: "k8s-gcrio.azureedge.net/", TillerImageBase: "gcrio.azureedge.net/kubernetes-helm/", ACIConnectorImageBase: "microsoft/", + ClusterAutoscalerImageBase: "http://gcrio.azureedge.net/google-containers/cluster-autoscaler/", EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", WindowsPackageSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", @@ -244,6 +245,24 @@ var ( }, } + // DefaultClusterAutoscalerAddonsConfig is the default cluster autoscaler addon config + DefaultClusterAutoscalerAddonsConfig = api.KubernetesAddon{ + Name: DefaultClusterAutoscalerAddonName, + Enabled: helpers.PointerToBool(api.DefaultClusterAutoscalerAddonEnabled), + Config: map[string]string{ + "foo": "bar", + }, + Containers: []api.KubernetesContainerSpec{ + { + Name: DefaultClusterAutoscalerAddonName, + CPURequests: "50m", + MemoryRequests: "150Mi", + CPULimits: "50m", + MemoryLimits: "150Mi", + }, + }, + } + // DefaultDashboardAddonsConfig is the default kubernetes-dashboard addon Config DefaultDashboardAddonsConfig = api.KubernetesAddon{ Name: DefaultDashboardAddonName, @@ -352,6 +371,7 @@ func setOrchestratorDefaults(cs *api.ContainerService) { o.KubernetesConfig.Addons = []api.KubernetesAddon{ DefaultTillerAddonsConfig, DefaultACIConnectorAddonsConfig, + DefaultClusterAutoscalerAddonsConfig, DefaultDashboardAddonsConfig, DefaultReschedulerAddonsConfig, DefaultMetricsServerAddonsConfig, @@ -369,6 +389,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) { // Provide default acs-engine config for ACI Connector o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultACIConnectorAddonsConfig) } + s := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + if s < 0 { + // Provide default acs-engine config for cluster autoscaler + o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonsConfig) + } d := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultDashboardAddonName) if d < 0 { // Provide default acs-engine config for Dashboard @@ -465,6 +490,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if a.OrchestratorProfile.KubernetesConfig.Addons[c].IsEnabled(api.DefaultACIConnectorAddonEnabled) { a.OrchestratorProfile.KubernetesConfig.Addons[c] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[c], DefaultACIConnectorAddonsConfig) } + s := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + if a.OrchestratorProfile.KubernetesConfig.Addons[s].IsEnabled(api.DefaultClusterAutoscalerAddonEnabled) { + a.OrchestratorProfile.KubernetesConfig.Addons[s] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[s], DefaultClusterAutoscalerAddonsConfig) + } d := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) if a.OrchestratorProfile.KubernetesConfig.Addons[d].IsEnabled(api.DefaultDashboardAddonEnabled) { a.OrchestratorProfile.KubernetesConfig.Addons[d] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[d], DefaultDashboardAddonsConfig) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index b6d2af215d..10d1496144 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -658,6 +658,24 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesACIConnectorSpec", cloudSpecConfig.KubernetesSpecConfig.ACIConnectorImageBase+KubeConfigs[k8sVersion][DefaultACIConnectorAddonName]) } } + clusterAutoscalerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + c = getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesClusterAutoscalerClientId", clusterAutoscalerAddon.Config["clientId"]) + addSecret(parametersMap, "kubernetesClusterAutoscalerClientKey", clusterAutoscalerAddon.Config["clientKey"], false) + addValue(parametersMap, "kubernetesClusterAutoscalerTenantId", clusterAutoscalerAddon.Config["tenantId"]) + addValue(parametersMap, "kubernetesClusterAutoscalerSubscriptionId", clusterAutoscalerAddon.Config["subscriptionId"]) + addValue(parametersMap, "kubernetesClusterAutoscalerResourceGroup", clusterAutoscalerAddon.Config["resourceGroup"]) + addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) + if clusterAutoscalerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.ClusterAutoscalerImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) + } + } dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) c = getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) if c > -1 { @@ -1563,6 +1581,8 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat tC := getAddonContainersIndexByName(tillerAddon.Containers, DefaultTillerAddonName) aciConnectorAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultACIConnectorAddonName) aC := getAddonContainersIndexByName(aciConnectorAddon.Containers, DefaultACIConnectorAddonName) + clusterAutoscalerAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + aS := getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) dashboardAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) dC := getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) reschedulerAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultReschedulerAddonName) @@ -1708,6 +1728,14 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } else { val = "" } + case "kubernetesClusterAutoscalerSpec": + if aS > -1 { + if clusterAutoscalerAddon.Containers[aS].Image != "" { + val = clusterAutoscalerAddon.Containers[aS].Image + } else { + val = cloudSpecConfig.KubernetesSpecConfig.ClusterAutoscalerImageBase + KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName] + } + } case "kubernetesTillerSpec": if tC > -1 { if tillerAddon.Containers[tC].Image != "" { diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 1e5d22f0b8..1bb32b40ef 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -36,58 +36,60 @@ var k8sComponentVersions = map[string]map[string]string{ "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.10": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.7", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + "cluster-autoscaler": "cluster-autoscaler:v1.2.0", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.9": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.7", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + "cluster-autoscaler": "cluster-autoscaler:v1.1.2", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.8": { "dockerEngine": "1.13.*", @@ -271,25 +273,26 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.10"]["addon-resizer"], "heapster": k8sComponentVersions["1.10"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.10"]["metrics-server"], - "dns": k8sComponentVersions["1.10"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.10"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.10"]["dnsmasq"], - "pause": k8sComponentVersions["1.10"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.10"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.10"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.10"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.10"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.10"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.10"]["podeviction"], - "routeperiod": k8sComponentVersions["1.10"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.10"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.10"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.10"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.10"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.10"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.10"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.10"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.10"]["gclowthreshold"], + "dns": k8sComponentVersions["1.10"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.10"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.10"]["dnsmasq"], + "pause": k8sComponentVersions["1.10"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.10"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.10"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.10"]["aci-connector"], + "nodestatusfreq": k8sComponentVersions["1.10"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.10"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.10"]["podeviction"], + "routeperiod": k8sComponentVersions["1.10"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.10"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.10"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.10"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.10"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.10"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.10"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.10"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.10"]["gclowthreshold"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.10"]["cluster-autoscaler"], } case "1.9": ret = map[string]string{ @@ -302,25 +305,26 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], } case "1.8": ret = map[string]string{ diff --git a/pkg/acsengine/types.go b/pkg/acsengine/types.go index 3aaf126b11..ae05cca116 100644 --- a/pkg/acsengine/types.go +++ b/pkg/acsengine/types.go @@ -50,6 +50,7 @@ type KubernetesSpecConfig struct { KubernetesImageBase string TillerImageBase string ACIConnectorImageBase string + ClusterAutoscalerImageBase string EtcdDownloadURLBase string KubeBinariesSASURLBase string WindowsPackageSASURLBase string diff --git a/pkg/api/const.go b/pkg/api/const.go index 039b497f94..4097111b9c 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -96,6 +96,8 @@ const ( DefaultTillerAddonEnabled = true // DefaultACIConnectorAddonEnabled determines the acs-engine provided default for enabling aci connector addon DefaultACIConnectorAddonEnabled = false + // DefaultClusterAutoscalerAddonEnabled determines the acs-engine provided default for enabling cluster autoscaler addon + DefaultClusterAutoscalerAddonEnabled = false // DefaultDashboardAddonEnabled determines the acs-engine provided default for enabling kubernetes-dashboard addon DefaultDashboardAddonEnabled = true // DefaultReschedulerAddonEnabled determines the acs-engine provided default for enabling kubernetes-rescheduler addon @@ -112,6 +114,8 @@ const ( DefaultTillerAddonName = "tiller" // DefaultACIConnectorAddonName is the name of the tiller addon deployment DefaultACIConnectorAddonName = "aci-connector" + // DefaultClusterAutoscalerAddonName is the name of the cluster autoscaler addon deployment + DefaultClusterAutoscalerAddonName = "cluster-autoscaler" // DefaultDashboardAddonName is the name of the kubernetes-dashboard addon deployment DefaultDashboardAddonName = "kubernetes-dashboard" // DefaultReschedulerAddonName is the name of the rescheduler addon deployment diff --git a/pkg/api/types.go b/pkg/api/types.go index 50a79a4e5d..81258a23fb 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -835,6 +835,17 @@ func (k *KubernetesConfig) IsACIConnectorEnabled() bool { return aciConnectorAddon.IsEnabled(DefaultACIConnectorAddonEnabled) } +// IsClusterAutoscalerEnabled checks if the cluster autoscaler addon is enabled +func (k *KubernetesConfig) IsClusterAutoscalerEnabled() bool { + var clusterAutoscalerAddon KubernetesAddon + for i := range k.Addons { + if k.Addons[i].Name == DefaultClusterAutoscalerAddonName { + clusterAutoscalerAddon = k.Addons[i] + } + } + return clusterAutoscalerAddon.IsEnabled(DefaultClusterAutoscalerAddonEnabled) +} + // IsDashboardEnabled checks if the kubernetes-dashboard addon is enabled func (k *KubernetesConfig) IsDashboardEnabled() bool { var dashboardAddon KubernetesAddon diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 0a07bba4df..5795b12c10 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -144,6 +144,36 @@ func TestIsACIConnectorEnabled(t *testing.T) { } } +func TestIsClusterAutoscalerEnabled(t *testing.T) { + c := KubernetesConfig{ + Addons: []KubernetesAddon{ + getMockAddon("addon"), + }, + } + enabled := c.IsClusterAutoscalerEnabled() + if enabled != DefaultClusterAutoscalerAddonEnabled { + t.Fatalf("KubernetesConfig.IsAutoscalerEnabled() should return %t when no cluster autoscaler addon has been specified, instead returned %t", DefaultClusterAutoscalerAddonEnabled, enabled) + } + c.Addons = append(c.Addons, getMockAddon(DefaultClusterAutoscalerAddonName)) + enabled = c.IsClusterAutoscalerEnabled() + if enabled { + t.Fatalf("KubernetesConfig.IsClusterAutoscalerEnabled() should return true when cluster autoscaler has been specified, instead returned %t", enabled) + } + b := true + c = KubernetesConfig{ + Addons: []KubernetesAddon{ + { + Name: DefaultClusterAutoscalerAddonName, + Enabled: &b, + }, + }, + } + enabled = c.IsClusterAutoscalerEnabled() + if !enabled { + t.Fatalf("KubernetesConfig.IsClusterAutoscalerEnabled() should return false when cluster autoscaler addon has been specified as disabled, instead returned %t", enabled) + } +} + func TestIsDashboardEnabled(t *testing.T) { c := KubernetesConfig{ Addons: []KubernetesAddon{ diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index 8edc55846b..886a9575c4 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -239,6 +239,23 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } }) + It("should have cluster-autoscaler running", func() { + if hasClusterAutoscaler, clusterAutoscalerAddon := eng.HasAddon("autoscaler"); hasClusterAutoscaler { + running, err := pod.WaitOnReady("cluster-autoscaler", "kube-system", 3, 30*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(running).To(Equal(true)) + By("Ensuring that the correct resources have been applied") + pods, err := pod.GetAllByPrefix("cluster-autoscaler", "kube-system") + Expect(err).NotTo(HaveOccurred()) + for i, c := range clusterAutoscalerAddon.Containers { + err := pods[0].Spec.Containers[i].ValidateResources(c) + Expect(err).NotTo(HaveOccurred()) + } + } else { + Skip("cluster autoscaler disabled for this cluster, will not test") + } + }) + It("should have rescheduler running", func() { if hasRescheduler, reschedulerAddon := eng.HasAddon("rescheduler"); hasRescheduler { running, err := pod.WaitOnReady("rescheduler", "kube-system", 3, 30*time.Second, cfg.Timeout) From d6b37474fa748f01d342ec8fa4b6b84871f7982b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Fri, 6 Apr 2018 22:25:18 -0700 Subject: [PATCH 04/17] add config variables --- ...raddons-cluster-autoscaler-deployment.yaml | 56 +++++++------------ parts/k8s/kubernetescustomscript.sh | 4 +- parts/k8s/kubernetesmastercustomdata.yml | 13 +++++ parts/k8s/kubernetesmastervars.t | 7 +++ parts/k8s/kubernetesparams.t | 49 ++++++++++++++++ pkg/acsengine/defaults.go | 12 ++-- pkg/acsengine/engine.go | 8 +-- 7 files changed, 102 insertions(+), 47 deletions(-) diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml index 55d031247d..6b19467474 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -4,6 +4,8 @@ metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" name: cluster-autoscaler namespace: kube-system --- @@ -14,6 +16,8 @@ metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" rules: - apiGroups: [""] resources: ["events","endpoints"] @@ -55,6 +59,8 @@ metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" rules: - apiGroups: [""] resources: ["configmaps"] @@ -71,6 +77,8 @@ metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -88,6 +96,8 @@ metadata: labels: k8s-addon: cluster-autoscaler.addons.k8s.io k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -105,17 +115,21 @@ data: SubscriptionID: TenantID: VMType: - Deployment: kind: Secret metadata: name: cluster-autoscaler-azure namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" --- apiVersion: extensions/v1beta1 kind: Deployment metadata: labels: app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" name: cluster-autoscaler namespace: kube-system spec: @@ -135,20 +149,18 @@ spec: name: cluster-autoscaler resources: limits: - cpu: 100m - memory: 300Mi + cpu: + memory: requests: - cpu: 100m - memory: 300Mi + cpu: + memory: command: - ./cluster-autoscaler - --v=3 - --logtostderr=true - --cloud-provider=azure - --skip-nodes-with-local-storage=false - {{}} - - --nodes=:: - {{}} + - --nodes=:: env: - name: ARM_SUBSCRIPTION_ID valueFrom: @@ -180,39 +192,13 @@ spec: secretKeyRef: key: VMType name: cluster-autoscaler-azure - -{{}} - - name: ARM_DEPLOYMENT - valueFrom: - secretKeyRef: - key: Deployment - name: cluster-autoscaler-azure -{{}} - volumeMounts: - mountPath: /etc/ssl/certs/ca-certificates.crt name: ssl-certs readOnly: true - -{{}} - - mountPath: /var/lib/azure/ - name: deploy-parameters - readOnly: true - dnsPolicy: ClusterFirst -{{}} - restartPolicy: Always volumes: - hostPath: path: /etc/ssl/certs/ca-certificates.crt type: "" - name: ssl-certs - -{{}} - - name: deploy-parameters - secret: - secretName: cluster-autoscaler-azure-deploy-parameters - items: - - key: deploy-parameters - path: azuredeploy.parameters.json -{{}} \ No newline at end of file + name: ssl-certs \ No newline at end of file diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index c5ed0186da..fb586fa095 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -251,8 +251,7 @@ function configClusterAutoscalerAddon() { sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - #TODO: this is for standard only, change this when we have VMSS PR (#) merged - sed -i "s||c3RhbmRhcmQ=|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" } function installClearContainersRuntime() { @@ -511,6 +510,7 @@ ensureDockerInstallCompleted ensureDocker echo `date`,`hostname`, configNetworkPluginStart>>/opt/m configNetworkPlugin +configAddons if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then # Ensure we can nest virtualization if grep -q vmx /proc/cpuinfo; then diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 1ee1771e14..bbafc237af 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -253,6 +253,19 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerClientId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerClientSecret"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerResourceGroup"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSubscriptionId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerTenantId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerVmType"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerVMSSName"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index b0485c9dc5..0d442e3939 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -110,6 +110,13 @@ "kubernetesACIConnectorCPULimit": "[parameters('kubernetesACIConnectorCPULimit')]", "kubernetesACIConnectorMemoryLimit": "[parameters('kubernetesACIConnectorMemoryLimit')]", "kubernetesClusterAutoscalerSpec": "[parameters('kubernetesClusterAutoscalerSpec')]", + "kubernetesClusterAutoscalerCPULimit": "[parameters('kubernetesClusterAutoscalerCPULimit')]", + "kubernetesClusterAutoscalerMemoryLimit": "[parameters('kubernetesClusterAutoscalerMemoryLimit')]", + "kubernetesClusterAutoscalerCPURequests": "[parameters('kubernetesClusterAutoscalerCPURequests')]", + "kubernetesClusterAutoscalerMemoryRequests": "[parameters('kubernetesClusterAutoscalerMemoryRequests')]", + "kubernetesClusterAutoscalerMinNodes": "[parameters('kubernetesClusterAutoscalerMinNodes')]", + "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", + "kubernetesClusterAutoscalerVMSSName": "[parameters('kubernetesClusterAutoscalerVMSSName')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 6f0dd010bd..fa8c3a4fb0 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -451,6 +451,55 @@ }, "type": "string" }, + "kubernetesClusterAutoscalerCPULimit": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerCPULimit"}} + "metadata": { + "description": "Cluster autoscaler cpu limit" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMemoryLimit": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMemoryLimit"}} + "metadata": { + "description": "Cluster autoscaler memory limit" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerCPURequests": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerCPURequests"}} + "metadata": { + "description": "Cluster autoscaler cpu requests" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMemoryRequests": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMemoryRequests"}} + "metadata": { + "description": "Cluster autoscaler memory requests" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMinNodes": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMinNodes"}} + "metadata": { + "description": "Cluster autoscaler min nodes" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMaxNodes": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMaxNodes"}} + "metadata": { + "description": "Cluster autoscaler max nodes" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerVMSSName": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerVMSSName"}} + "metadata": { + "description": "Cluster autoscaler VMSS name" + }, + "type": "string" + }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 89e0be55b2..c5734c3309 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -250,15 +250,17 @@ var ( Name: DefaultClusterAutoscalerAddonName, Enabled: helpers.PointerToBool(api.DefaultClusterAutoscalerAddonEnabled), Config: map[string]string{ - "foo": "bar", + "minNodes": "1", + "maxNodes": "5", + "vmssName": "vmss", }, Containers: []api.KubernetesContainerSpec{ { Name: DefaultClusterAutoscalerAddonName, - CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", + CPURequests: "100m", + MemoryRequests: "300Mi", + CPULimits: "100m", + MemoryLimits: "300Mi", }, }, } diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 10d1496144..d32bb44a61 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -661,15 +661,13 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s clusterAutoscalerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) c = getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) if c > -1 { - addValue(parametersMap, "kubernetesClusterAutoscalerClientId", clusterAutoscalerAddon.Config["clientId"]) - addSecret(parametersMap, "kubernetesClusterAutoscalerClientKey", clusterAutoscalerAddon.Config["clientKey"], false) - addValue(parametersMap, "kubernetesClusterAutoscalerTenantId", clusterAutoscalerAddon.Config["tenantId"]) - addValue(parametersMap, "kubernetesClusterAutoscalerSubscriptionId", clusterAutoscalerAddon.Config["subscriptionId"]) - addValue(parametersMap, "kubernetesClusterAutoscalerResourceGroup", clusterAutoscalerAddon.Config["resourceGroup"]) addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[c].CPURequests) addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[c].CPULimits) addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[c].MemoryRequests) addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerVMSSName", clusterAutoscalerAddon.Config["vmssName"]) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { From 92f42993292980134cb19f7786157e443b7d13e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Fri, 6 Apr 2018 23:30:33 -0700 Subject: [PATCH 05/17] update --- parts/k8s/kubernetescustomscript.sh | 20 ++++++++++++-------- parts/k8s/kubernetesmastercustomdata.yml | 21 +++++++-------------- parts/k8s/kubernetesmastervars.t | 3 +-- parts/k8s/kubernetesparams.t | 7 ------- pkg/acsengine/defaults.go | 2 +- pkg/acsengine/engine.go | 1 - 6 files changed, 21 insertions(+), 33 deletions(-) diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index fb586fa095..0168401d45 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -241,17 +241,21 @@ function configNetworkPlugin() { } function configAddons() { - if [[ "${CLUSTER_AUTOSCALER_ADDON}" = true ]]; then - configClusterAutoscalerAddon + configClusterAutoscalerAddon + + # if [[ "${CLUSTER_AUTOSCALER_ADDON}" = true ]]; then + # configClusterAutoscalerAddon + # fi } function configClusterAutoscalerAddon() { - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" } function installClearContainersRuntime() { diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index bbafc237af..54d50072c4 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -252,20 +252,13 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerClientId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerClientSecret"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerResourceGroup"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSubscriptionId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerTenantId"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - # sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerVmType"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" - sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerVMSSName"}}|g" "/etc/kubernetes/addons/kube-cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 0d442e3939..1d0a0c2f83 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -116,7 +116,6 @@ "kubernetesClusterAutoscalerMemoryRequests": "[parameters('kubernetesClusterAutoscalerMemoryRequests')]", "kubernetesClusterAutoscalerMinNodes": "[parameters('kubernetesClusterAutoscalerMinNodes')]", "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", - "kubernetesClusterAutoscalerVMSSName": "[parameters('kubernetesClusterAutoscalerVMSSName')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", @@ -147,7 +146,7 @@ "vnetCidr": "[parameters('vnetCidr')]", "gcHighThreshold":"[parameters('gcHighThreshold')]", "gcLowThreshold":"[parameters('gcLowThreshold')]", -{{if EnableDataEncryptionAtRest}} +{{if EnableDataEncryptionAtRest}} "etcdEncryptionKey": "[parameters('etcdEncryptionKey')]", {{end}} {{ if UseManagedIdentity }} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index fa8c3a4fb0..aeba35be94 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -493,13 +493,6 @@ }, "type": "string" }, - "kubernetesClusterAutoscalerVMSSName": { - {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerVMSSName"}} - "metadata": { - "description": "Cluster autoscaler VMSS name" - }, - "type": "string" - }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index c5734c3309..29f2923158 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -34,7 +34,7 @@ var ( KubernetesImageBase: "k8s-gcrio.azureedge.net/", TillerImageBase: "gcrio.azureedge.net/kubernetes-helm/", ACIConnectorImageBase: "microsoft/", - ClusterAutoscalerImageBase: "http://gcrio.azureedge.net/google-containers/cluster-autoscaler/", + ClusterAutoscalerImageBase: "gcrio.azureedge.net/google-containers/", EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", WindowsPackageSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index d32bb44a61..872b4defdd 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -667,7 +667,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) - addValue(parametersMap, "kubernetesClusterAutoscalerVMSSName", clusterAutoscalerAddon.Config["vmssName"]) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { From be907df4bee7a4f8baa418be912197c71066f6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Mon, 9 Apr 2018 11:59:01 -0700 Subject: [PATCH 06/17] update docs --- examples/addons/cluster-autoscaler/README.md | 61 +++++------- .../kubernetes-cluster-autoscaler.json | 96 ++++++++----------- pkg/acsengine/defaults.go | 1 - 3 files changed, 65 insertions(+), 93 deletions(-) diff --git a/examples/addons/cluster-autoscaler/README.md b/examples/addons/cluster-autoscaler/README.md index dd9b3f5657..6da94e3035 100644 --- a/examples/addons/cluster-autoscaler/README.md +++ b/examples/addons/cluster-autoscaler/README.md @@ -1,6 +1,13 @@ -# Cluster Autoscaler Add-on +# Cluster Autoscaler (VMSS) Add-on -This is the Cluster Autoscaler add-on. Add this add-on to your json file as shown below to automatically enable cluster autoscaler in your new Kubernetes cluster. +Cluster Autoscaler is a tool that automatically adjusts the size of the Kubernetes cluster when: + +* there are pods that failed to run in the cluster due to insufficient resources. +* some nodes in the cluster are so underutilized, for an extended period of time, that they can be deleted and their pods will be easily placed on some other, existing nodes. + +This is the Kubernetes Cluster Autoscaler add-on for Virtual Machine Scale Sets. Add this add-on to your json file as shown below to automatically enable cluster autoscaler in your new Kubernetes cluster. + +To use this add-on, make sure your cluster's Kubernetes version is 1.10 or above, and agent pool `availabilityProfile` is set to `VirtualMachineScaleSets`. This will automatically enable first agent pool to autoscale from 1 to 5 nodes by default. You can override these settings in `config` section of the `cluster-autoscaler` add-on. ``` { @@ -15,23 +22,9 @@ This is the Cluster Autoscaler add-on. Add this add-on to your json file as show "name": "cluster-autoscaler", "enabled" : true, "config": { - "clientId": "", - "clientKey": "", - "tenantId": "", - "subscriptionId": "", - "resourceGroup": "", - "region": "eastus" - }, - "containers": [ - { - "name": "cluster-autoscaler", - "image": "gcr.io/google_containers/cluster-autoscaler:1.2", - "cpuRequests": "50m", - "memoryRequests": "150Mi", - "cpuLimits": "50m", - "memoryLimits": "150Mi" - } - ] + "minNodes": "1", + "maxNodes": "5" + } } ] } @@ -44,9 +37,9 @@ This is the Cluster Autoscaler add-on. Add this add-on to your json file as show "agentPoolProfiles": [ { "name": "agentpool", - "count": 3, + "count": 1, "vmSize": "Standard_DS2_v2", - "availabilityProfile": "AvailabilitySet" + "availabilityProfile": "VirtualMachineScaleSets" } ], "linuxProfile": { @@ -77,22 +70,16 @@ Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluste # Configuration -| Name | Required | Description | Default Value | -| -------------- | -------- | ----------------------------------------------- | ---------------------------------- | -| clientId | yes | your client id | | -| clientKey | yes | your client key | | -| tenantId | yes | your tenant id | | -| resourceGroup | yes | your resource group | | -| region | no | Azure region | "westus" | -| nodeName | no | node name | "aci-connector" | -| os | no | operating system (Linux/Windows) | "Linux" | -| taint | no | apply taint to node, making scheduling explicit | "azure.com/aci" | -| name | no | container name | "aci-connector" | -| image | no | image | "microsoft/virtual-kubelet:latest" | -| cpuRequests | no | cpu requests for the container | "50m" | -| memoryRequests | no | memory requests for the container | "150Mi" | -| cpuLimits | no | cpu limits for the container | "50m" | -| memoryLimits | no | memory limits for the container | "150Mi" | +| Name | Required | Description | Default Value | +| -------------- | -------- | --------------------------------- | ---------------------------------------------------------- | +| minNodes | no | minimum node count | | +| maxNodes | no | maximum node count | | +| name | no | container name | "cluster-autoscaler" | +| image | no | image | "gcrio.azureedge.net/google-containers/cluster-autoscaler" | +| cpuRequests | no | cpu requests for the container | "100m" | +| memoryRequests | no | memory requests for the container | "300Mi" | +| cpuLimits | no | cpu limits for the container | "100m" | +| memoryLimits | no | memory limits for the container | "300Mi" | # Supported Orchestrators diff --git a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json index f9686e92ec..adf40bfba6 100644 --- a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json +++ b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json @@ -1,61 +1,47 @@ { - "apiVersion": "vlabs", - "properties": { - "orchestratorProfile": { - "orchestratorType": "Kubernetes", - "kubernetesConfig": { - "addons": [ - { - "name": "aci-connector", - "enabled" : true, - "config": { - "clientId": "", - "clientKey": "", - "tenantId": "", - "subscriptionId": "", - "resourceGroup": "", - "region": "eastus" - }, - "containers": [ - { - "name": "aci-connector", - "image": "microsoft/aci-connector-k8s:latest", - "cpuRequests": "50m", - "memoryRequests": "150Mi", - "cpuLimits": "50m", - "memoryLimits": "150Mi" - } - ] + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "kubernetesConfig": { + "addons": [ + { + "name": "cluster-autoscaler", + "enabled": true, + "config": { + "minNodes": "1", + "maxNodes": "5" } - ] - } - }, - "masterProfile": { + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", "count": 1, - "dnsPrefix": "", - "vmSize": "Standard_DS2_v2" - }, - "agentPoolProfiles": [ - { - "name": "agentpool", - "count": 3, - "vmSize": "Standard_DS2_v2", - "availabilityProfile": "AvailabilitySet" - } - ], - "linuxProfile": { - "adminUsername": "azureuser", - "ssh": { - "publicKeys": [ - { - "keyData": "" - } - ] - } - }, - "servicePrincipalProfile": { - "clientId": "", - "secret": "" + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "VirtualMachineScaleSets" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" } } +} diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 29f2923158..aeb04a33a4 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -252,7 +252,6 @@ var ( Config: map[string]string{ "minNodes": "1", "maxNodes": "5", - "vmssName": "vmss", }, Containers: []api.KubernetesContainerSpec{ { From 939e938f623ee2a18cb92b81f26f89a9de4425bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Mon, 9 Apr 2018 16:07:54 -0700 Subject: [PATCH 07/17] add validation --- pkg/api/vlabs/validate.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index f4e97f36a1..908979af89 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -676,6 +676,9 @@ func (a *Properties) Validate(isUpdate bool) error { if e := validateVNET(a); e != nil { return e } + if e := validateAddons(a); e != nil { + return e + } if a.AADProfile != nil { if a.OrchestratorProfile.OrchestratorType != Kubernetes { @@ -1099,6 +1102,25 @@ func validateVNET(a *Properties) error { return nil } +func validateAddons(a *Properties) error { + if a.OrchestratorProfile.KubernetesConfig != nil && a.OrchestratorProfile.KubernetesConfig.Addons != nil { + var isAvailabilitySets bool + + for _, agentPool := range a.AgentPoolProfiles { + if len(agentPool.AvailabilityProfile) == 0 || agentPool.IsAvailabilitySets() { + isAvailabilitySets = true + } + } + + for _, addon := range a.OrchestratorProfile.KubernetesConfig.Addons { + if addon.Name == "cluster-autoscaler" && *addon.Enabled && isAvailabilitySets { + return fmt.Errorf("Cluster Autoscaler add-on can only be used with VirtualMachineScaleSets. Please specify \"availabilityProfile\": \"%s\"", VirtualMachineScaleSets) + } + } + } + return nil +} + // GetVNETSubnetIDComponents extract subscription, resourcegroup, vnetname, subnetname from the vnetSubnetID func GetVNETSubnetIDComponents(vnetSubnetID string) (string, string, string, string, error) { vnetSubnetIDRegex := `^\/subscriptions\/([^\/]*)\/resourceGroups\/([^\/]*)\/providers\/Microsoft.Network\/virtualNetworks\/([^\/]*)\/subnets\/([^\/]*)$` From 1cda4464cc1009a65efa37ebf60b25972a72ff43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Mon, 9 Apr 2018 19:17:09 -0700 Subject: [PATCH 08/17] config --- parts/k8s/kubernetescustomscript.sh | 40 ++++++++++++++++------------- parts/k8s/kubernetesmastervars.t | 1 + parts/k8s/kubernetesparams.t | 7 +++++ pkg/acsengine/engine.go | 1 + 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 0168401d45..921ce22dec 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -52,6 +52,25 @@ ensureDockerInstallCompleted() fi } +configAddons() { + if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then + configClusterAutoscalerAddon + fi + echo `date`,`hostname`, configAddonsDone>>/opt/m +} + +configClusterAutoscalerAddon() { + echo `date`,`hostname`, configClusterAutoscalerAddonStart>>/opt/m + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + echo `date`,`hostname`, configClusterAutoscalerAddonDone>>/opt/m +} + echo `date`,`hostname`, startscript>>/opt/m if [ -f /var/run/reboot-required ]; then @@ -122,6 +141,9 @@ if [[ ! -z "${MASTER_NODE}" ]]; then echo `date`,`hostname`, endGettingEtcdCerts>>/opt/m mkdir -p /opt/azure/containers && touch /opt/azure/containers/certs.ready + + echo `date`,`hostname`, configAddonsStart>>/opt/m + configAddons else echo "skipping master node provision operations, this is an agent node" fi @@ -240,24 +262,6 @@ function configNetworkPlugin() { fi } -function configAddons() { - configClusterAutoscalerAddon - - # if [[ "${CLUSTER_AUTOSCALER_ADDON}" = true ]]; then - # configClusterAutoscalerAddon - # fi -} - -function configClusterAutoscalerAddon() { - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" - sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" -} - function installClearContainersRuntime() { # Add Clear Containers repository key echo "Adding Clear Containers repository key..." diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 1d0a0c2f83..4c2fbfca47 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -116,6 +116,7 @@ "kubernetesClusterAutoscalerMemoryRequests": "[parameters('kubernetesClusterAutoscalerMemoryRequests')]", "kubernetesClusterAutoscalerMinNodes": "[parameters('kubernetesClusterAutoscalerMinNodes')]", "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", + "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index aeba35be94..4b65bcda24 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -493,6 +493,13 @@ }, "type": "string" }, + "kubernetesClusterAutoscalerEnabled": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerEnabled"}} + "metadata": { + "description": "Cluster autoscaler status" + }, + "type": "bool" + }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 872b4defdd..c2c6408faf 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -667,6 +667,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", clusterAutoscalerAddon.Enabled) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { From 63b1fc4596ab12e307569bff707e935cede22d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Wed, 11 Apr 2018 16:07:53 -0700 Subject: [PATCH 09/17] add nodeselector --- .../kubernetesmasteraddons-cluster-autoscaler-deployment.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml index 6b19467474..6c25f2e432 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -201,4 +201,6 @@ spec: - hostPath: path: /etc/ssl/certs/ca-certificates.crt type: "" - name: ssl-certs \ No newline at end of file + name: ssl-certs + nodeSelector: + beta.kubernetes.io/os: linux From 2b7fd0557c83f36c5bbc9b9a9082be82fb66393f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Sun, 15 Apr 2018 23:40:20 -0700 Subject: [PATCH 10/17] update tag to v1.2.1 --- pkg/acsengine/k8s_versions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 1bb32b40ef..f251791131 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -49,7 +49,7 @@ var k8sComponentVersions = map[string]map[string]string{ "tiller": "tiller:v2.8.1", "rescheduler": "rescheduler:v0.3.1", "aci-connector": "virtual-kubelet:latest", - "cluster-autoscaler": "cluster-autoscaler:v1.2.0", + "cluster-autoscaler": "cluster-autoscaler:v1.2.1", "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, From 6317efcb5661e1e3583d61d7e784f6e1d4f99064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Tue, 17 Apr 2018 15:55:19 -0700 Subject: [PATCH 11/17] update test --- pkg/acsengine/k8s_versions_test.go | 78 +++++++++++++++--------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/pkg/acsengine/k8s_versions_test.go b/pkg/acsengine/k8s_versions_test.go index 62a12fcc18..199afeccab 100644 --- a/pkg/acsengine/k8s_versions_test.go +++ b/pkg/acsengine/k8s_versions_test.go @@ -61,25 +61,26 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range oneDotNineDotThree { @@ -180,25 +181,26 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range override { if expected[k] != v { From 8a0eafd0c1ca99a097feb9786882c6e7c425aca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Thu, 19 Apr 2018 10:47:34 -0700 Subject: [PATCH 12/17] update base image repo --- pkg/acsengine/defaults.go | 1 - pkg/acsengine/engine.go | 4 ++-- pkg/acsengine/types.go | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index aeb04a33a4..b3e680b947 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -34,7 +34,6 @@ var ( KubernetesImageBase: "k8s-gcrio.azureedge.net/", TillerImageBase: "gcrio.azureedge.net/kubernetes-helm/", ACIConnectorImageBase: "microsoft/", - ClusterAutoscalerImageBase: "gcrio.azureedge.net/google-containers/", EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", WindowsPackageSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index c2c6408faf..dc52a10c5e 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -671,7 +671,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { - addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.ClusterAutoscalerImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) } } dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) @@ -1731,7 +1731,7 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat if clusterAutoscalerAddon.Containers[aS].Image != "" { val = clusterAutoscalerAddon.Containers[aS].Image } else { - val = cloudSpecConfig.KubernetesSpecConfig.ClusterAutoscalerImageBase + KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName] + val = cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName] } } case "kubernetesTillerSpec": diff --git a/pkg/acsengine/types.go b/pkg/acsengine/types.go index ae05cca116..3aaf126b11 100644 --- a/pkg/acsengine/types.go +++ b/pkg/acsengine/types.go @@ -50,7 +50,6 @@ type KubernetesSpecConfig struct { KubernetesImageBase string TillerImageBase string ACIConnectorImageBase string - ClusterAutoscalerImageBase string EtcdDownloadURLBase string KubeBinariesSASURLBase string WindowsPackageSASURLBase string From 4423aab0b9ae6cda2f60192a23c81434fb901a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Tue, 1 May 2018 18:25:04 -0700 Subject: [PATCH 13/17] add managed identity support --- ...raddons-cluster-autoscaler-deployment.yaml | 5 +++ parts/k8s/kubernetescustomscript.sh | 18 +++++++++ parts/k8s/kubernetesmastercustomdata.yml | 3 +- parts/k8s/kubernetesmastervars.t | 1 + parts/k8s/kubernetesparams.t | 11 +++++- pkg/acsengine/engine.go | 39 +++++++++++++++++++ 6 files changed, 74 insertions(+), 3 deletions(-) diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml index 6c25f2e432..68415e0612 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -142,6 +142,7 @@ spec: labels: app: cluster-autoscaler spec: + serviceAccountName: cluster-autoscaler containers: - image: @@ -192,15 +193,19 @@ spec: secretKeyRef: key: VMType name: cluster-autoscaler-azure + - name: ARM_USE_MANAGED_IDENTITY_EXTENSION + value: "" volumeMounts: - mountPath: /etc/ssl/certs/ca-certificates.crt name: ssl-certs readOnly: true + restartPolicy: Always volumes: - hostPath: path: /etc/ssl/certs/ca-certificates.crt type: "" name: ssl-certs + nodeSelector: beta.kubernetes.io/os: linux diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 921ce22dec..214adeb710 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -60,7 +60,25 @@ configAddons() { } configClusterAutoscalerAddon() { + echo `date`,`hostname`, configClusterAutoscalerAddonStart>>/opt/m + + if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then + echo `date`,`hostname`, configClusterAutoscalerAddonManagedIdentityStart>>/opt/m + CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true" + CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent" + CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true" + + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + echo `date`,`hostname`, configClusterAutoscalerAddonManagedIdentityDone>>/opt/m + elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + fi + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 54d50072c4..7940c51ac2 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -259,6 +259,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" {{end}} {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} @@ -274,7 +275,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{if EnableDataEncryptionAtRest }} - sed -i "s||{{WrapAsVariable "etcdEncryptionKey"}}|g" "/etc/kubernetes/encryption-config.yaml" + sed -i "s||{{WrapAsVariable "etcdEncryptionKey"}}|g" "/etc/kubernetes/encryption-config.yaml" {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 4c2fbfca47..ef9c71c02f 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -117,6 +117,7 @@ "kubernetesClusterAutoscalerMinNodes": "[parameters('kubernetesClusterAutoscalerMinNodes')]", "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]", + "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 4b65bcda24..c3195cb48a 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -500,6 +500,13 @@ }, "type": "bool" }, + "kubernetesClusterAutoscalerUseManagedIdentity": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerUseManagedIdentity"}} + "metadata": { + "description": "Managed identity for the cluster autoscaler addon" + }, + "type": "string" + }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { @@ -762,9 +769,9 @@ }, "type": "string" }, - "etcdEncryptionKey": { + "etcdEncryptionKey": { "metadata": { - "description": "Encryption at rest key for etcd" + "description": "Encryption at rest key for etcd" }, "type": "string" } diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index dc52a10c5e..cf4ad46351 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -668,6 +668,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", clusterAutoscalerAddon.Enabled) + addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) if clusterAutoscalerAddon.Containers[c].Image != "" { addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) } else { @@ -1734,6 +1735,44 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat val = cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName] } } + case "kubernetesClusterAutoscalerCPURequests": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].CPURequests + } else { + val = "" + } + case "kubernetesClusterAutoscalerMemoryRequests": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].MemoryRequests + } else { + val = "" + } + case "kubernetesClusterAutoscalerCPULimit": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].CPULimits + } else { + val = "" + } + case "kubernetesClusterAutoscalerMemoryLimit": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].MemoryLimits + } else { + val = "" + } + case "kubernetesClusterAutoscalerEnabled": + if aS > -1 { + val = strconv.FormatBool(*clusterAutoscalerAddon.Enabled) + } else { + val = "false" + } + case "kubernetesClusterAutoscalerUseManagedIdentity": + if aS > -1 { + if cs.Properties.OrchestratorProfile.KubernetesConfig != nil && cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { + val = strings.ToLower(strconv.FormatBool(cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity)) + } else { + val = "false" + } + } case "kubernetesTillerSpec": if tC > -1 { if tillerAddon.Containers[tC].Image != "" { From 4d332d671109f742affbc53b21f668d97b1c5683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Tue, 1 May 2018 20:51:42 -0700 Subject: [PATCH 14/17] update example --- examples/addons/cluster-autoscaler/README.md | 87 +++++++++---------- .../kubernetes-cluster-autoscaler.json | 8 +- 2 files changed, 45 insertions(+), 50 deletions(-) diff --git a/examples/addons/cluster-autoscaler/README.md b/examples/addons/cluster-autoscaler/README.md index 6da94e3035..f728fdb98e 100644 --- a/examples/addons/cluster-autoscaler/README.md +++ b/examples/addons/cluster-autoscaler/README.md @@ -11,53 +11,50 @@ To use this add-on, make sure your cluster's Kubernetes version is 1.10 or above ``` { - "apiVersion": "vlabs", - "properties": { - "orchestratorProfile": { - "orchestratorType": "Kubernetes", - "orchestratorRelease": "1.10", - "kubernetesConfig": { - "addons": [ - { - "name": "cluster-autoscaler", - "enabled" : true, - "config": { - "minNodes": "1", - "maxNodes": "5" - } + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "kubernetesConfig": { + "useManagedIdentity": true, + "addons": [ + { + "name": "cluster-autoscaler", + "enabled": true, + "config": { + "minNodes": "1", + "maxNodes": "5" } - ] - } - }, - "masterProfile": { + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", "count": 1, - "dnsPrefix": "", - "vmSize": "Standard_DS2_v2" - }, - "agentPoolProfiles": [ - { - "name": "agentpool", - "count": 1, - "vmSize": "Standard_DS2_v2", - "availabilityProfile": "VirtualMachineScaleSets" - } - ], - "linuxProfile": { - "adminUsername": "azureuser", - "ssh": { - "publicKeys": [ - { - "keyData": "" - } - ] - } - }, - "servicePrincipalProfile": { - "clientId": "", - "secret": "" + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "VirtualMachineScaleSets", + "storageProfile": "ManagedDisks" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] } } } +} ``` You should see cluster autoscaler as running after running: @@ -66,9 +63,9 @@ You should see cluster autoscaler as running after running: $ kubectl get pods -n kube-system ``` -Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler for examples. +Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler for more information. -# Configuration +## Configuration | Name | Required | Description | Default Value | | -------------- | -------- | --------------------------------- | ---------------------------------------------------------- | @@ -81,6 +78,6 @@ Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluste | cpuLimits | no | cpu limits for the container | "100m" | | memoryLimits | no | memory limits for the container | "300Mi" | -# Supported Orchestrators +## Supported Orchestrators Kubernetes diff --git a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json index adf40bfba6..f147e02dd9 100644 --- a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json +++ b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json @@ -4,6 +4,7 @@ "orchestratorProfile": { "orchestratorType": "Kubernetes", "kubernetesConfig": { + "useManagedIdentity": true, "addons": [ { "name": "cluster-autoscaler", @@ -26,7 +27,8 @@ "name": "agentpool", "count": 1, "vmSize": "Standard_DS2_v2", - "availabilityProfile": "VirtualMachineScaleSets" + "availabilityProfile": "VirtualMachineScaleSets", + "storageProfile": "ManagedDisks" } ], "linuxProfile": { @@ -38,10 +40,6 @@ } ] } - }, - "servicePrincipalProfile": { - "clientId": "", - "secret": "" } } } From 477aeb3c26c1bfd79f2ac02d88d905be9958d6d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Mon, 7 May 2018 11:49:07 -0700 Subject: [PATCH 15/17] add tolerations --- ...etesmasteraddons-cluster-autoscaler-deployment.yaml | 10 ++++++++-- parts/k8s/kubernetescustomscript.sh | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml index 68415e0612..b95ffafdee 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -144,6 +144,14 @@ spec: spec: serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/master + nodeSelector: + kubernetes.io/role: master + beta.kubernetes.io/os: linux containers: - image: imagePullPolicy: Always @@ -207,5 +215,3 @@ spec: type: "" name: ssl-certs - nodeSelector: - beta.kubernetes.io/os: linux diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 214adeb710..d3e70d0de8 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -60,7 +60,6 @@ configAddons() { } configClusterAutoscalerAddon() { - echo `date`,`hostname`, configClusterAutoscalerAddonStart>>/opt/m if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then @@ -536,6 +535,7 @@ ensureDockerInstallCompleted ensureDocker echo `date`,`hostname`, configNetworkPluginStart>>/opt/m configNetworkPlugin +echo `date`,`hostname`, configAddonsStart>>/opt/m configAddons if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then # Ensure we can nest virtualization From 9cfc7603725979bcc695cc128a378c11f8d95ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Mon, 7 May 2018 15:13:05 -0700 Subject: [PATCH 16/17] update addon-resizer --- pkg/acsengine/k8s_versions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index f251791131..2eeaa81d96 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -39,7 +39,7 @@ var k8sComponentVersions = map[string]map[string]string{ "dockerEngine": "1.13.*", "dashboard": "kubernetes-dashboard-amd64:v1.8.3", "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.7", + "addon-resizer": "addon-resizer:1.8.1", "heapster": "heapster-amd64:v1.5.1", "metrics-server": "metrics-server-amd64:v0.2.1", "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", @@ -67,7 +67,7 @@ var k8sComponentVersions = map[string]map[string]string{ "dockerEngine": "1.13.*", "dashboard": "kubernetes-dashboard-amd64:v1.8.3", "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.7", + "addon-resizer": "addon-resizer:1.8.1", "heapster": "heapster-amd64:v1.5.1", "metrics-server": "metrics-server-amd64:v0.2.1", "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", From 90a3bcde267bab40382e640eb86284a9a8dfb713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sertac=CC=A7=20O=CC=88zercan?= Date: Wed, 9 May 2018 15:33:10 -0700 Subject: [PATCH 17/17] validate addons --- pkg/api/vlabs/validate.go | 44 +++++++++++++++++----------------- pkg/api/vlabs/validate_test.go | 26 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 908979af89..bdc8cac021 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -451,6 +451,9 @@ func (a *Properties) Validate(isUpdate bool) error { if e := a.validateContainerRuntime(); e != nil { return e } + if e := a.validateAddons(); e != nil { + return e + } if e := a.MasterProfile.Validate(); e != nil { return e } @@ -676,9 +679,6 @@ func (a *Properties) Validate(isUpdate bool) error { if e := validateVNET(a); e != nil { return e } - if e := validateAddons(a); e != nil { - return e - } if a.AADProfile != nil { if a.OrchestratorProfile.OrchestratorType != Kubernetes { @@ -989,6 +989,25 @@ func (a *Properties) validateContainerRuntime() error { return nil } +func (a *Properties) validateAddons() error { + if a.OrchestratorProfile.KubernetesConfig != nil && a.OrchestratorProfile.KubernetesConfig.Addons != nil { + var isAvailabilitySets bool + + for _, agentPool := range a.AgentPoolProfiles { + if len(agentPool.AvailabilityProfile) == 0 || agentPool.IsAvailabilitySets() { + isAvailabilitySets = true + } + } + + for _, addon := range a.OrchestratorProfile.KubernetesConfig.Addons { + if addon.Name == "cluster-autoscaler" && *addon.Enabled && isAvailabilitySets { + return fmt.Errorf("Cluster Autoscaler add-on can only be used with VirtualMachineScaleSets. Please specify \"availabilityProfile\": \"%s\"", VirtualMachineScaleSets) + } + } + } + return nil +} + func validateName(name string, label string) error { if name == "" { return fmt.Errorf("%s must be a non-empty value", label) @@ -1102,25 +1121,6 @@ func validateVNET(a *Properties) error { return nil } -func validateAddons(a *Properties) error { - if a.OrchestratorProfile.KubernetesConfig != nil && a.OrchestratorProfile.KubernetesConfig.Addons != nil { - var isAvailabilitySets bool - - for _, agentPool := range a.AgentPoolProfiles { - if len(agentPool.AvailabilityProfile) == 0 || agentPool.IsAvailabilitySets() { - isAvailabilitySets = true - } - } - - for _, addon := range a.OrchestratorProfile.KubernetesConfig.Addons { - if addon.Name == "cluster-autoscaler" && *addon.Enabled && isAvailabilitySets { - return fmt.Errorf("Cluster Autoscaler add-on can only be used with VirtualMachineScaleSets. Please specify \"availabilityProfile\": \"%s\"", VirtualMachineScaleSets) - } - } - } - return nil -} - // GetVNETSubnetIDComponents extract subscription, resourcegroup, vnetname, subnetname from the vnetSubnetID func GetVNETSubnetIDComponents(vnetSubnetID string) (string, string, string, string, error) { vnetSubnetIDRegex := `^\/subscriptions\/([^\/]*)\/resourceGroups\/([^\/]*)\/providers\/Microsoft.Network\/virtualNetworks\/([^\/]*)\/subnets\/([^\/]*)$` diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index afc79a6ff9..98660b53de 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/Azure/acs-engine/pkg/api/common" + "github.com/Azure/acs-engine/pkg/helpers" "github.com/Masterminds/semver" ) @@ -690,6 +691,31 @@ func Test_Properties_ValidateContainerRuntime(t *testing.T) { } } +func Test_Properties_ValidateAddons(t *testing.T) { + p := &Properties{} + p.OrchestratorProfile = &OrchestratorProfile{} + p.OrchestratorProfile.OrchestratorType = Kubernetes + + p.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{ + Addons: []KubernetesAddon{ + { + Name: "cluster-autoscaler", + Enabled: helpers.PointerToBool(true), + }, + }, + } + p.AgentPoolProfiles = []*AgentPoolProfile{ + { + AvailabilityProfile: AvailabilitySet, + }, + } + if err := p.validateAddons(); err == nil { + t.Errorf( + "should error on cluster-autoscaler with availability sets", + ) + } +} + func TestWindowsVersions(t *testing.T) { for _, version := range common.GetAllSupportedKubernetesVersionsWindows() { p := getK8sDefaultProperties(true)