diff --git a/examples/addons/cluster-autoscaler/README.md b/examples/addons/cluster-autoscaler/README.md new file mode 100644 index 0000000000..f728fdb98e --- /dev/null +++ b/examples/addons/cluster-autoscaler/README.md @@ -0,0 +1,83 @@ +# Cluster Autoscaler (VMSS) Add-on + +Cluster Autoscaler is a tool that automatically adjusts the size of the Kubernetes cluster when: + +* there are pods that failed to run in the cluster due to insufficient resources. +* some nodes in the cluster are so underutilized, for an extended period of time, that they can be deleted and their pods will be easily placed on some other, existing nodes. + +This is the Kubernetes Cluster Autoscaler add-on for Virtual Machine Scale Sets. Add this add-on to your json file as shown below to automatically enable cluster autoscaler in your new Kubernetes cluster. + +To use this add-on, make sure your cluster's Kubernetes version is 1.10 or above, and agent pool `availabilityProfile` is set to `VirtualMachineScaleSets`. This will automatically enable first agent pool to autoscale from 1 to 5 nodes by default. You can override these settings in `config` section of the `cluster-autoscaler` add-on. + +``` +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "kubernetesConfig": { + "useManagedIdentity": true, + "addons": [ + { + "name": "cluster-autoscaler", + "enabled": true, + "config": { + "minNodes": "1", + "maxNodes": "5" + } + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 1, + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "VirtualMachineScaleSets", + "storageProfile": "ManagedDisks" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + } + } +} +``` + +You should see cluster autoscaler as running after running: + +``` +$ kubectl get pods -n kube-system +``` + +Follow the README at https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler for more information. + +## Configuration + +| Name | Required | Description | Default Value | +| -------------- | -------- | --------------------------------- | ---------------------------------------------------------- | +| minNodes | no | minimum node count | | +| maxNodes | no | maximum node count | | +| name | no | container name | "cluster-autoscaler" | +| image | no | image | "gcrio.azureedge.net/google-containers/cluster-autoscaler" | +| cpuRequests | no | cpu requests for the container | "100m" | +| memoryRequests | no | memory requests for the container | "300Mi" | +| cpuLimits | no | cpu limits for the container | "100m" | +| memoryLimits | no | memory limits for the container | "300Mi" | + +## Supported Orchestrators + +Kubernetes diff --git a/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json new file mode 100644 index 0000000000..f147e02dd9 --- /dev/null +++ b/examples/addons/cluster-autoscaler/kubernetes-cluster-autoscaler.json @@ -0,0 +1,45 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "kubernetesConfig": { + "useManagedIdentity": true, + "addons": [ + { + "name": "cluster-autoscaler", + "enabled": true, + "config": { + "minNodes": "1", + "maxNodes": "5" + } + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_DS2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool", + "count": 1, + "vmSize": "Standard_DS2_v2", + "availabilityProfile": "VirtualMachineScaleSets", + "storageProfile": "ManagedDisks" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + } + } +} diff --git a/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml new file mode 100644 index 0000000000..b95ffafdee --- /dev/null +++ b/parts/k8s/addons/kubernetesmasteraddons-cluster-autoscaler-deployment.yaml @@ -0,0 +1,217 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["events","endpoints"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get","update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch","list","get","update"] +- apiGroups: [""] + resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] + verbs: ["watch","list","get"] +- apiGroups: ["extensions"] + resources: ["replicasets","daemonsets"] + verbs: ["watch","list","get"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch","list"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["watch","list","get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status"] + verbs: ["delete","get","update"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: v1 +data: + ClientID: + ClientSecret: + ResourceGroup: + SubscriptionID: + TenantID: + VMType: +kind: Secret +metadata: + name: cluster-autoscaler-azure + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: cluster-autoscaler + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: "EnsureExists" + name: cluster-autoscaler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/master + nodeSelector: + kubernetes.io/role: master + beta.kubernetes.io/os: linux + containers: + - image: + imagePullPolicy: Always + name: cluster-autoscaler + resources: + limits: + cpu: + memory: + requests: + cpu: + memory: + command: + - ./cluster-autoscaler + - --v=3 + - --logtostderr=true + - --cloud-provider=azure + - --skip-nodes-with-local-storage=false + - --nodes=:: + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: cluster-autoscaler-azure + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: cluster-autoscaler-azure + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: cluster-autoscaler-azure + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: cluster-autoscaler-azure + - name: ARM_USE_MANAGED_IDENTITY_EXTENSION + value: "" + volumeMounts: + - mountPath: /etc/ssl/certs/ca-certificates.crt + name: ssl-certs + readOnly: true + + restartPolicy: Always + volumes: + - hostPath: + path: /etc/ssl/certs/ca-certificates.crt + type: "" + name: ssl-certs + diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index 189d4afc99..c03062e69e 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -154,4 +154,4 @@ } }, "type": "Microsoft.Compute/virtualMachineScaleSets" - } \ No newline at end of file + } diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index c5e09c61f6..d3e70d0de8 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -52,6 +52,42 @@ ensureDockerInstallCompleted() fi } +configAddons() { + if [[ "${CLUSTER_AUTOSCALER_ADDON}" = True ]]; then + configClusterAutoscalerAddon + fi + echo `date`,`hostname`, configAddonsDone>>/opt/m +} + +configClusterAutoscalerAddon() { + echo `date`,`hostname`, configClusterAutoscalerAddonStart>>/opt/m + + if [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == true ]]; then + echo `date`,`hostname`, configClusterAutoscalerAddonManagedIdentityStart>>/opt/m + CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT="- mountPath: /var/lib/waagent/\n\ name: waagent\n\ readOnly: true" + CLUSTER_AUTOSCALER_MSI_VOLUME="- hostPath:\n\ path: /var/lib/waagent/\n\ name: waagent" + CLUSTER_AUTOSCALER_MSI_HOST_NETWORK="hostNetwork: true" + + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME_MOUNT}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||${CLUSTER_AUTOSCALER_MSI_VOLUME}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo "${CLUSTER_AUTOSCALER_MSI_HOST_NETWORK}")|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + echo `date`,`hostname`, configClusterAutoscalerAddonManagedIdentityDone>>/opt/m + elif [[ "${USE_MANAGED_IDENTITY_EXTENSION}" == false ]]; then + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||""|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + fi + + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $TENANT_ID | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $VM_TYPE | base64)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||$(echo $PRIMARY_SCALE_SET)|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + echo `date`,`hostname`, configClusterAutoscalerAddonDone>>/opt/m +} + echo `date`,`hostname`, startscript>>/opt/m if [ -f /var/run/reboot-required ]; then @@ -122,6 +158,9 @@ if [[ ! -z "${MASTER_NODE}" ]]; then echo `date`,`hostname`, endGettingEtcdCerts>>/opt/m mkdir -p /opt/azure/containers && touch /opt/azure/containers/certs.ready + + echo `date`,`hostname`, configAddonsStart>>/opt/m + configAddons else echo "skipping master node provision operations, this is an agent node" fi @@ -188,7 +227,7 @@ function ensureFilepath() { echo "Timeout waiting for $1" exit 6 fi - + } function setKubeletOpts () { @@ -496,6 +535,8 @@ ensureDockerInstallCompleted ensureDocker echo `date`,`hostname`, configNetworkPluginStart>>/opt/m configNetworkPlugin +echo `date`,`hostname`, configAddonsStart>>/opt/m +configAddons if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then # Ensure we can nest virtualization if grep -q vmx /proc/cpuinfo; then diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 9265ed0f63..7940c51ac2 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -251,6 +251,17 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sed -i "s||{{WrapAsVariable "kubernetesACIConnectorMemoryLimit"}}|g" "/etc/kubernetes/addons/aci-connector-deployment.yaml" {{end}} +{{if .OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled}} + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerSpec"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPULimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryLimit"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerCPURequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMemoryRequests"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMinNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerMaxNodes"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" + sed -i "s||{{WrapAsVariable "kubernetesClusterAutoscalerUseManagedIdentity"}}|g" "/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml" +{{end}} + {{if .OrchestratorProfile.KubernetesConfig.IsReschedulerEnabled}} sed -i "s||{{WrapAsVariable "kubernetesReschedulerSpec"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" sed -i "s||{{WrapAsVariable "kubernetesReschedulerCPURequests"}}|g" "/etc/kubernetes/addons/kube-rescheduler-deployment.yaml" @@ -264,7 +275,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{end}} {{if EnableDataEncryptionAtRest }} - sed -i "s||{{WrapAsVariable "etcdEncryptionKey"}}|g" "/etc/kubernetes/encryption-config.yaml" + sed -i "s||{{WrapAsVariable "etcdEncryptionKey"}}|g" "/etc/kubernetes/encryption-config.yaml" {{end}} {{if eq .OrchestratorProfile.KubernetesConfig.NetworkPolicy "calico"}} diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index be39cf51e3..ef9c71c02f 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -109,6 +109,15 @@ "kubernetesACIConnectorMemoryRequests": "[parameters('kubernetesACIConnectorMemoryRequests')]", "kubernetesACIConnectorCPULimit": "[parameters('kubernetesACIConnectorCPULimit')]", "kubernetesACIConnectorMemoryLimit": "[parameters('kubernetesACIConnectorMemoryLimit')]", + "kubernetesClusterAutoscalerSpec": "[parameters('kubernetesClusterAutoscalerSpec')]", + "kubernetesClusterAutoscalerCPULimit": "[parameters('kubernetesClusterAutoscalerCPULimit')]", + "kubernetesClusterAutoscalerMemoryLimit": "[parameters('kubernetesClusterAutoscalerMemoryLimit')]", + "kubernetesClusterAutoscalerCPURequests": "[parameters('kubernetesClusterAutoscalerCPURequests')]", + "kubernetesClusterAutoscalerMemoryRequests": "[parameters('kubernetesClusterAutoscalerMemoryRequests')]", + "kubernetesClusterAutoscalerMinNodes": "[parameters('kubernetesClusterAutoscalerMinNodes')]", + "kubernetesClusterAutoscalerMaxNodes": "[parameters('kubernetesClusterAutoscalerMaxNodes')]", + "kubernetesClusterAutoscalerEnabled": "[parameters('kubernetesClusterAutoscalerEnabled')]", + "kubernetesClusterAutoscalerUseManagedIdentity": "[parameters('kubernetesClusterAutoscalerUseManagedIdentity')]", "kubernetesReschedulerSpec": "[parameters('kubernetesReschedulerSpec')]", "kubernetesReschedulerCPURequests": "[parameters('kubernetesReschedulerCPURequests')]", "kubernetesReschedulerMemoryRequests": "[parameters('kubernetesReschedulerMemoryRequests')]", @@ -139,7 +148,7 @@ "vnetCidr": "[parameters('vnetCidr')]", "gcHighThreshold":"[parameters('gcHighThreshold')]", "gcLowThreshold":"[parameters('gcLowThreshold')]", -{{if EnableDataEncryptionAtRest}} +{{if EnableDataEncryptionAtRest}} "etcdEncryptionKey": "[parameters('etcdEncryptionKey')]", {{end}} {{ if UseManagedIdentity }} @@ -228,7 +237,7 @@ "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", {{if not IsOpenShift}} {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true CLUSTER_AUTOSCALER_ADDON=',variables('kubernetesClusterAutoscalerEnabled'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", {{if EnableEncryptionWithExternalKms}} {{ if not UseManagedIdentity}} "servicePrincipalObjectId": "[parameters('servicePrincipalObjectId')]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 35b8dad041..c3195cb48a 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -444,6 +444,69 @@ }, "type": "string" }, + "kubernetesClusterAutoscalerSpec": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerSpec"}} + "metadata": { + "description": "The container spec for the cluster autoscaler." + }, + "type": "string" + }, + "kubernetesClusterAutoscalerCPULimit": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerCPULimit"}} + "metadata": { + "description": "Cluster autoscaler cpu limit" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMemoryLimit": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMemoryLimit"}} + "metadata": { + "description": "Cluster autoscaler memory limit" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerCPURequests": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerCPURequests"}} + "metadata": { + "description": "Cluster autoscaler cpu requests" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMemoryRequests": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMemoryRequests"}} + "metadata": { + "description": "Cluster autoscaler memory requests" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMinNodes": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMinNodes"}} + "metadata": { + "description": "Cluster autoscaler min nodes" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerMaxNodes": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerMaxNodes"}} + "metadata": { + "description": "Cluster autoscaler max nodes" + }, + "type": "string" + }, + "kubernetesClusterAutoscalerEnabled": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerEnabled"}} + "metadata": { + "description": "Cluster autoscaler status" + }, + "type": "bool" + }, + "kubernetesClusterAutoscalerUseManagedIdentity": { + {{PopulateClassicModeDefaultValue "kubernetesClusterAutoscalerUseManagedIdentity"}} + "metadata": { + "description": "Managed identity for the cluster autoscaler addon" + }, + "type": "string" + }, "kubernetesReschedulerSpec": { {{PopulateClassicModeDefaultValue "kubernetesReschedulerSpec"}} "metadata": { @@ -706,9 +769,9 @@ }, "type": "string" }, - "etcdEncryptionKey": { + "etcdEncryptionKey": { "metadata": { - "description": "Encryption at rest key for etcd" + "description": "Encryption at rest key for etcd" }, "type": "string" } diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index 7494ca242e..814315774a 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -57,6 +57,11 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet "aci-connector-deployment.yaml", profile.OrchestratorProfile.KubernetesConfig.IsACIConnectorEnabled(), }, + { + "kubernetesmasteraddons-cluster-autoscaler-deployment.yaml", + "cluster-autoscaler-deployment.yaml", + profile.OrchestratorProfile.KubernetesConfig.IsClusterAutoscalerEnabled(), + }, { "kubernetesmasteraddons-kube-rescheduler-deployment.yaml", "kube-rescheduler-deployment.yaml", diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index fe33c1cdc4..6f4005f286 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -104,8 +104,8 @@ const ( DefaultACIConnectorAddonName = "aci-connector" // DefaultDashboardAddonName is the name of the kubernetes-dashboard addon deployment DefaultDashboardAddonName = "kubernetes-dashboard" - // DefaultACIConnectorImage defines the ACI Connector deployment version on Kubernetes Clusters - DefaultACIConnectorImage = "virtual-kubelet:latest" + // DefaultClusterAutoscalerAddonName is the name of the autoscaler addon deployment + DefaultClusterAutoscalerAddonName = "cluster-autoscaler" // DefaultKubernetesDNSServiceIP specifies the IP address that kube-dns // listens on by default. must by in the default Service CIDR range. DefaultKubernetesDNSServiceIP = "10.0.0.10" diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 9f30cea5b4..b3e680b947 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -244,6 +244,25 @@ var ( }, } + // DefaultClusterAutoscalerAddonsConfig is the default cluster autoscaler addon config + DefaultClusterAutoscalerAddonsConfig = api.KubernetesAddon{ + Name: DefaultClusterAutoscalerAddonName, + Enabled: helpers.PointerToBool(api.DefaultClusterAutoscalerAddonEnabled), + Config: map[string]string{ + "minNodes": "1", + "maxNodes": "5", + }, + Containers: []api.KubernetesContainerSpec{ + { + Name: DefaultClusterAutoscalerAddonName, + CPURequests: "100m", + MemoryRequests: "300Mi", + CPULimits: "100m", + MemoryLimits: "300Mi", + }, + }, + } + // DefaultDashboardAddonsConfig is the default kubernetes-dashboard addon Config DefaultDashboardAddonsConfig = api.KubernetesAddon{ Name: DefaultDashboardAddonName, @@ -352,6 +371,7 @@ func setOrchestratorDefaults(cs *api.ContainerService) { o.KubernetesConfig.Addons = []api.KubernetesAddon{ DefaultTillerAddonsConfig, DefaultACIConnectorAddonsConfig, + DefaultClusterAutoscalerAddonsConfig, DefaultDashboardAddonsConfig, DefaultReschedulerAddonsConfig, DefaultMetricsServerAddonsConfig, @@ -369,6 +389,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) { // Provide default acs-engine config for ACI Connector o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultACIConnectorAddonsConfig) } + s := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + if s < 0 { + // Provide default acs-engine config for cluster autoscaler + o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonsConfig) + } d := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultDashboardAddonName) if d < 0 { // Provide default acs-engine config for Dashboard @@ -465,6 +490,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if a.OrchestratorProfile.KubernetesConfig.Addons[c].IsEnabled(api.DefaultACIConnectorAddonEnabled) { a.OrchestratorProfile.KubernetesConfig.Addons[c] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[c], DefaultACIConnectorAddonsConfig) } + s := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + if a.OrchestratorProfile.KubernetesConfig.Addons[s].IsEnabled(api.DefaultClusterAutoscalerAddonEnabled) { + a.OrchestratorProfile.KubernetesConfig.Addons[s] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[s], DefaultClusterAutoscalerAddonsConfig) + } d := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) if a.OrchestratorProfile.KubernetesConfig.Addons[d].IsEnabled(api.DefaultDashboardAddonEnabled) { a.OrchestratorProfile.KubernetesConfig.Addons[d] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[d], DefaultDashboardAddonsConfig) diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index b6d2af215d..cf4ad46351 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -658,6 +658,23 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "kubernetesACIConnectorSpec", cloudSpecConfig.KubernetesSpecConfig.ACIConnectorImageBase+KubeConfigs[k8sVersion][DefaultACIConnectorAddonName]) } } + clusterAutoscalerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + c = getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", clusterAutoscalerAddon.Enabled) + addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) + if clusterAutoscalerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) + } + } dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) c = getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) if c > -1 { @@ -1563,6 +1580,8 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat tC := getAddonContainersIndexByName(tillerAddon.Containers, DefaultTillerAddonName) aciConnectorAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultACIConnectorAddonName) aC := getAddonContainersIndexByName(aciConnectorAddon.Containers, DefaultACIConnectorAddonName) + clusterAutoscalerAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + aS := getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) dashboardAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) dC := getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) reschedulerAddon := getAddonByName(cs.Properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultReschedulerAddonName) @@ -1708,6 +1727,52 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } else { val = "" } + case "kubernetesClusterAutoscalerSpec": + if aS > -1 { + if clusterAutoscalerAddon.Containers[aS].Image != "" { + val = clusterAutoscalerAddon.Containers[aS].Image + } else { + val = cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase + KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName] + } + } + case "kubernetesClusterAutoscalerCPURequests": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].CPURequests + } else { + val = "" + } + case "kubernetesClusterAutoscalerMemoryRequests": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].MemoryRequests + } else { + val = "" + } + case "kubernetesClusterAutoscalerCPULimit": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].CPULimits + } else { + val = "" + } + case "kubernetesClusterAutoscalerMemoryLimit": + if aS > -1 { + val = clusterAutoscalerAddon.Containers[aC].MemoryLimits + } else { + val = "" + } + case "kubernetesClusterAutoscalerEnabled": + if aS > -1 { + val = strconv.FormatBool(*clusterAutoscalerAddon.Enabled) + } else { + val = "false" + } + case "kubernetesClusterAutoscalerUseManagedIdentity": + if aS > -1 { + if cs.Properties.OrchestratorProfile.KubernetesConfig != nil && cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { + val = strings.ToLower(strconv.FormatBool(cs.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity)) + } else { + val = "false" + } + } case "kubernetesTillerSpec": if tC > -1 { if tillerAddon.Containers[tC].Image != "" { diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 1e5d22f0b8..2eeaa81d96 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -36,58 +36,60 @@ var k8sComponentVersions = map[string]map[string]string{ "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.10": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + "cluster-autoscaler": "cluster-autoscaler:v1.2.1", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.9": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + "cluster-autoscaler": "cluster-autoscaler:v1.1.2", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.8": { "dockerEngine": "1.13.*", @@ -271,25 +273,26 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.10"]["addon-resizer"], "heapster": k8sComponentVersions["1.10"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.10"]["metrics-server"], - "dns": k8sComponentVersions["1.10"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.10"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.10"]["dnsmasq"], - "pause": k8sComponentVersions["1.10"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.10"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.10"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.10"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.10"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.10"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.10"]["podeviction"], - "routeperiod": k8sComponentVersions["1.10"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.10"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.10"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.10"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.10"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.10"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.10"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.10"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.10"]["gclowthreshold"], + "dns": k8sComponentVersions["1.10"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.10"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.10"]["dnsmasq"], + "pause": k8sComponentVersions["1.10"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.10"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.10"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.10"]["aci-connector"], + "nodestatusfreq": k8sComponentVersions["1.10"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.10"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.10"]["podeviction"], + "routeperiod": k8sComponentVersions["1.10"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.10"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.10"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.10"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.10"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.10"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.10"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.10"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.10"]["gclowthreshold"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.10"]["cluster-autoscaler"], } case "1.9": ret = map[string]string{ @@ -302,25 +305,26 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], } case "1.8": ret = map[string]string{ diff --git a/pkg/acsengine/k8s_versions_test.go b/pkg/acsengine/k8s_versions_test.go index 62a12fcc18..199afeccab 100644 --- a/pkg/acsengine/k8s_versions_test.go +++ b/pkg/acsengine/k8s_versions_test.go @@ -61,25 +61,26 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range oneDotNineDotThree { @@ -180,25 +181,26 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range override { if expected[k] != v { diff --git a/pkg/api/const.go b/pkg/api/const.go index 039b497f94..4097111b9c 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -96,6 +96,8 @@ const ( DefaultTillerAddonEnabled = true // DefaultACIConnectorAddonEnabled determines the acs-engine provided default for enabling aci connector addon DefaultACIConnectorAddonEnabled = false + // DefaultClusterAutoscalerAddonEnabled determines the acs-engine provided default for enabling cluster autoscaler addon + DefaultClusterAutoscalerAddonEnabled = false // DefaultDashboardAddonEnabled determines the acs-engine provided default for enabling kubernetes-dashboard addon DefaultDashboardAddonEnabled = true // DefaultReschedulerAddonEnabled determines the acs-engine provided default for enabling kubernetes-rescheduler addon @@ -112,6 +114,8 @@ const ( DefaultTillerAddonName = "tiller" // DefaultACIConnectorAddonName is the name of the tiller addon deployment DefaultACIConnectorAddonName = "aci-connector" + // DefaultClusterAutoscalerAddonName is the name of the cluster autoscaler addon deployment + DefaultClusterAutoscalerAddonName = "cluster-autoscaler" // DefaultDashboardAddonName is the name of the kubernetes-dashboard addon deployment DefaultDashboardAddonName = "kubernetes-dashboard" // DefaultReschedulerAddonName is the name of the rescheduler addon deployment diff --git a/pkg/api/types.go b/pkg/api/types.go index 50a79a4e5d..81258a23fb 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -835,6 +835,17 @@ func (k *KubernetesConfig) IsACIConnectorEnabled() bool { return aciConnectorAddon.IsEnabled(DefaultACIConnectorAddonEnabled) } +// IsClusterAutoscalerEnabled checks if the cluster autoscaler addon is enabled +func (k *KubernetesConfig) IsClusterAutoscalerEnabled() bool { + var clusterAutoscalerAddon KubernetesAddon + for i := range k.Addons { + if k.Addons[i].Name == DefaultClusterAutoscalerAddonName { + clusterAutoscalerAddon = k.Addons[i] + } + } + return clusterAutoscalerAddon.IsEnabled(DefaultClusterAutoscalerAddonEnabled) +} + // IsDashboardEnabled checks if the kubernetes-dashboard addon is enabled func (k *KubernetesConfig) IsDashboardEnabled() bool { var dashboardAddon KubernetesAddon diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 0a07bba4df..5795b12c10 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -144,6 +144,36 @@ func TestIsACIConnectorEnabled(t *testing.T) { } } +func TestIsClusterAutoscalerEnabled(t *testing.T) { + c := KubernetesConfig{ + Addons: []KubernetesAddon{ + getMockAddon("addon"), + }, + } + enabled := c.IsClusterAutoscalerEnabled() + if enabled != DefaultClusterAutoscalerAddonEnabled { + t.Fatalf("KubernetesConfig.IsAutoscalerEnabled() should return %t when no cluster autoscaler addon has been specified, instead returned %t", DefaultClusterAutoscalerAddonEnabled, enabled) + } + c.Addons = append(c.Addons, getMockAddon(DefaultClusterAutoscalerAddonName)) + enabled = c.IsClusterAutoscalerEnabled() + if enabled { + t.Fatalf("KubernetesConfig.IsClusterAutoscalerEnabled() should return true when cluster autoscaler has been specified, instead returned %t", enabled) + } + b := true + c = KubernetesConfig{ + Addons: []KubernetesAddon{ + { + Name: DefaultClusterAutoscalerAddonName, + Enabled: &b, + }, + }, + } + enabled = c.IsClusterAutoscalerEnabled() + if !enabled { + t.Fatalf("KubernetesConfig.IsClusterAutoscalerEnabled() should return false when cluster autoscaler addon has been specified as disabled, instead returned %t", enabled) + } +} + func TestIsDashboardEnabled(t *testing.T) { c := KubernetesConfig{ Addons: []KubernetesAddon{ diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index f4e97f36a1..bdc8cac021 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -451,6 +451,9 @@ func (a *Properties) Validate(isUpdate bool) error { if e := a.validateContainerRuntime(); e != nil { return e } + if e := a.validateAddons(); e != nil { + return e + } if e := a.MasterProfile.Validate(); e != nil { return e } @@ -986,6 +989,25 @@ func (a *Properties) validateContainerRuntime() error { return nil } +func (a *Properties) validateAddons() error { + if a.OrchestratorProfile.KubernetesConfig != nil && a.OrchestratorProfile.KubernetesConfig.Addons != nil { + var isAvailabilitySets bool + + for _, agentPool := range a.AgentPoolProfiles { + if len(agentPool.AvailabilityProfile) == 0 || agentPool.IsAvailabilitySets() { + isAvailabilitySets = true + } + } + + for _, addon := range a.OrchestratorProfile.KubernetesConfig.Addons { + if addon.Name == "cluster-autoscaler" && *addon.Enabled && isAvailabilitySets { + return fmt.Errorf("Cluster Autoscaler add-on can only be used with VirtualMachineScaleSets. Please specify \"availabilityProfile\": \"%s\"", VirtualMachineScaleSets) + } + } + } + return nil +} + func validateName(name string, label string) error { if name == "" { return fmt.Errorf("%s must be a non-empty value", label) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index afc79a6ff9..98660b53de 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/Azure/acs-engine/pkg/api/common" + "github.com/Azure/acs-engine/pkg/helpers" "github.com/Masterminds/semver" ) @@ -690,6 +691,31 @@ func Test_Properties_ValidateContainerRuntime(t *testing.T) { } } +func Test_Properties_ValidateAddons(t *testing.T) { + p := &Properties{} + p.OrchestratorProfile = &OrchestratorProfile{} + p.OrchestratorProfile.OrchestratorType = Kubernetes + + p.OrchestratorProfile.KubernetesConfig = &KubernetesConfig{ + Addons: []KubernetesAddon{ + { + Name: "cluster-autoscaler", + Enabled: helpers.PointerToBool(true), + }, + }, + } + p.AgentPoolProfiles = []*AgentPoolProfile{ + { + AvailabilityProfile: AvailabilitySet, + }, + } + if err := p.validateAddons(); err == nil { + t.Errorf( + "should error on cluster-autoscaler with availability sets", + ) + } +} + func TestWindowsVersions(t *testing.T) { for _, version := range common.GetAllSupportedKubernetesVersionsWindows() { p := getK8sDefaultProperties(true) diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index 8edc55846b..886a9575c4 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -239,6 +239,23 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu } }) + It("should have cluster-autoscaler running", func() { + if hasClusterAutoscaler, clusterAutoscalerAddon := eng.HasAddon("autoscaler"); hasClusterAutoscaler { + running, err := pod.WaitOnReady("cluster-autoscaler", "kube-system", 3, 30*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(running).To(Equal(true)) + By("Ensuring that the correct resources have been applied") + pods, err := pod.GetAllByPrefix("cluster-autoscaler", "kube-system") + Expect(err).NotTo(HaveOccurred()) + for i, c := range clusterAutoscalerAddon.Containers { + err := pods[0].Spec.Containers[i].ValidateResources(c) + Expect(err).NotTo(HaveOccurred()) + } + } else { + Skip("cluster autoscaler disabled for this cluster, will not test") + } + }) + It("should have rescheduler running", func() { if hasRescheduler, reschedulerAddon := eng.HasAddon("rescheduler"); hasRescheduler { running, err := pod.WaitOnReady("rescheduler", "kube-system", 3, 30*time.Second, cfg.Timeout)