Merge branch 'master' into cloudprovider-hetzner

.github/workflows/ci.yaml

@@ -0,0 +1,36 @@
+name: Tests
+
+on:
+  - push
+  - pull_request
+
+env:
+  GOPATH: ${{ github.workspace }}/go
+
+jobs:
+  test-and-verify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - uses: actions/checkout@v2
+        with:
+          path: ${{ env.GOPATH }}/src/k8s.io/autoscaler
+
+      - name: Apt-get
+        run: sudo apt-get install libseccomp-dev -qq
+
+      - name: Prepare
+        working-directory: ${{ env.GOPATH }}/src/k8s.io/autoscaler
+        run: hack/install-verify-tools.sh
+
+      - name: Verify
+        working-directory: ${{ env.GOPATH }}/src/k8s.io/autoscaler
+        run: hack/verify-all.sh -v
+
+      - name: Test
+        working-directory: ${{ env.GOPATH }}/src/k8s.io/autoscaler
+        run: hack/for-go-proj.sh test

charts/cluster-autoscaler/Chart.yaml

@@ -17,4 +17,4 @@ name: cluster-autoscaler
 sources:
   - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
 type: application
-version: 9.3.2
+version: 9.4.0

charts/cluster-autoscaler/README.md

@@ -355,7 +355,9 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | extraEnv | object | `{}` | Additional container environment variables. |
 | extraEnvConfigMaps | object | `{}` | Additional container environment variables from ConfigMaps. |
 | extraEnvSecrets | object | `{}` | Additional container environment variables from Secrets. |
+| extraVolumeMounts | list | `[]` | Additional volumes to mount. |
 | extraVolumeSecrets | object | `{}` | Additional volumes to mount from Secrets. |
+| extraVolumes | list | `[]` | Additional volumes. |
 | fullnameOverride | string | `""` | String to fully override `cluster-autoscaler.fullname` template. |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.pullSecrets | list | `[]` | Image pull secrets |

charts/cluster-autoscaler/templates/deployment.yaml

@@ -182,7 +182,7 @@ spec:
           securityContext:
             {{ toYaml .Values.containerSecurityContext | nindent 12 | trim }}
           {{- end }}
-          {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets }}
+          {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumeMounts }}
           volumeMounts:
           {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }}
             - name: cloudconfig
@@ -199,6 +199,9 @@ spec:
               mountPath: {{ required "Must specify mountPath!" $value.mountPath }}
               readOnly: true
           {{- end }}
+          {{- if .Values.extraVolumeMounts }}
+            {{ toYaml .Values.extraVolumeMounts | nindent 12 }}
+          {{- end }}
           {{- end }}
     {{- if .Values.affinity }}
       affinity:
@@ -215,7 +218,7 @@ spec:
       securityContext:
         {{ toYaml .Values.securityContext | nindent 8 | trim }}
       {{- end }}
-      {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets }}
+      {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") .Values.extraVolumeSecrets .Values.extraVolumes }}
       volumes:
       {{- if or (eq .Values.cloudProvider "gce") (eq .Values.cloudProvider "magnum") }}
         - name: cloudconfig
@@ -236,6 +239,9 @@ spec:
             {{- toYaml $value.items | nindent 14 }}
             {{- end }}
       {{- end }}
+      {{- if .Values.extraVolumes }}
+        {{- toYaml .Values.extraVolumes | nindent 10 }}
+      {{- end }}
       {{- end }}
       {{- if .Values.image.pullSecrets }}
       imagePullSecrets:

charts/cluster-autoscaler/templates/role.yaml

@@ -12,14 +12,24 @@ rules:
       - configmaps
     verbs:
       - create
+{{- if eq (default "" .Values.extraArgs.expander) "priority" }}
+      - list
+      - watch
+{{- end }}
   - apiGroups:
       - ""
     resources:
       - configmaps
     resourceNames:
       - cluster-autoscaler-status
+{{- if eq (default "" .Values.extraArgs.expander) "priority" }}
+      - cluster-autoscaler-priority-expander
+{{- end }}
     verbs:
       - delete
       - get
       - update
+{{- if eq (default "" .Values.extraArgs.expander) "priority" }}
+      - watch
+{{- end }}
 {{- end -}}

charts/cluster-autoscaler/values.yaml

@@ -175,6 +175,18 @@ extraVolumeSecrets: {}
   #     - key: subkey
   #       path: mypath
 
+# extraVolumes -- Additional volumes.
+extraVolumes: []
+  # - name: ssl-certs
+  #   hostPath:
+  #     path: /etc/ssl/certs/ca-bundle.crt
+
+# extraVolumeMounts -- Additional volumes to mount.
+extraVolumeMounts: []
+  # - name: ssl-certs
+  #   mountPath: /etc/ssl/certs/ca-certificates.crt
+  #   readonly: true
+
 # fullnameOverride -- String to fully override `cluster-autoscaler.fullname` template.
 fullnameOverride: ""
 

cluster-autoscaler/FAQ.md

@@ -1,3 +1,4 @@
+<!--TODO: Remove "previously referred to as master" references from this doc once this terminology is fully removed from k8s-->
 # Frequently Asked Questions
 
 # Older versions
@@ -633,8 +634,8 @@ The following startup parameters are supported for cluster autoscaler:
 | --- | --- | --- |
 | `cluster-name` | Autoscaled cluster name, if available | ""
 | `address` | The address to expose prometheus metrics | :8085
-| `kubernetes` | Kubernetes master location. Leave blank for default | ""
-| `kubeconfig` | Path to kubeconfig file with authorization and master location information | ""
+| `kubernetes` | Kubernetes API Server location. Leave blank for default | ""
+| `kubeconfig` | Path to kubeconfig file with authorization and API Server location information | ""
 | `cloud-config` | The path to the cloud provider configuration file.  Empty string for no configuration file | ""
 | `namespace` | Namespace in which cluster-autoscaler run | "kube-system"
 | `scale-down-enabled` | Should CA scale down the cluster | true
@@ -674,7 +675,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `regional` | Cluster is regional | false
 | `leader-elect` | Start a leader election client and gain leadership before executing the main loop.<br>Enable this when running replicated components for high availability | true
 | `leader-elect-lease-duration` | The duration that non-leader candidates will wait after observing a leadership<br>renewal until attempting to acquire leadership of a led but unrenewed leader slot.<br>This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate.<br>This is only applicable if leader election is enabled | 15 seconds
-| `leader-elect-renew-deadline` | The interval between attempts by the acting master to renew a leadership slot before it stops leading.<br>This must be less than or equal to the lease duration.<br>This is only applicable if leader election is enabled | 10 seconds
+| `leader-elect-renew-deadline` | The interval between attempts by the active cluster-autoscaler to renew a leadership slot before it stops leading.<br>This must be less than or equal to the lease duration.<br>This is only applicable if leader election is enabled | 10 seconds
 | `leader-elect-retry-period` | The duration the clients should wait between attempting acquisition and renewal of a leadership.<br>This is only applicable if leader election is enabled | 2 seconds
 | `leader-elect-resource-lock` | The type of resource object that is used for locking during leader election.<br>Supported options are `endpoints` (default) and `configmaps` | "endpoints"
 | `aws-use-static-instance-list` | Should CA fetch instance types in runtime or use a static list. AWS only | false
@@ -775,7 +776,7 @@ If both the cluster and CA appear healthy:
 
 * If you expect some nodes to be added to make space for pending pods, but they are not added for a long time, check [I have a couple of pending pods, but there was no scale-up?](#i-have-a-couple-of-pending-pods-but-there-was-no-scale-up) section.
 
-* If you have access to the master machine, check Cluster Autoscaler logs in `/var/log/cluster-autoscaler.log`. Cluster Autoscaler logs a lot of useful information, including why it considers a pod unremovable or what was its scale-up plan.
+* If you have access to the control plane (previously referred to as master) machine, check Cluster Autoscaler logs in `/var/log/cluster-autoscaler.log`. Cluster Autoscaler logs a lot of useful information, including why it considers a pod unremovable or what was its scale-up plan.
 
 * Check events added by CA to the pod object.
 
@@ -787,7 +788,7 @@ If both the cluster and CA appear healthy:
 
 There are three options:
 
-* Logs on the master node, in `/var/log/cluster-autoscaler.log`.
+* Logs on the control plane (previously referred to as master) nodes, in `/var/log/cluster-autoscaler.log`.
 * Cluster Autoscaler 0.5 and later publishes kube-system/cluster-autoscaler-status config map.
   To see it, run `kubectl get configmap cluster-autoscaler-status -n kube-system
   -o yaml`.
@@ -862,7 +863,7 @@ Depending on how long scale-ups have been failing, it may wait up to 30 minutes
     ```
     This is the minimum number of nodes required for all e2e tests to pass. The tests should also pass if you set higher maximum nodes limit.
 3. Run `go run hack/e2e.go -- --verbose-commands --up` to bring up your cluster.
-4. SSH to the master node and edit `/etc/kubernetes/manifests/cluster-autoscaler.manifest` (you will need sudo for this).
+4. SSH to the control plane (previously referred to as master) node and edit `/etc/kubernetes/manifests/cluster-autoscaler.manifest` (you will need sudo for this).
     * If you want to test your custom changes set `image` to point at your own CA image.
     * Make sure `--scale-down-enabled` parameter in `command` is set to `true`.
 5. Run CA tests with:

cluster-autoscaler/README.md

@@ -1,3 +1,4 @@
+<!--TODO: Remove "previously referred to as master" references from this doc once this terminology is fully removed from k8s-->
 # Cluster Autoscaler
 
 # Introduction
@@ -22,10 +23,12 @@ You should also take a look at the notes and "gotchas" for your specific cloud p
 * [Packet](./cloudprovider/packet/README.md#notes) 
 * [IonosCloud](./cloudprovider/ionoscloud/README.md)
 * [OVHcloud](./cloudprovider/ovhcloud/README.md)
+* [Linode](./cloudprovider/linode/README.md)
+* [ClusterAPI](./cloudprovider/clusterapi/README.md)
 
 # Releases
 
-We recommend using Cluster Autoscaler with the Kubernetes master version for which it was meant. The below combinations have been tested on GCP. We don't do cross version testing or compatibility testing in other environments. Some user reports indicate successful use of a newer version of Cluster Autoscaler with older clusters, however, there is always a chance that it won't work as expected.
+We recommend using Cluster Autoscaler with the Kubernetes control plane (previously referred to as master) version for which it was meant. The below combinations have been tested on GCP. We don't do cross version testing or compatibility testing in other environments. Some user reports indicate successful use of a newer version of Cluster Autoscaler with older clusters, however, there is always a chance that it won't work as expected.
 
 Starting from Kubernetes 1.12, versioning scheme was changed to match Kubernetes minor releases exactly.
 
@@ -53,7 +56,7 @@ For CA 1.1.2 and later, please check [release
 notes.](https://github.com/kubernetes/autoscaler/releases)
 
 CA version 1.1.1:
-* Fixes around metrics in the multi-master configuration.
+* Fixes around metrics in the multiple kube apiserver configuration.
 * Fixes for unready nodes issues when quota is overrun.
 
 CA version 1.1.0:
@@ -132,7 +135,7 @@ CA Version 0.3:
 
 # Deployment
 
-Cluster Autoscaler is designed to run on Kubernetes master node. This is the
+Cluster Autoscaler is designed to run on Kubernetes control plane (previously referred to as master) node. This is the
 default deployment strategy on GCP.
 It is possible to run a customized deployment of Cluster Autoscaler on worker nodes, but extra care needs
 to be taken to ensure that Cluster Autoscaler remains up and running. Users can put it into kube-system
@@ -152,4 +155,5 @@ Supported cloud providers:
 * Exoscale https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/exoscale/README.md
 * Packet https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/packet/README.md
 * OVHcloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/ovhcloud/README.md
+* Linode https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/linode/README.md
 * Hetzner https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/hetzner/README.md

cluster-autoscaler/cloudprovider/alicloud/alicloud_auto_scaling_group.go

@@ -21,6 +21,7 @@ import (
 
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/config"
 	klog "k8s.io/klog/v2"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 )
@@ -211,3 +212,9 @@ func (asg *Asg) Autoprovisioned() bool {
 func (asg *Asg) Delete() error {
 	return cloudprovider.ErrNotImplemented
 }
+
+// GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular
+// NodeGroup. Returning a nil will result in using default options.
+func (asg *Asg) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}

cluster-autoscaler/cloudprovider/aws/README.md

@@ -208,16 +208,17 @@ kubectl apply -f examples/cluster-autoscaler-one-asg.yaml
 kubectl apply -f examples/cluster-autoscaler-multi-asg.yaml
 ```
 
-## Master Node Setup
+<!--TODO: Remove "previously referred to as master" references from this doc once this terminology is fully removed from k8s-->
+## Control Plane (previously referred to as master) Node Setup
 
 **NOTE**: This setup is not compatible with Amazon EKS.
 
-To run a CA pod in master node - CA deployment should tolerate the master
-`taint` and `nodeSelector` should be used to schedule the pods in master node.
+To run a CA pod on a control plane node the CA deployment should tolerate the `master`
+taint and `nodeSelector` should be used to schedule the pods on a control plane node.
 Please replace `{{ node_asg_min }}`, `{{ node_asg_max }}` and `{{ name }}` with
 your ASG setting in the yaml file.
 ```
-kubectl apply -f examples/cluster-autoscaler-run-on-master.yaml
+kubectl apply -f examples/cluster-autoscaler-run-on-control-plane.yaml
 ```
 
 ## Using Mixed Instances Policies and Spot Instances

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go

@@ -215,6 +215,12 @@ func (ng *AwsNodeGroup) Delete() error {
 	return cloudprovider.ErrNotImplemented
 }
 
+// GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular
+// NodeGroup. Returning a nil will result in using default options.
+func (ng *AwsNodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
 // IncreaseSize increases Asg size
 func (ng *AwsNodeGroup) IncreaseSize(delta int) error {
 	if delta <= 0 {

cluster-autoscaler/cloudprovider/azure/README.md

@@ -133,7 +133,8 @@ Save the updated deployment manifest, then deploy cluster-autoscaler by running:
 kubectl create -f cluster-autoscaler-vmss.yaml
 ```
 
-To run a cluster autoscaler pod on a master node, the deployment should tolerate the `master` taint, and `nodeSelector` should be used to schedule pods. Use [cluster-autoscaler-vmss-master.yaml](examples/cluster-autoscaler-vmss-master.yaml) in this case.
+<!--TODO: Remove "previously referred to as master" references from this doc once this terminology is fully removed from k8s-->
+To run a cluster autoscaler pod on a control plane (previously referred to as master) node, the deployment should tolerate the `master` taint, and `nodeSelector` should be used to schedule pods. Use [cluster-autoscaler-vmss-control-plane.yaml](examples/cluster-autoscaler-vmss-control-plane.yaml) in this case.
 
 To run a cluster autoscaler pod with Azure managed service identity (MSI), use [cluster-autoscaler-vmss-msi.yaml](examples/cluster-autoscaler-vmss-msi.yaml) instead.
 
@@ -172,7 +173,7 @@ Prerequisites:
 - Get Azure credentials from the [**Permissions**](#permissions) step above.
 - Get the name of the initial Azure deployment resource for the cluster. You can find this in the [Azure Portal](https://portal.azure.com) or with the `az deployment list` command. If there are multiple deployments, get the name of the first one.
 
-Make a copy of [cluster-autoscaler-standard-master.yaml](examples/cluster-autoscaler-standard-master.yaml). Fill in the placeholder values for the `cluster-autoscaler-azure` secret data by base64-encoding each of your Azure credential fields.
+Make a copy of [cluster-autoscaler-standard-control-plane.yaml](examples/cluster-autoscaler-standard-control-plane.yaml). Fill in the placeholder values for the `cluster-autoscaler-azure` secret data by base64-encoding each of your Azure credential fields.
 
 - ClientID: `<base64-encoded-client-id>`
 - ClientSecret: `<base64-encoded-client-secret>`
@@ -208,7 +209,7 @@ kubectl -n kube-system create secret generic cluster-autoscaler-azure-deploy-par
 Then deploy cluster-autoscaler by running:
 
 ```sh
-kubectl create -f cluster-autoscaler-standard-master.yaml
+kubectl create -f cluster-autoscaler-standard-control-plane.yaml
 ```
 
 To run a cluster autoscaler pod with Azure managed service identity (MSI), use [cluster-autoscaler-standard-msi.yaml](examples/cluster-autoscaler-standard-msi.yaml) instead.

cluster-autoscaler/cloudprovider/azure/azure_agent_pool.go

@@ -32,6 +32,7 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
 	klog "k8s.io/klog/v2"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
@@ -119,6 +120,12 @@ func (as *AgentPool) Autoprovisioned() bool {
 	return false
 }
 
+// GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular
+// NodeGroup. Returning a nil will result in using default options.
+func (as *AgentPool) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
 // MaxSize returns maximum size of the node group.
 func (as *AgentPool) MaxSize() int {
 	return as.maxSize

cluster-autoscaler/cloudprovider/azure/azure_config_test.go

@@ -235,11 +235,3 @@ func TestOverrideDefaultRateLimitConfig(t *testing.T) {
 	newconfig = overrideDefaultRateLimitConfig(&defaultConfigWithRateLimits.RateLimitConfig, &falseCloudProviderRateLimit.RateLimitConfig)
 	assert.Equal(t, &falseCloudProviderRateLimit.RateLimitConfig, newconfig)
 }
-
-func TestGetSubscriptionIdFromInstanceMetadata(t *testing.T) {
-	// metadataURL in azure_manager.go is not available for our tests, expect fail.
-	result, err := getSubscriptionIdFromInstanceMetadata()
-	expected := ""
-	assert.NotNil(t, err.Error())
-	assert.Equal(t, expected, result, "Verify return result failed, expected: %v, actual: %v", expected, result)
-}

cluster-autoscaler/cloudprovider/azure/azure_kubernetes_service_pool.go

@@ -27,6 +27,7 @@ import (
 
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
 	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 )
@@ -433,3 +434,9 @@ func (agentPool *AKSAgentPool) Delete() error {
 func (agentPool *AKSAgentPool) Autoprovisioned() bool {
 	return false
 }
+
+// GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular
+// NodeGroup. Returning a nil will result in using default options.
+func (agentPool *AKSAgentPool) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*config.NodeGroupAutoscalingOptions, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}