Merge branch 'master' into cloudprovider-hetzner

charts/cluster-autoscaler/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.18.1
+appVersion: 1.20.0
 description: Scales Kubernetes worker nodes within autoscaling groups.
 engine: gotpl
 home: https://github.com/kubernetes/autoscaler
@@ -17,4 +17,4 @@ name: cluster-autoscaler
 sources:
   - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler
 type: application
-version: 9.4.0
+version: 9.7.0

charts/cluster-autoscaler/README.md

@@ -362,7 +362,7 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
 | image.pullSecrets | list | `[]` | Image pull secrets |
 | image.repository | string | `"us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler"` | Image repository |
-| image.tag | string | `"v1.18.1"` | Image tag |
+| image.tag | string | `"v1.20.0"` | Image tag |
 | kubeTargetVersionOverride | string | `""` | Allow overriding the `.Capabilities.KubeVersion.GitVersion` check. Useful for `helm template` commands. |
 | magnumCABundlePath | string | `"/etc/kubernetes/ca-bundle.crt"` | Path to the host's CA bundle, from `ca-file` in the cloud-config file. |
 | magnumClusterName | string | `""` | Cluster name or ID in Magnum. Required if `cloudProvider=magnum` and not setting `autoDiscovery.clusterName`. |
@@ -372,9 +372,15 @@ Though enough for the majority of installations, the default PodSecurityPolicy _
 | podDisruptionBudget | object | `{"maxUnavailable":1}` | Pod disruption budget. |
 | podLabels | object | `{}` | Labels to add to each pod. |
 | priorityClassName | string | `""` | priorityClassName |
+| prometheusRule.additionalLabels | object | `{}` | Additional labels to be set in metadata. |
+| prometheusRule.enabled | bool | `false` | If true, creates a Prometheus Operator PrometheusRule. |
+| prometheusRule.interval | string | `nil` | How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set). |
+| prometheusRule.namespace | string | `"monitoring"` | Namespace which Prometheus is running in. |
+| prometheusRule.rules | list | `[]` | Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule). |
 | rbac.create | bool | `true` | If `true`, create and use RBAC resources. |
 | rbac.pspEnabled | bool | `false` | If `true`, creates and uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. Must be used with `rbac.create` set to `true`. |
 | rbac.serviceAccount.annotations | object | `{}` | Additional Service Account annotations. |
+| rbac.serviceAccount.automountServiceAccountToken | bool | `true` | Automount API credentials for a Service Account. |
 | rbac.serviceAccount.create | bool | `true` | If `true` and `rbac.create` is also true, a Service Account will be created. |
 | rbac.serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template. |
 | replicaCount | int | `1` | Desired number of pods |

charts/cluster-autoscaler/templates/prometheusrule.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.prometheusRule.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "cluster-autoscaler.fullname" . }}
+  {{- if .Values.prometheusRule.namespace }}
+  namespace: {{ .Values.prometheusRule.namespace }}
+  {{- end }}
+  labels: {{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }}
+spec:
+  groups:
+    - name: {{ include "cluster-autoscaler.fullname" . }}
+      interval: {{ .Values.prometheusRule.interval }}
+      rules: {{- tpl (toYaml .Values.prometheusRule.rules) . | nindent 8 }}
+{{- end }}

charts/cluster-autoscaler/templates/serviceaccount.yaml

@@ -8,4 +8,5 @@ metadata:
 {{- if .Values.rbac.serviceAccount.annotations }}
   annotations: {{ toYaml .Values.rbac.serviceAccount.annotations | nindent 4 }}
 {{- end }}
+automountServiceAccountToken: {{ .Values.rbac.serviceAccount.automountServiceAccountToken }}
 {{- end }}

charts/cluster-autoscaler/values.yaml

@@ -194,7 +194,7 @@ image:
   # image.repository -- Image repository
   repository: us.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler
   # image.tag -- Image tag
-  tag: v1.18.1
+  tag: v1.20.0
   # image.pullPolicy -- Image pull policy
   pullPolicy: IfNotPresent
   ## Optionally specify an array of imagePullSecrets.
@@ -244,6 +244,8 @@ rbac:
     create: true
     # rbac.serviceAccount.name -- The name of the ServiceAccount to use. If not set and create is `true`, a name is generated using the fullname template.
     name: ""
+    # rbac.serviceAccount.automountServiceAccountToken -- Automount API credentials for a Service Account.
+    automountServiceAccountToken: true
 
 # replicaCount -- Desired number of pods
 replicaCount: 1
@@ -298,6 +300,21 @@ serviceMonitor:
   # serviceMonitor.path -- The path to scrape for metrics; autoscaler exposes `/metrics` (this is standard)
   path: /metrics
 
+## Custom PrometheusRule to be defined
+## The value is evaluated as a template, so, for example, the value can depend on .Release or .Chart
+## ref: https://github.com/coreos/prometheus-operator#customresourcedefinitions
+prometheusRule:
+  # prometheusRule.enabled -- If true, creates a Prometheus Operator PrometheusRule.
+  enabled: false
+  # prometheusRule.additionalLabels -- Additional labels to be set in metadata.
+  additionalLabels: {}
+  # prometheusRule.namespace -- Namespace which Prometheus is running in.
+  namespace: monitoring
+  # prometheusRule.interval -- How often rules in the group are evaluated (falls back to `global.evaluation_interval` if not set).
+  interval: null
+  # prometheusRule.rules -- Rules spec template (see https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule).
+  rules: []
+
 # tolerations -- List of node taints to tolerate (requires Kubernetes >= 1.6).
 tolerations: []
 

cluster-autoscaler/.gitignore

@@ -1,4 +1,6 @@
 cluster-autoscaler
+cluster-autoscaler-amd64
+cluster-autoscaler-arm64
 cluster_autoscaler
 .cover
 

cluster-autoscaler/FAQ.md

@@ -918,40 +918,16 @@ Cluster Autoscaler imports a huge chunk of internal k8s code as it calls out to
 Therefore we want to keep set of libraries used in CA as close to one used by k8s, to avoid
 unexpected problems coming from version incompatibilities.
 
-Cluster Autoscaler depends on `go modules` mechanism for dependency management, but do not use it directly
-during build process. `go.mod` file is just used to generate the `vendor` directory and further compilation
-is run against set of libraries stored in `vendor`. `vendor` directory can be regenerated using [`update-vendor.sh`](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/hack/update-vendor.sh) script.
-The `update-vendor.sh` script is responsible for autogenerating `go.mod` file used by Cluster Autoscaler. The base
-of the file is `go.mod` file coming from [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) repository.
-On top of that script adds modifications as defined
-locally in [`go.mod-extra`](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/go.mod-extra) file.
-
-Note: It is important that one should **never manually edit** `go.mod` file as it is regenerated
-on each `update-vendor.sh` call. Any extra libraries or version overrides should be put in `go.mod-extra` file (syntax of the file
-is same as syntax of `go.mod` file).
-
-Finally `vendor` directry is materialized and validation tests are run.
-
-If everything completes correctly a commit with updated `vendor` directory is created automatically. The pull-request with changed vendor
-must be sent out manually. The PR should include the auto-generated commit as well as commits containing any manual changes/fixes that need to
-go together.
-
-Execution of `update-vendor.sh` can be parametrized using command line argumets:
- - `-f` - kubernetes/kubernetes fork to use. On `master` it defaults to `[email protected]:kubernetes/kubernetes.git`
- - `-r` - revision in kubernetes/kubernetes which should be used to get base `go.mod` file
- - `-d` - specifies script workdir; useful to speed up execution if script needs to be run multiple times, because updating vendor resulted in some compilation errors on Cluster-Autoscaler side which need to be fixed
- - `-o` - overrides go version check, which may be useful if CA needs to use a different go version than the one in kubernetes go.mod file
+To sync the repositories' vendored k8s libraries, we have a script that takes a
+released version of k8s and updates the `replace` directives of each k8s
+sub-library.
 
 Example execution looks like this:
 ```
-./hack/update-vendor.sh -d/tmp/ca-update-vendor.ou1l [email protected]:kubernetes/kubernetes.git -rmaster
+./hack/update-vendor.sh 1.20.0-alpha.1
 ```
 
-Caveats:
- - `update-vendor.sh` is called directly in shell (no docker is used) therefore its operation may differ from environment to environment.
- - It is important that go version, which isn in use in the shell in which `update-vendor.sh` is called, matches the `go <version>` directive specified in `go.mod` file
-   in `kubernetes/kubernetes` revision against which revendoring is done.
- - `update-vendor.sh` automatically runs unit tests as part of verification process. If one needs to suppress that, it can be done by overriding `VERIFY_COMMAND` variable (`VERIFY_COMMAND=true ./hack/update-vendor.sh ...`)
- - If one wants to only add new libraries to `go.mod-extra`, but not change the base `go.mod`, `-r` should be used with kubernetes/kubernets revision, which was used last time `update-vendor.sh` was called. One can determine that revision by looking at `git log` in Cluster Autoscaler repository. Following command will do the trick `git log | grep "Updating vendor against"`.
-
-
+If you need to update vendor to an unreleased commit of Kubernetes, you can use the breakglass script:
+```
+./hack/submodule-k8s.sh <k8s commit sha>
+```

cluster-autoscaler/Makefile

@@ -4,7 +4,7 @@ all: $(addprefix build-arch-,$(ALL_ARCH))
 TAG?=dev
 FLAGS=
 LDFLAGS?=-s
-ENVVAR=CGO_ENABLED=0 GO111MODULE=off
+ENVVAR=CGO_ENABLED=0
 GOOS?=linux
 GOARCH?=$(shell go env GOARCH)
 REGISTRY?=staging-k8s.gcr.io
@@ -44,7 +44,7 @@ build-binary-arch-%: clean-arch-%
 	$(ENVVAR) GOOS=$(GOOS) GOARCH=$* go build -o cluster-autoscaler-$* ${LDFLAGS_FLAG} ${TAGS_FLAG}
 
 test-unit: clean build
-	GO111MODULE=off go test --test.short -race ./... ${TAGS_FLAG}
+	go test --test.short -race ./... ${TAGS_FLAG}
 
 dev-release: dev-release-arch-$(GOARCH)
 
@@ -108,7 +108,6 @@ container-arch-%: build-in-docker-arch-% make-image-arch-%
 	@echo "Full in-docker image ${TAG}${FOR_PROVIDER}-$* completed"
 
 test-in-docker: clean docker-builder
-	docker run ${RM_FLAG} -v `pwd`:/gopath/src/k8s.io/autoscaler/cluster-autoscaler/:Z autoscaling-builder:latest \
-		bash -c 'cd /gopath/src/k8s.io/autoscaler/cluster-autoscaler && GO111MODULE=off go test -race ./... ${TAGS_FLAG}'
+	docker run ${RM_FLAG} -v `pwd`:/cluster-autoscaler/:Z autoscaling-builder:latest bash -c 'cd /cluster-autoscaler && go test -race ./... ${TAGS_FLAG}'
 
 .PHONY: all build test-unit clean format execute-release dev-release docker-builder build-in-docker release generate push-image push-manifest

cluster-autoscaler/README.md

@@ -157,3 +157,4 @@ Supported cloud providers:
 * OVHcloud https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/ovhcloud/README.md
 * Linode https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/linode/README.md
 * Hetzner https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/hetzner/README.md
+* Cluster API https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/clusterapi/README.md

cluster-autoscaler/cloudprovider/alicloud/alicloud_manager.go

@@ -27,7 +27,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess"
 	klog "k8s.io/klog/v2"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 	"math/rand"
 	"time"
 )

cluster-autoscaler/cloudprovider/aws/OWNERS

@@ -1,3 +1,4 @@
 approvers:
 - Jeffwan
 - jaypipes
+- gjtempleton

cluster-autoscaler/cloudprovider/aws/aws_manager.go

@@ -41,7 +41,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
 	klog "k8s.io/klog/v2"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 	provider_aws "k8s.io/legacy-cloud-providers/aws"
 )
 

cluster-autoscaler/cloudprovider/aws/aws_manager_test.go

@@ -38,7 +38,7 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 	provider_aws "k8s.io/legacy-cloud-providers/aws"
 )
 

cluster-autoscaler/cloudprovider/aws/aws_util.go

@@ -76,7 +76,7 @@ func GenerateEC2InstanceTypes(region string) (map[string]*InstanceType, error) {
 			klog.V(1).Infof("fetching %s\n", url)
 			res, err := http.Get(url)
 			if err != nil {
-				klog.Warningf("Error fetching %s skipping...\n", url)
+				klog.Warningf("Error fetching %s skipping...\n%s\n", url, err)
 				continue
 			}
 

cluster-autoscaler/cloudprovider/azure/azure_template.go

@@ -26,7 +26,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
 	cloudvolume "k8s.io/cloud-provider/volume"
 	"k8s.io/klog/v2"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 	"math/rand"
 	"regexp"
 	"strings"

cluster-autoscaler/cloudprovider/gce/autoscaling_gce_client.go

@@ -168,6 +168,9 @@ func (client *autoscalingGceClientV1) FetchMigBasename(migRef GceRef) (string, e
 	registerRequest("instance_group_managers", "get")
 	igm, err := client.gceService.InstanceGroupManagers.Get(migRef.Project, migRef.Zone, migRef.Name).Do()
 	if err != nil {
+		if err, ok := err.(*googleapi.Error); ok && err.Code == http.StatusNotFound {
+			return "", errors.NewAutoscalerError(errors.NodeGroupDoesNotExistError, "%s", err.Error())
+		}
 		return "", err
 	}
 	return igm.BaseInstanceName, nil

cluster-autoscaler/cloudprovider/gce/gce_price_model.go

@@ -101,6 +101,7 @@ var (
 		"e2-highcpu-4":     0.09894,
 		"e2-highcpu-8":     0.19788,
 		"e2-highcpu-16":    0.39576,
+		"e2-highcpu-32":    0.79149,
 		"e2-highmem-2":     0.09040,
 		"e2-highmem-4":     0.18080,
 		"e2-highmem-8":     0.36160,
@@ -209,6 +210,7 @@ var (
 		"e2-highcpu-4":     0.02968,
 		"e2-highcpu-8":     0.05936,
 		"e2-highcpu-16":    0.11873,
+		"e2-highcpu-32":    0.23744,
 		"e2-highmem-2":     0.02712,
 		"e2-highmem-4":     0.05424,
 		"e2-highmem-8":     0.10848,

cluster-autoscaler/cloudprovider/gce/templates.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/units"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 
 	"github.com/ghodss/yaml"
 	klog "k8s.io/klog/v2"

cluster-autoscaler/cloudprovider/gce/templates_test.go

@@ -29,7 +29,7 @@ import (
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	quota "k8s.io/apiserver/pkg/quota/v1"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 
 	"github.com/stretchr/testify/assert"
 )

cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_service_manager.go

@@ -35,7 +35,7 @@ import (
 	huaweicloudsdkecsmodel "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3/services/ecs/v2/model"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
 	"k8s.io/klog/v2"
-	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	kubeletapis "k8s.io/kubelet/pkg/apis"
 )
 
 // ElasticCloudServerService represents the elastic cloud server interfaces.

cluster-autoscaler/core/static_autoscaler.go

@@ -619,6 +619,7 @@ func removeOldUnregisteredNodes(unregisteredNodes []clusterstate.UnregisteredNod
 			}
 			logRecorder.Eventf(apiv1.EventTypeNormal, "DeleteUnregistered",
 				"Removed unregistered node %v", unregisteredNode.Node.Name)
+			metrics.RegisterOldUnregisteredNodesRemoved(1)
 			removedAny = true
 		}
 	}