New expander: priority expander

cluster-autoscaler/.gitignore

@@ -8,3 +8,4 @@ cluster_autoscaler
 *.un~
 Session.vim
 .netrwhist
+.vscode

cluster-autoscaler/FAQ.md

@@ -590,7 +590,7 @@ new nodes will be added.
 Expanders can be selected by passing the name to the `--expander` flag, i.e.
 `./cluster-autoscaler --expander=random`.
 
-Currently Cluster Autoscaler has 4 expanders:
+Currently Cluster Autoscaler has 5 expanders:
 
 * `random` - this is the default expander, and should be used when you don't have a particular
 need for the node groups to scale differently.
@@ -607,12 +607,15 @@ after scale-up. This is useful when you have different classes of nodes, for exa
 would match the cluster size. This expander is described in more details
 [HERE](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/pricing.md). Currently it works only for GCE and GKE (patches welcome.)
 
+* `priority` - selects the node group that has the highest priority assigned by the user. It's configuration is described in more details [here](expander/priority/readme.md)
+
 ### Does CA respect node affinity when selecting node groups to scale up?
+
 CA respects `nodeSelector` and `requiredDuringSchedulingIgnoredDuringExecution` in nodeAffinity given that you have labelled your node groups accordingly. If there is a pod that cannot be scheduled with either `nodeSelector` or `requiredDuringSchedulingIgnoredDuringExecution` specified, CA will only consider node groups that satisfy those requirements for expansion.
 
 However, CA does not consider "soft" constraints like `preferredDuringSchedulingIgnoredDuringExecution` when selecting node groups. That means that if CA has two or more node groups available for expansion, it will not use soft constraints to pick one node group over another.
 
-************
+****************
 
 ### What are the parameters to CA?
 

cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1

cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-one-asg.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-master.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-containerservice.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-master.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-msi.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-master.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-msi.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss.yaml

@@ -66,8 +66,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-one-asg.yaml

@@ -63,8 +63,8 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["configmaps"]
-  resourceNames: ["cluster-autoscaler-status"]
-  verbs: ["delete","get","update"]
+  resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+  verbs: ["delete","get","update","watch"]
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

cluster-autoscaler/core/autoscaler.go

@@ -94,7 +94,7 @@ func initializeDefaultOptions(opts *AutoscalerOptions) error {
 	}
 	if opts.ExpanderStrategy == nil {
 		expanderStrategy, err := factory.ExpanderStrategyFromString(opts.ExpanderName,
-			opts.CloudProvider, opts.AutoscalingKubeClients.AllNodeLister())
+			opts.CloudProvider, opts.AutoscalingKubeClients, opts.KubeClient, opts.ConfigNamespace)
 		if err != nil {
 			return err
 		}

cluster-autoscaler/expander/expander.go

@@ -24,7 +24,7 @@ import (
 
 var (
 	// AvailableExpanders is a list of available expander options
-	AvailableExpanders = []string{RandomExpanderName, MostPodsExpanderName, LeastWasteExpanderName, PriceBasedExpanderName}
+	AvailableExpanders = []string{RandomExpanderName, MostPodsExpanderName, LeastWasteExpanderName, PriceBasedExpanderName, PriorityBasedExpanderName}
 	// RandomExpanderName selects a node group at random
 	RandomExpanderName = "random"
 	// MostPodsExpanderName selects a node group that fits the most pods
@@ -34,6 +34,8 @@ var (
 	// PriceBasedExpanderName selects a node group that is the most cost-effective and consistent with
 	// the preferred node size for the cluster
 	PriceBasedExpanderName = "price"
+	// PriorityBasedExpanderName selects a node group based on a user-configured priorities assigned to group names
+	PriorityBasedExpanderName = "priority"
 )
 
 // Option describes an option to expand the cluster.

cluster-autoscaler/expander/factory/expander_factory.go

@@ -18,19 +18,21 @@ package factory
 
 import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/context"
 	"k8s.io/autoscaler/cluster-autoscaler/expander"
 	"k8s.io/autoscaler/cluster-autoscaler/expander/mostpods"
 	"k8s.io/autoscaler/cluster-autoscaler/expander/price"
+	"k8s.io/autoscaler/cluster-autoscaler/expander/priority"
 	"k8s.io/autoscaler/cluster-autoscaler/expander/random"
 	"k8s.io/autoscaler/cluster-autoscaler/expander/waste"
 	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
-
-	kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes"
+	kube_client "k8s.io/client-go/kubernetes"
 )
 
 // ExpanderStrategyFromString creates an expander.Strategy according to its name
 func ExpanderStrategyFromString(expanderFlag string, cloudProvider cloudprovider.CloudProvider,
-	nodeLister kube_util.NodeLister) (expander.Strategy, errors.AutoscalerError) {
+	autoscalingKubeClients *context.AutoscalingKubeClients, kubeClient kube_client.Interface,
+	configNamespace string) (expander.Strategy, errors.AutoscalerError) {
 	switch expanderFlag {
 	case expander.RandomExpanderName:
 		return random.NewStrategy(), nil
@@ -44,8 +46,15 @@ func ExpanderStrategyFromString(expanderFlag string, cloudProvider cloudprovider
 			return nil, err
 		}
 		return price.NewStrategy(pricing,
-			price.NewSimplePreferredNodeProvider(nodeLister),
+			price.NewSimplePreferredNodeProvider(autoscalingKubeClients.AllNodeLister()),
 			price.SimpleNodeUnfitness), nil
+	case expander.PriorityBasedExpanderName:
+		maps := kubeClient.CoreV1().ConfigMaps(configNamespace)
+		initialPriorities, priorityChangesChan, err := priority.InitPriorityConfigMap(maps, configNamespace)
+		if err != nil {
+			return nil, errors.ToAutoscalerError(errors.InternalError, err)
+		}
+		return priority.NewStrategy(initialPriorities, priorityChangesChan, autoscalingKubeClients.LogRecorder)
 	}
 	return nil, errors.NewAutoscalerError(errors.InternalError, "Expander %s not supported", expanderFlag)
 }

cluster-autoscaler/expander/priority/configmap_handler.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package priority
+
+import (
+	"errors"
+	"fmt"
+
+	kube_errors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/watch"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/klog"
+	api "k8s.io/kubernetes/pkg/apis/core"
+)
+
+const (
+	// PriorityConfigMapName defines a name of the ConfigMap used to store priority expander configuration
+	PriorityConfigMapName = "cluster-autoscaler-priority-expander"
+	// ConfigMapKey defines the key used in the ConfigMap to configure priorities
+	ConfigMapKey = "priorities"
+)
+
+// InitPriorityConfigMap initializes ConfigMap with priority expander configurations. It checks if the map exists
+// and has the correct top level key. If it doesn't, it returns error or Exits. If the value is found,
+// the current value is fetched and a Watcher is started to watch for changes. It returns the current value of
+// the config map, the channel with value updates and an error.
+func InitPriorityConfigMap(maps v1.ConfigMapInterface, namespace string) (string, <-chan watch.Event, error) {
+	errMsg := ""
+	priorities := ""
+
+	configMap, getStatusError := maps.Get(PriorityConfigMapName, metav1.GetOptions{})
+	if getStatusError == nil {
+		priorities = configMap.Data[ConfigMapKey]
+	} else if kube_errors.IsNotFound(getStatusError) {
+		errMsg = fmt.Sprintf("Priority expander config map %s/%s not found. You have to create it before starting cluster-autoscaler "+
+			"with priority expander.", namespace, PriorityConfigMapName)
+	} else {
+		errMsg = fmt.Sprintf("Failed to retrieve priority expander configmap %s/%s: %v", namespace, PriorityConfigMapName,
+			getStatusError)
+	}
+	if errMsg != "" {
+		return "", nil, errors.New(errMsg)
+	}
+
+	watcher, err := maps.Watch(metav1.ListOptions{
+		FieldSelector: fields.OneTermEqualSelector(api.ObjectNameField, PriorityConfigMapName).String(),
+		Watch:         true,
+	})
+	if err != nil {
+		errMsg = fmt.Sprintf("Error when starting a watcher for changes of the priority expander configmap %s/%s: %v",
+			namespace, PriorityConfigMapName, err)
+		klog.Errorf(errMsg)
+		return "", nil, errors.New(errMsg)
+	}
+
+	return priorities, watcher.ResultChan(), nil
+}

cluster-autoscaler/expander/priority/priority-expander-configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-autoscaler-priority-expander
+data:
+  priorities: |-
+    10: 
+      - .*t2\.large.*
+      - .*t3\.large.*
+    50: 
+      - .*m4\.4xlarge.*