Merge branch 'kubernetes:master' into vpa-gc-controller-two

cluster-autoscaler/FAQ.md

@@ -728,7 +728,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `leader-elect-lease-duration` | The duration that non-leader candidates will wait after observing a leadership<br>renewal until attempting to acquire leadership of a led but unrenewed leader slot.<br>This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate.<br>This is only applicable if leader election is enabled | 15 seconds
 | `leader-elect-renew-deadline` | The interval between attempts by the active cluster-autoscaler to renew a leadership slot before it stops leading.<br>This must be less than or equal to the lease duration.<br>This is only applicable if leader election is enabled | 10 seconds
 | `leader-elect-retry-period` | The duration the clients should wait between attempting acquisition and renewal of a leadership.<br>This is only applicable if leader election is enabled | 2 seconds
-| `leader-elect-resource-lock` | The type of resource object that is used for locking during leader election.<br>Supported options are `endpoints` (default) and `configmaps` | "endpoints"
+| `leader-elect-resource-lock` | The type of resource object that is used for locking during leader election.<br>Supported options are `leases` (default), `endpoints`, `endpointsleases`, `configmaps`, and `configmapsleases` | "leases"
 | `aws-use-static-instance-list` | Should CA fetch instance types in runtime or use a static list. AWS only | false
 | `skip-nodes-with-system-pods` | If true cluster autoscaler will never delete nodes with pods from kube-system (except for DaemonSet or mirror pods) | true
 | `skip-nodes-with-local-storage`| If true cluster autoscaler will never delete nodes with pods with local storage, e.g. EmptyDir or HostPath | true

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go

@@ -38,8 +38,8 @@ var testAwsManager = &AwsManager{
 	awsService: testAwsService,
 }
 
-func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2I, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
-	awsService := awsWrapper{mockAutoScaling, mockEC2}
+func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2I, mockEKS eksI, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
+	awsService := awsWrapper{mockAutoScaling, mockEC2, mockEKS}
 	return &AwsManager{
 		awsService: awsService,
 		asgCache: &asgCache{
@@ -57,13 +57,13 @@ func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2
 }
 
 func newTestAwsManagerWithAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string) *AwsManager {
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, nil)
 	m.asgCache.parseExplicitAsgs(specs)
 	return m
 }
 
 func newTestAwsManagerWithAutoAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager {
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, autoDiscoverySpecs)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, autoDiscoverySpecs)
 	m.asgCache.parseExplicitAsgs(specs)
 	return m
 }
@@ -525,7 +525,7 @@ func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) {
 func TestGetResourceLimiter(t *testing.T) {
 	mockAutoScaling := &autoScalingMock{}
 	mockEC2 := &ec2Mock{}
-	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil)
+	m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil, nil)
 
 	provider := testProvider(t, m)
 	_, err := provider.GetResourceLimiter()

cluster-autoscaler/cloudprovider/aws/aws_manager.go

@@ -35,6 +35,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
 	"gopkg.in/gcfg.v1"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -197,7 +198,7 @@ func createAWSManagerInternal(
 			return nil, err
 		}
 
-		awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess)}
+		awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess), eks.New(sess)}
 	}
 
 	specs, err := parseASGAutoDiscoverySpecs(discoveryOpts)

cluster-autoscaler/cloudprovider/aws/aws_manager_test.go

@@ -403,7 +403,7 @@ func TestFetchExplicitAsgs(t *testing.T) {
 	defer resetAWSRegion(os.LookupEnv("AWS_REGION"))
 	os.Setenv("AWS_REGION", "fanghorn")
 	instanceTypes, _ := GetStaticEC2InstanceTypes()
-	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes)
+	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil, nil}, instanceTypes)
 	assert.NoError(t, err)
 
 	asgs := m.asgCache.Get()
@@ -468,7 +468,7 @@ func TestGetASGTemplate(t *testing.T) {
 			instanceTypes, _ := GetStaticEC2InstanceTypes()
 			do := cloudprovider.NodeGroupDiscoveryOptions{}
 
-			m, err := createAWSManagerInternal(nil, do, &awsWrapper{nil, e}, instanceTypes)
+			m, err := createAWSManagerInternal(nil, do, &awsWrapper{nil, e, nil}, instanceTypes)
 			origGetInstanceTypeFunc := getInstanceTypeForAsg
 			defer func() { getInstanceTypeForAsg = origGetInstanceTypeFunc }()
 			getInstanceTypeForAsg = func(m *asgCache, asg *asg) (string, error) {
@@ -556,7 +556,7 @@ func TestFetchAutoAsgs(t *testing.T) {
 	os.Setenv("AWS_REGION", "fanghorn")
 	// fetchAutoASGs is called at manager creation time, via forceRefresh
 	instanceTypes, _ := GetStaticEC2InstanceTypes()
-	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes)
+	m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil, nil}, instanceTypes)
 	assert.NoError(t, err)
 
 	asgs := m.asgCache.Get()

cluster-autoscaler/cloudprovider/aws/aws_wrapper.go

@@ -18,11 +18,14 @@ package aws
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	apiv1 "k8s.io/api/core/v1"
 	klog "k8s.io/klog/v2"
 )
 
@@ -40,10 +43,73 @@ type ec2I interface {
 	DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error)
 }
 
+// eksI is the interface that represents a specific aspect of EKS (Elastic Kubernetes Service) which is provided by AWS SDK for use in CA
+type eksI interface {
+	DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error)
+}
+
 // awsWrapper provides several utility methods over the services provided by the AWS SDK
 type awsWrapper struct {
 	autoScalingI
 	ec2I
+	eksI
+}
+
+func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName string) ([]apiv1.Taint, map[string]string, error) {
+	params := &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}
+	start := time.Now()
+	r, err := m.DescribeNodegroup(params)
+	observeAWSRequest("DescribeNodegroup", err, start)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	taints := make([]apiv1.Taint, 0)
+	labels := make(map[string]string)
+
+	// Labels will include diskSize, amiType, capacityType, version
+	if r.Nodegroup.DiskSize != nil {
+		labels["diskSize"] = strconv.FormatInt(*r.Nodegroup.DiskSize, 10)
+	}
+
+	if r.Nodegroup.AmiType != nil && len(*r.Nodegroup.AmiType) > 0 {
+		labels["amiType"] = *r.Nodegroup.AmiType
+	}
+
+	if r.Nodegroup.CapacityType != nil && len(*r.Nodegroup.CapacityType) > 0 {
+		labels["capacityType"] = *r.Nodegroup.CapacityType
+	}
+
+	if r.Nodegroup.Version != nil && len(*r.Nodegroup.Version) > 0 {
+		labels["k8sVersion"] = *r.Nodegroup.Version
+	}
+
+	if r.Nodegroup.Labels != nil && len(r.Nodegroup.Labels) > 0 {
+		labelsMap := r.Nodegroup.Labels
+		for k, v := range labelsMap {
+			if v != nil {
+				labels[k] = *v
+			}
+		}
+	}
+
+	if r.Nodegroup.Taints != nil && len(r.Nodegroup.Taints) > 0 {
+		taintList := r.Nodegroup.Taints
+		for _, taint := range taintList {
+			if taint != nil && taint.Effect != nil && taint.Key != nil && taint.Value != nil {
+				taints = append(taints, apiv1.Taint{
+					Key:    *taint.Key,
+					Value:  *taint.Value,
+					Effect: apiv1.TaintEffect(*taint.Effect),
+				})
+			}
+		}
+	}
+
+	return taints, labels, nil
 }
 
 func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*string) (map[string]string, error) {

cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go

@@ -19,13 +19,16 @@ package aws
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+	apiv1 "k8s.io/api/core/v1"
 )
 
 type autoScalingMock struct {
@@ -66,13 +69,178 @@ func (e *ec2Mock) DescribeLaunchTemplateVersions(i *ec2.DescribeLaunchTemplateVe
 	return args.Get(0).(*ec2.DescribeLaunchTemplateVersionsOutput), nil
 }
 
-var testAwsService = awsWrapper{&autoScalingMock{}, &ec2Mock{}}
+type eksMock struct {
+	mock.Mock
+}
+
+func (k *eksMock) DescribeNodegroup(i *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) {
+	args := k.Called(i)
+	return args.Get(0).(*eks.DescribeNodegroupOutput), nil
+}
+
+var testAwsService = awsWrapper{&autoScalingMock{}, &ec2Mock{}, &eksMock{}}
+
+func TestGetManagedNodegroup(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	labelKey1 := "labelKey 1"
+	labelKey2 := "labelKey 2"
+	labelValue1 := "testValue 1"
+	labelValue2 := "testValue 2"
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	taintEffect1 := "effect 1"
+	taintKey1 := "key 1"
+	taintValue1 := "value 1"
+	taint1 := eks.Taint{
+		Effect: &taintEffect1,
+		Key:    &taintKey1,
+		Value:  &taintValue1,
+	}
+
+	taintEffect2 := "effect 2"
+	taintKey2 := "key 2"
+	taintValue2 := "value 2"
+	taint2 := eks.Taint{
+		Effect: &taintEffect2,
+		Key:    &taintKey2,
+		Value:  &taintValue2,
+	}
+
+	amiType := "testAmiType"
+	diskSize := int64(100)
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      &diskSize,
+		Labels:        map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2},
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        []*eks.Taint{&taint1, &taint2},
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 2)
+	assert.Equal(t, taintList[0].Effect, apiv1.TaintEffect(taintEffect1))
+	assert.Equal(t, taintList[0].Key, taintKey1)
+	assert.Equal(t, taintList[0].Value, taintValue1)
+	assert.Equal(t, taintList[1].Effect, apiv1.TaintEffect(taintEffect2))
+	assert.Equal(t, taintList[1].Key, taintKey2)
+	assert.Equal(t, taintList[1].Value, taintValue2)
+	assert.Equal(t, len(labelMap), 6)
+	assert.Equal(t, labelMap[labelKey1], labelValue1)
+	assert.Equal(t, labelMap[labelKey2], labelValue2)
+	assert.Equal(t, labelMap["diskSize"], strconv.FormatInt(diskSize, 10))
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
+
+func TestGetManagedNodegroupWithNilValues(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	amiType := "testAmiType"
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      nil,
+		Labels:        nil,
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        nil,
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 0)
+	assert.Equal(t, len(labelMap), 3)
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
+
+func TestGetManagedNodegroupWithEmptyValues(t *testing.T) {
+	k := &eksMock{}
+	awsWrapper := &awsWrapper{
+		autoScalingI: nil,
+		ec2I:         nil,
+		eksI:         k,
+	}
+
+	nodegroupName := "testNodegroup"
+	clusterName := "testCluster"
+
+	amiType := "testAmiType"
+	capacityType := "testCapacityType"
+	k8sVersion := "1.19"
+
+	// Create test nodegroup
+	testNodegroup := eks.Nodegroup{
+		AmiType:       &amiType,
+		ClusterName:   &clusterName,
+		DiskSize:      nil,
+		Labels:        make(map[string]*string),
+		NodegroupName: &nodegroupName,
+		CapacityType:  &capacityType,
+		Version:       &k8sVersion,
+		Taints:        make([]*eks.Taint, 0),
+	}
+
+	k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{
+		ClusterName:   &clusterName,
+		NodegroupName: &nodegroupName,
+	}).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup})
+
+	taintList, labelMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName)
+	assert.Nil(t, err)
+	assert.Equal(t, len(taintList), 0)
+	assert.Equal(t, len(labelMap), 3)
+	assert.Equal(t, labelMap["amiType"], amiType)
+	assert.Equal(t, labelMap["capacityType"], capacityType)
+	assert.Equal(t, labelMap["k8sVersion"], k8sVersion)
+}
 
 func TestMoreThen100Groups(t *testing.T) {
 	a := &autoScalingMock{}
 	awsWrapper := &awsWrapper{
 		autoScalingI: a,
 		ec2I:         nil,
+		eksI:         nil,
 	}
 
 	// Generate 101 ASG names
@@ -152,6 +320,7 @@ func TestGetInstanceTypesForAsgs(t *testing.T) {
 	awsWrapper := &awsWrapper{
 		autoScalingI: a,
 		ec2I:         e,
+		eksI:         nil,
 	}
 
 	cases := []struct {

cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go

@@ -76,7 +76,7 @@ func TestLTVersionChange(t *testing.T) {
 		},
 		fakeClock,
 	)
-	m := newAsgInstanceTypeCacheWithClock(&awsWrapper{a, e}, fakeClock, fakeStore)
+	m := newAsgInstanceTypeCacheWithClock(&awsWrapper{a, e, nil}, fakeClock, fakeStore)
 
 	for i := 0; i < 2; i++ {
 		err := m.populate([]*asg{