Load AWS EC2 Instance Types dynamically

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go

@@ -49,13 +49,16 @@ var (
 type awsCloudProvider struct {
 	awsManager      *AwsManager
 	resourceLimiter *cloudprovider.ResourceLimiter
+	// InstanceTypes is a map of ec2 resources
+	instanceTypes map[string]*InstanceType
 }
 
 // BuildAwsCloudProvider builds CloudProvider implementation for AWS.
-func BuildAwsCloudProvider(awsManager *AwsManager, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) {
+func BuildAwsCloudProvider(awsManager *AwsManager, instanceTypes map[string]*InstanceType, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) {
 	aws := &awsCloudProvider{
 		awsManager:      awsManager,
 		resourceLimiter: resourceLimiter,
+		instanceTypes:   instanceTypes,
 	}
 	return aws, nil
 }
@@ -348,7 +351,15 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover
 		klog.Fatalf("Failed to create AWS Manager: %v", err)
 	}
 
-	provider, err := BuildAwsCloudProvider(manager, rl)
+	// Generate EC2 list
+	instanceTypes, err := GenerateEC2InstanceTypes()
+	if err != nil {
+		klog.Warningf("Failed to generate AWS EC2 Instance Types: %v", err)
+		klog.Warning("Use static EC2 Instance Types, list could be outdated")
+		instanceTypes = GetStaticEC2InstanceTypes()
+	}
+
+	provider, err := BuildAwsCloudProvider(manager, instanceTypes, rl)
 	if err != nil {
 		klog.Fatalf("Failed to create AWS cloud provider: %v", err)
 	}

cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go

@@ -133,7 +133,7 @@ func testProvider(t *testing.T, m *AwsManager) *awsCloudProvider {
 		map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000},
 		map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000})
 
-	provider, err := BuildAwsCloudProvider(m, resourceLimiter)
+	provider, err := BuildAwsCloudProvider(m, GetStaticEC2InstanceTypes(), resourceLimiter)
 	assert.NoError(t, err)
 	return provider.(*awsCloudProvider)
 }
@@ -143,7 +143,7 @@ func TestBuildAwsCloudProvider(t *testing.T) {
 		map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000},
 		map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000})
 
-	_, err := BuildAwsCloudProvider(testAwsManager, resourceLimiter)
+	_, err := BuildAwsCloudProvider(testAwsManager, GetStaticEC2InstanceTypes(), resourceLimiter)
 	assert.NoError(t, err)
 }
 

cluster-autoscaler/cloudprovider/aws/aws_manager.go

@@ -62,7 +62,7 @@ type AwsManager struct {
 }
 
 type asgTemplate struct {
-	InstanceType *instanceType
+	InstanceType *InstanceType
 	Region       string
 	Zone         string
 	Tags         []*autoscaling.TagDescription

cluster-autoscaler/cloudprovider/aws/aws_manager_test.go

@@ -70,7 +70,7 @@ func TestGetRegion(t *testing.T) {
 
 func TestBuildGenericLabels(t *testing.T) {
 	labels := buildGenericLabels(&asgTemplate{
-		InstanceType: &instanceType{
+		InstanceType: &InstanceType{
 			InstanceType: "c4.large",
 			VCPU:         2,
 			MemoryMb:     3840,

cluster-autoscaler/cloudprovider/aws/aws_util.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"encoding/json"
+	"errors"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"io/ioutil"
+	"k8s.io/klog"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type response struct {
+	Products map[string]product `json:"products"`
+}
+
+type product struct {
+	Attributes productAttributes `json:"attributes"`
+}
+
+type productAttributes struct {
+	InstanceType string `json:"instanceType"`
+	VCPU         string `json:"vcpu"`
+	Memory       string `json:"memory"`
+	GPU          string `json:"gpu"`
+}
+
+// GenerateEC2InstanceTypes returns a map of ec2 resources
+func GenerateEC2InstanceTypes() (map[string]*InstanceType, error) {
+	instanceTypes := make(map[string]*InstanceType)
+
+	resolver := endpoints.DefaultResolver()
+	partitions := resolver.(endpoints.EnumPartitions).Partitions()
+
+	for _, p := range partitions {
+		for _, r := range p.Regions() {
+			url := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/" + r.ID() + "/index.json"
+			klog.V(1).Infof("fetching %s\n", url)
+			res, err := http.Get(url)
+			if err != nil {
+				klog.Warningf("Error fetching %s skipping...\n", url)
+				continue
+			}
+
+			defer res.Body.Close()
+
+			body, err := ioutil.ReadAll(res.Body)
+			if err != nil {
+				klog.Warningf("Error parsing %s skipping...\n", url)
+				continue
+			}
+
+			var unmarshalled = response{}
+			err = json.Unmarshal(body, &unmarshalled)
+			if err != nil {
+				klog.Warningf("Error unmarshalling %s, skip...\n", url)
+				continue
+			}
+
+			for _, product := range unmarshalled.Products {
+				attr := product.Attributes
+				if attr.InstanceType != "" {
+					instanceTypes[attr.InstanceType] = &InstanceType{
+						InstanceType: attr.InstanceType,
+					}
+					if attr.Memory != "" && attr.Memory != "NA" {
+						instanceTypes[attr.InstanceType].MemoryMb = parseMemory(attr.Memory)
+					}
+					if attr.VCPU != "" {
+						instanceTypes[attr.InstanceType].VCPU = parseCPU(attr.VCPU)
+					}
+					if attr.GPU != "" {
+						instanceTypes[attr.InstanceType].GPU = parseCPU(attr.GPU)
+					}
+				}
+			}
+		}
+	}
+
+	if len(instanceTypes) == 0 {
+		return nil, errors.New("unable to load EC2 Instance Type list")
+	}
+
+	return instanceTypes, nil
+}
+
+// GetStaticEC2InstanceTypes return pregenerated ec2 instance type list
+func GetStaticEC2InstanceTypes() map[string]*InstanceType {
+	return InstanceTypes
+}
+
+func parseMemory(memory string) int64 {
+	reg, err := regexp.Compile("[^0-9\\.]+")
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	parsed := strings.TrimSpace(reg.ReplaceAllString(memory, ""))
+	mem, err := strconv.ParseFloat(parsed, 64)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	return int64(mem * float64(1024))
+}
+
+func parseCPU(cpu string) int64 {
+	i, err := strconv.ParseInt(cpu, 10, 64)
+	if err != nil {
+		klog.Fatal(err)
+	}
+	return i
+}

cluster-autoscaler/cloudprovider/aws/aws_util_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"github.com/stretchr/testify/assert"
+	"strconv"
+	"testing"
+)
+
+func TestGetStaticEC2InstanceTypes(t *testing.T) {
+	result := GetStaticEC2InstanceTypes()
+	assert.True(t, len(result) != 0)
+}
+
+func TestParseMemory(t *testing.T) {
+	expectedResultInMiB := int64(3.75 * 1024)
+	tests := []struct {
+		input  string
+		expect int64
+	}{
+		{
+			input:  "3.75 GiB",
+			expect: expectedResultInMiB,
+		},
+		{
+			input:  "3.75 Gib",
+			expect: expectedResultInMiB,
+		},
+		{
+			input:  "3.75GiB",
+			expect: expectedResultInMiB,
+		},
+		{
+			input:  "3.75",
+			expect: expectedResultInMiB,
+		},
+	}
+
+	for _, test := range tests {
+		got := parseMemory(test.input)
+		assert.Equal(t, test.expect, got)
+	}
+}
+
+func TestParseCPU(t *testing.T) {
+	tests := []struct {
+		input  string
+		expect int64
+	}{
+		{
+			input:  strconv.FormatInt(8, 10),
+			expect: int64(8),
+		},
+	}
+
+	for _, test := range tests {
+		got := parseCPU(test.input)
+		assert.Equal(t, test.expect, got)
+	}
+}

cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go

@@ -18,15 +18,16 @@ limitations under the License.
 
 package aws
 
-type instanceType struct {
+// InstanceType represents ec2 resource
+type InstanceType struct {
 	InstanceType string
 	VCPU         int64
 	MemoryMb     int64
 	GPU          int64
 }
 
 // InstanceTypes is a map of ec2 resources
-var InstanceTypes = map[string]*instanceType{
+var InstanceTypes = map[string]*InstanceType{
 	"a1": {
 		InstanceType: "a1",
 		VCPU:         16,