Skip to content

Commit

Permalink
add support for 'MinimizeUsage' resource registration
Browse files Browse the repository at this point in the history
  • Loading branch information
@tzneal
tzneal committed Mar 18, 2022
1 parent 56d23c2 commit 85cd89a
Show file tree
Hide file tree
Showing 13 changed files with 525 additions and 174 deletions.
26 changes: 3 additions & 23 deletions pkg/cloudprovider/aws/cloudprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
"fmt"
"time"

"github.com/aws/karpenter/pkg/utils/resources"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
Expand Down Expand Up @@ -60,6 +58,9 @@ const (

func init() {
v1alpha5.NormalizedLabels = functional.UnionStringMaps(v1alpha5.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone})
cloudprovider.RegisterResource(v1alpha1.ResourceAWSNeuron, cloudprovider.ResourceFlagMinimizeUsage)
cloudprovider.RegisterResource(v1alpha1.ResourceNVIDIAGPU, cloudprovider.ResourceFlagMinimizeUsage)
cloudprovider.RegisterResource(v1alpha1.ResourceAMDGPU, cloudprovider.ResourceFlagMinimizeUsage)
}

type CloudProvider struct {
Expand Down Expand Up @@ -159,27 +160,6 @@ func (c *CloudProvider) Name() string {
return "aws"
}

func (c *CloudProvider) LessThan(lhs, rhs cloudprovider.InstanceType) bool {
for _, rn := range []v1.ResourceName{
v1alpha1.ResourceNVIDIAGPU,
v1alpha1.ResourceAWSNeuron,
v1alpha1.ResourceAMDGPU,
v1.ResourceMemory,
v1.ResourceCPU,
} {
cmp := resources.Cmp(lhs.Resources()[rn], rhs.Resources()[rn])
if cmp < 0 {
return true
} else if cmp > 0 {
return false
}
}

// provide a consistent ordering for the unlikely occurrence of two instance types having
// the same GPU, memory and CPU resources
return lhs.Name() < rhs.Name()
}

// get the current region from EC2 IMDS
func getRegionFromIMDS(sess *session.Session) string {
region, err := ec2metadata.New(sess).Region()
Expand Down
25 changes: 23 additions & 2 deletions pkg/cloudprovider/aws/instancetype.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,27 @@ func (i *InstanceType) Resources() v1.ResourceList {
}
}

func (i *InstanceType) Price() float64 {
const MbToGb = 1 / 1024.0

gpuMemoryMiB := 0.0
gpuCount := 0.0
if i.GpuInfo != nil {
for _, gpu := range i.GpuInfo.Gpus {
if gpu.MemoryInfo != nil && gpu.MemoryInfo.SizeInMiB != nil {
gpuMemoryMiB += float64(*gpu.MemoryInfo.SizeInMiB)
}
if gpu.Count != nil {
gpuCount += float64(*gpu.Count)
}
}
}

// These are meant to give some relative weighting
return float64(*i.VCpuInfo.DefaultVCpus) +
float64(*i.MemoryInfo.SizeInMiB)/MbToGb +
5*gpuCount + 10*gpuMemoryMiB/MbToGb
}
func (i *InstanceType) cpu() resource.Quantity {
return *resources.Quantity(fmt.Sprint(*i.VCpuInfo.DefaultVCpus))
}
Expand Down Expand Up @@ -104,7 +125,7 @@ func (i *InstanceType) nvidiaGPUs() resource.Quantity {
count := int64(0)
if i.GpuInfo != nil {
for _, gpu := range i.GpuInfo.Gpus {
if *i.GpuInfo.Gpus[0].Manufacturer == "NVIDIA" {
if *gpu.Manufacturer == "NVIDIA" {
count += *gpu.Count
}
}
Expand All @@ -116,7 +137,7 @@ func (i *InstanceType) amdGPUs() resource.Quantity {
count := int64(0)
if i.GpuInfo != nil {
for _, gpu := range i.GpuInfo.Gpus {
if *i.GpuInfo.Gpus[0].Manufacturer == "AMD" {
if *gpu.Manufacturer == "NVIDIA" {
count += *gpu.Count
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/cloudprovider/aws/instancetypes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func (p *InstanceTypeProvider) Get(ctx context.Context, provider *v1alpha1.AWS)
if err != nil {
return nil, err
}
result := []cloudprovider.InstanceType{}
var result []cloudprovider.InstanceType
for _, instanceType := range instanceTypes {
offerings := p.createOfferings(instanceType, subnetZones, instanceTypeZones[instanceType.Name()])
if len(offerings) > 0 {
Expand Down
111 changes: 18 additions & 93 deletions pkg/cloudprovider/aws/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,37 +183,6 @@ var _ = Describe("Allocation", func() {
Expect(supportsPodENI()).To(Equal(true))
}
})
It("should pack nodes tightly", func() {
var nodes []*v1.Node
for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("31")},
},
}),
test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")},
},
})) {
node := ExpectScheduled(ctx, env.Client, pod)
nodes = append(nodes, node)
}
// the first pod consumes a p3.8xlarge with no room for the second pod, the second pod is much smaller in
// terms of resources and should get a smaller node
Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable]))
})
It("should handle zero-quantity resource requests", func() {
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{
Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")},
Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")},
},
}))
// requesting a resource of quantity zero of a type unsupported by any instance is fine
ExpectScheduled(ctx, env.Client, pod[0])
})
It("should launch instances for Nvidia GPU resource requests", func() {
nodeNames := sets.NewString()
for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
Expand Down Expand Up @@ -244,48 +213,6 @@ var _ = Describe("Allocation", func() {
}
Expect(nodeNames.Len()).To(Equal(2))
})
It("should not schedule a non-GPU workload on a node w/GPU", func() {
Skip("enable after scheduling and binpacking are merged into the same process")
nodeNames := sets.NewString()
for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1alpha1.ResourceNVIDIAGPU: resource.MustParse("1"),
"cpu": resource.MustParse("31"),
},
Limits: v1.ResourceList{
v1alpha1.ResourceNVIDIAGPU: resource.MustParse("1"),
"cpu": resource.MustParse("31"),
},
},
}),
// Can't pack onto the same instance due to consuming too much CPU
test.UnschedulablePod(test.PodOptions{
ResourceRequirements: v1.ResourceRequirements{
Requests: v1.ResourceList{
"cpu": resource.MustParse("3"),
},
Limits: v1.ResourceList{
"cpu": resource.MustParse("3"),
},
},
}),
) {
node := ExpectScheduled(ctx, env.Client, pod)

// This test has a GPU workload that nearly maxes out the test instance type. It's intended to ensure
// that the second pod won't get a GPU node since it doesn't require one, even though it's compatible
// with the first pod that does require a GPU.
if _, isGpuPod := pod.Spec.Containers[0].Resources.Requests[v1alpha1.ResourceNVIDIAGPU]; isGpuPod {
Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "p3.8xlarge"))
} else {
Expect(node.Labels).ToNot(HaveKeyWithValue(v1.LabelInstanceTypeStable, "p3.8xlarge"))
}
nodeNames.Insert(node.Name)
}
Expect(nodeNames.Len()).To(Equal(2))
})
It("should launch instances for AWS Neuron resource requests", func() {
nodeNames := sets.NewString()
for _, pod := range ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner,
Expand Down Expand Up @@ -548,24 +475,22 @@ var _ = Describe("Allocation", func() {
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)

Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))

ltNames := sets.NewString()
firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
ltNames.Insert(*firstLt.LaunchTemplateName)
ltNames.Insert(*fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput).LaunchTemplateName)
ltNames.Insert(*fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput).LaunchTemplateName)

Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1))

createFleetInput := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput)
launchTemplate := createFleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification
Expect(createFleetInput.LaunchTemplateConfigs).To(HaveLen(3))
Expect(createFleetInput.LaunchTemplateConfigs).To(HaveLen(2))

createFleetLtNames := sets.NewString()
createFleetLtNames.Insert(*createFleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName)
createFleetLtNames.Insert(*createFleetInput.LaunchTemplateConfigs[1].LaunchTemplateSpecification.LaunchTemplateName)
createFleetLtNames.Insert(*createFleetInput.LaunchTemplateConfigs[2].LaunchTemplateSpecification.LaunchTemplateName)

Expect(ltNames).To(Equal(createFleetLtNames))
Expect(firstLt.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Encrypted).To(Equal(aws.Bool(true)))
Expand All @@ -591,7 +516,7 @@ var _ = Describe("Allocation", func() {
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1))
input := fakeEC2API.CalledWithCreateFleetInput.Pop().(*ec2.CreateFleetInput)
Expect(input.LaunchTemplateConfigs).To(HaveLen(2))
Expect(input.LaunchTemplateConfigs).To(HaveLen(1))

foundNonGPULT := false
for _, v := range input.LaunchTemplateConfigs {
Expand Down Expand Up @@ -626,7 +551,7 @@ var _ = Describe("Allocation", func() {
It("should default to the clusters security groups", func() {
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(input.LaunchTemplateData.SecurityGroupIds).To(ConsistOf(
aws.String("test-security-group-1"),
Expand All @@ -641,7 +566,7 @@ var _ = Describe("Allocation", func() {
}}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(input.LaunchTemplateData.SecurityGroupIds).To(ConsistOf(
aws.String("test-sg-1"),
Expand All @@ -655,7 +580,7 @@ var _ = Describe("Allocation", func() {
localCtx := injection.WithOptions(ctx, opts)
pod := ExpectProvisioned(localCtx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(localCtx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
Expect(string(userData)).NotTo(ContainSubstring("--use-max-pods false"))
Expand All @@ -665,7 +590,7 @@ var _ = Describe("Allocation", func() {
localCtx := injection.WithOptions(ctx, opts)
pod := ExpectProvisioned(localCtx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(localCtx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
Expect(string(userData)).To(ContainSubstring("--use-max-pods false"))
Expand All @@ -676,8 +601,8 @@ var _ = Describe("Allocation", func() {
provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
for i := 0; i < 3; i++ { // AMD64, ARM and GPU
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
for i := 0; i < 2; i++ { // amd64 and arm
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData)
Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip '10.0.10.100'"))
Expand All @@ -689,7 +614,7 @@ var _ = Describe("Allocation", func() {
provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("test-instance-profile"))
})
Expand All @@ -700,7 +625,7 @@ var _ = Describe("Allocation", func() {
ProvisionerWithProvider(&v1alpha5.Provisioner{ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}}, provider)
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("overridden-profile"))
})
Expand All @@ -710,7 +635,7 @@ var _ = Describe("Allocation", func() {
It("should default metadata options on generated launch template", func() {
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled))
Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled))
Expand All @@ -728,7 +653,7 @@ var _ = Describe("Allocation", func() {
}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateDisabled))
Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled))
Expand All @@ -742,8 +667,8 @@ var _ = Describe("Allocation", func() {
provider.AMIFamily = &v1alpha1.AMIFamilyAL2
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
// A non-GPU and GPU template
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
// An amd64 and arm template
for i := 0; i < 2; i++ {
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1))
Expand All @@ -770,7 +695,7 @@ var _ = Describe("Allocation", func() {
}
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
for i := 0; i < 2; i++ {
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1))
Expand All @@ -788,8 +713,8 @@ var _ = Describe("Allocation", func() {
provider.AMIFamily = &v1alpha1.AMIFamilyBottlerocket
pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0]
ExpectScheduled(ctx, env.Client, pod)
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(3))
for i := 0; i < 3; i++ {
Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(2))
for i := 0; i < 2; i++ {
input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput)
Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(2))
// Bottlerocket control volume
Expand Down
Loading

0 comments on commit 85cd89a

Please sign in to comment.