diff --git a/pkg/cloudprovider/aws/fake/ec2api.go b/pkg/cloudprovider/aws/fake/ec2api.go index 42bea1e3f6e2..17bcb6d98eb4 100644 --- a/pkg/cloudprovider/aws/fake/ec2api.go +++ b/pkg/cloudprovider/aws/fake/ec2api.go @@ -218,12 +218,34 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 } fn(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: []*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String("t3.large"), + SupportedUsageClasses: DefaultSupportedUsageClasses, + SupportedVirtualizationTypes: []*string{aws.String("hvm")}, + BurstablePerformanceSupported: aws.Bool(true), + BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), + }, + VCpuInfo: &ec2.VCpuInfo{ + DefaultVCpus: aws.Int64(2), + }, + MemoryInfo: &ec2.MemoryInfo{ + SizeInMiB: aws.Int64(8 * 1024), + }, + NetworkInfo: &ec2.NetworkInfo{ + MaximumNetworkInterfaces: aws.Int64(3), + Ipv4AddressesPerInterface: aws.Int64(12), + }, + }, { InstanceType: aws.String("m5.large"), SupportedUsageClasses: DefaultSupportedUsageClasses, SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, @@ -244,6 +266,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, @@ -264,6 +287,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, @@ -290,6 +314,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{v1alpha5.ArchitectureArm64}), }, @@ -310,6 +335,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, @@ -335,6 +361,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2 SupportedVirtualizationTypes: []*string{aws.String("hvm")}, BurstablePerformanceSupported: aws.Bool(false), BareMetal: aws.Bool(false), + Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, @@ -402,6 +429,14 @@ func (e *EC2API) DescribeInstanceTypeOfferingsPagesWithContext(_ context.Context InstanceType: aws.String("p3.8xlarge"), Location: aws.String("test-zone-1b"), }, + { + InstanceType: aws.String("t3.large"), + Location: aws.String("test-zone-1a"), + }, + { + InstanceType: aws.String("t3.large"), + Location: aws.String("test-zone-1b"), + }, { InstanceType: aws.String("inf1.2xlarge"), Location: aws.String("test-zone-1a"), diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 1c7c2cfb381c..1ddc2a188686 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -16,6 +16,7 @@ package aws import ( "fmt" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -70,6 +71,15 @@ func (i *InstanceType) Pods() *resource.Quantity { return resources.Quantity(fmt.Sprint(*i.NetworkInfo.MaximumNetworkInterfaces*(*i.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) } +func (i *InstanceType) AWSPodENI() *resource.Quantity { + // Pod ENI is supported by Bare Metal & all Nitro except T3 + // https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types + if aws.BoolValue(i.BareMetal) || (aws.StringValue(i.Hypervisor) == "nitro" && !strings.HasPrefix(*i.InstanceType, "t3")) { + return i.Pods() + } + return resources.Quantity("0") +} + func (i *InstanceType) NvidiaGPUs() *resource.Quantity { count := int64(0) if i.GpuInfo != nil { diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 130bc6a491f9..33c3fd8f1be2 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -157,7 +157,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, optio } // needsDocker returns true if the instance type is unable to use -// conatinerd directly +// containerd directly func needsDocker(is []cloudprovider.InstanceType) bool { for _, i := range is { if !i.AWSNeurons().IsZero() || !i.NvidiaGPUs().IsZero() { diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index 3592894c4a6b..36f9202ed907 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -121,6 +121,20 @@ var _ = Describe("Allocation", func() { Context("Reconciliation", func() { Context("Specialized Hardware", func() { + It("should not launch AWS Pod ENI on a t3", func() { + for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{ + v1.LabelInstanceTypeStable: "t3.large", + }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{resources.AWSPodENI: resource.MustParse("1")}, + Limits: v1.ResourceList{resources.AWSPodENI: resource.MustParse("1")}, + }, + })) { + ExpectNotScheduled(ctx, env.Client, pod) + } + }) It("should launch instances for Nvidia GPU resource requests", func() { nodeNames := sets.NewString() for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner, diff --git a/pkg/cloudprovider/fake/cloudprovider.go b/pkg/cloudprovider/fake/cloudprovider.go index b43404c3d111..ab8ef950ec8c 100644 --- a/pkg/cloudprovider/fake/cloudprovider.go +++ b/pkg/cloudprovider/fake/cloudprovider.go @@ -82,6 +82,10 @@ func (c *CloudProvider) GetInstanceTypes(_ context.Context, _ *v1alpha5.Constrai NewInstanceType(InstanceTypeOptions{ name: "default-instance-type", }), + NewInstanceType(InstanceTypeOptions{ + name: "pod-eni-instance-type", + awsPodENI: resource.MustParse("1"), + }), NewInstanceType(InstanceTypeOptions{ name: "small-instance-type", cpu: resource.MustParse("2"), diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 640eedb1e3e4..7510f8c1e35c 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -53,6 +53,7 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType { nvidiaGPUs: options.nvidiaGPUs, amdGPUs: options.amdGPUs, awsNeurons: options.awsNeurons, + awsPodENI: options.awsPodENI, }, } } @@ -67,6 +68,7 @@ type InstanceTypeOptions struct { nvidiaGPUs resource.Quantity amdGPUs resource.Quantity awsNeurons resource.Quantity + awsPodENI resource.Quantity } type InstanceType struct { @@ -109,6 +111,10 @@ func (i *InstanceType) AWSNeurons() *resource.Quantity { return &i.awsNeurons } +func (i *InstanceType) AWSPodENI() *resource.Quantity { + return &i.awsPodENI +} + func (i *InstanceType) Overhead() v1.ResourceList { return v1.ResourceList{ v1.ResourceCPU: resource.MustParse("100m"), diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 5c62a7ac5d4d..46f216d2e821 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -61,6 +61,7 @@ type InstanceType interface { NvidiaGPUs() *resource.Quantity AMDGPUs() *resource.Quantity AWSNeurons() *resource.Quantity + AWSPodENI() *resource.Quantity Overhead() v1.ResourceList } diff --git a/pkg/controllers/provisioning/binpacking/packable.go b/pkg/controllers/provisioning/binpacking/packable.go index affac7d84e51..a4069d2fd0c6 100644 --- a/pkg/controllers/provisioning/binpacking/packable.go +++ b/pkg/controllers/provisioning/binpacking/packable.go @@ -54,6 +54,7 @@ func PackablesFor(ctx context.Context, instanceTypes []cloudprovider.InstanceTyp packable.validateInstanceType(schedule), packable.validateArchitecture(schedule), packable.validateCapacityTypes(schedule), + packable.validateAWSPodENI(schedule), // Although this will remove instances that have GPUs when // not required, removal of instance types that *lack* // GPUs will be done later. @@ -87,6 +88,7 @@ func PackableFor(i cloudprovider.InstanceType) *Packable { resources.NvidiaGPU: *i.NvidiaGPUs(), resources.AMDGPU: *i.AMDGPUs(), resources.AWSNeuron: *i.AWSNeurons(), + resources.AWSPodENI: *i.AWSPodENI(), v1.ResourcePods: *i.Pods(), }, } @@ -229,6 +231,20 @@ func (p *Packable) validateAWSNeurons(schedule *scheduling.Schedule) error { return fmt.Errorf("aws neuron is not required") } +func (p *Packable) validateAWSPodENI(schedule *scheduling.Schedule) error { + for _, pod := range schedule.Pods { + for _, container := range pod.Spec.Containers { + if _, ok := container.Resources.Requests[resources.AWSPodENI]; ok { + if p.InstanceType.AWSPodENI().IsZero() { + return fmt.Errorf("aws pod eni is required") + } + return nil + } + } + } + return nil +} + func packableNames(instanceTypes []*Packable) []string { names := []string{} for _, instanceType := range instanceTypes { diff --git a/pkg/utils/resources/resources.go b/pkg/utils/resources/resources.go index 2110044ab4dc..eaf7d49879be 100644 --- a/pkg/utils/resources/resources.go +++ b/pkg/utils/resources/resources.go @@ -23,6 +23,7 @@ const ( NvidiaGPU = "nvidia.com/gpu" AMDGPU = "amd.com/gpu" AWSNeuron = "aws.amazon.com/neuron" + AWSPodENI = "vpc.amazonaws.com/pod-eni" ) // RequestsForPods returns the total resources of a variadic list of podspecs.