Add a filter for checking Pod ENI support

pkg/cloudprovider/aws/fake/ec2api.go

@@ -218,12 +218,34 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 	}
 	fn(&ec2.DescribeInstanceTypesOutput{
 		InstanceTypes: []*ec2.InstanceTypeInfo{
+			{
+				InstanceType:                  aws.String("t3.large"),
+				SupportedUsageClasses:         DefaultSupportedUsageClasses,
+				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
+				BurstablePerformanceSupported: aws.Bool(true),
+				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
+				ProcessorInfo: &ec2.ProcessorInfo{
+					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
+				},
+				VCpuInfo: &ec2.VCpuInfo{
+					DefaultVCpus: aws.Int64(2),
+				},
+				MemoryInfo: &ec2.MemoryInfo{
+					SizeInMiB: aws.Int64(8 * 1024),
+				},
+				NetworkInfo: &ec2.NetworkInfo{
+					MaximumNetworkInterfaces:  aws.Int64(3),
+					Ipv4AddressesPerInterface: aws.Int64(12),
+				},
+			},
 			{
 				InstanceType:                  aws.String("m5.large"),
 				SupportedUsageClasses:         DefaultSupportedUsageClasses,
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
 				},
@@ -244,6 +266,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
 				},
@@ -264,6 +287,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
 				},
@@ -290,6 +314,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{v1alpha5.ArchitectureArm64}),
 				},
@@ -310,6 +335,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
 				},
@@ -335,6 +361,7 @@ func (e *EC2API) DescribeInstanceTypesPagesWithContext(_ context.Context, _ *ec2
 				SupportedVirtualizationTypes:  []*string{aws.String("hvm")},
 				BurstablePerformanceSupported: aws.Bool(false),
 				BareMetal:                     aws.Bool(false),
+				Hypervisor:                    aws.String("nitro"),
 				ProcessorInfo: &ec2.ProcessorInfo{
 					SupportedArchitectures: aws.StringSlice([]string{"x86_64"}),
 				},
@@ -402,6 +429,14 @@ func (e *EC2API) DescribeInstanceTypeOfferingsPagesWithContext(_ context.Context
 				InstanceType: aws.String("p3.8xlarge"),
 				Location:     aws.String("test-zone-1b"),
 			},
+			{
+				InstanceType: aws.String("t3.large"),
+				Location:     aws.String("test-zone-1a"),
+			},
+			{
+				InstanceType: aws.String("t3.large"),
+				Location:     aws.String("test-zone-1b"),
+			},
 			{
 				InstanceType: aws.String("inf1.2xlarge"),
 				Location:     aws.String("test-zone-1a"),

pkg/cloudprovider/aws/instancetype.go

@@ -16,6 +16,7 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -70,6 +71,15 @@ func (i *InstanceType) Pods() *resource.Quantity {
 	return resources.Quantity(fmt.Sprint(*i.NetworkInfo.MaximumNetworkInterfaces*(*i.NetworkInfo.Ipv4AddressesPerInterface-1) + 2))
 }
 
+func (i *InstanceType) AWSPodENI() *resource.Quantity {
+	// Pod ENI is supported by Bare Metal & all Nitro except T3
+	// https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types
+	if aws.BoolValue(i.BareMetal) || (aws.StringValue(i.Hypervisor) == "nitro" && !strings.HasPrefix(*i.InstanceType, "t3")) {
+		return i.Pods()
+	}
+	return resources.Quantity("0")
+}
+
 func (i *InstanceType) NvidiaGPUs() *resource.Quantity {
 	count := int64(0)
 	if i.GpuInfo != nil {

pkg/cloudprovider/aws/launchtemplate.go

@@ -157,7 +157,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, optio
 }
 
 // needsDocker returns true if the instance type is unable to use
-// conatinerd directly
+// containerd directly
 func needsDocker(is []cloudprovider.InstanceType) bool {
 	for _, i := range is {
 		if !i.AWSNeurons().IsZero() || !i.NvidiaGPUs().IsZero() {

pkg/cloudprovider/aws/suite_test.go

@@ -121,6 +121,20 @@ var _ = Describe("Allocation", func() {
 
 	Context("Reconciliation", func() {
 		Context("Specialized Hardware", func() {
+			It("should not launch AWS Pod ENI on a t3", func() {
+				for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner,
+					test.UnschedulablePod(test.PodOptions{
+						NodeSelector: map[string]string{
+							v1.LabelInstanceTypeStable: "t3.large",
+						},
+						ResourceRequirements: v1.ResourceRequirements{
+							Requests: v1.ResourceList{resources.AWSPodENI: resource.MustParse("1")},
+							Limits:   v1.ResourceList{resources.AWSPodENI: resource.MustParse("1")},
+						},
+					})) {
+					ExpectNotScheduled(ctx, env.Client, pod)
+				}
+			})
 			It("should launch instances for Nvidia GPU resource requests", func() {
 				nodeNames := sets.NewString()
 				for _, pod := range ExpectProvisioned(ctx, env.Client, scheduler, provisioners, provisioner,

pkg/cloudprovider/fake/cloudprovider.go

@@ -82,6 +82,10 @@ func (c *CloudProvider) GetInstanceTypes(_ context.Context, _ *v1alpha5.Constrai
 		NewInstanceType(InstanceTypeOptions{
 			name: "default-instance-type",
 		}),
+		NewInstanceType(InstanceTypeOptions{
+			name:      "pod-eni-instance-type",
+			awsPodENI: resource.MustParse("1"),
+		}),
 		NewInstanceType(InstanceTypeOptions{
 			name:   "small-instance-type",
 			cpu:    resource.MustParse("2"),

pkg/cloudprovider/fake/instancetype.go

@@ -53,6 +53,7 @@ func NewInstanceType(options InstanceTypeOptions) *InstanceType {
 			nvidiaGPUs:   options.nvidiaGPUs,
 			amdGPUs:      options.amdGPUs,
 			awsNeurons:   options.awsNeurons,
+			awsPodENI:    options.awsPodENI,
 		},
 	}
 }
@@ -67,6 +68,7 @@ type InstanceTypeOptions struct {
 	nvidiaGPUs   resource.Quantity
 	amdGPUs      resource.Quantity
 	awsNeurons   resource.Quantity
+	awsPodENI    resource.Quantity
 }
 
 type InstanceType struct {
@@ -109,6 +111,10 @@ func (i *InstanceType) AWSNeurons() *resource.Quantity {
 	return &i.awsNeurons
 }
 
+func (i *InstanceType) AWSPodENI() *resource.Quantity {
+	return &i.awsPodENI
+}
+
 func (i *InstanceType) Overhead() v1.ResourceList {
 	return v1.ResourceList{
 		v1.ResourceCPU:    resource.MustParse("100m"),

pkg/cloudprovider/types.go

@@ -61,6 +61,7 @@ type InstanceType interface {
 	NvidiaGPUs() *resource.Quantity
 	AMDGPUs() *resource.Quantity
 	AWSNeurons() *resource.Quantity
+	AWSPodENI() *resource.Quantity
 	Overhead() v1.ResourceList
 }
 

pkg/controllers/provisioning/binpacking/packable.go

@@ -54,6 +54,7 @@ func PackablesFor(ctx context.Context, instanceTypes []cloudprovider.InstanceTyp
 			packable.validateInstanceType(schedule),
 			packable.validateArchitecture(schedule),
 			packable.validateCapacityTypes(schedule),
+			packable.validateAWSPodENI(schedule),
 			// Although this will remove instances that have GPUs when
 			// not required, removal of instance types that *lack*
 			// GPUs will be done later.
@@ -87,6 +88,7 @@ func PackableFor(i cloudprovider.InstanceType) *Packable {
 			resources.NvidiaGPU: *i.NvidiaGPUs(),
 			resources.AMDGPU:    *i.AMDGPUs(),
 			resources.AWSNeuron: *i.AWSNeurons(),
+			resources.AWSPodENI: *i.AWSPodENI(),
 			v1.ResourcePods:     *i.Pods(),
 		},
 	}
@@ -229,6 +231,20 @@ func (p *Packable) validateAWSNeurons(schedule *scheduling.Schedule) error {
 	return fmt.Errorf("aws neuron is not required")
 }
 
+func (p *Packable) validateAWSPodENI(schedule *scheduling.Schedule) error {
+	for _, pod := range schedule.Pods {
+		for _, container := range pod.Spec.Containers {
+			if _, ok := container.Resources.Requests[resources.AWSPodENI]; ok {
+				if p.InstanceType.AWSPodENI().IsZero() {
+					return fmt.Errorf("aws pod eni is required")
+				}
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
 func packableNames(instanceTypes []*Packable) []string {
 	names := []string{}
 	for _, instanceType := range instanceTypes {

pkg/utils/resources/resources.go

@@ -23,6 +23,7 @@ const (
 	NvidiaGPU = "nvidia.com/gpu"
 	AMDGPU    = "amd.com/gpu"
 	AWSNeuron = "aws.amazon.com/neuron"
+	AWSPodENI = "vpc.amazonaws.com/pod-eni"
 )
 
 // RequestsForPods returns the total resources of a variadic list of podspecs.