diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index f769ecee9704..01582e9bbbd5 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -478,19 +478,12 @@ spec: type items: description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. properties: key: description: The label key that the selector applies to. type: string - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer operator: description: |- Represents a key's relationship to a set of values. diff --git a/pkg/apis/v1beta1/ec2nodeclass_status.go b/pkg/apis/v1beta1/ec2nodeclass_status.go index 309183bc0c8e..e0c1b7ff322b 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_status.go +++ b/pkg/apis/v1beta1/ec2nodeclass_status.go @@ -16,7 +16,7 @@ package v1beta1 import ( op "github.com/awslabs/operatorpkg/status" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + v1 "k8s.io/api/core/v1" ) // Subnet contains resolved Subnet selector values utilized for node launch @@ -49,7 +49,7 @@ type AMI struct { Name string `json:"name,omitempty"` // Requirements of the AMI to be utilized on an instance type // +required - Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"` + Requirements []v1.NodeSelectorRequirement `json:"requirements"` } // EC2NodeClassStatus contains the resolved state of the EC2NodeClass diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index dc9487431676..f248b480be5b 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -20,8 +20,8 @@ package v1beta1 import ( "github.com/awslabs/operatorpkg/status" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - apisv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -29,7 +29,7 @@ func (in *AMI) DeepCopyInto(out *AMI) { *out = *in if in.Requirements != nil { in, out := &in.Requirements, &out.Requirements - *out = make([]apisv1beta1.NodeSelectorRequirementWithMinValues, len(*in)) + *out = make([]v1.NodeSelectorRequirement, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index f40455ad3837..5c87fdb62da9 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -77,14 +77,10 @@ func (c *CloudProvider) isAMIDrifted(ctx context.Context, nodeClaim *corev1beta1 if !found { return "", fmt.Errorf(`finding node instance type "%s"`, nodeClaim.Labels[v1.LabelInstanceTypeStable]) } - amis, err := c.amiProvider.Get(ctx, nodeClass, &amifamily.Options{}) - if err != nil { - return "", fmt.Errorf("getting amis, %w", err) - } - if len(amis) == 0 { + if len(nodeClass.Status.AMIs) == 0 { return "", fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}) + mappedAMIs := amifamily.MapToInstanceTypes([]*cloudprovider.InstanceType{nodeInstanceType}, nodeClass.Status.AMIs) if !lo.Contains(lo.Keys(mappedAMIs), instance.ImageID) { return AMIDrift, nil } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 76b7c06d718e..1b84be8050e3 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -115,41 +115,7 @@ var _ = Describe("CloudProvider", func() { var nodePool *corev1beta1.NodePool var nodeClaim *corev1beta1.NodeClaim var _ = BeforeEach(func() { - nodeClass = test.EC2NodeClass( - v1beta1.EC2NodeClass{ - Status: v1beta1.EC2NodeClassStatus{ - InstanceProfile: "test-profile", - SecurityGroups: []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - Name: "securityGroup-test1", - }, - { - ID: "sg-test2", - Name: "securityGroup-test2", - }, - { - ID: "sg-test3", - Name: "securityGroup-test3", - }, - }, - Subnets: []v1beta1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - }, - }, - }, - ) + nodeClass = test.EC2NodeClass() nodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeNodeClassReady) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ @@ -631,19 +597,36 @@ var _ = Describe("CloudProvider", func() { }, }, }) - nodeClass.Status.Subnets = []v1beta1.Subnet{ - { - ID: validSubnet1, - Zone: "zone-1", + nodeClass.Status = v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + Subnets: []v1beta1.Subnet{ + { + ID: validSubnet1, + Zone: "zone-1", + }, + { + ID: validSubnet2, + Zone: "zone-2", + }, }, - { - ID: validSubnet2, - Zone: "zone-2", + SecurityGroups: []v1beta1.SecurityGroup{ + { + ID: validSecurityGroup, + }, }, - } - nodeClass.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: validSecurityGroup, + AMIs: []v1beta1.AMI{ + { + ID: armAMIID, + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureArm64}}, + }, + }, + { + ID: amdAMIID, + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + }, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -794,6 +777,14 @@ var _ = Describe("CloudProvider", func() { }) It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: amdAMIID}} + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: amdAMIID, + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + }, + }, + } ExpectApplied(ctx, env.Client, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).ToNot(HaveOccurred()) @@ -801,6 +792,12 @@ var _ = Describe("CloudProvider", func() { }) Context("Static Drift Detection", func() { BeforeEach(func() { + armRequirements := []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureArm64}}, + } + amdRequirements := []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + } nodeClass = &v1beta1.EC2NodeClass{ ObjectMeta: nodeClass.ObjectMeta, Spec: v1beta1.EC2NodeClassSpec{ @@ -839,6 +836,7 @@ var _ = Describe("CloudProvider", func() { }, }, Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", Subnets: []v1beta1.Subnet{ { ID: validSubnet1, @@ -854,6 +852,16 @@ var _ = Describe("CloudProvider", func() { ID: validSecurityGroup, }, }, + AMIs: []v1beta1.AMI{ + { + ID: armAMIID, + Requirements: armRequirements, + }, + { + ID: amdAMIID, + Requirements: amdRequirements, + }, + }, }, } nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) @@ -1100,6 +1108,7 @@ var _ = Describe("CloudProvider", func() { }, }, Status: v1beta1.EC2NodeClassStatus{ + AMIs: nodeClass.Status.AMIs, SecurityGroups: []v1beta1.SecurityGroup{ { ID: "sg-test1", diff --git a/pkg/controllers/nodeclass/status/ami.go b/pkg/controllers/nodeclass/status/ami.go index 0759f4419712..b6f92ab700b7 100644 --- a/pkg/controllers/nodeclass/status/ami.go +++ b/pkg/controllers/nodeclass/status/ami.go @@ -21,10 +21,12 @@ import ( "time" "github.com/samber/lo" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) type AMI struct { @@ -32,7 +34,7 @@ type AMI struct { } func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { - amis, err := a.amiProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := a.amiProvider.List(ctx, nodeClass) if err != nil { return reconcile.Result{}, fmt.Errorf("getting amis, %w", err) } @@ -41,7 +43,10 @@ func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (r return reconcile.Result{}, nil } nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1beta1.AMI { - reqs := ami.Requirements.NodeSelectorRequirements() + reqs := lo.Map(ami.Requirements.NodeSelectorRequirements(), func(item corev1beta1.NodeSelectorRequirementWithMinValues, _ int) v1.NodeSelectorRequirement { + return item.NodeSelectorRequirement + }) + sort.Slice(reqs, func(i, j int) bool { if len(reqs[i].Key) != len(reqs[j].Key) { return len(reqs[i].Key) < len(reqs[j].Key) diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go index 0e442cbf33a1..6aa7843072ea 100644 --- a/pkg/controllers/nodeclass/status/ami_test.go +++ b/pkg/controllers/nodeclass/status/ami_test.go @@ -142,88 +142,68 @@ var _ = Describe("NodeClass AMI Status Controller", func() { { Name: "test-ami-3", ID: "ami-id-789", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureArm64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpExists, - }, + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpExists, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpExists, - }, + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpExists, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, }, }, @@ -270,50 +250,38 @@ var _ = Describe("NodeClass AMI Status Controller", func() { { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureArm64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []v1.NodeSelectorRequirement{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }, + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, - }, + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, }, }, }, @@ -328,16 +296,11 @@ var _ = Describe("NodeClass AMI Status Controller", func() { { Name: "test-ami-3", ID: "ami-test3", - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: "kubernetes.io/arch", - Operator: "In", - Values: []string{ - "amd64", - }, - }, - }, + Requirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, }, }, diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index 6542817b0c84..446b40d5679b 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "sort" - "strings" "sync" "time" @@ -42,7 +41,7 @@ import ( ) type Provider interface { - Get(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (AMIs, error) + List(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (AMIs, error) } type DefaultProvider struct { @@ -76,25 +75,13 @@ func (a AMIs) Sort() { }) } -func (a AMIs) String() string { - var sb strings.Builder - ids := lo.Map(a, func(a AMI, _ int) string { return a.AmiID }) - if len(a) > 25 { - sb.WriteString(strings.Join(ids[:25], ", ")) - sb.WriteString(fmt.Sprintf(" and %d other(s)", len(a)-25)) - } else { - sb.WriteString(strings.Join(ids, ", ")) - } - return sb.String() -} - // MapToInstanceTypes returns a map of AMIIDs that are the most recent on creationDate to compatible instancetypes -func (a AMIs) MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType) map[string][]*cloudprovider.InstanceType { +func MapToInstanceTypes(instanceTypes []*cloudprovider.InstanceType, amis []v1beta1.AMI) map[string][]*cloudprovider.InstanceType { amiIDs := map[string][]*cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - for _, ami := range a { - if err := instanceType.Requirements.Compatible(ami.Requirements, scheduling.AllowUndefinedWellKnownLabels); err == nil { - amiIDs[ami.AmiID] = append(amiIDs[ami.AmiID], instanceType) + for _, ami := range amis { + if err := instanceType.Requirements.Compatible(scheduling.NewNodeSelectorRequirements(ami.Requirements...), scheduling.AllowUndefinedWellKnownLabels); err == nil { + amiIDs[ami.ID] = append(amiIDs[ami.ID], instanceType) break } } @@ -113,14 +100,14 @@ func NewDefaultProvider(versionProvider version.Provider, ssm ssmiface.SSMAPI, e } // Get Returning a list of AMIs with its associated requirements -func (p *DefaultProvider) Get(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (AMIs, error) { +func (p *DefaultProvider) List(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (AMIs, error) { p.Lock() defer p.Unlock() var err error var amis AMIs if len(nodeClass.Spec.AMISelectorTerms) == 0 { - amis, err = p.getDefaultAMIs(ctx, nodeClass, options) + amis, err = p.getDefaultAMIs(ctx, nodeClass) if err != nil { return nil, err } @@ -137,13 +124,13 @@ func (p *DefaultProvider) Get(ctx context.Context, nodeClass *v1beta1.EC2NodeCla return amis, nil } -func (p *DefaultProvider) getDefaultAMIs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, options *Options) (res AMIs, err error) { +func (p *DefaultProvider) getDefaultAMIs(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (res AMIs, err error) { if images, ok := p.cache.Get(lo.FromPtr(nodeClass.Spec.AMIFamily)); ok { // Ensure what's returned from this function is a deep-copy of AMIs so alterations // to the data don't affect the original return append(AMIs{}, images.(AMIs)...), nil } - amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) + amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, &Options{}) kubernetesVersion, err := p.versionProvider.Get(ctx) if err != nil { return nil, fmt.Errorf("getting kubernetes version %w", err) diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 65adec3a9abb..e55aafc254e2 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -15,7 +15,6 @@ limitations under the License. package amifamily import ( - "context" "fmt" "net" @@ -27,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/utils/pretty" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" @@ -120,18 +120,14 @@ func NewResolver(amiProvider Provider) *Resolver { // Resolve generates launch templates using the static options and dynamically generates launch template parameters. // Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. -func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { +func (r Resolver) Resolve(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) - amis, err := r.amiProvider.Get(ctx, nodeClass, options) - if err != nil { - return nil, err - } - if len(amis) == 0 { + if len(nodeClass.Status.AMIs) == 0 { return nil, fmt.Errorf("no amis exist given constraints") } - mappedAMIs := amis.MapToInstanceTypes(instanceTypes) + mappedAMIs := MapToInstanceTypes(instanceTypes, nodeClass.Status.AMIs) if len(mappedAMIs) == 0 { - return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", amis) + return nil, fmt.Errorf("no instance types satisfy requirements of amis %v", pretty.Slice(lo.Map(nodeClass.Status.AMIs, func(a v1beta1.AMI, _ int) string { return a.ID }), 25)) } var resolvedTemplates []*LaunchTemplate for amiID, instanceTypes := range mappedAMIs { diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go index cc67884508a4..599476c09787 100644 --- a/pkg/providers/amifamily/suite_test.go +++ b/pkg/providers/amifamily/suite_test.go @@ -137,7 +137,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): amd64NvidiaAMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version): arm64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(4)) }) @@ -147,7 +147,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version): arm64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) @@ -159,7 +159,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): arm64AMI, fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/arm64/latest/image_id", version): arm64NvidiaAMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(6)) }) @@ -169,7 +169,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version): amd64AMI, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) @@ -178,7 +178,7 @@ var _ = Describe("AMIProvider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) @@ -187,13 +187,13 @@ var _ = Describe("AMIProvider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) It("should succeed to resolve AMIs (Custom)", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(0)) }) @@ -212,7 +212,7 @@ var _ = Describe("AMIProvider", func() { go func() { defer wg.Done() defer GinkgoRecover() - images, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + images, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(images).To(HaveLen(2)) @@ -249,7 +249,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version): arm64AMI, } // Only 2 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) @@ -258,7 +258,7 @@ var _ = Describe("AMIProvider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) @@ -271,7 +271,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): arm64AMI, } // Only 4 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(4)) }) @@ -282,7 +282,7 @@ var _ = Describe("AMIProvider", func() { fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, } // Only 1 of the requirements sets for the SSM aliases will resolve - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) }) @@ -314,7 +314,7 @@ var _ = Describe("AMIProvider", func() { Tags: map[string]string{"*": "*"}, }, } - amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + amis, err := awsEnv.AMIProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(1)) Expect(amis).To(ConsistOf(amifamily.AMI{ diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 723cfcd8c16b..3a4e4d8ee0da 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -82,38 +82,7 @@ var _ = Describe("InstanceProvider", func() { var nodePool *corev1beta1.NodePool var nodeClaim *corev1beta1.NodeClaim BeforeEach(func() { - nodeClass = test.EC2NodeClass( - v1beta1.EC2NodeClass{ - Status: v1beta1.EC2NodeClassStatus{ - InstanceProfile: "test-profile", - SecurityGroups: []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - }, - Subnets: []v1beta1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - }, - }, - }, - ) + nodeClass = test.EC2NodeClass() nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 70c677961450..c2bc31662991 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -109,38 +109,7 @@ var _ = Describe("InstanceTypeProvider", func() { var nodeClass, windowsNodeClass *v1beta1.EC2NodeClass var nodePool, windowsNodePool *corev1beta1.NodePool BeforeEach(func() { - nodeClass = test.EC2NodeClass( - v1beta1.EC2NodeClass{ - Status: v1beta1.EC2NodeClassStatus{ - InstanceProfile: "test-profile", - SecurityGroups: []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - }, - Subnets: []v1beta1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - }, - }, - }, - ) + nodeClass = test.EC2NodeClass() nodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeNodeClassReady) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ @@ -171,6 +140,16 @@ var _ = Describe("InstanceTypeProvider", func() { InstanceProfile: "test-profile", SecurityGroups: nodeClass.Status.SecurityGroups, Subnets: nodeClass.Status.Subnets, + AMIs: []v1beta1.AMI{ + { + ID: "ami-window-test1", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + {Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Windows)}}, + {Key: v1.LabelWindowsBuild, Operator: v1.NodeSelectorOpIn, Values: []string{v1beta1.Windows2022Build}}, + }, + }, + }, }, }) windowsNodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeNodeClassReady) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index b90949530f6f..56ac98a6aacc 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -119,7 +119,7 @@ func (p *DefaultProvider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2N if err != nil { return nil, err } - resolvedLaunchTemplates, err := p.amiFamily.Resolve(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, options) + resolvedLaunchTemplates, err := p.amiFamily.Resolve(nodeClass, nodeClaim, instanceTypes, capacityType, options) if err != nil { return nil, err } diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 99b5d4c8b8d8..2103d184890c 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -121,38 +121,7 @@ var _ = Describe("LaunchTemplate Provider", func() { var nodePool *corev1beta1.NodePool var nodeClass *v1beta1.EC2NodeClass BeforeEach(func() { - nodeClass = test.EC2NodeClass( - v1beta1.EC2NodeClass{ - Status: v1beta1.EC2NodeClassStatus{ - InstanceProfile: "test-profile", - SecurityGroups: []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - }, - Subnets: []v1beta1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - }, - }, - }, - ) + nodeClass = test.EC2NodeClass() nodeClass.StatusConditions().SetTrue(v1beta1.ConditionTypeNodeClassReady) nodePool = coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ @@ -185,7 +154,16 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(awsEnv.InstanceTypesProvider.UpdateInstanceTypeOfferings(ctx)).To(Succeed()) }) It("should create unique launch templates for multiple identical nodeClasses", func() { - nodeClass2 := test.EC2NodeClass() + nodeClass2 := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Status: v1beta1.EC2NodeClassStatus{ + InstanceProfile: "test-profile", + Subnets: nodeClass.Status.Subnets, + SecurityGroups: nodeClass.Status.SecurityGroups, + AMIs: nodeClass.Status.AMIs, + }, + }) + _, err := awsEnv.SubnetProvider.List(ctx, nodeClass2) // Hydrate the subnet cache + Expect(err).To(BeNil()) nodePool2 := coretest.NodePool(corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ @@ -206,31 +184,6 @@ var _ = Describe("LaunchTemplate Provider", func() { }, }, }) - nodeClass2.Status.SecurityGroups = []v1beta1.SecurityGroup{ - { - ID: "sg-test1", - }, - { - ID: "sg-test2", - }, - { - ID: "sg-test3", - }, - } - nodeClass2.Status.Subnets = []v1beta1.Subnet{ - { - ID: "subnet-test1", - Zone: "test-zone-1a", - }, - { - ID: "subnet-test2", - Zone: "test-zone-1b", - }, - { - ID: "subnet-test3", - Zone: "test-zone-1c", - }, - } nodeClass2.StatusConditions().SetTrue(v1beta1.ConditionTypeNodeClassReady) pods := []*v1.Pod{ @@ -1827,14 +1780,14 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("Custom AMI Selector", func() { It("should use ami selector specified in EC2NodeClass", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1848,14 +1801,14 @@ var _ = Describe("LaunchTemplate Provider", func() { nodeClass.Spec.UserData = aws.String("special user data") nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1884,6 +1837,8 @@ var _ = Describe("LaunchTemplate Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) + _, err := awsEnv.AMIProvider.List(ctx, nodeClass) + Expect(err).To(BeNil()) Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 2)) actualFilter := awsEnv.EC2API.CalledWithDescribeImagesInput.Pop().Filters expectedFilter := []*ec2.Filter{ @@ -1895,21 +1850,21 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(actualFilter).To(Equal(expectedFilter)) }) It("should create multiple launch templates when multiple amis are discovered with non-equivalent requirements", func() { - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "ami-123", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + }, }, { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("ami-456"), - Architecture: aws.String("arm64"), - CreationDate: aws.String("2022-08-10T12:00:00Z"), + ID: "ami-456", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureArm64}}, + }, }, - }}) - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1946,6 +1901,8 @@ var _ = Describe("LaunchTemplate Provider", func() { }}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass) + controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeClass)) nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { NodeSelectorRequirement: v1.NodeSelectorRequirement{ @@ -1968,6 +1925,7 @@ var _ = Describe("LaunchTemplate Provider", func() { It("should fail if no amis match selector.", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{} ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1978,6 +1936,14 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ {Name: aws.String(coretest.RandomName()), ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}}}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} + nodeClass.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-123", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"newnew"}}, + }, + }, + } ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1989,14 +1955,14 @@ var _ = Describe("LaunchTemplate Provider", func() { awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): "test-ami-123", } - awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + nodeClass.Status.AMIs = []v1beta1.AMI{ { - Name: aws.String(coretest.RandomName()), - ImageId: aws.String("test-ami-123"), - Architecture: aws.String("x86_64"), - CreationDate: aws.String("2022-08-15T12:00:00Z"), + ID: "test-ami-123", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(corev1beta1.ArchitectureAmd64)}}, + }, }, - }}) + } ExpectApplied(ctx, env.Client, nodeClass) ExpectApplied(ctx, env.Client, nodePool) pod := coretest.UnschedulablePod() diff --git a/pkg/test/nodeclass.go b/pkg/test/nodeclass.go index 4aa58d4a75d3..df81e6640a95 100644 --- a/pkg/test/nodeclass.go +++ b/pkg/test/nodeclass.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/imdario/mergo" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,6 +38,38 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { } if options.Spec.AMIFamily == nil { options.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + options.Status.AMIs = []v1beta1.AMI{ + { + ID: "ami-test1", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: v1.NodeSelectorOpDoesNotExist}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: v1.NodeSelectorOpDoesNotExist}, + }, + }, + { + ID: "ami-test2", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: v1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test3", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureAmd64}}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: v1.NodeSelectorOpExists}, + }, + }, + { + ID: "ami-test4", + Requirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.ArchitectureArm64}}, + {Key: v1beta1.LabelInstanceGPUCount, Operator: v1.NodeSelectorOpDoesNotExist}, + {Key: v1beta1.LabelInstanceAcceleratorCount, Operator: v1.NodeSelectorOpDoesNotExist}, + }, + }, + } } if options.Spec.Role == "" { options.Spec.Role = "test-role" @@ -50,6 +83,17 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { }, }, } + options.Status.SecurityGroups = []v1beta1.SecurityGroup{ + { + ID: "sg-test1", + }, + { + ID: "sg-test2", + }, + { + ID: "sg-test3", + }, + } } if len(options.Spec.SubnetSelectorTerms) == 0 { options.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ @@ -59,6 +103,20 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { }, }, } + options.Status.Subnets = []v1beta1.Subnet{ + { + ID: "subnet-test1", + Zone: "test-zone-1a", + }, + { + ID: "subnet-test2", + Zone: "test-zone-1b", + }, + { + ID: "subnet-test3", + Zone: "test-zone-1c", + }, + } } return &v1beta1.EC2NodeClass{ ObjectMeta: test.ObjectMeta(options.ObjectMeta),