Skip to content

Commit

Permalink
test: migrate e2e tests to use AL2023 (#5941)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmdeal
jmdeal authored Mar 29, 2024
1 parent 286071e commit 8a4c558
Show file tree
Hide file tree
Showing 7 changed files with 61 additions and 36 deletions.
7 changes: 4 additions & 3 deletions test/pkg/environment/aws/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,9 @@ func init() {
const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9"

// ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatibility
// issues with versions of the VPR Resource Controller
var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i"}
// issues with versions of the VPR Resource Controller.
// TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible)
var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i", "a1"}

type Environment struct {
*common.Environment
Expand Down Expand Up @@ -122,7 +123,7 @@ func GetTimeStreamAPI(session *session.Session) timestreamwriteiface.TimestreamW

func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass {
nodeClass := test.EC2NodeClass()
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023
nodeClass.Spec.Tags = map[string]string{
"testing/cluster": env.ClusterName,
}
Expand Down
8 changes: 8 additions & 0 deletions test/pkg/environment/common/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,14 @@ func (env *Environment) DefaultNodePool(nodeClass *v1beta1.EC2NodeClass) *corev1
Values: []string{"2"},
},
},
// Filter out a1 instance types, which are incompatible with AL2023 AMIs
{
NodeSelectorRequirement: v1.NodeSelectorRequirement{
Key: v1beta1.LabelInstanceFamily,
Operator: v1.NodeSelectorOpNotIn,
Values: []string{"a1"},
},
},
}
nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{}
nodePool.Spec.Disruption.ExpireAfter.Duration = nil
Expand Down
21 changes: 20 additions & 1 deletion test/suites/consolidation/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,8 @@ var _ = Describe("Consolidation", func() {
Operator: v1.NodeSelectorOpNotIn,
// remove some cheap burstable and the odd c1 instance types so we have
// more control over what gets provisioned
Values: []string{"t2", "t3", "c1", "t3a", "t4g"},
// TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible)
Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"},
},
},
},
Expand Down Expand Up @@ -500,6 +501,15 @@ var _ = Describe("Consolidation", func() {
Values: []string{"large", "2xlarge"},
},
},
{
NodeSelectorRequirement: v1.NodeSelectorRequirement{
Key: v1beta1.LabelInstanceFamily,
Operator: v1.NodeSelectorOpNotIn,
// remove some cheap burstable and the odd c1 / a1 instance types so we have
// more control over what gets provisioned
Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"},
},
},
},
NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name},
},
Expand Down Expand Up @@ -627,6 +637,15 @@ var _ = Describe("Consolidation", func() {
Values: []string{"large"},
},
},
{
NodeSelectorRequirement: v1.NodeSelectorRequirement{
Key: v1beta1.LabelInstanceFamily,
Operator: v1.NodeSelectorOpNotIn,
// remove some cheap burstable and the odd c1 / a1 instance types so we have
// more control over what gets provisioned
Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"},
},
},
},
NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name},
},
Expand Down
28 changes: 15 additions & 13 deletions test/suites/drift/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ var _ = Describe("Drift", func() {
var selector labels.Selector
var numPods int
BeforeEach(func() {
amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1)
amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1)
numPods = 1
// Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt
dep = coretest.Deployment(coretest.DeploymentOptions{
Expand Down Expand Up @@ -399,15 +399,14 @@ var _ = Describe("Drift", func() {
It("should disrupt nodes that have drifted due to AMIs", func() {
// Choose and old, static image. The 1.23 image is incompatible with EKS 1.29 so fallback to a newer image.
parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) >= 29,
"/aws/service/eks/optimized-ami/1.27/amazon-linux-2/amazon-eks-node-1.27-v20240129/image_id",
"/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id",
"/aws/service/eks/optimized-ami/1.27/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-1.27-v20240307/image_id",
"/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id",
)
parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: awssdk.String(parameterName)})
Expect(err).To(BeNil())
oldCustomAMI := *parameter.Parameter.Value
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}}
nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))

env.ExpectCreated(dep, nodeClass, nodePool)
pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0]
Expand All @@ -428,11 +427,11 @@ var _ = Describe("Drift", func() {
It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() {
version := env.GetK8sVersion(1)
armParameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{
Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)),
Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version)),
})
Expect(err).To(BeNil())
armAMI := *armParameter.Parameter.Value
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: armAMI}}

env.ExpectCreated(dep, nodeClass, nodePool)
Expand All @@ -454,15 +453,18 @@ var _ = Describe("Drift", func() {
It("should not disrupt nodes that have drifted without the featureGate enabled", func() {
version := env.GetK8sVersion(1)
env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"})
// choose an old static image
// Choose an old static image (AL2023 AMIs don't exist for 1.22)
parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) == 23,
"/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id",
fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version),
)
parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{
Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)),
Name: awssdk.String(parameterName),
})
Expect(err).To(BeNil())
oldCustomAMI := *parameter.Parameter.Value
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}}
nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))

env.ExpectCreated(dep, nodeClass, nodePool)
env.EventuallyExpectHealthyPodCount(selector, numPods)
Expand Down Expand Up @@ -674,7 +676,7 @@ var _ = Describe("Drift", func() {
env.EventuallyExpectNotFound(pod, node)
env.EventuallyExpectHealthyPodCount(selector, numPods)
},
Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\n/etc/eks/bootstrap.sh")}),
Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\necho \"Hello, AL2023\"")}),
Entry("Tags", v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}),
Entry("MetadataOptions", v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: awssdk.String("required"), HTTPPutResponseHopLimit: awssdk.Int64(10)}}),
Entry("BlockDeviceMappings", v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{
Expand Down Expand Up @@ -737,7 +739,7 @@ var _ = Describe("Drift", func() {
node := env.ExpectCreatedNodeCount("==", 1)[0]

nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resources.Quantity("100Gi")
env.ExpectCreatedOrUpdated()
env.ExpectCreatedOrUpdated(nodeClass)

By("validating the drifted status condition has propagated")
Eventually(func(g Gomega) {
Expand Down
24 changes: 8 additions & 16 deletions test/suites/integration/ami_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import (
var _ = Describe("AMI", func() {
var customAMI string
BeforeEach(func() {
customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1)
customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1)
})

It("should use the AMI defined by the AMI Selector Terms", func() {
Expand All @@ -61,12 +61,7 @@ var _ = Describe("AMI", func() {
})
It("should use the most recent AMI when discovering multiple", func() {
// choose an old static image
parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{
Name: aws.String("/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id"),
})
Expect(err).To(BeNil())
oldCustomAMI := *parameter.Parameter.Value
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
oldCustomAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%[1]s/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-%[1]s-v20240307/image_id", 1)
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
{
ID: customAMI,
Expand All @@ -75,7 +70,6 @@ var _ = Describe("AMI", func() {
ID: oldCustomAMI,
},
}
nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))
pod := coretest.Pod()

env.ExpectCreated(pod, nodeClass, nodePool)
Expand All @@ -90,14 +84,12 @@ var _ = Describe("AMI", func() {
})
Expect(err).To(BeNil())
Expect(output.Images).To(HaveLen(1))
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
{
Name: *output.Images[0].Name,
Owner: "fakeOwnerValue",
},
}
nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))
pod := coretest.Pod()

env.ExpectCreated(pod, nodeClass, nodePool)
Expand All @@ -111,13 +103,11 @@ var _ = Describe("AMI", func() {
Expect(err).To(BeNil())
Expect(output.Images).To(HaveLen(1))

nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
{
Name: *output.Images[0].Name,
},
}
nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))
pod := coretest.Pod()

env.ExpectCreated(pod, nodeClass, nodePool)
Expand All @@ -127,13 +117,11 @@ var _ = Describe("AMI", func() {
env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI))))
})
It("should support ami selector ids", func() {
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
{
ID: customAMI,
},
}
nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))
pod := coretest.Pod()

env.ExpectCreated(pod, nodeClass, nodePool)
Expand All @@ -146,6 +134,7 @@ var _ = Describe("AMI", func() {
Context("AMIFamily", func() {
It("should provision a node using the AL2 family", func() {
pod := coretest.Pod()
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
env.ExpectCreated(nodeClass, nodePool, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
Expand Down Expand Up @@ -195,9 +184,10 @@ var _ = Describe("AMI", func() {
})
It("should support Custom AMIFamily with AMI Selectors", func() {
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom
al2AMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1)
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
{
ID: customAMI,
ID: al2AMI,
},
}
nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName))
Expand All @@ -207,7 +197,7 @@ var _ = Describe("AMI", func() {
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)

env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI))))
env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(al2AMI))))
})
It("should have the EC2NodeClass status for AMIs using wildcard", func() {
nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{
Expand Down Expand Up @@ -236,6 +226,7 @@ var _ = Describe("AMI", func() {
It("should merge UserData contents for AL2 AMIFamily", func() {
content, err := os.ReadFile("testdata/al2_userdata_input.sh")
Expect(err).ToNot(HaveOccurred())
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
nodeClass.Spec.UserData = aws.String(string(content))
nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}}
nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}}
Expand All @@ -256,6 +247,7 @@ var _ = Describe("AMI", func() {
It("should merge non-MIME UserData contents for AL2 AMIFamily", func() {
content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh")
Expect(err).ToNot(HaveOccurred())
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
nodeClass.Spec.UserData = aws.String(string(content))
nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}}
nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}}
Expand Down
4 changes: 2 additions & 2 deletions test/suites/integration/block_device_mappings_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ var _ = Describe("BlockDeviceMappings", func() {
{
DeviceName: aws.String("/dev/xvda"),
EBS: &v1beta1.BlockDevice{
VolumeSize: resources.Quantity("10G"),
VolumeSize: resources.Quantity("20Gi"),
VolumeType: aws.String("io2"),
IOPS: aws.Int64(1000),
Encrypted: aws.Bool(true),
Expand All @@ -51,7 +51,7 @@ var _ = Describe("BlockDeviceMappings", func() {
Expect(instance.BlockDeviceMappings[0].Ebs).To(HaveField("DeleteOnTermination", HaveValue(BeTrue())))
volume := env.GetVolume(instance.BlockDeviceMappings[0].Ebs.VolumeId)
Expect(volume).To(HaveField("Encrypted", HaveValue(BeTrue())))
Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(10))))) // Convert G -> Gib (rounded up)
Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(20)))))
Expect(volume).To(HaveField("Iops", HaveValue(Equal(int64(1000)))))
Expect(volume).To(HaveField("VolumeType", HaveValue(Equal("io2"))))
})
Expand Down
5 changes: 4 additions & 1 deletion test/suites/integration/extended_resources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ import (
var _ = Describe("Extended Resources", func() {
It("should provision nodes for a deployment that requests nvidia.com/gpu", func() {
ExpectNvidiaDevicePluginCreated()

// TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2
numPods := 1
dep := test.Deployment(test.DeploymentOptions{
Replicas: int32(numPods),
Expand Down Expand Up @@ -232,6 +233,8 @@ var _ = Describe("Extended Resources", func() {
}
// Only select private subnets since instances with multiple network instances at launch won't get a public IP.
nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*"
// TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available
nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2

numPods := 1
dep := test.Deployment(test.DeploymentOptions{
Expand Down

0 comments on commit 8a4c558

Please sign in to comment.