diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 4e4b7fd92b16..38baec47e819 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -82,6 +82,13 @@ func IsAlreadyExists(err error) bool { return false } +func IgnoreAlreadyExists(err error) error { + if IsAlreadyExists(err) { + return nil + } + return err +} + // IsUnfulfillableCapacity returns true if the Fleet err means // capacity is temporarily unavailable for launching. // This could be due to account limits, insufficient ec2 capacity, etc. diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index f53522cb3133..4d50a2bfa125 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -115,24 +115,20 @@ func (env *Environment) ExpectExperimentTemplateDeleted(id string) { Expect(err).ToNot(HaveOccurred()) } -func (env *Environment) EventuallyExpectInstanceProfileExists(profileName string) { +func (env *Environment) EventuallyExpectInstanceProfileExists(profileName string) iam.InstanceProfile { GinkgoHelper() - By(fmt.Sprintf("expecting instance profile %s to exist", profileName)) + By(fmt.Sprintf("eventually expecting instance profile %s to exist", profileName)) var instanceProfile iam.InstanceProfile Eventually(func(g Gomega) { - instanceProfile = env.ExpectInstanceProfileExists(profileName) + out, err := env.IAMAPI.GetInstanceProfileWithContext(env.Context, &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(profileName), + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out.InstanceProfile).ToNot(BeNil()) + g.Expect(out.InstanceProfile.InstanceProfileName).ToNot(BeNil()) + instanceProfile = lo.FromPtr(out.InstanceProfile) }).WithTimeout(20 * time.Second).Should(Succeed()) - Expect(instanceProfile.InstanceProfileName).ToNot(BeNil()) -} - -func (env *Environment) ExpectInstanceProfileExists(profileName string) iam.InstanceProfile { - GinkgoHelper() - out, err := env.IAMAPI.GetInstanceProfileWithContext(env.Context, &iam.GetInstanceProfileInput{ - InstanceProfileName: aws.String(profileName), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(out.InstanceProfile).ToNot(BeNil()) - return lo.FromPtr(out.InstanceProfile) + return instanceProfile } // GetInstanceProfileName gets the string for the profile name based on the cluster name, region and the NodeClass name. @@ -316,18 +312,19 @@ func (env *Environment) GetCustomAMI(amiPath string, versionOffset int) string { return *parameter.Parameter.Value } -func (env *Environment) ExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation { +func (env *Environment) EventuallyExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation { GinkgoHelper() - env.EventuallyExpectInstanceProfileExists(aws.StringValue(instanceInput.IamInstanceProfile.Name)) // implement IMDSv2 instanceInput.MetadataOptions = &ec2.InstanceMetadataOptionsRequest{ HttpEndpoint: aws.String("enabled"), HttpTokens: aws.String("required"), } - - out, err := env.EC2API.RunInstances(instanceInput) - Expect(err).ToNot(HaveOccurred()) - + var out *ec2.Reservation + var err error + Eventually(func(g Gomega) { + out, err = env.EC2API.RunInstances(instanceInput) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(Succeed()) return out } diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index 0d34be68507e..6a6519a39dc5 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -36,7 +36,7 @@ var _ = Describe("InstanceProfile Generation", func() { Expect(instance.IamInstanceProfile).ToNot(BeNil()) Expect(lo.FromPtr(instance.IamInstanceProfile.Arn)).To(ContainSubstring(nodeClass.Status.InstanceProfile)) - instanceProfile := env.ExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass)) + instanceProfile := env.EventuallyExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass)) Expect(instanceProfile.Roles).To(HaveLen(1)) Expect(lo.FromPtr(instanceProfile.Roles[0].RoleName)).To(Equal(nodeClass.Spec.Role)) }) diff --git a/test/suites/nodeclaim/garbage_collection_test.go b/test/suites/nodeclaim/garbage_collection_test.go index 703b4edd505b..5297d4fbd8a9 100644 --- a/test/suites/nodeclaim/garbage_collection_test.go +++ b/test/suites/nodeclaim/garbage_collection_test.go @@ -18,10 +18,12 @@ import ( "encoding/base64" "fmt" "os" + "strings" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/iam" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" @@ -35,7 +37,7 @@ import ( environmentaws "github.com/aws/karpenter/test/pkg/environment/aws" ) -var _ = Describe("NodeClaimGarbageCollection", func() { +var _ = Describe("GarbageCollection", func() { var customAMI string var instanceInput *ec2.RunInstancesInput @@ -93,8 +95,13 @@ var _ = Describe("NodeClaimGarbageCollection", func() { instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, env.ClusterEndpoint, env.ExpectCABundle(), nodePool.Name)))) + instanceProfileName := fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName) + ExpectInstanceProfileCreated(instanceProfileName) + DeferCleanup(func() { + ExpectInstanceProfileDeleted(instanceProfileName) + }) // Create an instance manually to mock Karpenter launching an instance - out := env.ExpectRunInstances(instanceInput) + out := env.EventuallyExpectRunInstances(instanceInput) Expect(out.Instances).To(HaveLen(1)) // Always ensure that we cleanup the instance @@ -147,3 +154,50 @@ var _ = Describe("NodeClaimGarbageCollection", func() { env.EventuallyExpectNotFound(node) }) }) + +func ExpectInstanceProfileCreated(instanceProfileName string) { + By("creating an instance profile") + createInstanceProfile := &iam.CreateInstanceProfileInput{ + InstanceProfileName: aws.String(instanceProfileName), + Tags: []*iam.Tag{ + { + Key: aws.String(coretest.DiscoveryLabel), + Value: aws.String(env.ClusterName), + }, + }, + } + By("adding the karpenter role to new instance profile") + _, err := env.IAMAPI.CreateInstanceProfile(createInstanceProfile) + Expect(awserrors.IgnoreAlreadyExists(err)).ToNot(HaveOccurred()) + addInstanceProfile := &iam.AddRoleToInstanceProfileInput{ + InstanceProfileName: aws.String(instanceProfileName), + RoleName: aws.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)), + } + _, err = env.IAMAPI.AddRoleToInstanceProfile(addInstanceProfile) + Expect(ignoreAlreadyContainsRole(err)).ToNot(HaveOccurred()) +} + +func ExpectInstanceProfileDeleted(instanceProfileName string) { + By("deleting an instance profile") + removeRoleFromInstanceProfile := &iam.RemoveRoleFromInstanceProfileInput{ + InstanceProfileName: aws.String(instanceProfileName), + RoleName: aws.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)), + } + _, err := env.IAMAPI.RemoveRoleFromInstanceProfile(removeRoleFromInstanceProfile) + Expect(awserrors.IgnoreNotFound(err)).To(BeNil()) + + deleteInstanceProfile := &iam.DeleteInstanceProfileInput{ + InstanceProfileName: aws.String(instanceProfileName), + } + _, err = env.IAMAPI.DeleteInstanceProfile(deleteInstanceProfile) + Expect(awserrors.IgnoreNotFound(err)).ToNot(HaveOccurred()) +} + +func ignoreAlreadyContainsRole(err error) error { + if err != nil { + if strings.Contains(err.Error(), "Cannot exceed quota for InstanceSessionsPerInstanceProfile") { + return nil + } + } + return err +}