test: fix flake for nodeclaim test (#5037)

pkg/errors/errors.go

@@ -82,6 +82,13 @@ func IsAlreadyExists(err error) bool {
 	return false
 }
 
+func IgnoreAlreadyExists(err error) error {
+	if IsAlreadyExists(err) {
+		return nil
+	}
+	return err
+}
+
 // IsUnfulfillableCapacity returns true if the Fleet err means
 // capacity is temporarily unavailable for launching.
 // This could be due to account limits, insufficient ec2 capacity, etc.

test/pkg/environment/aws/expectations.go

@@ -115,24 +115,20 @@ func (env *Environment) ExpectExperimentTemplateDeleted(id string) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
-func (env *Environment) EventuallyExpectInstanceProfileExists(profileName string) {
+func (env *Environment) EventuallyExpectInstanceProfileExists(profileName string) iam.InstanceProfile {
 	GinkgoHelper()
-	By(fmt.Sprintf("expecting instance profile %s to exist", profileName))
+	By(fmt.Sprintf("eventually expecting instance profile %s to exist", profileName))
 	var instanceProfile iam.InstanceProfile
 	Eventually(func(g Gomega) {
-		instanceProfile = env.ExpectInstanceProfileExists(profileName)
+		out, err := env.IAMAPI.GetInstanceProfileWithContext(env.Context, &iam.GetInstanceProfileInput{
+			InstanceProfileName: aws.String(profileName),
+		})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(out.InstanceProfile).ToNot(BeNil())
+		g.Expect(out.InstanceProfile.InstanceProfileName).ToNot(BeNil())
+		instanceProfile = lo.FromPtr(out.InstanceProfile)
 	}).WithTimeout(20 * time.Second).Should(Succeed())
-	Expect(instanceProfile.InstanceProfileName).ToNot(BeNil())
-}
-
-func (env *Environment) ExpectInstanceProfileExists(profileName string) iam.InstanceProfile {
-	GinkgoHelper()
-	out, err := env.IAMAPI.GetInstanceProfileWithContext(env.Context, &iam.GetInstanceProfileInput{
-		InstanceProfileName: aws.String(profileName),
-	})
-	Expect(err).ToNot(HaveOccurred())
-	Expect(out.InstanceProfile).ToNot(BeNil())
-	return lo.FromPtr(out.InstanceProfile)
+	return instanceProfile
 }
 
 // GetInstanceProfileName gets the string for the profile name based on the cluster name, region and the NodeClass name.
@@ -316,18 +312,19 @@ func (env *Environment) GetCustomAMI(amiPath string, versionOffset int) string {
 	return *parameter.Parameter.Value
 }
 
-func (env *Environment) ExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation {
+func (env *Environment) EventuallyExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation {
 	GinkgoHelper()
-	env.EventuallyExpectInstanceProfileExists(aws.StringValue(instanceInput.IamInstanceProfile.Name))
 	// implement IMDSv2
 	instanceInput.MetadataOptions = &ec2.InstanceMetadataOptionsRequest{
 		HttpEndpoint: aws.String("enabled"),
 		HttpTokens:   aws.String("required"),
 	}
-
-	out, err := env.EC2API.RunInstances(instanceInput)
-	Expect(err).ToNot(HaveOccurred())
-
+	var out *ec2.Reservation
+	var err error
+	Eventually(func(g Gomega) {
+		out, err = env.EC2API.RunInstances(instanceInput)
+		g.Expect(err).ToNot(HaveOccurred())
+	}).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(Succeed())
 	return out
 }
 

test/suites/integration/instance_profile_test.go

@@ -36,7 +36,7 @@ var _ = Describe("InstanceProfile Generation", func() {
 		Expect(instance.IamInstanceProfile).ToNot(BeNil())
 		Expect(lo.FromPtr(instance.IamInstanceProfile.Arn)).To(ContainSubstring(nodeClass.Status.InstanceProfile))
 
-		instanceProfile := env.ExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass))
+		instanceProfile := env.EventuallyExpectInstanceProfileExists(env.GetInstanceProfileName(nodeClass))
 		Expect(instanceProfile.Roles).To(HaveLen(1))
 		Expect(lo.FromPtr(instanceProfile.Roles[0].RoleName)).To(Equal(nodeClass.Spec.Role))
 	})

test/suites/nodeclaim/garbage_collection_test.go

@@ -18,10 +18,12 @@ import (
 	"encoding/base64"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/iam"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/samber/lo"
@@ -35,7 +37,7 @@ import (
 	environmentaws "github.com/aws/karpenter/test/pkg/environment/aws"
 )
 
-var _ = Describe("NodeClaimGarbageCollection", func() {
+var _ = Describe("GarbageCollection", func() {
 	var customAMI string
 	var instanceInput *ec2.RunInstancesInput
 
@@ -93,8 +95,13 @@ var _ = Describe("NodeClaimGarbageCollection", func() {
 		instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName,
 			env.ClusterEndpoint, env.ExpectCABundle(), nodePool.Name))))
 
+		instanceProfileName := fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName)
+		ExpectInstanceProfileCreated(instanceProfileName)
+		DeferCleanup(func() {
+			ExpectInstanceProfileDeleted(instanceProfileName)
+		})
 		// Create an instance manually to mock Karpenter launching an instance
-		out := env.ExpectRunInstances(instanceInput)
+		out := env.EventuallyExpectRunInstances(instanceInput)
 		Expect(out.Instances).To(HaveLen(1))
 
 		// Always ensure that we cleanup the instance
@@ -147,3 +154,50 @@ var _ = Describe("NodeClaimGarbageCollection", func() {
 		env.EventuallyExpectNotFound(node)
 	})
 })
+
+func ExpectInstanceProfileCreated(instanceProfileName string) {
+	By("creating an instance profile")
+	createInstanceProfile := &iam.CreateInstanceProfileInput{
+		InstanceProfileName: aws.String(instanceProfileName),
+		Tags: []*iam.Tag{
+			{
+				Key:   aws.String(coretest.DiscoveryLabel),
+				Value: aws.String(env.ClusterName),
+			},
+		},
+	}
+	By("adding the karpenter role to new instance profile")
+	_, err := env.IAMAPI.CreateInstanceProfile(createInstanceProfile)
+	Expect(awserrors.IgnoreAlreadyExists(err)).ToNot(HaveOccurred())
+	addInstanceProfile := &iam.AddRoleToInstanceProfileInput{
+		InstanceProfileName: aws.String(instanceProfileName),
+		RoleName:            aws.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)),
+	}
+	_, err = env.IAMAPI.AddRoleToInstanceProfile(addInstanceProfile)
+	Expect(ignoreAlreadyContainsRole(err)).ToNot(HaveOccurred())
+}
+
+func ExpectInstanceProfileDeleted(instanceProfileName string) {
+	By("deleting an instance profile")
+	removeRoleFromInstanceProfile := &iam.RemoveRoleFromInstanceProfileInput{
+		InstanceProfileName: aws.String(instanceProfileName),
+		RoleName:            aws.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)),
+	}
+	_, err := env.IAMAPI.RemoveRoleFromInstanceProfile(removeRoleFromInstanceProfile)
+	Expect(awserrors.IgnoreNotFound(err)).To(BeNil())
+
+	deleteInstanceProfile := &iam.DeleteInstanceProfileInput{
+		InstanceProfileName: aws.String(instanceProfileName),
+	}
+	_, err = env.IAMAPI.DeleteInstanceProfile(deleteInstanceProfile)
+	Expect(awserrors.IgnoreNotFound(err)).ToNot(HaveOccurred())
+}
+
+func ignoreAlreadyContainsRole(err error) error {
+	if err != nil {
+		if strings.Contains(err.Error(), "Cannot exceed quota for InstanceSessionsPerInstanceProfile") {
+			return nil
+		}
+	}
+	return err
+}