*: remove CFN dependency from MNG

Signed-off-by: Gyuho Lee <[email protected]>

CHANGELOG/CHANGELOG-1.6.md

@@ -18,6 +18,7 @@ See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.6.0...v1.6.1
   - after, `AWS_K8S_TESTER_EKS_ADD_ON_NODE_GROUPS_ASGS='{"GetRef.Name-ng-for-cni":{"name":"GetRef.Name-ng-for-cni","remote-access-user-name":"ec2-user","ami-type":"AL2_x86_64","asg-min-size":30,"asg-max-size":35,"asg-desired-capacity":34, "instance-type":"type-2", "image-id":"my-ami",  "ssm":{"document-create":true,    "document-name":"GetRef.Name-document"}, "kubelet-extra-args":"aaa aa", "cluster-autoscaler": {"enable" : true}, "volume-size":500}}'`
 - Rename [`AddOnNodeGroups.ASG.InstanceTypes` to `InstanceType`](https://github.com/aws/aws-k8s-tester/commit/a4a3e3635466731519a38f411a1035318fecec59).
   - Rename `"instance-types"` to `"instance-type"`.
+- [Remove CloudFormation dependency from `AddOnManagedNodeGroups`](https://github.com/aws/aws-k8s-tester/commit/).
 
 ### `k8s-tester`
 

eks/app-mesh/app-mesh.go

@@ -227,8 +227,8 @@ func (ts *tester) createPolicy() error {
 	if ts.cfg.EKSConfig.AddOnNodeGroups != nil && ts.cfg.EKSConfig.Role.Name != "" {
 		roleNames = append(roleNames, ts.cfg.EKSConfig.Role.Name)
 	}
-	if ts.cfg.EKSConfig.AddOnManagedNodeGroups != nil && ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleName != "" {
-		roleNames = append(roleNames, ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleName)
+	if ts.cfg.EKSConfig.AddOnManagedNodeGroups != nil && ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name != "" {
+		roleNames = append(roleNames, ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name)
 	}
 	if len(roleNames) == 0 {
 		return errors.New("roles not found from node group or managed node group")

eks/cluster/role.go

@@ -54,8 +54,8 @@ func (ts *tester) createRole() error {
 	}
 
 	ts.cfg.Logger.Info("created a new role and attached policy",
-		zap.String("cluster-role-arn", ts.cfg.EKSConfig.Role.ARN),
-		zap.String("cluster-role-name", ts.cfg.EKSConfig.Role.Name),
+		zap.String("role-arn", ts.cfg.EKSConfig.Role.ARN),
+		zap.String("role-name", ts.cfg.EKSConfig.Role.Name),
 	)
 	return nil
 }
@@ -85,8 +85,8 @@ func (ts *tester) deleteRole() error {
 
 	if len(errs) == 0 {
 		ts.cfg.Logger.Info("successfully deleted role",
-			zap.String("cluster-role-arn", ts.cfg.EKSConfig.Role.ARN),
-			zap.String("cluster-role-name", ts.cfg.EKSConfig.Role.Name),
+			zap.String("role-arn", ts.cfg.EKSConfig.Role.ARN),
+			zap.String("role-name", ts.cfg.EKSConfig.Role.Name),
 		)
 		return nil
 	}
@@ -517,5 +517,23 @@ func createStatementEntriesForRolePolicyDocument(partition string, bucketName st
 				"s3:*",
 			},
 		},
+		{ // arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
+			Effect:   "Allow",
+			Resource: "*",
+			Action: []string{
+				"ecr:GetAuthorizationToken",
+				"ecr:BatchCheckLayerAvailability",
+				"ecr:GetDownloadUrlForLayer",
+				"ecr:GetRepositoryPolicy",
+				"ecr:DescribeRepositories",
+				"ecr:ListImages",
+				"ecr:DescribeImages",
+				"ecr:BatchGetImage",
+				"ecr:GetLifecyclePolicy",
+				"ecr:GetLifecyclePolicyPreview",
+				"ecr:ListTagsForResource",
+				"ecr:DescribeImageScanFindings",
+			},
+		},
 	}
 }

eks/cluster/vpc.go

@@ -1063,6 +1063,7 @@ func (ts *tester) deletePublicNATGateways() (err error) {
 		if _, ok := ts.cfg.EKSConfig.Status.DeletedResources[id]; ok {
 			continue
 		}
+
 		_, err := ts.cfg.EC2APIV2.DeleteNatGateway(
 			context.Background(),
 			&aws_ec2_v2.DeleteNatGatewayInput{
@@ -1084,19 +1085,29 @@ func (ts *tester) deletePublicNATGateways() (err error) {
 			}
 			continue
 		}
-		for i := 0; i < 6; i++ {
+		for i := 0; i < 10; i++ {
 			time.Sleep(10 * time.Second)
-			_, err = ts.cfg.EC2APIV2.DeleteNatGateway(
+			_, err1 := ts.cfg.EC2APIV2.DeleteNatGateway(
 				context.Background(),
 				&aws_ec2_v2.DeleteNatGatewayInput{
 					NatGatewayId: aws_v2.String(id),
 				},
 			)
-			if err == nil {
+			_, err2 := ts.cfg.EC2APIV2.DescribeNatGateways(
+				context.Background(),
+				&aws_ec2_v2.DescribeNatGatewaysInput{
+					NatGatewayIds: []string{id},
+				},
+			)
+			if err2 == nil {
 				continue
 			}
+			ts.cfg.Logger.Warn("failed to describe NAT gateway during deletion",
+				zap.String("delete-error", fmt.Sprintf("%v", err1)),
+				zap.Error(err2),
+			)
 			var apiErr smithy.APIError
-			if errors.As(err, &apiErr) {
+			if errors.As(err2, &apiErr) {
 				if strings.Contains(apiErr.ErrorCode(), "NotFound") {
 					ts.cfg.EKSConfig.Status.DeletedResources[id] = "VPC.NATGatewayID"
 					ts.cfg.EKSConfig.Sync()

eks/eks.go

@@ -176,9 +176,10 @@ type Tester struct {
 	ecrAPIV2         *aws_ecr_v2.Client
 
 	// used for EKS + EKS MNG API calls
-	eksSession *session.Session
-	eksAPI     eksiface.EKSAPI
-	eksAPIV2   *aws_eks_v2.Client
+	eksAPIForCluster   eksiface.EKSAPI
+	eksAPIForClusterV2 *aws_eks_v2.Client
+	eksAPIForMNG       eksiface.EKSAPI
+	eksAPIForMNGV2     *aws_eks_v2.Client
 
 	s3Uploaded bool
 
@@ -496,7 +497,8 @@ func New(cfg *eksconfig.Config) (ts *Tester, err error) {
 	}
 
 	// create a separate session for EKS (for resolver endpoint)
-	ts.eksSession, _, ts.cfg.Status.AWSCredentialPath, err = pkg_aws.New(&pkg_aws.Config{
+	var eksSessionForCluster *session.Session
+	eksSessionForCluster, _, ts.cfg.Status.AWSCredentialPath, err = pkg_aws.New(&pkg_aws.Config{
 		Logger:        ts.lg,
 		DebugAPICalls: ts.cfg.LogLevel == "debug",
 		Partition:     ts.cfg.Partition,
@@ -507,9 +509,8 @@ func New(cfg *eksconfig.Config) (ts *Tester, err error) {
 	if err != nil {
 		return nil, err
 	}
-	ts.eksAPI = aws_eks.New(ts.eksSession)
+	ts.eksAPIForCluster = aws_eks.New(eksSessionForCluster)
 
-	ts.lg.Info("checking AWS SDK Go v2 for EKS")
 	awsCfgV2EKS, err := pkg_aws.NewV2(&pkg_aws.Config{
 		Logger:        ts.lg,
 		DebugAPICalls: ts.cfg.LogLevel == "debug",
@@ -521,11 +522,40 @@ func New(cfg *eksconfig.Config) (ts *Tester, err error) {
 	if err != nil {
 		return nil, err
 	}
-	ts.eksAPIV2 = aws_eks_v2.NewFromConfig(awsCfgV2EKS)
+	ts.eksAPIForClusterV2 = aws_eks_v2.NewFromConfig(awsCfgV2EKS)
+
+	if ts.cfg.IsEnabledAddOnManagedNodeGroups() {
+		var eksSessionForMNG *session.Session
+		eksSessionForMNG, _, ts.cfg.Status.AWSCredentialPath, err = pkg_aws.New(&pkg_aws.Config{
+			Logger:        ts.lg,
+			DebugAPICalls: ts.cfg.LogLevel == "debug",
+			Partition:     ts.cfg.Partition,
+			Region:        ts.cfg.Region,
+			ResolverURL:   ts.cfg.AddOnManagedNodeGroups.ResolverURL,
+			SigningName:   ts.cfg.AddOnManagedNodeGroups.SigningName,
+		})
+		if err != nil {
+			return nil, err
+		}
+		ts.eksAPIForMNG = aws_eks.New(eksSessionForMNG)
+
+		awsCfgV2EKS, err := pkg_aws.NewV2(&pkg_aws.Config{
+			Logger:        ts.lg,
+			DebugAPICalls: ts.cfg.LogLevel == "debug",
+			Partition:     ts.cfg.Partition,
+			Region:        ts.cfg.Region,
+			ResolverURL:   ts.cfg.AddOnManagedNodeGroups.ResolverURL,
+			SigningName:   ts.cfg.AddOnManagedNodeGroups.SigningName,
+		})
+		if err != nil {
+			return nil, err
+		}
+		ts.eksAPIForMNGV2 = aws_eks_v2.NewFromConfig(awsCfgV2EKS)
+	}
 
 	ts.lg.Info("checking EKS API v1 availability; listing clusters")
 	var eksListResp *aws_eks.ListClustersOutput
-	eksListResp, err = ts.eksAPI.ListClusters(&aws_eks.ListClustersInput{
+	eksListResp, err = ts.eksAPIForCluster.ListClusters(&aws_eks.ListClustersInput{
 		MaxResults: aws.Int64(20),
 	})
 	if err != nil {
@@ -539,7 +569,7 @@ func New(cfg *eksconfig.Config) (ts *Tester, err error) {
 	ts.lg.Info("checking EKS API v2 availability; listing clusters")
 	var eksListRespV2 *aws_eks_v2.ListClustersOutput
 	cctx, ccancel := context.WithTimeout(context.Background(), 10*time.Second)
-	eksListRespV2, err = ts.eksAPIV2.ListClusters(
+	eksListRespV2, err = ts.eksAPIForClusterV2.ListClusters(
 		cctx,
 		&aws_eks_v2.ListClustersInput{
 			MaxResults: aws.Int32(20),
@@ -618,8 +648,8 @@ func (ts *Tester) createTesters() (err error) {
 		KMSAPIV2:   ts.kmsAPIV2,
 		CFNAPI:     ts.cfnAPI,
 		EC2APIV2:   ts.ec2APIV2,
-		EKSAPI:     ts.eksAPI,
-		EKSAPIV2:   ts.eksAPIV2,
+		EKSAPI:     ts.eksAPIForCluster,
+		EKSAPIV2:   ts.eksAPIForClusterV2,
 		ELBV2APIV2: ts.elbv2APIV2,
 	})
 
@@ -654,12 +684,10 @@ func (ts *Tester) createTesters() (err error) {
 		IAMAPIV2: ts.iamAPIV2,
 		EC2APIV2: ts.ec2APIV2,
 		ASGAPIV2: ts.asgAPIV2,
-		EKSAPI:   ts.eksAPI,
+		EKSAPI:   ts.eksAPIForMNG,
+		EKSAPIV2: ts.eksAPIForMNGV2,
 
-		IAMAPI: ts.iamAPI,
 		CFNAPI: ts.cfnAPI,
-		EC2API: ts.ec2API,
-		ASGAPI: ts.asgAPI,
 	})
 	ts.gpuTester = gpu.New(gpu.Config{
 		Logger:    ts.lg,
@@ -873,7 +901,7 @@ func (ts *Tester) createTesters() (err error) {
 			S3API:     ts.s3API,
 			IAMAPI:    ts.iamAPI,
 			CFNAPI:    ts.cfnAPI,
-			EKSAPI:    ts.eksAPI,
+			EKSAPI:    ts.eksAPIForCluster,
 			ECRAPI:    ecr.New(ts.awsSession, aws.NewConfig().WithRegion(ts.cfg.GetAddOnFargateRepositoryRegion())),
 		}),
 		irsa.New(irsa.Config{
@@ -896,7 +924,7 @@ func (ts *Tester) createTesters() (err error) {
 			S3API:     ts.s3API,
 			IAMAPI:    ts.iamAPI,
 			CFNAPI:    ts.cfnAPI,
-			EKSAPI:    ts.eksAPI,
+			EKSAPI:    ts.eksAPIForCluster,
 			ECRAPI:    ecr.New(ts.awsSession, aws.NewConfig().WithRegion(ts.cfg.GetAddOnIRSAFargateRepositoryRegion())),
 		}),
 		wordpress.New(wordpress.Config{
@@ -995,7 +1023,7 @@ func (ts *Tester) createTesters() (err error) {
 			Stopc:     ts.stopCreationCh,
 			EKSConfig: ts.cfg,
 			K8SClient: ts.k8sClient,
-			EKSAPI:    ts.eksAPI,
+			EKSAPI:    ts.eksAPIForCluster,
 		}),
 		ami_soft_lockup_issue_454.New(ami_soft_lockup_issue_454.Config{
 			Logger:    ts.lg,

eks/kubeflow/kubeflow.go

@@ -196,7 +196,7 @@ func (ts *tester) writeKfctlConfig() error {
 		nodeInstanceRoleName = ts.cfg.EKSConfig.Role.Name
 	}
 	if ts.cfg.EKSConfig.IsEnabledAddOnManagedNodeGroups() {
-		nodeInstanceRoleName = ts.cfg.EKSConfig.AddOnManagedNodeGroups.RoleName
+		nodeInstanceRoleName = ts.cfg.EKSConfig.AddOnManagedNodeGroups.Role.Name
 	}
 
 	tpl := template.Must(template.New("kfctlConfigTmpl").Parse(kfctlConfigTmpl))