OCM-6318 | ci: Build up deprovision step

tests/e2e/e2e_tear_down_test.go

@@ -0,0 +1,22 @@
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/openshift/rosa/tests/ci/labels"
+	"github.com/openshift/rosa/tests/utils/exec/rosacli"
+	ph "github.com/openshift/rosa/tests/utils/profilehandler"
+)
+
+var _ = Describe("ROSA CLI Test", func() {
+	It("DestroyClusterByProfile",
+		labels.Critical,
+		labels.Destroy,
+		func() {
+			client := rosacli.NewClient()
+			profile := ph.LoadProfileYamlFileByENV()
+			var errs = ph.DestroyResourceByProfile(profile, client)
+			Expect(len(errs)).To(Equal(0))
+		})
+})

tests/utils/exec/rosacli/cluster_service.go

@@ -20,6 +20,7 @@ type ClusterService interface {
 	DescribeClusterAndReflect(clusterID string) (*ClusterDescription, error)
 	List() (bytes.Buffer, error)
 	Create(clusterName string, flags ...string) (bytes.Buffer, error, string)
+	DeleteCluster(clusterID string, flags ...string) (bytes.Buffer, error)
 	CreateDryRun(clusterName string, flags ...string) (bytes.Buffer, error)
 	EditCluster(clusterID string, flags ...string) (bytes.Buffer, error)
 	DeleteUpgrade(flags ...string) (bytes.Buffer, error)
@@ -150,6 +151,7 @@ func (c *clusterService) CreateDryRun(clusterName string, flags ...string) (byte
 		CmdFlags(combflags...)
 	return createDryRun.Run()
 }
+
 func (c *clusterService) Create(clusterName string, flags ...string) (bytes.Buffer, error, string) {
 	combflags := append([]string{"-c", clusterName}, flags...)
 	createCommand := c.client.Runner.
@@ -159,6 +161,14 @@ func (c *clusterService) Create(clusterName string, flags ...string) (bytes.Buff
 	return output, err, createCommand.CMDString()
 }
 
+func (c *clusterService) DeleteCluster(clusterID string, flags ...string) (bytes.Buffer, error) {
+	combflags := append([]string{"-c", clusterID}, flags...)
+	deleteCluster := c.client.Runner.
+		Cmd("delete", "cluster").
+		CmdFlags(combflags...)
+	return deleteCluster.Run()
+}
+
 func (c *clusterService) EditCluster(clusterID string, flags ...string) (bytes.Buffer, error) {
 	combflags := append([]string{"-c", clusterID}, flags...)
 	editCluster := c.client.Runner.

tests/utils/profilehandler/data_cleaner.go

@@ -0,0 +1,27 @@
+package profilehandler
+
+import (
+	"github.com/openshift-online/ocm-common/pkg/aws/aws_client"
+	"github.com/openshift-online/ocm-common/pkg/test/kms_key"
+	"github.com/openshift-online/ocm-common/pkg/test/vpc_client"
+)
+
+func DeleteVPCChain(vpcID string, region string) error {
+	vpcClient, err := vpc_client.GenerateVPCByID(vpcID, region)
+	if err != nil {
+		return err
+	}
+	return vpcClient.DeleteVPCChain()
+}
+
+func ScheduleKMSDesiable(kmsKey string, region string) error {
+	return kms_key.ScheduleKeyDeletion(kmsKey, region)
+}
+
+func DeleteAuditLogRoleArn(arn string, region string) error {
+	awsClent, err := aws_client.CreateAWSClient("", region)
+	if err != nil {
+		return err
+	}
+	return awsClent.DeleteRoleAndPolicy(arn, false)
+}

tests/utils/profilehandler/parse_info.go

@@ -0,0 +1,49 @@
+package profilehandler
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/openshift/rosa/tests/ci/config"
+	"github.com/openshift/rosa/tests/utils/common"
+	"github.com/openshift/rosa/tests/utils/log"
+)
+
+// ParseUserData Get user data from resources.json file
+func ParseUserData() (*UserData, error) {
+	var ud *UserData
+
+	udContent, err := common.ReadFileContent(config.Test.UserDataFile)
+	if err != nil {
+		log.Logger.Errorf("Error happened when read user data: %s", err.Error())
+		return nil, err
+	}
+	err = json.Unmarshal([]byte(udContent), &ud)
+	if err != nil {
+		log.Logger.Errorf("Error happend when parse resource file data to UserData struct: %s", err.Error())
+		return nil, err
+	}
+	return ud, err
+}
+
+// ParserClusterDetail Get the cluster info from cluster-detail.json file
+func ParserClusterDetail() (*ClusterDetail, error) {
+	var cd *ClusterDetail
+
+	_, err := os.Stat(config.Test.ClusterDetailFile)
+	if err != nil {
+		log.Logger.Warn("Cluster detail file not exists")
+		return nil, nil
+	}
+	cdContent, err := common.ReadFileContent(config.Test.ClusterDetailFile)
+	if err != nil {
+		log.Logger.Errorf("Error happened when read cluster detail: %s", err.Error())
+		return nil, err
+	}
+	err = json.Unmarshal([]byte(cdContent), &cd)
+	if err != nil {
+		log.Logger.Errorf("Error happend when parse cluster detail file to ClusterDetail struct: %s", err.Error())
+		return nil, err
+	}
+	return cd, err
+}

tests/utils/profilehandler/profile_handler.go

@@ -685,3 +685,209 @@ func CreateClusterByProfile(profile *Profile, client *rosacli.Client, waitForClu
 	}
 	return description, err
 }
+
+func WaitForClusterUninstalled(client *rosacli.Client, cluster string, timeoutMin int) error {
+
+	endTime := time.Now().Add(time.Duration(timeoutMin) * time.Minute)
+	for time.Now().Before(endTime) {
+		output, err := client.Cluster.DescribeCluster(cluster)
+		desc, err := client.Cluster.ReflectClusterDescription(output)
+
+		if err != nil {
+			return err
+		}
+		if strings.Contains(output.String(), fmt.Sprintf("There is no cluster with identifier or name '%s'", cluster)) {
+			log.Logger.Infof("Cluster %s has been deleted.", cluster)
+			return nil
+		}
+		if strings.Contains(desc.State, con.Uninstalling) {
+			time.Sleep(2 * time.Minute)
+			continue
+		}
+		return fmt.Errorf("Cluster %s is in status of %s which won't be deleted, stop waiting", cluster, desc.State)
+	}
+	return fmt.Errorf("timeout for waiting for cluster deletion finished after %d mins", timeoutMin)
+}
+
+func DestroyCluster(client *rosacli.Client) (*ClusterDetail, []error) {
+	var (
+		cd             *ClusterDetail
+		clusterService rosacli.ClusterService
+		errors         []error
+	)
+	// get cluster info from cluster detail file
+	cd, err := ParserClusterDetail()
+	if err != nil {
+		errors = append(errors, err)
+		return nil, errors
+	}
+
+	// delete cluster
+	if cd != nil && cd.ClusterID != "" {
+		clusterService = client.Cluster
+		output, errDeleteCluster := clusterService.DeleteCluster(cd.ClusterID, "-y")
+		if errDeleteCluster != nil {
+			if strings.Contains(output.String(), fmt.Sprintf("There is no cluster with identifier or name '%s'", cd.ClusterID)) {
+				log.Logger.Infof("Cluster %s not exists.", cd.ClusterID)
+			} else {
+				log.Logger.Errorf("Error happened when delete cluster: %s", output.String())
+				errors = append(errors, errDeleteCluster)
+				return nil, errors
+			}
+		} else {
+			log.Logger.Infof("Waiting for the cluster %s to be uninstalled", cd.ClusterID)
+
+			err = WaitForClusterUninstalled(client, cd.ClusterID, config.Test.GlobalENV.ClusterWaitingTime)
+			if err != nil {
+				log.Logger.Errorf("Error happened when waiting cluster uninstall: %s", err.Error())
+				errors = append(errors, err)
+				return nil, errors
+			} else {
+				log.Logger.Infof("Delete cluster %s successfully.", cd.ClusterID)
+			}
+		}
+	}
+	return cd, errors
+}
+
+func DestroyPreparedUserData(client *rosacli.Client, clusterID string, region string, isSTS bool) []error {
+	var (
+		ud                 *UserData
+		ocmResourceService rosacli.OCMResourceService
+		errors             []error
+	)
+	ocmResourceService = client.OCMResource
+
+	// get user data from resource file
+	ud, err := ParseUserData()
+	if err != nil {
+		errors = append(errors, err)
+		return errors
+	}
+	defer func() {
+		log.Logger.Info("Rewrite User data file")
+		// rewrite user data
+		_, err = common.CreateFileWithContent(config.Test.UserDataFile, ud)
+	}()
+
+	if ud != nil {
+		// schedule KMS key
+		if ud.KMSKey != "" {
+			log.Logger.Infof("Find prepared kms key: %s. Going to schedule the deletion.", ud.KMSKey)
+			err = ScheduleKMSDesiable(ud.KMSKey, region)
+			if err != nil {
+				log.Logger.Errorf("Error happened when schedule kms key: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.KMSKey = ""
+				log.Logger.Info("Schedule kms key deletion successfully")
+			}
+		}
+		// schedule Etcd KMS key
+		if ud.EtcdKMSKey != "" {
+			log.Logger.Infof("Find prepared etcd kms key: %s. Going to schedule the deletion", ud.EtcdKMSKey)
+			err = ScheduleKMSDesiable(ud.EtcdKMSKey, region)
+			if err != nil {
+				log.Logger.Errorf("Error happened when schedule etcd kms key deletion: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.EtcdKMSKey = ""
+				log.Logger.Infof("Schedule etcd kms key deletion successfully for cluster: %s", clusterID)
+			}
+		}
+		// delete audit log arn
+		if ud.AuditLogArn != "" {
+			log.Logger.Infof("Find prepared audit log arn: %s", ud.AuditLogArn)
+			roleName := strings.Split(ud.AuditLogArn, "/")[1]
+			err = DeleteAuditLogRoleArn(roleName, region)
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete audit log arn: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.AuditLogArn = ""
+				log.Logger.Infof("Delete audit log arn successfully for cluster: %s", clusterID)
+			}
+		}
+		// delete vpc chain
+		if ud.VpcID != "" {
+			log.Logger.Infof("Find prepared vpc id: %s", ud.VpcID)
+			err = DeleteVPCChain(ud.VpcID, region)
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete vpc chain: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.VpcID = ""
+				log.Logger.Infof("Delete vpc chain successfully for cluster: %s", clusterID)
+			}
+		}
+		// delete operator roles
+		if ud.OperatorRolesPrefix != "" {
+			log.Logger.Infof("Find prepared operator roles with prefix: %s", ud.OperatorRolesPrefix)
+			_, err := ocmResourceService.DeleteOperatorRoles("--prefix", ud.OperatorRolesPrefix, "--mode", "auto", "-y")
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete operator role: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.OperatorRolesPrefix = ""
+				log.Logger.Info("Delete operator roles successfully")
+			}
+		}
+		// delete oidc config id
+		if ud.OIDCConfigID != "" {
+			log.Logger.Infof("Find prepared oidc config id: %s", ud.OIDCConfigID)
+			_, err := ocmResourceService.DeleteOIDCConfig("--oidc-config-id", ud.OIDCConfigID, "--mode", "auto", "-y")
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete oidc config id: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.OIDCConfigID = ""
+				log.Logger.Info("Delete oidc config id successfully ")
+			}
+		}
+		// delete oidc provider of sts cluster
+		if clusterID != "" && isSTS && ud.OIDCConfigID == "" {
+			_, err = ocmResourceService.DeleteOIDCProvider("-c", clusterID, "-y", "--mode", "auto")
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete oidc provider: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				log.Logger.Info("Delete oidc provider successfully ")
+			}
+		}
+		// delete account roles
+		if ud.AccountRolesPrefix != "" {
+			log.Logger.Infof("Find prepared accout roles with prefix: %s", ud.AccountRolesPrefix)
+			_, err := ocmResourceService.DeleteAccountRole("--mode", "auto", "--prefix", ud.AccountRolesPrefix, "-y")
+			if err != nil {
+				log.Logger.Errorf("Error happened when delete account roles: %s", err.Error())
+				errors = append(errors, err)
+			} else {
+				ud.AccountRolesPrefix = ""
+				log.Logger.Info("Delete account roles successfully ")
+			}
+		}
+	}
+	return errors
+}
+
+func DestroyResourceByProfile(profile *Profile, client *rosacli.Client) (errors [][]error) {
+	// destroy cluster
+	cd, errDestroyCluster := DestroyCluster(client)
+	if len(errDestroyCluster) > 0 {
+		errors = append(errors, errDestroyCluster)
+		return errors
+	}
+
+	// destroy prepared user data
+	clusterId := ""
+	if cd != nil {
+		clusterId = cd.ClusterID
+	}
+	region := profile.Region
+	isSTS := profile.ClusterConfig.STS
+	errDestroyUserData := DestroyPreparedUserData(client, clusterId, region, isSTS)
+	if len(errDestroyUserData) > 0 {
+		errors = append(errors, errDestroyUserData)
+	}
+	return errors
+}