From cbfe957633f9529e880f5e4b3fce14ad83127d84 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Fri, 15 Sep 2023 18:49:52 -0700 Subject: [PATCH 01/12] Adding support for 1.26, 1.27 --- .../pkg/controllers/master/kubeapiserver.go | 13 ++++++++ .../master/kubecontrollermanager.go | 30 +++++++++++++++++-- .../pkg/controllers/master/kubescheduler.go | 27 +++++++++++++++-- .../pkg/utils/imageprovider/imageprovider.go | 4 +++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/operator/pkg/controllers/master/kubeapiserver.go b/operator/pkg/controllers/master/kubeapiserver.go index 4eb5bc50..2c0f5ef2 100644 --- a/operator/pkg/controllers/master/kubeapiserver.go +++ b/operator/pkg/controllers/master/kubeapiserver.go @@ -455,6 +455,10 @@ var ( disabledFlagsForAPIServer = map[string]struct{}{"--feature-gates": {}} ) +var ( + disabledFlagsForApi126 = map[string]struct{}{"--feature-gates": {}, "--logtostderr": {}} +) + func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.25": @@ -466,6 +470,15 @@ func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodS args = append(args, arg) } defaultSpec.Containers[0].Args = args + case "1.26", "1.27": + args := []string{} + for _, arg := range defaultSpec.Containers[0].Args { + if _, skip := disabledFlagsForApi126[strings.Split(arg, "=")[0]]; skip { + continue + } + args = append(args, arg) + } + defaultSpec.Containers[0].Args = args } return *defaultSpec } diff --git a/operator/pkg/controllers/master/kubecontrollermanager.go b/operator/pkg/controllers/master/kubecontrollermanager.go index bbd24379..67f3c2a6 100644 --- a/operator/pkg/controllers/master/kubecontrollermanager.go +++ b/operator/pkg/controllers/master/kubecontrollermanager.go @@ -269,6 +269,14 @@ var ( disabledFlagsForKube122 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}} ) +var ( + disabledFlagsForKube126 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}, "--logtostderr": {}} +) + +var ( + disabledFlagsForKube127 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}, "--logtostderr": {}, "--cloud-provider": {}} +) + func kcmPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.22", "1.23", "1.24", "1.25": @@ -280,13 +288,31 @@ func kcmPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { args = append(args, arg) } defaultSpec.Containers[0].Args = args + case "1.26": + args := []string{} + for _, arg := range defaultSpec.Containers[0].Args { + if _, skip := disabledFlagsForKube126[strings.Split(arg, "=")[0]]; skip { + continue + } + args = append(args, arg) + } + defaultSpec.Containers[0].Args = args + case "1.27": + args := []string{} + for _, arg := range defaultSpec.Containers[0].Args { + if _, skip := disabledFlagsForKube127[strings.Split(arg, "=")[0]]; skip { + continue + } + args = append(args, arg) + } + defaultSpec.Containers[0].Args = args } return *defaultSpec } func kcmHealthCheckPortForVersion(version string) intstr.IntOrString { switch version { - case "1.22", "1.23", "1.24", "1.25": + case "1.22", "1.23", "1.24", "1.25", "1.26", "1.27": return intstr.FromInt(10257) } return intstr.FromInt(10252) @@ -294,7 +320,7 @@ func kcmHealthCheckPortForVersion(version string) intstr.IntOrString { func kcmHealthCheckSchemeForVersion(version string) v1.URIScheme { switch version { - case "1.22", "1.23", "1.24", "1.25": + case "1.22", "1.23", "1.24", "1.25", "1.26", "1.27": return v1.URISchemeHTTPS } return v1.URISchemeHTTP diff --git a/operator/pkg/controllers/master/kubescheduler.go b/operator/pkg/controllers/master/kubescheduler.go index 6e0729a0..149ce5f2 100644 --- a/operator/pkg/controllers/master/kubescheduler.go +++ b/operator/pkg/controllers/master/kubescheduler.go @@ -17,6 +17,7 @@ package master import ( "context" "fmt" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/apis/controlplane/v1alpha1" @@ -70,7 +71,7 @@ func schedulerLabels(clusterName string) map[string]string { func schedulerPodSpecFor(controlPlane *v1alpha1.ControlPlane) v1.PodSpec { hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate - return v1.PodSpec{ + return kschPodSpecForVersion(controlPlane.Spec.KubernetesVersion, &v1.PodSpec{ TerminationGracePeriodSeconds: aws.Int64(1), HostNetwork: true, DNSPolicy: v1.DNSClusterFirstWithHostNet, @@ -144,19 +145,39 @@ func schedulerPodSpecFor(controlPlane *v1alpha1.ControlPlane) v1.PodSpec { }, }, }}, + }) +} + +var ( + disabledFlagsForKsh126 = map[string]struct{}{"--logtostderr": {}} +) + +//Specific flags to be disabled for a particular version +func kschPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { + switch version { + case "1.26", "1.27": + args := []string{} + for _, arg := range defaultSpec.Containers[0].Args { + if _, skip := disabledFlagsForKsh126[strings.Split(arg, "=")[0]]; skip { + continue + } + args = append(args, arg) + } + defaultSpec.Containers[0].Args = args } + return *defaultSpec } func kschHealthCheckPortForVersion(version string) int { switch version { - case "1.23", "1.24", "1.25": + case "1.23", "1.24", "1.25", "1.26", "1.27": return 10259 } return 10251 } func kschHealthCheckSchemeForVersion(version string) v1.URIScheme { switch version { - case "1.23", "1.24", "1.25": + case "1.23", "1.24", "1.25", "1.26", "1.27": return v1.URISchemeHTTPS } return v1.URISchemeHTTP diff --git a/operator/pkg/utils/imageprovider/imageprovider.go b/operator/pkg/utils/imageprovider/imageprovider.go index 8cffd31c..db5cf5b3 100644 --- a/operator/pkg/utils/imageprovider/imageprovider.go +++ b/operator/pkg/utils/imageprovider/imageprovider.go @@ -23,6 +23,8 @@ var ( "1.23": kubeVersion123Tag, "1.24": kubeVersion124Tag, "1.25": kubeVersion125Tag, + "1.26": kubeVersion126Tag, + "1.27": kubeVersion127Tag, } ) @@ -40,6 +42,8 @@ const ( kubeVersion123Tag = "v1.23.13-eks-1-23-9" kubeVersion124Tag = "v1.24.8-eks-1-24-5" kubeVersion125Tag = "v1.25.5-eks-1-25-3" + kubeVersion126Tag = "v1.26.7-eks-1-26-17" + kubeVersion127Tag = "v1.27.4-eks-1-27-10" repositoryName = "public.ecr.aws/eks-distro/" busyBoxImage = "public.ecr.aws/docker/library/busybox:stable" ) From 25c439709205f34b464e36a946eaea8e98da282b Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Sun, 24 Sep 2023 14:21:26 -0700 Subject: [PATCH 02/12] Code refactor --- .../pkg/controllers/master/kubeapiserver.go | 30 +++++++++---------- .../master/kubecontrollermanager.go | 29 ++---------------- .../pkg/controllers/master/kubescheduler.go | 11 +------ 3 files changed, 18 insertions(+), 52 deletions(-) diff --git a/operator/pkg/controllers/master/kubeapiserver.go b/operator/pkg/controllers/master/kubeapiserver.go index 2c0f5ef2..87af1d8d 100644 --- a/operator/pkg/controllers/master/kubeapiserver.go +++ b/operator/pkg/controllers/master/kubeapiserver.go @@ -462,23 +462,21 @@ var ( func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.25": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForAPIServer[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForAPIServer) case "1.26", "1.27": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForApi126[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForApi126) } return *defaultSpec } + +//Method to disable flags from default specs for a k8s version +func disableFlags(defaultSpec *interface{}, disabledFlags map[string]struct{}) { + args := []string{} + for _, arg := range defaultSpec.Containers[0].Args { + if _, skip := disabledFlags[strings.Split(arg, "=")[0]]; skip { + continue + } + args = append(args, arg) + } + defaultSpec.Containers[0].Args = args +} diff --git a/operator/pkg/controllers/master/kubecontrollermanager.go b/operator/pkg/controllers/master/kubecontrollermanager.go index 67f3c2a6..0e2de9af 100644 --- a/operator/pkg/controllers/master/kubecontrollermanager.go +++ b/operator/pkg/controllers/master/kubecontrollermanager.go @@ -17,8 +17,6 @@ package master import ( "context" "fmt" - "strings" - "github.com/aws/aws-sdk-go/aws" "github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/apis/controlplane/v1alpha1" "github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/utils/imageprovider" @@ -280,32 +278,11 @@ var ( func kcmPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.22", "1.23", "1.24", "1.25": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForKube122[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForKube122) case "1.26": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForKube126[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForKube126) case "1.27": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForKube127[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForKube127) } return *defaultSpec } diff --git a/operator/pkg/controllers/master/kubescheduler.go b/operator/pkg/controllers/master/kubescheduler.go index 149ce5f2..cb49339f 100644 --- a/operator/pkg/controllers/master/kubescheduler.go +++ b/operator/pkg/controllers/master/kubescheduler.go @@ -17,8 +17,6 @@ package master import ( "context" "fmt" - "strings" - "github.com/aws/aws-sdk-go/aws" "github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/apis/controlplane/v1alpha1" "github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/utils/imageprovider" @@ -156,14 +154,7 @@ var ( func kschPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.26", "1.27": - args := []string{} - for _, arg := range defaultSpec.Containers[0].Args { - if _, skip := disabledFlagsForKsh126[strings.Split(arg, "=")[0]]; skip { - continue - } - args = append(args, arg) - } - defaultSpec.Containers[0].Args = args + disableFlags(defaultSpec, disabledFlagsForKsh126) } return *defaultSpec } From 8b397bb27abb96f7b2a06cad765baed843b5b68b Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Tue, 26 Sep 2023 10:31:48 -0700 Subject: [PATCH 03/12] Refactor code --- operator/pkg/controllers/master/kubeapiserver.go | 9 +++------ operator/pkg/controllers/master/kubecontrollermanager.go | 6 ------ 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/operator/pkg/controllers/master/kubeapiserver.go b/operator/pkg/controllers/master/kubeapiserver.go index 87af1d8d..4e2f1df0 100644 --- a/operator/pkg/controllers/master/kubeapiserver.go +++ b/operator/pkg/controllers/master/kubeapiserver.go @@ -452,17 +452,14 @@ func nodeSelector(clusterName string, colocateWithEtcd bool) map[string]string { } var ( - disabledFlagsForAPIServer = map[string]struct{}{"--feature-gates": {}} -) - -var ( + disabledFlagsForAPI125 = map[string]struct{}{"--feature-gates": {}} disabledFlagsForApi126 = map[string]struct{}{"--feature-gates": {}, "--logtostderr": {}} ) func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec { switch version { case "1.25": - disableFlags(defaultSpec, disabledFlagsForAPIServer) + disableFlags(defaultSpec, disabledFlagsForAPI125) case "1.26", "1.27": disableFlags(defaultSpec, disabledFlagsForApi126) } @@ -470,7 +467,7 @@ func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodS } //Method to disable flags from default specs for a k8s version -func disableFlags(defaultSpec *interface{}, disabledFlags map[string]struct{}) { +func disableFlags(defaultSpec interface{}, disabledFlags map[string]struct{}) { args := []string{} for _, arg := range defaultSpec.Containers[0].Args { if _, skip := disabledFlags[strings.Split(arg, "=")[0]]; skip { diff --git a/operator/pkg/controllers/master/kubecontrollermanager.go b/operator/pkg/controllers/master/kubecontrollermanager.go index 0e2de9af..13282b52 100644 --- a/operator/pkg/controllers/master/kubecontrollermanager.go +++ b/operator/pkg/controllers/master/kubecontrollermanager.go @@ -265,13 +265,7 @@ data: var ( disabledFlagsForKube122 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}} -) - -var ( disabledFlagsForKube126 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}, "--logtostderr": {}} -) - -var ( disabledFlagsForKube127 = map[string]struct{}{"--horizontal-pod-autoscaler-use-rest-clients": {}, "--logtostderr": {}, "--cloud-provider": {}} ) From 607ddde3046c68ac5d3f2000396b05ef9e4d198a Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Tue, 26 Sep 2023 10:40:11 -0700 Subject: [PATCH 04/12] Correcting method signature --- operator/pkg/controllers/master/kubeapiserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/pkg/controllers/master/kubeapiserver.go b/operator/pkg/controllers/master/kubeapiserver.go index 4e2f1df0..9b49d0b4 100644 --- a/operator/pkg/controllers/master/kubeapiserver.go +++ b/operator/pkg/controllers/master/kubeapiserver.go @@ -467,7 +467,7 @@ func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodS } //Method to disable flags from default specs for a k8s version -func disableFlags(defaultSpec interface{}, disabledFlags map[string]struct{}) { +func disableFlags(defaultSpec *v1.PodSpec, disabledFlags map[string]struct{}) { args := []string{} for _, arg := range defaultSpec.Containers[0].Args { if _, skip := disabledFlags[strings.Split(arg, "=")[0]]; skip { From 6c00d562c6f871db377e9401cc9205f5fa85f289 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Wed, 11 Oct 2023 16:53:39 -0700 Subject: [PATCH 05/12] PVRE reporting for short lived instances --- .../eks/awscli-cl2-load-with-addons.yaml | 4 ++ tests/pipelines/eks/awscli-eks-cl2-load.yaml | 4 ++ tests/tasks/setup/eks/awscli-mng.yaml | 55 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml index 66cc3bd8..32d54be8 100644 --- a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml +++ b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml @@ -97,6 +97,8 @@ spec: value: $(params.cluster-name)-node-role - name: endpoint value: $(params.endpoint) + - name: kubernetes-version + value: $(params.kubernetes-version) - name: desired-nodes value: "1" - name: max-nodes @@ -137,6 +139,8 @@ spec: value: $(params.cluster-name)-node-role - name: endpoint value: $(params.endpoint) + - name: kubernetes-version + value: $(params.kubernetes-version) runAfter: - install-fluentbit-addon taskRef: diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml index aaef0c83..9b8692ab 100644 --- a/tests/pipelines/eks/awscli-eks-cl2-load.yaml +++ b/tests/pipelines/eks/awscli-eks-cl2-load.yaml @@ -96,6 +96,8 @@ spec: value: "$(params.cluster-name)-node-role" - name: endpoint value: $(params.endpoint) + - name: kubernetes-version + value: "$(params.kubernetes-version)" - name: desired-nodes value: "1" - name: max-nodes @@ -136,6 +138,8 @@ spec: value: "$(params.cluster-name)-node-role" - name: endpoint value: $(params.endpoint) + - name: kubernetes-version + value: "$(params.kubernetes-version)" runAfter: - install-fluentbit-addon taskRef: diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml index 0a2561b7..b39c0837 100644 --- a/tests/tasks/setup/eks/awscli-mng.yaml +++ b/tests/tasks/setup/eks/awscli-mng.yaml @@ -14,6 +14,9 @@ spec: - name: region default: "us-west-2" description: The region where the cluster is in. + - name: kubernetes-version + default: "1.21" + description: The EKS version to install. - name: desired-nodes default: "10" description: The desired number of nodes in the cluster. @@ -75,9 +78,60 @@ spec: asgs=$((nodes/max_nodes)) echo "asgs: $asgs" node_group=$(params.nodegroup-prefix)$(params.cluster-name)-nodes + + create_and_validate_launchTemplate_resource() + { + launch_template_name=$node_group-launchTemplate + iam_instance_profile=$NODE_ROLE_ARN + imageId=/aws/service/eks/optimized-ami/$(params.kubernetes-version)/amazon-linux-2/recommended/image_id + + cat sed "s/\$LAUNCH_TEMPLATE_NAME/$launch_template_name/g; \ + s/\$NODE_INSTANCE_PROFILE/$iam_instance_profile/g; \ + s/\$IMAGE_ID/$imageId/g;" > "$(workspaces.config.path)/cfnConfig.yaml" < Date: Thu, 12 Oct 2023 15:36:03 -0700 Subject: [PATCH 06/12] Correcting CFN template --- tests/tasks/setup/eks/awscli-mng.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml index b39c0837..b92294d0 100644 --- a/tests/tasks/setup/eks/awscli-mng.yaml +++ b/tests/tasks/setup/eks/awscli-mng.yaml @@ -113,7 +113,7 @@ spec: HttpPutResponseHopLimit : 2 HttpEndpoint: enabled HttpTokens: !If - - IMDSv1Disabled + - true - required - optional EOL From 6fb725cb612609de91bbdfd4d07c155080a6d7b6 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Thu, 26 Oct 2023 23:31:35 -0700 Subject: [PATCH 07/12] Refactor code --- .../assets/eks_nodeGroup_LaunchTemplate.yaml | 29 ++++++++ .../eks/awscli-cl2-load-with-addons.yaml | 5 ++ tests/pipelines/eks/awscli-eks-cl2-load.yaml | 5 ++ tests/tasks/setup/eks/awscli-mng.yaml | 67 +++++-------------- 4 files changed, 57 insertions(+), 49 deletions(-) create mode 100644 tests/assets/eks_nodeGroup_LaunchTemplate.yaml diff --git a/tests/assets/eks_nodeGroup_LaunchTemplate.yaml b/tests/assets/eks_nodeGroup_LaunchTemplate.yaml new file mode 100644 index 00000000..985ca988 --- /dev/null +++ b/tests/assets/eks_nodeGroup_LaunchTemplate.yaml @@ -0,0 +1,29 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Create an EKS Node Group Launch Template +Resources: + NodeGroupLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateName: !Ref LaunchTemplateName + LaunchTemplateData: + InstanceType: t3.medium + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: 20 + VolumeType: gp2 + IamInstanceProfile: + Arn: arn:aws:iam::953421922360:instance-profile/sushima-CodeDeployDemo-EC2-Instance-Profile + UserData: !Base64 + "Fn::Sub": | + #!/bin/bash + yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + MetadataOptions: + HttpPutResponseHopLimit : 2 + HttpEndpoint: enabled + HttpTokens: required + +Outputs: + NodeGroupLaunchTemplateName: + Description: Name of the Node Group Launch Template + Value: !Ref NodeGroupLaunchTemplate \ No newline at end of file diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml index 32d54be8..e666bac7 100644 --- a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml +++ b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml @@ -16,6 +16,7 @@ spec: - name: slack-message - name: amp-workspace-id - name: vpc-cfn-url + - name: ng-cfn-url - name: kubernetes-version - name: service-role-cfn-url default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_service_role.json" @@ -99,6 +100,8 @@ spec: value: $(params.endpoint) - name: kubernetes-version value: $(params.kubernetes-version) + - name: ng-cfn-url + value: $(params.ng-cfn-url) - name: desired-nodes value: "1" - name: max-nodes @@ -141,6 +144,8 @@ spec: value: $(params.endpoint) - name: kubernetes-version value: $(params.kubernetes-version) + - name: ng-cfn-url + value: $(params.ng-cfn-url) runAfter: - install-fluentbit-addon taskRef: diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml index 9b8692ab..249de763 100644 --- a/tests/pipelines/eks/awscli-eks-cl2-load.yaml +++ b/tests/pipelines/eks/awscli-eks-cl2-load.yaml @@ -18,6 +18,7 @@ spec: - name: slack-hook - name: slack-message - name: vpc-cfn-url + - name: ng-cfn-url - name: kubernetes-version default: "1.23" - name: amp-workspace-id @@ -98,6 +99,8 @@ spec: value: $(params.endpoint) - name: kubernetes-version value: "$(params.kubernetes-version)" + - name: ng-cfn-url + value: "$(params.ng-cfn-url)" - name: desired-nodes value: "1" - name: max-nodes @@ -140,6 +143,8 @@ spec: value: $(params.endpoint) - name: kubernetes-version value: "$(params.kubernetes-version)" + - name: ng-cfn-url + value: "$(params.ng-cfn-url)" runAfter: - install-fluentbit-addon taskRef: diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml index b92294d0..abbd5030 100644 --- a/tests/tasks/setup/eks/awscli-mng.yaml +++ b/tests/tasks/setup/eks/awscli-mng.yaml @@ -15,7 +15,7 @@ spec: default: "us-west-2" description: The region where the cluster is in. - name: kubernetes-version - default: "1.21" + default: "1.28" description: The EKS version to install. - name: desired-nodes default: "10" @@ -33,6 +33,8 @@ spec: - name: host-instance-types description: Instance types for managed nodes. default: "c5.large m5.large r5.large t3.large t3a.large c5a.large m5a.large r5a.large" + - name: ng-cfn-url + description: The url of the CFN YAML/JSON to create CFN stack dor NG launch template - name: host-taints description: Taints to be added to managed nodes default: "" @@ -78,56 +80,23 @@ spec: asgs=$((nodes/max_nodes)) echo "asgs: $asgs" node_group=$(params.nodegroup-prefix)$(params.cluster-name)-nodes + launch_template_name=$node_group-launchTemplate - create_and_validate_launchTemplate_resource() - { - launch_template_name=$node_group-launchTemplate - iam_instance_profile=$NODE_ROLE_ARN - imageId=/aws/service/eks/optimized-ami/$(params.kubernetes-version)/amazon-linux-2/recommended/image_id - - cat sed "s/\$LAUNCH_TEMPLATE_NAME/$launch_template_name/g; \ - s/\$NODE_INSTANCE_PROFILE/$iam_instance_profile/g; \ - s/\$IMAGE_ID/$imageId/g;" > "$(workspaces.config.path)/cfnConfig.yaml" < Date: Thu, 26 Oct 2023 23:36:04 -0700 Subject: [PATCH 08/12] Refactor code --- tests/tasks/setup/eks/awscli-mng.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml index abbd5030..079b66ac 100644 --- a/tests/tasks/setup/eks/awscli-mng.yaml +++ b/tests/tasks/setup/eks/awscli-mng.yaml @@ -83,6 +83,7 @@ spec: launch_template_name=$node_group-launchTemplate curl -s $(params.ng-cfn-url) -o ./amazon-ng-cfn + sed -i "s/!Ref LaunchTemplateName/$LaunchTemplateName/g" ./amazon-ng-cfn STACK_NAME=$node_group-stack STACK_STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].StackStatus' --output text --region $(params.region)) From 8322e1015a2baf5342cfde2238d2839e008e15b3 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Fri, 3 Nov 2023 15:38:23 -0700 Subject: [PATCH 09/12] Updating the node group launch template --- .../assets/eks_nodeGroup_LaunchTemplate.yaml | 86 +++++++++++++------ .../eks/awscli-cl2-load-with-addons.yaml | 24 +++++- tests/pipelines/eks/awscli-eks-cl2-load.yaml | 28 ++++-- tests/tasks/setup/eks/awscli-cfn-lt.yaml | 69 +++++++++++++++ tests/tasks/setup/eks/awscli-mng.yaml | 26 +----- tests/tasks/teardown/awscli-eks.yaml | 2 + 6 files changed, 181 insertions(+), 54 deletions(-) create mode 100644 tests/tasks/setup/eks/awscli-cfn-lt.yaml diff --git a/tests/assets/eks_nodeGroup_LaunchTemplate.yaml b/tests/assets/eks_nodeGroup_LaunchTemplate.yaml index 985ca988..896acab1 100644 --- a/tests/assets/eks_nodeGroup_LaunchTemplate.yaml +++ b/tests/assets/eks_nodeGroup_LaunchTemplate.yaml @@ -1,29 +1,67 @@ AWSTemplateFormatVersion: '2010-09-09' Description: Create an EKS Node Group Launch Template + +Parameters: + LaunchTemplateName: + Type: String + Description: Name of the Launch Template + + ClusterName: + Type: String + Description: Name of the Cluster + + ImageId: + Type: String + Description: Latest AMI for a given kubernetes version + + ClusterCA: + Type: String + Description: Base64-encoded cluster CA certificate + + APIServerEndpoint: + Type: String + Description: API server endpoint URL + + DNSClusterIPRange: + Type: String + Description: DNS cluster IP range + Resources: - NodeGroupLaunchTemplate: - Type: AWS::EC2::LaunchTemplate - Properties: - LaunchTemplateName: !Ref LaunchTemplateName - LaunchTemplateData: - InstanceType: t3.medium - BlockDeviceMappings: - - DeviceName: /dev/xvda - Ebs: - VolumeSize: 20 - VolumeType: gp2 - IamInstanceProfile: - Arn: arn:aws:iam::953421922360:instance-profile/sushima-CodeDeployDemo-EC2-Instance-Profile - UserData: !Base64 - "Fn::Sub": | - #!/bin/bash - yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm - MetadataOptions: - HttpPutResponseHopLimit : 2 - HttpEndpoint: enabled - HttpTokens: required + NodeGroupLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateName: !Ref LaunchTemplateName + LaunchTemplateData: + ImageId: !Ref ImageId + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeSize: 20 + VolumeType: gp2 + UserData: !Base64 + "Fn::Sub": | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="==MYBOUNDARY==" + + --==MYBOUNDARY== + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + set -ex + + /etc/eks/bootstrap.sh ${ClusterName} \ + --b64-cluster-ca ${ClusterCA} \ + --apiserver-endpoint ${APIServerEndpoint} \ + --dns-cluster-ip ${DNSClusterIPRange} \ + --container-runtime containerd + + --==MYBOUNDARY==-- + MetadataOptions: + HttpPutResponseHopLimit: 2 + HttpEndpoint: enabled + HttpTokens: required Outputs: - NodeGroupLaunchTemplateName: - Description: Name of the Node Group Launch Template - Value: !Ref NodeGroupLaunchTemplate \ No newline at end of file + NodeGroupLaunchTemplateName: + Description: Name of the Node Group Launch Template + Value: !Ref NodeGroupLaunchTemplate \ No newline at end of file diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml index e666bac7..f3eb958c 100644 --- a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml +++ b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: awscli-eks-cl2loadtest-with-addons + name: sushima-awscli-eks-cl2loadtest-with-addons namespace: scalability spec: params: @@ -90,6 +90,26 @@ spec: workspaces: - name: config workspace: config + - name: create-launch-template + params: + - name: cluster-name + value: $(params.cluster-name) + - name: stack-name + value: $(params.cluster-name)-launch-template + - name: kubernetes-version + value: "$(params.kubernetes-version)" + - name: ng-cfn-url + value: "$(params.ng-cfn-url)" + - name: endpoint + value: $(params.endpoint) + runAfter: + - create-eks-cluster + taskRef: + kind: Task + name: awscli-eks-cfn-launch-template-create + workspaces: + - name: config + workspace: config - name: create-mng-monitoring-nodes params: - name: cluster-name @@ -223,6 +243,8 @@ spec: value: $(params.cluster-name)-service-role - name: node-role-stack-name value: $(params.cluster-name)-node-role + - name: launch-template-stack-name + value: $(params.cluster-name)-launch-template taskRef: kind: Task name: awscli-eks-cluster-teardown diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml index 249de763..033be32e 100644 --- a/tests/pipelines/eks/awscli-eks-cl2-load.yaml +++ b/tests/pipelines/eks/awscli-eks-cl2-load.yaml @@ -89,6 +89,26 @@ spec: workspaces: - name: config workspace: config + - name: create-cfn-launch-template + params: + - name: cluster-name + value: $(params.cluster-name) + - name: stack-name + value: $(params.cluster-name)-launch-template + - name: kubernetes-version + value: "$(params.kubernetes-version)" + - name: ng-cfn-url + value: "$(params.ng-cfn-url)" + - name: endpoint + value: $(params.endpoint) + runAfter: + - create-eks-cluster + taskRef: + kind: Task + name: awscli-eks-cfn-launch-template-create + workspaces: + - name: config + workspace: config - name: create-mng-monitoring-nodes params: - name: cluster-name @@ -112,7 +132,7 @@ spec: - name: nodegroup-prefix value: "monitoring-" runAfter: - - create-eks-cluster + - create-cfn-launch-template taskRef: kind: Task name: awscli-eks-nodegroup-create @@ -141,10 +161,6 @@ spec: value: "$(params.cluster-name)-node-role" - name: endpoint value: $(params.endpoint) - - name: kubernetes-version - value: "$(params.kubernetes-version)" - - name: ng-cfn-url - value: "$(params.ng-cfn-url)" runAfter: - install-fluentbit-addon taskRef: @@ -210,6 +226,8 @@ spec: value: $(params.cluster-name)-service-role - name: node-role-stack-name value: $(params.cluster-name)-node-role + - name: launch-template-stack-name + value: $(params.cluster-name)-launch-template taskRef: kind: Task name: awscli-eks-cluster-teardown diff --git a/tests/tasks/setup/eks/awscli-cfn-lt.yaml b/tests/tasks/setup/eks/awscli-cfn-lt.yaml new file mode 100644 index 00000000..f0c97bbc --- /dev/null +++ b/tests/tasks/setup/eks/awscli-cfn-lt.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: awscli-eks-cfn-launch-template-create + namespace: scalability +spec: + description: | + Create an EKS CFN stack to output a launch template. + This Task can be used to create an EKS CFN stack that outputs a launch template with prod ami for a given cluster to ensure PVRE reporting of instances. + params: + - name: cluster-name + description: The name of the EKS cluster you want to spin managed nodegroups for. + - name: stack-name + description: The name of the VPC name you want to spin. + - name: region + default: "us-west-2" + description: The region where the cluster is in. + - name: kubernetes-version + default: "1.28" + description: The EKS version to install. + - name: ng-cfn-url + description: The url of the CFN YAML/JSON to create CFN stack for NG launch template + - name: endpoint + default: "" + workspaces: + - name: config + mountPath: /config/ + stepTemplate: + env: + - name: KUBECONFIG + value: /config/kubeconfig + steps: + - name: create-launch-template + image: alpine/k8s:1.23.7 + script: | + ENDPOINT_FLAG="" + if [ -n "$(params.endpoint)" ]; then + ENDPOINT_FLAG="--endpoint $(params.endpoint)" + fi + + curl -s $(params.ng-cfn-url) -o ./amazon-ng-cfn + + launch_template_name=$(params.cluster-name)-launchTemplate + certificate_authority=$(aws eks $ENDPOINT_FLAG describe-cluster --query "cluster.certificateAuthority.data" --output text --name $(params.cluster-name) --region $(params.region)) + api_server_endpoint=$(aws eks $ENDPOINT_FLAG describe-cluster --query "cluster.endpoint" --output text --name $(params.cluster-name) --region $(params.region)) + dns_cluster_ip=$(aws eks $ENDPOINT_FLAG describe-cluster --query "cluster.kubernetesNetworkConfig.serviceIpv4Cidr" --output text --name $(params.cluster-name) --region $(params.region)) + image_id=$(aws ssm get-parameter --name "/aws/service/eks/optimized-ami/$(params.kubernetes-version)/amazon-linux-2/recommended/image_id" --query "Parameter.Value" --output text) + + STACK_NAME=$(params.stack-name) + STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text --region $(params.region)) + + if [[ "$STACK_STATUS" == "" ]]; then + aws cloudformation create-stack \ + --stack-name $STACK_NAME \ + --template-body file://$(pwd)/amazon-ng-cfn \ + --parameters ParameterKey=LaunchTemplateName,ParameterValue=$launch_template_name\ + ParameterKey=ImageId,ParameterValue=$image_id\ + ParameterKey=ClusterName,ParameterValue=$(params.cluster-name)\ + ParameterKey=ClusterCA,ParameterValue=$certificate_authority\ + ParameterKey=APIServerEndpoint,ParameterValue=$api_server_endpoint\ + ParameterKey=DNSClusterIPRange,ParameterValue=$dns_cluster_ip\ + --region $(params.region) + + aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region) + echo "CREATED_CFN_STACK=$STACK_NAME" + else + echo "$STACK_NAME Already exists" + fi \ No newline at end of file diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml index 079b66ac..ab38b7f5 100644 --- a/tests/tasks/setup/eks/awscli-mng.yaml +++ b/tests/tasks/setup/eks/awscli-mng.yaml @@ -14,9 +14,6 @@ spec: - name: region default: "us-west-2" description: The region where the cluster is in. - - name: kubernetes-version - default: "1.28" - description: The EKS version to install. - name: desired-nodes default: "10" description: The desired number of nodes in the cluster. @@ -33,8 +30,6 @@ spec: - name: host-instance-types description: Instance types for managed nodes. default: "c5.large m5.large r5.large t3.large t3a.large c5a.large m5a.large r5a.large" - - name: ng-cfn-url - description: The url of the CFN YAML/JSON to create CFN stack dor NG launch template - name: host-taints description: Taints to be added to managed nodes default: "" @@ -80,28 +75,11 @@ spec: asgs=$((nodes/max_nodes)) echo "asgs: $asgs" node_group=$(params.nodegroup-prefix)$(params.cluster-name)-nodes - launch_template_name=$node_group-launchTemplate - - curl -s $(params.ng-cfn-url) -o ./amazon-ng-cfn - sed -i "s/!Ref LaunchTemplateName/$LaunchTemplateName/g" ./amazon-ng-cfn - - STACK_NAME=$node_group-stack - STACK_STATUS=$(aws cloudformation describe-stacks --stack-name $STACK_NAME --query 'Stacks[0].StackStatus' --output text --region $(params.region)) - - if [[ "$STACK_STATUS" == "" ]]; then - aws cloudformation create-stack \ - --stack-name $STACK_NAME \ - --template-body ./amazon-ng-cfn \ - --region $(params.region) - fi - - aws cloudformation wait stack-create-complete --stack-name $node_group-stack --region $(params.region) - echo "CREATED_CFN_STACK=$STACK_NAME" create_and_validate_dp_nodes() { node_group_name=$node_group-$1 - launch_template_name=$node_group-launchTemplate + launch_template_name=$(params.cluster-name)-launchTemplate CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text) EC2_INSTANCES=$3 if [ "$CREATED_NODEGROUP" == "" ]; then @@ -110,7 +88,7 @@ spec: --cluster-name $(params.cluster-name) \ --nodegroup-name $node_group_name \ --node-role $NODE_ROLE_ARN \ - --launch-template (aws ec2 describe-launch-templates --launch-template-names $launch_template_name) + --launch-template name=$launch_template_name\ --region $(params.region) \ --instance-types $EC2_INSTANCES \ --scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \ diff --git a/tests/tasks/teardown/awscli-eks.yaml b/tests/tasks/teardown/awscli-eks.yaml index 3d6714c7..579eb6de 100644 --- a/tests/tasks/teardown/awscli-eks.yaml +++ b/tests/tasks/teardown/awscli-eks.yaml @@ -22,6 +22,7 @@ spec: default: "Job is completed" - name: service-role-stack-name - name: node-role-stack-name + - name: launch-template-stack-name steps: - name: delete-cluster image: alpine/k8s:1.23.7 @@ -42,6 +43,7 @@ spec: script: | aws cloudformation delete-stack --stack-name $(params.service-role-stack-name) aws cloudformation delete-stack --stack-name $(params.node-role-stack-name) + aws cloudformation delete-stack --stack-name $(params.launch-template-stack-name) - name: send-slack-notification image: alpine/k8s:1.23.7 script: | From 8537b7678c4b85def1f3957831cba5d3694af4e4 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Fri, 3 Nov 2023 15:50:31 -0700 Subject: [PATCH 10/12] Refactor code --- tests/pipelines/eks/awscli-cl2-load-with-addons.yaml | 12 ++---------- tests/pipelines/eks/awscli-eks-cl2-load.yaml | 4 ---- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml index f3eb958c..6364642a 100644 --- a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml +++ b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Pipeline metadata: - name: sushima-awscli-eks-cl2loadtest-with-addons + name: awscli-eks-cl2loadtest-with-addons namespace: scalability spec: params: @@ -118,10 +118,6 @@ spec: value: $(params.cluster-name)-node-role - name: endpoint value: $(params.endpoint) - - name: kubernetes-version - value: $(params.kubernetes-version) - - name: ng-cfn-url - value: $(params.ng-cfn-url) - name: desired-nodes value: "1" - name: max-nodes @@ -133,7 +129,7 @@ spec: - name: nodegroup-prefix value: "monitoring-" runAfter: - - create-eks-cluster + - create-launch-template taskRef: kind: Task name: awscli-eks-nodegroup-create @@ -162,10 +158,6 @@ spec: value: $(params.cluster-name)-node-role - name: endpoint value: $(params.endpoint) - - name: kubernetes-version - value: $(params.kubernetes-version) - - name: ng-cfn-url - value: $(params.ng-cfn-url) runAfter: - install-fluentbit-addon taskRef: diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml index 033be32e..f4db6672 100644 --- a/tests/pipelines/eks/awscli-eks-cl2-load.yaml +++ b/tests/pipelines/eks/awscli-eks-cl2-load.yaml @@ -117,10 +117,6 @@ spec: value: "$(params.cluster-name)-node-role" - name: endpoint value: $(params.endpoint) - - name: kubernetes-version - value: "$(params.kubernetes-version)" - - name: ng-cfn-url - value: "$(params.ng-cfn-url)" - name: desired-nodes value: "1" - name: max-nodes From e57a24656400de99a1199c103f62d12c57901800 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Fri, 3 Nov 2023 16:30:15 -0700 Subject: [PATCH 11/12] Refactor code --- tests/pipelines/eks/awscli-cl2-load-with-addons.yaml | 2 +- tests/pipelines/eks/awscli-eks-cl2-load.yaml | 2 +- tests/tasks/setup/eks/awscli-cfn-lt.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml index 6364642a..52f1a6c3 100644 --- a/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml +++ b/tests/pipelines/eks/awscli-cl2-load-with-addons.yaml @@ -106,7 +106,7 @@ spec: - create-eks-cluster taskRef: kind: Task - name: awscli-eks-cfn-launch-template-create + name: awscli-eks-cfn-launch-template workspaces: - name: config workspace: config diff --git a/tests/pipelines/eks/awscli-eks-cl2-load.yaml b/tests/pipelines/eks/awscli-eks-cl2-load.yaml index f4db6672..64a0f1a3 100644 --- a/tests/pipelines/eks/awscli-eks-cl2-load.yaml +++ b/tests/pipelines/eks/awscli-eks-cl2-load.yaml @@ -105,7 +105,7 @@ spec: - create-eks-cluster taskRef: kind: Task - name: awscli-eks-cfn-launch-template-create + name: awscli-eks-cfn-launch-template workspaces: - name: config workspace: config diff --git a/tests/tasks/setup/eks/awscli-cfn-lt.yaml b/tests/tasks/setup/eks/awscli-cfn-lt.yaml index f0c97bbc..235dfe37 100644 --- a/tests/tasks/setup/eks/awscli-cfn-lt.yaml +++ b/tests/tasks/setup/eks/awscli-cfn-lt.yaml @@ -2,17 +2,17 @@ apiVersion: tekton.dev/v1beta1 kind: Task metadata: - name: awscli-eks-cfn-launch-template-create + name: awscli-eks-cfn-launch-template namespace: scalability spec: description: | Create an EKS CFN stack to output a launch template. - This Task can be used to create an EKS CFN stack that outputs a launch template with prod ami for a given cluster to ensure PVRE reporting of instances. + This Task can be used to create an EKS CFN stack that outputs a launch template. params: - name: cluster-name - description: The name of the EKS cluster you want to spin managed nodegroups for. + description: EKS cluster you want to create CFN stack for. - name: stack-name - description: The name of the VPC name you want to spin. + description: Stack name you want to spin. - name: region default: "us-west-2" description: The region where the cluster is in. From f9691d28030f273dc7b0ea8585049901b699a9d1 Mon Sep 17 00:00:00 2001 From: Sushmita Mahendarkar Date: Wed, 8 Nov 2023 15:18:55 -0800 Subject: [PATCH 12/12] Refactor code --- tests/tasks/teardown/awscli-vpc-delete.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tasks/teardown/awscli-vpc-delete.yaml b/tests/tasks/teardown/awscli-vpc-delete.yaml index 328d90e6..d4173aa7 100644 --- a/tests/tasks/teardown/awscli-vpc-delete.yaml +++ b/tests/tasks/teardown/awscli-vpc-delete.yaml @@ -29,4 +29,4 @@ spec: aws cloudformation delete-stack --region $(params.region) --stack-name $(params.stack-name) # Wait for the stack to be deleted aws cloudformation wait stack-delete-complete --region $(params.region) --stack-name $(params.stack-name) - echo "Stack deleted successfully!" + echo "Stack deleted successfully!" \ No newline at end of file