doc update and clean up script

aws-observability · Sep 20, 2023 · 85d57c9 · 85d57c9
1 parent 107fe83
commit 85d57c9
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 55 deletions.
diff --git a/...lti-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md b/...lti-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md
@@ -169,9 +169,7 @@ aws ssm put-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION}
 1. Run `scripts/multi-acc-new-eks-mixed-observability-pattern/amg-preconfig.sh` script to
 
    1. create SSM SecureString parameter `/cdk-accelerator/amg-info` in `pipelineEnv` region of `pipelineEnv` account. This will be used by CDK for Grafana Operator resources configuration.
-
    2. create Grafana workspace API key.
-
    3. create SSM SecureString parameter `/cdk-accelerator/grafana-api-key` in `monitoringEnv` region of `monitoringEnv` account. This will be used by [External Secrets Operator](https://github.com/external-secrets/external-secrets/tree/main/deploy/charts/external-secrets).
 
 ```bash
@@ -183,7 +181,6 @@ eval bash `git rev-parse --show-toplevel`/scripts/multi-acc-new-eks-mixed-observ
 1. Run `scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh` script to
 
    1. create SSM SecureString Parameter `/cdk-accelerator/pipeline-git-info` in `pipelineEnv` region of `pipelineEnv` account. This parameter contains GitHub owner name, repository name (`cdk-aws-observability-accelerator`) and branch (`main`) which will be used as source for CodePipeline. [`cdk-aws-observability-accelerator`](https://github.com/aws-observability/cdk-aws-observability-accelerator) repository should be available in this GitHub source, ideally through forking.
-
    2. create secret `github-ssh-key` in `monitoringEnv` region of `monitoringEnv` account. This secret must contain GitHub SSH private key as a JSON structure containing fields `sshPrivateKey` and `url` in AWS Secrets Manager. This will be used by ArgoCD addon to authenticate against any GitHub repository (private or public). This secret is expected to be defined in the region where the pipeline will be deployed to. For more information on SSH credentials setup see [ArgoCD Secrets Support](https://aws-quickstart.github.io/cdk-eks-blueprints/addons/argo-cd/#secrets-support).
 
 ```bash { promptEnv=true }
@@ -213,7 +210,6 @@ unset $COA_GIT_PAT
 ## Deployment
 
 1. Fork [`cdk-aws-observability-accelerator`](https://github.com/aws-observability/cdk-aws-observability-accelerator) repository to your CodePioeline source GitHub organisation/user.
-
 2. Install the AWS CDK Toolkit globally on host machine.
 
 ```bash
@@ -266,13 +262,10 @@ make pattern multi-acc-new-eks-mixed-observability deploy multi-account-central-
 ```
 
 6. Login to `pipelineEnv` account and navigate to [AWS CodePipeline console](https://console.aws.amazon.com/codesuite/codepipeline/pipelines) at `pipelineEnv` region. Check status of pipeline that deploys multiple Amazon EKS clusters to different environments.
-
 7. The deployment also creates
 
    - `ampPrometheusDataSourceRole` with permissions to retrieve metrics from AMP in `prod1Env` account,
-
    - `cloudwatchDataSourceRole` with permissions to retrieve metrics from CloudWatch in `prod2Env` account and
-
    - Updates Amazon Grafana workspace IAM role in `monitoringEnv` account to assume roles in `prod1Env` and `prod2Env` accounts for retrieving and visualizing metrics in Grafana
 
 ## Post Deployment
@@ -396,8 +389,12 @@ done
 
 1. Run this command to destroy this pattern. This will delete pipeline.
 
-```bash
+```bash { promptEnv=false }
+source `git rev-parse --show-toplevel`/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh
 AWS_PROFILE='pipeline-account'
+export AWS_REGION=${COA_PIPELINE_REGION}
+cd `git rev-parse --show-toplevel`
+
 make pattern multi-acc-new-eks-mixed-observability destroy multi-account-central-pipeline
 ```
 

diff --git a/output.txt b/output.txt
diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh
@@ -1,19 +1,27 @@
 #!/bin/bash
 
-#set -e # exit when any command fails
-
-SCRIPT_PATH=$(pwd)/$(dirname $0)
+# set -e # exit when any command fails
+
+NC='\033[0m'       # Text Reset
+R='\033[0;31m'          # Red
+G='\033[0;32m'        # Green
+Y='\033[0;33m'       # Yellow
+echo -e "${R}"
+
+read -p "This script will clean up all resources deployed as part of this pattern. Are you sure you want to proceed [y/N]? " -n 2
+echo -e "\n"
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    echo -e "${Y}proceeding with clean up steps.${NC}"
+    echo -e "\n"
+else
+    exit 1
+fi
+
+SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern
 
 source ${SCRIPT_PATH}/format-display.sh # format display
-source ${SCRIPT_PATH}/source-envs.sh # sets required environment variables
-
-# if [[ $# -lt 1 ]]; then
-#     log 'R' "Usage: clean-up.sh <ARG 1>"
-#     exit 1
-# fi
-
-# clean up apps from all envs
-
+source ${SCRIPT_PATH}/post-deployment-source-envs.sh # sets required environment variables
 
 pipeline=(pipeline-account COA_PIPELINE_ACCOUNT_ID COA_PIPELINE_REGION)
 prod1=(prod1-account COA_PROD1_ACCOUNT_ID COA_PROD1_REGION)
@@ -41,61 +49,95 @@ for profile in "${!profiles[@]}"; do
         nGRole=$(aws cloudformation describe-stack-resources --profile ${env[0]} --region ${!env[2]} \
         --stack-name ${stackName} \
         --query "StackResources[?ResourceType=='AWS::IAM::Role' && contains(LogicalResourceId,'NodeGroupRole')].PhysicalResourceId" \
-        --output text)
+        --output text)        
 
         ClusterName=$(aws cloudformation describe-stacks --profile ${env[0]} --region ${!env[2]} \
             --stack-name ${stackName} \
             --query "Stacks[0].Outputs[?contains(OutputKey,'blueprintClusterName')].OutputValue" \
             --output text)
 
-        kubeContext="arn:aws:eks:${!env[2]}:${!env[1]}:cluster/${ClusterName}"  
+        kubeContext="arn:aws:eks:${!env[2]}:${!env[1]}:cluster/${ClusterName}"
 
         log 'O' "Initiating clean up of argocd apps in ${profile} account.."
-        argocd --kube-context ${kubeContext} app delete argocd/bootstrap-apps
 
-        log 'O' "Initiating deletion of cloudformation stack in ${profile} account.."
-
+        kubectl delete applications.argoproj.io bootstrap-apps -n argocd
+
+        appNames=($(kubectl --context ${kubeContext} get applications.argoproj.io -n argocd -o custom-columns=":metadata.name" --no-headers))
+
+        for appName in "${appNames[@]}"; do
+            kubectl --context ${kubeContext} delete applications.argoproj.io "$appName" -n argocd
+        done
+
+        log 'O' "deleting nodegroup IAM Role for ${env[0]}.."
+        aws iam delete-role --profile ${env[0]} \
+            --role-name ${nGRole}
+
+        log 'O' "Initiating deletion of cloudformation stack in ${profile} account.."        
         aws cloudformation delete-stack --profile ${env[0]} --region ${!env[2]} \
             --stack-name ${stackName}
 
         log 'O' "Removing kubecontext ${kubeContext}.."
-        kubectl config delete-context ${kubeContext}
+        kubectl config delete-context ${kubeContext}        
     fi
 
-    log 'O' "Cleaning CDK bootstrap for ${env[0]}.."
-    cdk bootstrap --destroy --profile ${env[0]}
-    cdk boostrap --clean --profile ${env[0]}
-done
+    log 'O' "cleaning CDK bootstrap for ${env[0]}.."
+
+    BUCKET_TO_DELETE=$(aws s3 --profile ${env[0]} ls | grep cdk-.*"${!env[2]}" | cut -d' ' -f3)
+    if [[ ! -z $BUCKET_TO_DELETE ]]
+    then
+        OBJECT_COUNT=$(aws s3api --profile ${env[0]} list-object-versions --region ${!env[2]} \
+            --bucket ${BUCKET_TO_DELETE} --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}' \
+            --output text | grep -v ^None | wc -l)
+
+        if [[ $OBJECT_COUNT > 0 ]]
+        then
+            aws s3api --profile ${env[0]} delete-objects --region ${!env[2]} \
+                --bucket ${BUCKET_TO_DELETE} \
+                --delete "$(aws s3api list-object-versions --region ${!env[2]} \
+                --bucket ${BUCKET_TO_DELETE} --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')"
+        fi
+
+        DELETE_MARKER_COUNT=$(aws s3api --profile ${env[0]} list-object-versions --region ${!env[2]} \
+            --bucket ${BUCKET_TO_DELETE} --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}' \
+            --output text  | grep -v ^None | wc -l)
+        if [[ $DELETE_MARKER_COUNT > 0 ]]
+        then
+            aws s3api --profile ${env[0]} delete-objects --region ${!env[2]} \
+                --bucket ${BUCKET_TO_DELETE} \
+                --delete "$(aws s3api list-object-versions --region ${!env[2]} \
+                --bucket ${BUCKET_TO_DELETE} --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')"
+        fi
+
+        aws s3 --profile ${env[0]} rb --region ${!env[2]} s3://${BUCKET_TO_DELETE} --force
+    fi
 
-# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/cdk-context"
+    aws cloudformation --profile ${env[0]} delete-stack --region ${!env[2]} --stack-name CDKToolkit
 
-# aws secretsmanager delete-secret --profile pipeline-account --region ${COA_PIPELINE_REGION} --secret-id "github-token" --force-delete-without-recovery
-# aws secretsmanager delete-secret --profile monitoring-account --region ${COA_MON_REGION} --secret-id "github-ssh-key" --force-delete-without-recovery
+done
 
-# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/pipeline-git-info"
+log 'O' "deleting Amazon Grafana API key, Secrets and SSM SecureString Parameters.."
 
-# aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/grafana-api-key" 
+aws secretsmanager delete-secret --profile pipeline-account --region ${COA_PIPELINE_REGION} --secret-id "github-token" --force-delete-without-recovery
+aws secretsmanager delete-secret --profile monitoring-account --region ${COA_MON_REGION} --secret-id "github-ssh-key" --force-delete-without-recovery
 
-# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/amg-info"
+aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/pipeline-git-info"
 
-# COA_AMG_WORKSPACE_NAME=$(aws ssm get-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} \
-#     --name "/cdk-accelerator/amg-info" --with-decryption \
-#     --query "Parameter.Value" --output text | jq .amg.workspaceName | sed 's/"//g')
+aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/grafana-api-key" 
 
-# COA_AMG_WORKSPACE_ID=$(aws grafana list-workspaces --profile monitoring-account --region ${COA_MON_REGION} \
-#     --query "workspaces[?name=='${COA_AMG_WORKSPACE_NAME}'].id" \
-#     --output text)
+COA_AMG_WORKSPACE_NAME=$(aws ssm get-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} \
+    --name "/cdk-accelerator/amg-info" --with-decryption \
+    --query "Parameter.Value" --output text | jq .amg.workspaceName | sed 's/"//g')
 
-# aws grafana delete-workspace-api-key --profile monitoring-account --region ${COA_MON_REGION} \
-#     --key-name "grafana-operator-key"
-#     --workspace-id $COA_AMG_WORKSPACE_ID
+COA_AMG_WORKSPACE_ID=$(aws grafana list-workspaces --profile monitoring-account --region ${COA_MON_REGION} \
+    --query "workspaces[?name=='${COA_AMG_WORKSPACE_NAME}'].id" \
+    --output text)
 
-# aws iam delete-role-policy --profile monitoring-account \
-#   --policy-name "AssumePROD1RolePolicy" \
-#   --role-name "crossAccAMPInfoFromPROD1Role"
+aws grafana delete-workspace-api-key --profile monitoring-account --region ${COA_MON_REGION} \
+    --key-name "grafana-operator-key"
+    --workspace-id $COA_AMG_WORKSPACE_ID
 
-# aws iam delete-role --profile monitoring-account \
-#   --role-name "crossAccAMPInfoFromPROD1Role"
+aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/amg-info"
 
+aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/cdk-context"
 
 log 'G' "DONE!"
diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh
@@ -42,7 +42,7 @@ if [ $existingParam -eq 0 ]; then
         "pipelineSource": {
             "gitOwner": "'${gitOwner}'",
             "gitRepoName": "'${gitRepoName}'",
-            "gitBranch": "'${gitBranch}'
+            "gitBranch": "'${gitBranch}'"
         }
     }'
 else

diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -e # exit when any command fails
-
 SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern
 
 source ${SCRIPT_PATH}/format-display.sh # format display

diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -e # exit when any command fails
-
 SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern
 
 source ${SCRIPT_PATH}/format-display.sh # format display