From 85d57c9ddc1e0e6aad4bbabf034a552c1ffc6e02 Mon Sep 17 00:00:00 2001 From: iamprakkie Date: Tue, 19 Sep 2023 23:34:34 -0600 Subject: [PATCH] doc update and clean up script --- .../multi-acc-new-eks-mixed-observability.md | 13 +- output.txt | 0 .../clean-up.sh | 126 ++++++++++++------ .../gitsource-preconfig.sh | 2 +- .../post-deployment-source-envs.sh | 2 - .../source-envs.sh | 2 - 6 files changed, 90 insertions(+), 55 deletions(-) delete mode 100644 output.txt diff --git a/docs/patterns/multi-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md b/docs/patterns/multi-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md index 807a2575..7d7c517e 100644 --- a/docs/patterns/multi-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md +++ b/docs/patterns/multi-new-eks-observability-accelerators/multi-acc-new-eks-mixed-observability.md @@ -169,9 +169,7 @@ aws ssm put-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} 1. Run `scripts/multi-acc-new-eks-mixed-observability-pattern/amg-preconfig.sh` script to 1. create SSM SecureString parameter `/cdk-accelerator/amg-info` in `pipelineEnv` region of `pipelineEnv` account. This will be used by CDK for Grafana Operator resources configuration. - 2. create Grafana workspace API key. - 3. create SSM SecureString parameter `/cdk-accelerator/grafana-api-key` in `monitoringEnv` region of `monitoringEnv` account. This will be used by [External Secrets Operator](https://github.com/external-secrets/external-secrets/tree/main/deploy/charts/external-secrets). ```bash @@ -183,7 +181,6 @@ eval bash `git rev-parse --show-toplevel`/scripts/multi-acc-new-eks-mixed-observ 1. Run `scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh` script to 1. create SSM SecureString Parameter `/cdk-accelerator/pipeline-git-info` in `pipelineEnv` region of `pipelineEnv` account. This parameter contains GitHub owner name, repository name (`cdk-aws-observability-accelerator`) and branch (`main`) which will be used as source for CodePipeline. [`cdk-aws-observability-accelerator`](https://github.com/aws-observability/cdk-aws-observability-accelerator) repository should be available in this GitHub source, ideally through forking. - 2. create secret `github-ssh-key` in `monitoringEnv` region of `monitoringEnv` account. This secret must contain GitHub SSH private key as a JSON structure containing fields `sshPrivateKey` and `url` in AWS Secrets Manager. This will be used by ArgoCD addon to authenticate against any GitHub repository (private or public). This secret is expected to be defined in the region where the pipeline will be deployed to. For more information on SSH credentials setup see [ArgoCD Secrets Support](https://aws-quickstart.github.io/cdk-eks-blueprints/addons/argo-cd/#secrets-support). ```bash { promptEnv=true } @@ -213,7 +210,6 @@ unset $COA_GIT_PAT ## Deployment 1. Fork [`cdk-aws-observability-accelerator`](https://github.com/aws-observability/cdk-aws-observability-accelerator) repository to your CodePioeline source GitHub organisation/user. - 2. Install the AWS CDK Toolkit globally on host machine. ```bash @@ -266,13 +262,10 @@ make pattern multi-acc-new-eks-mixed-observability deploy multi-account-central- ``` 6. Login to `pipelineEnv` account and navigate to [AWS CodePipeline console](https://console.aws.amazon.com/codesuite/codepipeline/pipelines) at `pipelineEnv` region. Check status of pipeline that deploys multiple Amazon EKS clusters to different environments. - 7. The deployment also creates - `ampPrometheusDataSourceRole` with permissions to retrieve metrics from AMP in `prod1Env` account, - - `cloudwatchDataSourceRole` with permissions to retrieve metrics from CloudWatch in `prod2Env` account and - - Updates Amazon Grafana workspace IAM role in `monitoringEnv` account to assume roles in `prod1Env` and `prod2Env` accounts for retrieving and visualizing metrics in Grafana ## Post Deployment @@ -396,8 +389,12 @@ done 1. Run this command to destroy this pattern. This will delete pipeline. -```bash +```bash { promptEnv=false } +source `git rev-parse --show-toplevel`/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh AWS_PROFILE='pipeline-account' +export AWS_REGION=${COA_PIPELINE_REGION} +cd `git rev-parse --show-toplevel` + make pattern multi-acc-new-eks-mixed-observability destroy multi-account-central-pipeline ``` diff --git a/output.txt b/output.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh index 03032842..aa21f6a8 100644 --- a/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh +++ b/scripts/multi-acc-new-eks-mixed-observability-pattern/clean-up.sh @@ -1,19 +1,27 @@ #!/bin/bash -#set -e # exit when any command fails - -SCRIPT_PATH=$(pwd)/$(dirname $0) +# set -e # exit when any command fails + +NC='\033[0m' # Text Reset +R='\033[0;31m' # Red +G='\033[0;32m' # Green +Y='\033[0;33m' # Yellow +echo -e "${R}" + +read -p "This script will clean up all resources deployed as part of this pattern. Are you sure you want to proceed [y/N]? " -n 2 +echo -e "\n" +if [[ $REPLY =~ ^[Yy]$ ]] +then + echo -e "${Y}proceeding with clean up steps.${NC}" + echo -e "\n" +else + exit 1 +fi + +SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern source ${SCRIPT_PATH}/format-display.sh # format display -source ${SCRIPT_PATH}/source-envs.sh # sets required environment variables - -# if [[ $# -lt 1 ]]; then -# log 'R' "Usage: clean-up.sh " -# exit 1 -# fi - -# clean up apps from all envs - +source ${SCRIPT_PATH}/post-deployment-source-envs.sh # sets required environment variables pipeline=(pipeline-account COA_PIPELINE_ACCOUNT_ID COA_PIPELINE_REGION) prod1=(prod1-account COA_PROD1_ACCOUNT_ID COA_PROD1_REGION) @@ -41,61 +49,95 @@ for profile in "${!profiles[@]}"; do nGRole=$(aws cloudformation describe-stack-resources --profile ${env[0]} --region ${!env[2]} \ --stack-name ${stackName} \ --query "StackResources[?ResourceType=='AWS::IAM::Role' && contains(LogicalResourceId,'NodeGroupRole')].PhysicalResourceId" \ - --output text) + --output text) ClusterName=$(aws cloudformation describe-stacks --profile ${env[0]} --region ${!env[2]} \ --stack-name ${stackName} \ --query "Stacks[0].Outputs[?contains(OutputKey,'blueprintClusterName')].OutputValue" \ --output text) - kubeContext="arn:aws:eks:${!env[2]}:${!env[1]}:cluster/${ClusterName}" + kubeContext="arn:aws:eks:${!env[2]}:${!env[1]}:cluster/${ClusterName}" log 'O' "Initiating clean up of argocd apps in ${profile} account.." - argocd --kube-context ${kubeContext} app delete argocd/bootstrap-apps - log 'O' "Initiating deletion of cloudformation stack in ${profile} account.." - + kubectl delete applications.argoproj.io bootstrap-apps -n argocd + + appNames=($(kubectl --context ${kubeContext} get applications.argoproj.io -n argocd -o custom-columns=":metadata.name" --no-headers)) + + for appName in "${appNames[@]}"; do + kubectl --context ${kubeContext} delete applications.argoproj.io "$appName" -n argocd + done + + log 'O' "deleting nodegroup IAM Role for ${env[0]}.." + aws iam delete-role --profile ${env[0]} \ + --role-name ${nGRole} + + log 'O' "Initiating deletion of cloudformation stack in ${profile} account.." aws cloudformation delete-stack --profile ${env[0]} --region ${!env[2]} \ --stack-name ${stackName} log 'O' "Removing kubecontext ${kubeContext}.." - kubectl config delete-context ${kubeContext} + kubectl config delete-context ${kubeContext} fi - log 'O' "Cleaning CDK bootstrap for ${env[0]}.." - cdk bootstrap --destroy --profile ${env[0]} - cdk boostrap --clean --profile ${env[0]} -done + log 'O' "cleaning CDK bootstrap for ${env[0]}.." + + BUCKET_TO_DELETE=$(aws s3 --profile ${env[0]} ls | grep cdk-.*"${!env[2]}" | cut -d' ' -f3) + if [[ ! -z $BUCKET_TO_DELETE ]] + then + OBJECT_COUNT=$(aws s3api --profile ${env[0]} list-object-versions --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}' \ + --output text | grep -v ^None | wc -l) + + if [[ $OBJECT_COUNT > 0 ]] + then + aws s3api --profile ${env[0]} delete-objects --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} \ + --delete "$(aws s3api list-object-versions --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" + fi + + DELETE_MARKER_COUNT=$(aws s3api --profile ${env[0]} list-object-versions --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}' \ + --output text | grep -v ^None | wc -l) + if [[ $DELETE_MARKER_COUNT > 0 ]] + then + aws s3api --profile ${env[0]} delete-objects --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} \ + --delete "$(aws s3api list-object-versions --region ${!env[2]} \ + --bucket ${BUCKET_TO_DELETE} --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')" + fi + + aws s3 --profile ${env[0]} rb --region ${!env[2]} s3://${BUCKET_TO_DELETE} --force + fi -# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/cdk-context" + aws cloudformation --profile ${env[0]} delete-stack --region ${!env[2]} --stack-name CDKToolkit -# aws secretsmanager delete-secret --profile pipeline-account --region ${COA_PIPELINE_REGION} --secret-id "github-token" --force-delete-without-recovery -# aws secretsmanager delete-secret --profile monitoring-account --region ${COA_MON_REGION} --secret-id "github-ssh-key" --force-delete-without-recovery +done -# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/pipeline-git-info" +log 'O' "deleting Amazon Grafana API key, Secrets and SSM SecureString Parameters.." -# aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/grafana-api-key" +aws secretsmanager delete-secret --profile pipeline-account --region ${COA_PIPELINE_REGION} --secret-id "github-token" --force-delete-without-recovery +aws secretsmanager delete-secret --profile monitoring-account --region ${COA_MON_REGION} --secret-id "github-ssh-key" --force-delete-without-recovery -# aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/amg-info" +aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/pipeline-git-info" -# COA_AMG_WORKSPACE_NAME=$(aws ssm get-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} \ -# --name "/cdk-accelerator/amg-info" --with-decryption \ -# --query "Parameter.Value" --output text | jq .amg.workspaceName | sed 's/"//g') +aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/grafana-api-key" -# COA_AMG_WORKSPACE_ID=$(aws grafana list-workspaces --profile monitoring-account --region ${COA_MON_REGION} \ -# --query "workspaces[?name=='${COA_AMG_WORKSPACE_NAME}'].id" \ -# --output text) +COA_AMG_WORKSPACE_NAME=$(aws ssm get-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} \ + --name "/cdk-accelerator/amg-info" --with-decryption \ + --query "Parameter.Value" --output text | jq .amg.workspaceName | sed 's/"//g') -# aws grafana delete-workspace-api-key --profile monitoring-account --region ${COA_MON_REGION} \ -# --key-name "grafana-operator-key" -# --workspace-id $COA_AMG_WORKSPACE_ID +COA_AMG_WORKSPACE_ID=$(aws grafana list-workspaces --profile monitoring-account --region ${COA_MON_REGION} \ + --query "workspaces[?name=='${COA_AMG_WORKSPACE_NAME}'].id" \ + --output text) -# aws iam delete-role-policy --profile monitoring-account \ -# --policy-name "AssumePROD1RolePolicy" \ -# --role-name "crossAccAMPInfoFromPROD1Role" +aws grafana delete-workspace-api-key --profile monitoring-account --region ${COA_MON_REGION} \ + --key-name "grafana-operator-key" + --workspace-id $COA_AMG_WORKSPACE_ID -# aws iam delete-role --profile monitoring-account \ -# --role-name "crossAccAMPInfoFromPROD1Role" +aws ssm delete-parameter --profile monitoring-account --region ${COA_MON_REGION} --name "/cdk-accelerator/amg-info" +aws ssm delete-parameter --profile pipeline-account --region ${COA_PIPELINE_REGION} --name "/cdk-accelerator/cdk-context" log 'G' "DONE!" \ No newline at end of file diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh index 8154c8c8..97ce7c17 100644 --- a/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh +++ b/scripts/multi-acc-new-eks-mixed-observability-pattern/gitsource-preconfig.sh @@ -42,7 +42,7 @@ if [ $existingParam -eq 0 ]; then "pipelineSource": { "gitOwner": "'${gitOwner}'", "gitRepoName": "'${gitRepoName}'", - "gitBranch": "'${gitBranch}' + "gitBranch": "'${gitBranch}'" } }' else diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh index 7ea597ab..566a8dce 100644 --- a/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh +++ b/scripts/multi-acc-new-eks-mixed-observability-pattern/post-deployment-source-envs.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -e # exit when any command fails - SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern source ${SCRIPT_PATH}/format-display.sh # format display diff --git a/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh b/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh index d1da8cd5..94e4f650 100644 --- a/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh +++ b/scripts/multi-acc-new-eks-mixed-observability-pattern/source-envs.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -e # exit when any command fails - SCRIPT_PATH=$(git rev-parse --show-toplevel)/scripts/multi-acc-new-eks-mixed-observability-pattern source ${SCRIPT_PATH}/format-display.sh # format display