From a922de2729079a9fa9282cce9fdc75da0d82fa8b Mon Sep 17 00:00:00 2001 From: "Stephen.James" Date: Wed, 11 Sep 2024 11:51:45 +0100 Subject: [PATCH] Added new scripts for the DB access Add connection files to gitignore we don't want to commit those. Created new Make targets for DB connections Make it really easy for an engineer to get the information for connecting to the RDS DB from the bastion when debugging issues. Update the documentation for RDS Bastion reflect new changes since AWS Secrets have been used and the explain to use the new make targets. ND-510 --- .gitignore | 2 + Makefile | 16 +++++++- documentation/rds-bastion.md | 41 ++++++++++++++------- scripts/create_db_connection_details.sh | 49 +++++++++++++++++++++++++ scripts/get_db_credentials.sh | 28 ++++++++++++++ scripts/get_db_parameters.sh | 20 ---------- 6 files changed, 120 insertions(+), 36 deletions(-) create mode 100755 scripts/create_db_connection_details.sh create mode 100755 scripts/get_db_credentials.sh delete mode 100755 scripts/get_db_parameters.sh diff --git a/.gitignore b/.gitignore index cc61db5..135a083 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ VpnCerts # ignore tfenv pinned version file .terraform-version + +/.db_connection* diff --git a/Makefile b/Makefile index c0540ad..ad49c26 100644 --- a/Makefile +++ b/Makefile @@ -120,8 +120,20 @@ output: ## terraform output (make output OUTPUT_ARGUMENT='--raw dns_dhcp_vpc_id' output-bastion-rds-admin: ## terraform output (make output-bastion-rds-admin) $(DOCKER_RUN) /bin/bash -c "terraform output -no-color -json rds_bastion | jq -r .admin[][]" -.PHONY: output-bastion-rds-server -output-bastion-rds-server: ## terraform output (make output-bastion-rds-server) +.PHONY: rds-admin +rds-admin: ## Get RDS admin connection details (make rds-admin) + $(DOCKER_RUN) /bin/bash -c "./scripts/create_db_connection_details.sh admin" + +.PHONY: rds-admin-password +rds-admin-password: ## Get RDS admin password (make rds-admin-password) + $(DOCKER_RUN) /bin/bash -c "./scripts/get_db_credentials.sh admin" + +.PHONY: instanceid-bastion-rds-admin +instanceid-bastion-rds-admin: ## Get RDS Admin bastion Instance ID (make instanceid-bastion-rds-admin) + $(DOCKER_RUN) /bin/bash -c "terraform output -no-color -json rds_bastion | jq -r .admin[][]" + +.PHONY: instanceid-bastion-rds-server +instanceid-bastion-rds-server: ## Get RDS server bastion Instance ID (make instanceid-bastion-rds-server) $(DOCKER_RUN) /bin/bash -c "terraform output -no-color -json rds_bastion | jq -r .server[][]" .PHONY: apply diff --git a/documentation/rds-bastion.md b/documentation/rds-bastion.md index c24d0ea..8f3b33b 100644 --- a/documentation/rds-bastion.md +++ b/documentation/rds-bastion.md @@ -30,8 +30,21 @@ run the pipeline ### Get environment details for the target env +We will need to query the Terraform state for the environment we need to run the init command, which will get then necessary env vars and terraform providers and modules. +For development we do need to add an ENV_ARGUMENT + +``` +make clean +make init +make init +``` + +For pre-production and production we do add the ENV_ARGUMENT as shown below. + ``` -make gen-env ENV_ARGUMENT=production +make clean +make init ENV_ARGUMENT=production +make init ENV_ARGUMENT=production ``` ### run the script to identify the bastion instance id @@ -46,6 +59,12 @@ Then identify the running bastion host i-019174128cf7b4563| t3a.small | None | running | mojo-production-rds-admin-bastion ``` +Alternatively there is another make target that will return the bastion's instance_id if it exists. + +```shell +make instanceid-bastion-rds-admin +``` + ### Start session on bastion Run make command with instance id @@ -94,18 +113,6 @@ make shell the issue a terraform command to get the database details -Admin (dhcp & dns) - -```shell -terraform output -json terraform_outputs | jq '.admin.db' -``` - -DHCP - -```shell -terraform output -json terraform_outputs | jq '.dhcp.db' -``` - Admin (NAC)\* note: NAC code used `rds` as module name. ```shell @@ -115,7 +122,7 @@ terraform output -json terraform_outputs | jq '.admin.rds' To get the password run ```shell -./scripts/get_db_parameters.sh +make rds-admin-password ``` ## DHCP Database Backup and Restore @@ -126,6 +133,12 @@ In order to connect to the database the following items will be needed. - username e.g. `"username": "adminuser"` - password +Connection strings for testing connectivity and accessing the DBs are described below, however you can obtain ready baked dynamically created versions by running: + +```shell +make rds-admin +``` + ### Test connection ```shell diff --git a/scripts/create_db_connection_details.sh b/scripts/create_db_connection_details.sh new file mode 100755 index 0000000..480407b --- /dev/null +++ b/scripts/create_db_connection_details.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +db_type=${1} +file_name=".db_connection.${ENV}.${db_type}" +terraform_outputs=$(terraform output -json) + +if [ ${db_type} == "admin" ]; then + ## Admin RDS + admin_db_username=admin + admin_db_fqdn=$(echo ${terraform_outputs} | jq -r '.terraform_outputs.value.admin.rds.fqdn') + admin_db_port=$(echo ${terraform_outputs} | jq -r '.terraform_outputs.value.admin.rds.port') + admin_db_name=$(echo ${terraform_outputs} | jq -r '.terraform_outputs.value.admin.rds.name') + +cat << EOF > ./${file_name} +Connections strings for ${ENV} environment RDS + +NAC Admin RDS: +Test connection: +Copy command below to test RDS DB access from Admin RDS Bastion. +---- +curl -v telnet://${admin_db_fqdn}:${admin_db_port} --output rds.admin.txt + + + +Connect to DB with MySQL client: +Copy command below to test RDS DB access from Admin RDS Bastion. +----- +mysql --user=${admin_db_username} --host=${admin_db_fqdn} --port=${admin_db_port} --ssl --password + + +Create DB dump and push to S3 +-------- +filename="\`date "+%Y_%m_%d-%H_%M_%S"\`_${ENV}_${admin_db_name}_rds-dump.sql"; \\ +mysqldump \\ + -u "${admin_db_username}" \\ + -p \\ + --ssl \\ + --set-gtid-purged=OFF \\ + --triggers --routines --events \\ + -h "${admin_db_fqdn}" \\ + "${admin_db_name}" > ~/${filename}; \\ + ls -al; \\ +aws s3 cp ~/\${filename} s3://mojo-file-transfer/ --profile s3-role; \\ +aws s3 ls s3://mojo-file-transfer/ --profile s3-role; + +EOF +fi + +cat ./${file_name} diff --git a/scripts/get_db_credentials.sh b/scripts/get_db_credentials.sh new file mode 100755 index 0000000..c34b2be --- /dev/null +++ b/scripts/get_db_credentials.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +aws_secretsmanager_get_secret_value() { + db_type=${1} + + if [ ${db_type} == "admin" ]; then + aws secretsmanager get-secret-value \ + --secret-id /moj-network-access-control/${ENV}/admin/db | jq --raw-output '.SecretString' | jq -r .password + aws secretsmanager get-secret-value \ + --secret-id /moj-network-access-control/${ENV}/admin/db | jq --raw-output '.SecretString' | jq -r .username + fi +} + +assume_role_in_environment() { + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn "${TF_VAR_assume_role}" \ + --role-session-name MySessionName \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --output text)) +} + +main() { + assume_role_in_environment + aws_secretsmanager_get_secret_value "${1}" +} + +main "${1}" diff --git a/scripts/get_db_parameters.sh b/scripts/get_db_parameters.sh deleted file mode 100755 index db7f486..0000000 --- a/scripts/get_db_parameters.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -export PARAM=$(aws ssm get-parameters --region eu-west-2 --with-decryption --names \ - "/moj-network-access-control/$ENV/admin_db_username" \ - "/moj-network-access-control/$ENV/admin_db_password" \ - --query Parameters) - -echo $ENV -echo $PARAM - -declare -A params - -params["admin_db_password"]="$(echo $PARAM | jq '.[] | select(.Name | test("admin_db_password")) | .Value' --raw-output)" -params["admin_db_username"]="$(echo $PARAM | jq '.[] | select(.Name | test("admin_db_username")) | .Value' --raw-output)" - - -for key in "${!params[@]}" -do - echo "${key}=${params[${key}]}" -done