From c810aec0f6187f508369b0ea25112a76a45e0c43 Mon Sep 17 00:00:00 2001 From: Rishi Kumar <96657880+k3yss@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:01:59 +0100 Subject: [PATCH] fix: Migration scripts and guide (#229) * feat: Add functionality for OUTPUT_PREFIX * chore: Introduce output name * chore: Make migration_sql_commands clear * fix: Destination db version to 16 * fix: postgres-perms script * chore: Add sanity check to postgres-perms-update.sh * docs: Mention warning * chore: remove name string --- docs/pg-migration-guide/README.md | 23 ++++--- modules/postgresql/gcp/bin/create-dms.sh | 9 ++- .../gcp/bin/postgres-perms-update.sh | 60 ++++++++++++++++--- modules/postgresql/gcp/outputs.tf | 4 +- modules/postgresql/gcp/variables.tf | 2 +- 5 files changed, 77 insertions(+), 21 deletions(-) diff --git a/docs/pg-migration-guide/README.md b/docs/pg-migration-guide/README.md index 7c832a68..593acf61 100644 --- a/docs/pg-migration-guide/README.md +++ b/docs/pg-migration-guide/README.md @@ -77,7 +77,9 @@ $ tofu apply Before proceeding with the DMS creation we will expose the required things by gcloud using the `output` block, add these output blocks to your main tofu file. ```sh # run the create-dms.sh script located in modules/postgresql/gcp/bin -$ ./create-dms.sh +# to be used for output automation +# this is the module name of the current project we are performing migration +$ ./create-dms.sh Enter the region: us-east1 Enter the job name: test-migration Creating migration job 'test-migration' in region 'us-east1'... @@ -134,9 +136,14 @@ $ gcloud database-migration migration-jobs start "test-migration" --region="us-e ```sh $ gcloud database-migration migration-jobs start "test-migration" --region="us-east1" -# Use the describe command to check the status of the migration-job +# Use the describe command to check the status of the migration-job !IMPORTANT $ gcloud database-migration migration-jobs describe "test-job" --region=us-east1 + +## NOTE: + +BEFORE PROMOTING VERIFY THAT THE DMS HAS BEEN SUCCESSFUL BY RUNNING THE `describe` command from earlier. + ``` # Step 3: Pre-promotion @@ -148,11 +155,13 @@ $ gcloud database-migration migration-jobs describe "test-job" --region=us-east1 ### Step 3.5: Handing the non-migrated settings and syncing state via `tofu` #### Step 3.5.1 -Log in to the `destination instance` as the `postgres` user and change the name of the `cloudsqlexternalsync` user to the ``. -The value of `` and `destination-connection-string` can be found by running +- Log in to the `destination instance` as the `postgres` user. +- Change the name of the `cloudsqlexternalsync` user to the ``. +- The commands to do this can be found by running the following command: ```sh -$ tf output -json migration_sql_command | jq -r '.sql_command' | bash +# get the value here +$ tf output -json migration_sql_command ``` #### Step 3.5.2 @@ -178,7 +187,7 @@ module "postgresql" { #source = "git::https://github.com/GaloyMoney/galoy-infra.git//modules/postgresql/gcp?ref=689daa7" source = "../../../modules/postgresql/gcp" - instance_name = "rishi-pg" + instance_name = "test-pg" vpc_name = "${var.name_prefix}-vpc" gcp_project = var.gcp_project destroyable = var.destroyable_postgres @@ -238,7 +247,7 @@ module "postgresql" { #source = "git::https://github.com/GaloyMoney/galoy-infra.git//modules/postgresql/gcp?ref=689daa7" source = "../../../modules/postgresql/gcp" - instance_name = "rishi-pg" + instance_name = "test-pg" vpc_name = "${var.name_prefix}-vpc" gcp_project = var.gcp_project destroyable = var.destroyable_postgres diff --git a/modules/postgresql/gcp/bin/create-dms.sh b/modules/postgresql/gcp/bin/create-dms.sh index 5c2a41ae..71e1a071 100755 --- a/modules/postgresql/gcp/bin/create-dms.sh +++ b/modules/postgresql/gcp/bin/create-dms.sh @@ -9,6 +9,9 @@ PROJECT=${2} REGION=${3} # the migration job name JOB_NAME=${4} +# tofu output prefix to be used for output automation +# this is the module name of the current project we are performing migration +OUTPUT_PREFIX=${5} TYPE="CONTINUOUS" @@ -43,9 +46,9 @@ else cmd="terraform" fi # Get Terraform outputs -SOURCE_ID=$($cmd output -raw source_connection_profile_id) -DEST_ID=$($cmd output -raw destination_connection_profile_id) -VPC=$($cmd output -raw vpc) +SOURCE_ID=$($cmd output -raw "${OUTPUT_PREFIX}source_connection_profile_id") +DEST_ID=$($cmd output -raw "${OUTPUT_PREFIX}destination_connection_profile_id") +VPC=$($cmd output -raw "${OUTPUT_PREFIX}vpc") # Construct and run the gcloud command to create the migration job echo "Creating migration job '$JOB_NAME' in region '$REGION'..." diff --git a/modules/postgresql/gcp/bin/postgres-perms-update.sh b/modules/postgresql/gcp/bin/postgres-perms-update.sh index 94cb7619..3d90741c 100755 --- a/modules/postgresql/gcp/bin/postgres-perms-update.sh +++ b/modules/postgresql/gcp/bin/postgres-perms-update.sh @@ -1,22 +1,63 @@ #!/usr/bin/env bash set -e -dir=${1} -DB_NAME=${2} +# Function to validate input parameters +validate_inputs() { + if [ -z "$1" ] || [ -z "$2" ]; then + echo "Usage: $0 " + echo "Error: Both directory and database name are required" + exit 1 + fi + + if [ ! -d "$1" ]; then + echo "Error: Directory '$1' does not exist" + exit 1 + fi + + if [ ! -f "$1/pg_connection.txt" ]; then + echo "Error: pg_connection.txt not found in $1" + exit 1 + fi +} -pushd ${dir} +# Function to test database connection +test_connection() { + local connection=$1 + local db_name=$2 + + if ! psql "$connection/$db_name" -c '\q' >/dev/null 2>&1; then + echo "Error: Could not connect to database $db_name" + exit 1 + fi +} + +validate_inputs "$1" "$2" + +dir=$1 +DB_NAME=$2 +pushd "${dir}" || exit 1 NEW_OWNER=${DB_NAME}-user # READ PG_CON from a file PG_CON=$(cat pg_connection.txt) -PSQL_CMD="psql $PG_CON -At -c" +# Test connections before proceeding +test_connection "$PG_CON" "postgres" +test_connection "$PG_CON" "$DB_NAME" -$PSQL_CMD "ALTER DATABASE postgres OWNER TO cloudsqlsuperuser;" -$PSQL_CMD "ALTER SCHEMA public OWNER TO cloudsqlsuperuser;" +# Command for database owner change needs to connect to postgres database +PSQL_CMD_POSTGRES="psql $PG_CON/postgres -At -c" +# Command for all other operations needs to connect to target database +PSQL_CMD="psql $PG_CON/$DB_NAME -At -c" +echo "Starting ownership transfer process..." + +# Perform ownership changes +$PSQL_CMD_POSTGRES "ALTER DATABASE postgres OWNER TO cloudsqlsuperuser;" +$PSQL_CMD "ALTER SCHEMA public OWNER TO cloudsqlsuperuser;" $PSQL_CMD "GRANT \"$NEW_OWNER\" TO \"postgres\";" -# Get list of all tables in the database + +# Get and process tables tables=$($PSQL_CMD "SELECT tablename FROM pg_tables WHERE schemaname = 'public';") # Loop through each table and change the owner @@ -32,6 +73,7 @@ for sequence in $sequences; do $PSQL_CMD "ALTER SEQUENCE public.\"$sequence\" OWNER TO \"$NEW_OWNER\";" done -echo "Ownership of all tables in $DB_NAME has been granted to $NEW_OWNER." +echo "Ownership transfer process completed for $DB_NAME" +echo "Please review any warnings above" -popd +popd || exit 1 \ No newline at end of file diff --git a/modules/postgresql/gcp/outputs.tf b/modules/postgresql/gcp/outputs.tf index 3ca0f21e..45d03b00 100644 --- a/modules/postgresql/gcp/outputs.tf +++ b/modules/postgresql/gcp/outputs.tf @@ -61,6 +61,8 @@ output "source_instance" { } output "migration_sql_command" { value = local.prep_upgrade_as_source_db ? { - sql_command = "psql postgres://postgres:${module.migration[0].postgres_user_password}@${module.migration[0].destination_instance_private_ip_address}:5432/postgres -c \"ALTER ROLE cloudsqlexternalsync RENAME TO \\\"${google_sql_user.admin.name}\\\"; ALTER ROLE \\\"${google_sql_user.admin.name}\\\" PASSWORD '${random_password.admin.result}';\"" + destination_connection_string = "psql postgres://postgres:${module.migration[0].postgres_user_password}@${module.migration[0].destination_instance_private_ip_address}:5432/postgres" + alter_admin_name = "ALTER ROLE cloudsqlexternalsync RENAME TO ${google_sql_user.admin.name}" + alter_admin_password = "ALTER ROLE ${google_sql_user.admin.name} PASSWORD ${random_password.admin.result}" } : {} } diff --git a/modules/postgresql/gcp/variables.tf b/modules/postgresql/gcp/variables.tf index 5d0fe5d2..1cf9c9a2 100644 --- a/modules/postgresql/gcp/variables.tf +++ b/modules/postgresql/gcp/variables.tf @@ -26,7 +26,7 @@ variable "database_version" { default = "POSTGRES_14" } variable "destination_database_version" { - default = "POSTGRES_15" + default = "POSTGRES_16" } variable "big_query_viewers" { default = []