VOTE-2874 downsync workflow improvements (#1027)

usagov · Oct 4, 2024 · 0a314a0 · 0a314a0
1 parent b9ff9d0
commit 0a314a0
Show file tree

Hide file tree

Showing 5 changed files with 232 additions and 58 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,7 +6,7 @@ parameters:
   restore:
     type: string
     default: "none"
-  scheduled_backup:
+  scheduled-backup:
     type: boolean
     default: false
   upkeep:
@@ -15,7 +15,7 @@ parameters:
 jobs:
   upkeep:
     docker:
-      - image: cimg/base:stable
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - run:
@@ -88,7 +88,7 @@ jobs:
           path: testing/cypress/screenshots
   deploy:
     docker:
-      - image: cimg/base:stable
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - attach_workspace:
@@ -106,7 +106,7 @@ jobs:
             source ./scripts/pipeline/cloud-gov-deploy.sh
   post-deploy:
     docker:
-      - image: cimg/base:stable
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - attach_workspace:
@@ -124,7 +124,7 @@ jobs:
             source ./scripts/pipeline/cloud-gov-post-deploy.sh
   post-deploy-upkeep:
     docker:
-      - image: cimg/base:stable
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - attach_workspace:
@@ -141,12 +141,9 @@ jobs:
             source ./scripts/pipeline/exports.sh ${CIRCLE_BRANCH}
             source ./scripts/pipeline/cloud-gov-login.sh
             source ./scripts/pipeline/cloud-gov-post-deploy-upkeep.sh
-  downsync:
+  downsync-backup:
     docker:
-      - image: cimg/base:stable
-    environment:
-      BACKUP_ENV: << pipeline.parameters.backup>>
-      RESTORE_ENV: << pipeline.parameters.restore>>
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - run:
@@ -157,19 +154,38 @@ jobs:
             source ./scripts/pipeline/deb-mysql-client-install.sh
             source ./scripts/pipeline/deb-awscli.sh
 
-            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.3/cf-service-connect_linux_amd64
+            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.4/cf-service-connect_linux_amd64
       - run:
-          name: "Run downsync"
+          name: "Run downsync-backup"
           no_output_timeout: 30m
           command: |
-            source ./scripts/pipeline/exports.sh "${BACKUP_ENV}"
+            source ./scripts/pipeline/exports.sh "prod"
             source ./scripts/pipeline/cloud-gov-login.sh
-            source ./scripts/pipeline/downsync-backup.sh
+            source ./scripts/pipeline/prod-db-backup.sh
             cf logout
+  downsync-restore:
+    docker:
+      - image: cimg/base:current-22.04
+    environment:
+      RESTORE_ENV: << pipeline.parameters.restore>>
+    steps:
+      - checkout
+      - run:
+          name: "Install cloudfoundry & dependencies"
+          command: |
+            source ./scripts/pipeline/deb-basic-deps.sh
+            source ./scripts/pipeline/deb-cf-install.sh
+            source ./scripts/pipeline/deb-mysql-client-install.sh
+            source ./scripts/pipeline/deb-awscli.sh
 
+            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.4/cf-service-connect_linux_amd64
+      - run:
+          name: "Run downsync"
+          no_output_timeout: 30m
+          command: |
             source ./scripts/pipeline/exports.sh "${RESTORE_ENV}"
             source ./scripts/pipeline/cloud-gov-login.sh
-            source ./scripts/pipeline/downsync-restore.sh
+            source ./scripts/pipeline/downsync-preprod.sh
   php-lint:
     docker:
       - image: php:8.3-alpine
@@ -199,7 +215,7 @@ jobs:
           destination: phpcs_<<pipeline.number>>.csv
   scheduled-backup:
     docker:
-      - image: cimg/base:stable
+      - image: cimg/base:current-22.04
     steps:
       - checkout
       - run:
@@ -210,16 +226,16 @@ jobs:
             source ./scripts/pipeline/deb-mysql-client-install.sh
             source ./scripts/pipeline/deb-awscli.sh
 
-            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.3/cf-service-connect_linux_amd64
+            cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.4/cf-service-connect_linux_amd64
       - run:
           name: "Run backup"
           no_output_timeout: 30m
           command: |
             export BACKUP_ENV=${CIRCLE_BRANCH}
             source ./scripts/pipeline/exports.sh "${CIRCLE_BRANCH}"
             source ./scripts/pipeline/cloud-gov-login.sh
-            source ./scripts/pipeline/downsync-backup.sh
-            ./scripts/pipeline/scheduled_backup.sh "${prod_cf_space}"
+            source ./scripts/pipeline/app-backup.sh
+            ./scripts/pipeline/scheduled-backup.sh "${prod_cf_space}"
       - run:
           name: "Run backup"
           no_output_timeout: 30m
@@ -228,8 +244,8 @@ jobs:
             export BACKUP_ENV=${CIRCLE_BRANCH}
             source ./scripts/pipeline/exports.sh "${CIRCLE_BRANCH}"
             source ./scripts/pipeline/cloud-gov-login.sh
-            source ./scripts/pipeline/downsync-backup.sh
-            ./scripts/pipeline/scheduled_backup.sh "${prod_cf_space}"
+            source ./scripts/pipeline/app-backup.sh
+            ./scripts/pipeline/scheduled-backup.sh "${prod_cf_space}"
           when: on_fail
   link-validation:
     docker:
@@ -383,22 +399,24 @@ workflows:
     when:
       and:
         - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
-        - equal: [ true, << pipeline.parameters.scheduled_backup >> ]
+        - equal: [ true, << pipeline.parameters.scheduled-backup >> ]
     jobs:
       - scheduled-backup
-  downsync-workflow:
+  downsync-backup:
     when:
       and:
         - equal:  [api, << pipeline.trigger_source >> ]
         - or:
-          - equal: [ dev, << pipeline.parameters.backup >> ]
           - equal: [ prod, << pipeline.parameters.backup >> ]
-          - equal:  [stage, << pipeline.parameters.backup >> ]
-          - equal: [ test, << pipeline.parameters.backup >> ]
+    jobs:
+      - downsync-backup
+  downsync-restore:
+    when:
+      and:
+        - equal:  [api, << pipeline.trigger_source >> ]
         - or:
           - equal: [ dev, << pipeline.parameters.restore >> ]
           - equal: [ stage, << pipeline.parameters.restore >> ]
           - equal: [ test, << pipeline.parameters.restore >> ]
     jobs:
-      - downsync
-
+      - downsync-restore
diff --git a/scripts/pipeline/downsync-backup.sh → scripts/pipeline/app-backup.sh b/scripts/pipeline/downsync-backup.sh → scripts/pipeline/app-backup.sh
diff --git a/scripts/pipeline/downsync-restore.sh → scripts/pipeline/downsync-preprod.sh b/scripts/pipeline/downsync-restore.sh → scripts/pipeline/downsync-preprod.sh
@@ -1,4 +1,8 @@
 #!/bin/bash
+if [ ${RESTORE_ENV} = 'prod' ]; then
+  echo "Restoring to prod is not allowed."
+  exit 1
+fi
 
 kill_pids() {
   app=$1
@@ -19,16 +23,48 @@ wait_for_tunnel() {
 
 date
 
+## Download latest prod backup.
+echo "Downloading latest prod database backup..."
+{
+  cf target -s "${project}-prod" >/dev/null 2>&1
+
+  export service="${project}-backup"
+  export service_key="${service}-key"
+  cf delete-service-key "${service}" "${service_key}" -f >/dev/null 2>&1
+  cf create-service-key "${service}" "${service_key}" >/dev/null 2>&1
+  sleep 2
+
+  export s3_credentials=$(cf service-key "${service}" "${service_key}" | tail -n +2)
+
+  export AWS_ACCESS_KEY_ID=$(echo "${s3_credentials}" | jq -r '.credentials.access_key_id')
+  export bucket=$(echo "${s3_credentials}" | jq -r '.credentials.bucket')
+  export AWS_DEFAULT_REGION=$(echo "${s3_credentials}" | jq -r '.credentials.region')
+  export AWS_SECRET_ACCESS_KEY=$(echo "${s3_credentials}" | jq -r '.credentials.secret_access_key')
+
+  # copy latest database from top level
+  aws s3 cp s3://${bucket}/prod/latest.sql.gz ./latest.sql.gz --no-verify-ssl >/dev/null 2>&1 && echo "Successfully copied latest.sql.gz from S3!" || echo "Failed to copy latest.sql.gz from S3!"
+  gunzip latest.sql.gz
+
+  cf delete-service-key "${service}" "${service_key}" -f >/dev/null 2>&1
+}
+
+date
+
 ## Create a tunnel through the application to restore the database.
 echo "Creating tunnel to database..."
+if [ ${RESTORE_ENV} = 'test' ]; then
+  cf target -s "${project}-dev" >/dev/null 2>&1
+else
+  cf target -s "${project}-${RESTORE_ENV}" >/dev/null 2>&1
+fi
 cf connect-to-service --no-client ${project}-drupal-${RESTORE_ENV} ${project}-mysql-${RESTORE_ENV} > restore.txt &
 
 wait_for_tunnel
 
 date
 
 ## Create variables and credential file for MySQL login.
-echo "Restoring '${BACKUP_ENV}' database to '${RESTORE_ENV}'..."
+echo "Restoring 'prod' database to '${RESTORE_ENV}'..."
 {
   host=$(cat restore.txt | grep -i host | awk '{print $2}')
   port=$(cat restore.txt | grep -i port | awk '{print $2}')
@@ -48,7 +84,7 @@ echo "Restoring '${BACKUP_ENV}' database to '${RESTORE_ENV}'..."
   --host=${host} \
   --port=${port} \
   --protocol=TCP \
-  --database=${dbname} < backup_${BACKUP_ENV}.sql
+  --database=${dbname} < latest.sql
 
 } >/dev/null 2>&1
 
@@ -61,7 +97,7 @@ echo "Cleaning up old connections..."
 } >/dev/null 2>&1
 
 ## Clean up.
-rm -rf restore.txt ~/.mysql backup_${BACKUP_ENV}.sql
+rm -rf restore.txt ~/.mysql latest.sql
 
 date
 
@@ -70,33 +106,6 @@ source $(pwd $(dirname $0))/scripts/pipeline/cloud-gov-remote-command.sh "${proj
 
 date
 
-# Upload media files.
-backup_media="cms/public/media"
-
-echo "Uploading media files..."
-{
-  cf target -s "${cf_space}"
-
-  service="${project}-storage-${RESTORE_ENV}"
-  service_key="${service}-key"
-  cf delete-service-key "${service}" "${service_key}" -f
-  cf create-service-key "${service}" "${service_key}"
-  sleep 2
-  s3_credentials=$(cf service-key "${service}" "${service_key}" | tail -n +2)
-
-  export AWS_ACCESS_KEY_ID=$(echo "${s3_credentials}" | jq -r '.credentials.access_key_id')
-  export bucket=$(echo "${s3_credentials}" | jq -r '.credentials.bucket')
-  export AWS_DEFAULT_REGION=$(echo "${s3_credentials}" | jq -r '.credentials.region')
-  export AWS_SECRET_ACCESS_KEY=$(echo "${s3_credentials}" | jq -r '.credentials.secret_access_key')
-
-  # Sync files to restore env, deleting those not found in backup env.
-  aws s3 sync --no-verify-ssl --delete ${backup_media}/ s3://${bucket}/${backup_media} 2>/dev/null
-
-  cf delete-service-key "${service}" "${service_key}" -f
-} >/dev/null 2>&1
-
-date
-
 echo "Running 'drush image-flush --all' on '${RESTORE_ENV}'..."
 source $(pwd $(dirname $0))/scripts/pipeline/cloud-gov-remote-command.sh "${project}-drupal-${RESTORE_ENV}" "drush image-flush --all"