fix: re-organize deployment pipeline for better failure handling

.github/workflows/deploy-stack.yml

@@ -58,19 +58,6 @@ jobs:
       - invoke-and-check-cdn
     secrets: inherit
 
-  down: # the backend services
-    needs:
-      - set-maintenance-mode
-    strategy:
-     fail-fast: false
-     matrix:
-      service: [ "pinga", "rebaser", "sdf" ]
-    uses: ./.github/workflows/down-service.yml
-    with:
-      environment: ${{ inputs.environment }}
-      service: ${{ matrix.service }}
-    secrets: inherit
-
   upgrade-web:
     needs:
       - set-maintenance-mode
@@ -82,33 +69,20 @@ jobs:
   upgrade: # the backend services
     needs:
       - set-service-versions
-      - down
+      - set-maintenance-mode
     strategy:
      fail-fast: false
      matrix:
-       service: [ "pinga", "rebaser", "sdf", "veritech" ]
+       service: [ "pinga", "rebaser", "veritech" ]
     uses: ./.github/workflows/upgrade-service.yml
     with:
       environment: ${{ inputs.environment }}
       service: ${{ matrix.service }}
     secrets: inherit
 
-  up: # the backend services
+  upgrade-and-migrate-sdf:
     needs:
       - upgrade
-    strategy:
-     fail-fast: false
-     matrix:
-      service: [ "pinga", "rebaser", "veritech" ]
-    uses: ./.github/workflows/up-service.yml
-    with:
-      environment: ${{ inputs.environment }}
-      service: ${{ matrix.service }}
-    secrets: inherit
-
-  migrate-and-up-sdf:
-    needs:
-      - up
     uses: ./.github/workflows/migrate-sdf.yml
     with:
       environment: ${{ inputs.environment }}
@@ -122,7 +96,7 @@ jobs:
       cancel-in-progress: true
     needs:
       - upgrade-web
-      - migrate-and-up-sdf
+      - upgrade-and-migrate-sdf
     if: always()
     uses: ./.github/workflows/e2e-validation.yml
     with:

.github/workflows/migrate-sdf.yml

@@ -24,7 +24,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Migrate and up service
+      - name: Upgrade service
+        run: |
+          component/toolbox/awsi.sh upgrade -p pull-from-env -r us-east-1 -a y -s sdf -e ${{ inputs.environment }}
+
+      - name: Migrate service
         run: |
           component/toolbox/awsi.sh migrate -p pull-from-env -r us-east-1 -a y -s sdf
+
+      - name: Up service
+        run: |
           component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s sdf -S up

.github/workflows/upgrade-service.yml

@@ -17,6 +17,7 @@ jobs:
     environment: ${{ inputs.environment }}
     name: ${{ inputs.service }} Upgrade
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - name: Configure AWS credentials for ${{ inputs.environment }}
         uses: aws-actions/configure-aws-credentials@v4
@@ -28,10 +29,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Down service
+        run: |
+          component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -S down
+
       - name: Upgrade service
         run: |
           component/toolbox/awsi.sh upgrade -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -e ${{ inputs.environment }}
 
+      - name: Up service
+        run: |
+          component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -S up
+
   on-failure:
     uses: ./.github/workflows/instance-refresh.yml
     needs: upgrade

component/toolbox/scripts/migrate

@@ -107,21 +107,20 @@ stop_results_file=stop_results.json
 upgrade_results_file=upgrade_results.json
 mkdir -p "$results_directory/"
 
-# get the first SDF and go do the thing
+# attempt on each SDF until one succeeds or they all fail
 while read -r line; do
     instance_id=$(echo "$line" | awk '{print $2}')
     start_and_track_ssm_session "$instance_id" "$sdf_migrate_script" "$results_directory" "InstanceId=$instance_id"
-    break
-done <<< "$instances"
 
-await_file_results "$results_directory" 1
-concat_and_output_json "$results_directory" "$check_results_file"
+    await_file_results "$results_directory" 1
+    concat_and_output_json "$results_directory" "$check_results_file"
 
-if jq -e 'all(.[]; .status == "success")' "$results_directory/$check_results_file" > /dev/null; then
-  echo "SDF database has been migrated"
-  echo "----------------------------------------"
-  exit 0
-else
-  echo "Error: One or more of the checks failed to push a node into maintenance mode, try again later or look at the logs"
-  exit 2
-fi
+    if jq -e 'all(.[]; .status == "success")' "$results_directory/$check_results_file" > /dev/null; then
+      echo "SDF database has been migrated"
+      echo "----------------------------------------"
+      exit 0
+    fi
+done <<< "$instances"
+
+echo "Error: One or more of the checks failed to push a node into maintenance mode, try again later or look at the logs"
+exit 2