diff --git a/.github/workflows/deploy-on-main.yml b/.github/workflows/deploy-on-main.yml index f3bbfca372..6558b57d5b 100644 --- a/.github/workflows/deploy-on-main.yml +++ b/.github/workflows/deploy-on-main.yml @@ -1,3 +1,5 @@ +name: Deploy Main + on: push: tags: diff --git a/.github/workflows/deploy-stack.yml b/.github/workflows/deploy-stack.yml index a618069abc..4b7211fab7 100644 --- a/.github/workflows/deploy-stack.yml +++ b/.github/workflows/deploy-stack.yml @@ -46,29 +46,15 @@ jobs: environment: ${{ inputs.environment }} service: ${{ matrix.service }} version: ${{ inputs.version }} - needs: - - invoke-and-check-cdn secrets: inherit set-maintenance-mode: - uses: ./.github/workflows/set-maintenance-mode.yml - with: - environment: ${{ inputs.environment }} needs: - invoke-and-check-cdn - secrets: inherit - - down: # the backend services - needs: - - set-maintenance-mode - strategy: - fail-fast: false - matrix: - service: [ "pinga", "rebaser", "sdf" ] - uses: ./.github/workflows/down-service.yml + - set-service-versions + uses: ./.github/workflows/set-maintenance-mode.yml with: environment: ${{ inputs.environment }} - service: ${{ matrix.service }} secrets: inherit upgrade-web: @@ -82,33 +68,20 @@ jobs: upgrade: # the backend services needs: - set-service-versions - - down + - set-maintenance-mode strategy: fail-fast: false matrix: - service: [ "pinga", "rebaser", "sdf", "veritech" ] + service: [ "pinga", "rebaser", "veritech" ] uses: ./.github/workflows/upgrade-service.yml with: environment: ${{ inputs.environment }} service: ${{ matrix.service }} secrets: inherit - up: # the backend services + upgrade-and-migrate-sdf: needs: - upgrade - strategy: - fail-fast: false - matrix: - service: [ "pinga", "rebaser", "veritech" ] - uses: ./.github/workflows/up-service.yml - with: - environment: ${{ inputs.environment }} - service: ${{ matrix.service }} - secrets: inherit - - migrate-and-up-sdf: - needs: - - up uses: ./.github/workflows/migrate-sdf.yml with: environment: ${{ inputs.environment }} @@ -122,8 +95,7 @@ jobs: cancel-in-progress: true needs: - upgrade-web - - migrate-and-up-sdf - if: always() + - upgrade-and-migrate-sdf uses: ./.github/workflows/e2e-validation.yml with: environment: ${{ inputs.environment }} diff --git a/.github/workflows/migrate-sdf.yml b/.github/workflows/migrate-sdf.yml index 66a7a2f4c9..b033a992b7 100644 --- a/.github/workflows/migrate-sdf.yml +++ b/.github/workflows/migrate-sdf.yml @@ -24,7 +24,14 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Migrate and up service + - name: Upgrade service + run: | + component/toolbox/awsi.sh upgrade -p pull-from-env -r us-east-1 -a y -s sdf -e ${{ inputs.environment }} + + - name: Migrate service run: | component/toolbox/awsi.sh migrate -p pull-from-env -r us-east-1 -a y -s sdf + + - name: Up service + run: | component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s sdf -S up diff --git a/.github/workflows/set-maintenance-mode.yml b/.github/workflows/set-maintenance-mode.yml index ca24ec952b..0b0332946f 100644 --- a/.github/workflows/set-maintenance-mode.yml +++ b/.github/workflows/set-maintenance-mode.yml @@ -24,6 +24,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Down service + - name: Toggle maintenance run: | - component/toolbox/awsi.sh toggle-maintenance -p pull-from-env -r us-east-1 -s sdf -m n -a y + component/toolbox/awsi.sh toggle-maintenance -p pull-from-env -r us-east-1 -s sdf -m y -a y diff --git a/.github/workflows/upgrade-service.yml b/.github/workflows/upgrade-service.yml index a8243ad8cd..8e4ec7d631 100644 --- a/.github/workflows/upgrade-service.yml +++ b/.github/workflows/upgrade-service.yml @@ -17,6 +17,7 @@ jobs: environment: ${{ inputs.environment }} name: ${{ inputs.service }} Upgrade runs-on: ubuntu-latest + continue-on-error: true steps: - name: Configure AWS credentials for ${{ inputs.environment }} uses: aws-actions/configure-aws-credentials@v4 @@ -28,10 +29,18 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Down service + run: | + component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -S down + - name: Upgrade service run: | component/toolbox/awsi.sh upgrade -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -e ${{ inputs.environment }} + - name: Up service + run: | + component/toolbox/awsi.sh service-state -p pull-from-env -r us-east-1 -a y -s ${{ inputs.service }} -S up + on-failure: uses: ./.github/workflows/instance-refresh.yml needs: upgrade diff --git a/.github/workflows/upgrade-web.yml b/.github/workflows/upgrade-web.yml index 0238e7a8b0..b4acb5f4fd 100644 --- a/.github/workflows/upgrade-web.yml +++ b/.github/workflows/upgrade-web.yml @@ -23,11 +23,12 @@ jobs: - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: '18.18.2' - - - uses: pnpm/action-setup@v4 + cache: 'pnpm' - name: Install dependencies working-directory: app/web diff --git a/component/toolbox/scripts/migrate b/component/toolbox/scripts/migrate index ee1c94d613..7c3cf13b29 100755 --- a/component/toolbox/scripts/migrate +++ b/component/toolbox/scripts/migrate @@ -107,21 +107,20 @@ stop_results_file=stop_results.json upgrade_results_file=upgrade_results.json mkdir -p "$results_directory/" -# get the first SDF and go do the thing +# attempt on each SDF until one succeeds or they all fail while read -r line; do instance_id=$(echo "$line" | awk '{print $2}') start_and_track_ssm_session "$instance_id" "$sdf_migrate_script" "$results_directory" "InstanceId=$instance_id" - break -done <<< "$instances" -await_file_results "$results_directory" 1 -concat_and_output_json "$results_directory" "$check_results_file" + await_file_results "$results_directory" 1 + concat_and_output_json "$results_directory" "$check_results_file" -if jq -e 'all(.[]; .status == "success")' "$results_directory/$check_results_file" > /dev/null; then - echo "SDF database has been migrated" - echo "----------------------------------------" - exit 0 -else - echo "Error: One or more of the checks failed to push a node into maintenance mode, try again later or look at the logs" - exit 2 -fi + if jq -e 'all(.[]; .status == "success")' "$results_directory/$check_results_file" > /dev/null; then + echo "SDF database has been migrated" + echo "----------------------------------------" + exit 0 + fi +done <<< "$instances" + +echo "Error: One or more of the checks failed to push a node into maintenance mode, try again later or look at the logs" +exit 2 diff --git a/component/toolbox/scripts/ssm-scripts/si-service-maintenance b/component/toolbox/scripts/ssm-scripts/si-service-maintenance index b900a1b63d..5346f248fc 100644 --- a/component/toolbox/scripts/ssm-scripts/si-service-maintenance +++ b/component/toolbox/scripts/ssm-scripts/si-service-maintenance @@ -22,17 +22,17 @@ mainSteps: - | flip_or_stick() { requested_state=$1 - response_code=$(curl -s -o /dev/null -w "%{http_code}" localhost:5156/api/) - [[ "$response_code" == "200" ]] && [[ "$requested_state" == "y" ]] && killall -s USR2 {{ Service }} + response_code=$(curl -s -o /dev/null -w "%{http_code}" localhost:5156/) + [[ "$response_code" == "404" ]] && [[ "$requested_state" == "y" ]] && killall -s USR2 {{ Service }} [[ "$response_code" == "503" ]] && [[ "$requested_state" == "n" ]] && killall -s USR2 {{ Service }} } check_state() { requested_state=$1 - response_code=$(curl -s -o /dev/null -w "%{http_code}" localhost:5156/api/) + response_code=$(curl -s -o /dev/null -w "%{http_code}" localhost:5156/) if [[ "$response_code" == "503" ]] && [[ "$requested_state" == "y" ]]; then echo "{\"instance_id\": \"{{ InstanceId }}\", \"status\": \"success\", \"service\": \"{{ Service }}\", \"mode\": \"maintenance\" }" - elif [[ "$response_code" == "200" ]] && [[ "$requested_state" == "n" ]]; then + elif [[ "$response_code" == "404" ]] && [[ "$requested_state" == "n" ]]; then echo "{\"instance_id\": \"{{ InstanceId }}\", \"status\": \"success\", \"service\": \"{{ Service }}\", \"mode\": \"running\" }" else echo "{\"instance_id\": \"{{ InstanceId }}\", \"status\": \"failure\", \"service\": \"{{ Service }}\", \"mode\": \"Status code from API not valid for requested action. Response Code: $response_code, Maintenance Mode Requested: {{ Action }}\" }" diff --git a/component/toolbox/scripts/toggle-maintenance b/component/toolbox/scripts/toggle-maintenance index 0dbbf4146f..ac3217a18a 100755 --- a/component/toolbox/scripts/toggle-maintenance +++ b/component/toolbox/scripts/toggle-maintenance @@ -125,8 +125,7 @@ i=1 while read -r line; do instance_id=$(echo "$line" | awk '{print $2}') service_name=$(echo "$line" | awk '{print $1}' | awk -F- '{print $2}') - start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$service_name" "$maintenance" "$results_directory" & - start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$results_directory" "InstanceId=$instance_id,Action=$maintenance" & + start_and_track_ssm_session "$instance_id" "$service_maintenance_script" "$results_directory" "InstanceId=$instance_id,Service=$service_name,Action=$maintenance" & ((i++)) done <<< "$instances"