create_destroy_test_vm.yaml

name: Create/Destroy Test VM

on:
  pull_request_target:
    types: [opened, reopened, synchronize, labeled, unlabeled, closed]

jobs:
##################################################################################
# The following job is for creating a new DO droplet, creating DNS records for it
# and copying the project files to it.
##################################################################################
  create_test_vm:
    if: |
      (
        (
          github.event.action == 'opened' ||
          github.event.action == 'reopened' ||
          github.event.action == 'synchronize'
        ) && contains(github.event.pull_request.labels.*.name, 'create-test-vm')
      ) ||
      (
        github.event.action == 'labeled' &&
        github.event.label.name == 'create-test-vm'
      )
    runs-on: ubuntu-latest
    steps:
      - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue_number: ${{ github.event.pull_request.number }}
          owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          message: |
            @${{ github.actor }} is creating a test VM for this PR 🚀🚀🚀
            This may take a few minutes so relax and grab a cup of coffee ☕
            We will notify you when the VM is ready.

      - name: Get checkout commit ref and repo
        run: |

          # for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250
          merge_commit_sha=${{ github.event.pull_request.merge_commit_sha }}
          head_ref=${{ github.event.pull_request.head.ref }}
          target_repo=${{ github.repository }}
          pr_repo=${{ github.event.pull_request.head.repo.full_name }}

          CHECKOUT_COMMIT_REF=$([ "$merge_commit_sha" != "" ] && \
          echo "$merge_commit_sha" || echo "$head_ref")
          CHECKOUT_REPO=$([ "$merge_commit_sha" != "" ] && \
          echo "$target_repo" || echo "$pr_repo")

          echo "CHECKOUT_COMMIT_REF=$CHECKOUT_COMMIT_REF" >> $GITHUB_ENV
          echo "CHECKOUT_REPO=$CHECKOUT_REPO" >> $GITHUB_ENV

# careful with this as you are checking out the source branch of the PR
# which is a potential security risk. Only do this when you know what you are doing
# See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques
# See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
# See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
      - uses: unstructuredstudio/zubhub/.github/actions/checkout@master
        with:
          ref: ${{ env.CHECKOUT_COMMIT_REF }}
          repository: ${{ env.CHECKOUT_REPO }}

      - name: Create new droplet and DNS records
        id: create_droplet
        uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
        with:
          token: ${{ secrets.DO_ACCESS_TOKEN }}
          script: |

            # This script will be executed inside a composite action.
            # rename GITHUB_OUTPUT so that it's clear that it's a
            # composite output and should be handled differently from $GITHUB_OUTPUT
            COMPOSITE_OUTPUT=$GITHUB_OUTPUT
            NEW_DROPLET_NAME=zubhub-test-${{ github.event.pull_request.number }}
            FRONTEND_DOMAIN=${NEW_DROPLET_NAME}
            API_DOMAIN=api.${FRONTEND_DOMAIN}
            MEDIA_DOMAIN=media.${FRONTEND_DOMAIN}
            echo "NEW_DROPLET_NAME=$NEW_DROPLET_NAME" >> $COMPOSITE_OUTPUT
            echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $COMPOSITE_OUTPUT
            echo "API_DOMAIN=$API_DOMAIN" >> $COMPOSITE_OUTPUT
            echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $COMPOSITE_OUTPUT
            # check if droplet already exists and exit script if it does
            NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \
              --format PublicIPv4 --no-header 2>/dev/null || true)
            if [[ -n "$NEW_DROPLET_IP" ]] ; then
              echo "Droplet already exists. Save droplet IP to env variable and exit..."
              echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT
              exit 0
            fi

            # create new droplet
            doctl compute droplet create $NEW_DROPLET_NAME --image \
              ${{ secrets.ZUBHUB_TEST_SNAPSHOT_ID }} --tag-name zubhub-test --size s-1vcpu-1gb \
              --region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait
            sleep 30s
            NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \
              --format PublicIPv4 --no-header)
            echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT

            # we only need records for frontend, media server and api server
            doctl compute domain records create unstructured.studio --record-type A --record-name \
              $FRONTEND_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600
            doctl compute domain records create unstructured.studio --record-type A --record-name \
              $API_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600
            doctl compute domain records create unstructured.studio --record-type A --record-name \
              $MEDIA_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600

      - uses: unstructuredstudio/zubhub/.github/actions/scp_action@master
        env:
          NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }}
        with:
          host: ${{env.NEW_DROPLET_IP}}
          username: ${{ vars.DO_BACKEND_USERNAME }}
          key: ${{ secrets.DO_SSHKEY }}
          source: "."
          target: "/home/zubhub"

      - name: Set output
        id: set_output
        env:
          NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }}
          FRONTEND_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).FRONTEND_DOMAIN }}
          API_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).API_DOMAIN }}
          MEDIA_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).MEDIA_DOMAIN }}
        run: |
          echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $GITHUB_OUTPUT
          echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $GITHUB_OUTPUT
          echo "API_DOMAIN=$API_DOMAIN" >> $GITHUB_OUTPUT
          echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $GITHUB_OUTPUT
          echo "CHECKOUT_COMMIT_REF=${{ env.CHECKOUT_COMMIT_REF }}" >> $GITHUB_OUTPUT
          echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT

    outputs:
      NEW_DROPLET_IP: ${{ steps.set_output.outputs.NEW_DROPLET_IP }}
      FRONTEND_DOMAIN: ${{ steps.set_output.outputs.FRONTEND_DOMAIN }}
      API_DOMAIN: ${{ steps.set_output.outputs.API_DOMAIN }}
      MEDIA_DOMAIN: ${{ steps.set_output.outputs.MEDIA_DOMAIN }}
      CHECKOUT_COMMIT_REF: ${{ steps.set_output.outputs.CHECKOUT_COMMIT_REF }}
      CHECKOUT_REPO: ${{ steps.set_output.outputs.CHECKOUT_REPO }}

#################################################################################

#################################################################################
# The following job is for building docker images and pushing them to dockerhub
##################################################################################
  build_and_push:
    needs: create_test_vm
    runs-on: ubuntu-latest
    strategy:
      matrix:
        service: ['frontend', 'web', 'celery', 'media']
    steps:
      - name: Use output
        id: use_output
        run: |
          echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV
          echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV
          echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV
          echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV
          echo "CHECKOUT_COMMIT_REF=${{ needs.create_test_vm.outputs.CHECKOUT_COMMIT_REF }}" >> $GITHUB_ENV
          echo "CHECKOUT_REPO=${{ needs.create_test_vm.outputs.CHECKOUT_REPO }}" >> $GITHUB_ENV

# careful with this as you are checking out the source branch of the PR
# which is a potential security risk. Only do this when you know what you are doing
# See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques
# See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
# See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
      - uses: unstructuredstudio/zubhub/.github/actions/checkout@master
        with:
          ref: ${{ env.CHECKOUT_COMMIT_REF }}
          repository: ${{ env.CHECKOUT_REPO }}

      - name: create a .env file in the zubhub_frontend/zubhub/ directory
        run: |

          # create env file for frontend.
          # This vm is only for testing so we don't need to worry about secrets being exposed
          # (provided we don't put sensitive info in the env file)
          cat << EOF > ./zubhub_frontend/zubhub/.env
          GENERATE_SOURCEMAP=false
          DISABLE_ESLINT_PLUGIN=true
          REACT_APP_NODE_ENV=development
          REACT_APP_BACKEND_DEVELOPMENT_URL=https://${{env.API_DOMAIN}}.unstructured.studio
          REACT_APP_BACKEND_PRODUCTION_URL=https://${{env.API_DOMAIN}}.unstructured.studio
          REACT_APP_DOSPACE_ACCESS_KEY_ID=
          REACT_APP_DOSPACE_ACCESS_SECRET_KEY=
          REACT_APP_VIDEO_UPLOAD_URL=
          REACT_APP_VIDEO_FOLDER_NAME=videos
          REACT_APP_DEV_VIDEO_FOLDER_NAME=dev_videos
          REACT_APP_VIDEO_UPLOAD_PRESET_NAME=video_upload_preset
          REACT_APP_DEV_VIDEO_UPLOAD_PRESET_NAME=dev_video_upload_preset
          EOF

      - name: Build and push ${{ matrix.service }}
        uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master
        with:
          username: ${{ vars.DOCKERHUB_USERNAME }}
          token: ${{ secrets.DOCKERHUB_TOKEN }}
          # for frontend, context is ./zubhub_frontend/zubhub/
          # for backend services, context is ./zubhub_backend/
          context: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || 'backend' }}/
          # for frontend, Dockerfile is ./zubhub_frontend/zubhub/Dockerfile.prod
          # for backend services, Dockerfile is ./zubhub_backend/compose/<service>/prod/Dockerfile
          file: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || format('backend/compose/{0}', matrix.service) }}/${{ matrix.service != 'frontend' && 'prod/' || '' }}Dockerfile${{ matrix.service == 'frontend' && '.prod' || '' }}
          push: true
          tags: unstructuredstudio/zubhub-test_${{ matrix.service }}:latest
##################################################################################

#################################################################################
# The following job is for deploying the project on the test droplet
##################################################################################
  deploy:
    needs:
      - create_test_vm
      - build_and_push
    runs-on: ubuntu-latest
    steps:
      - name: Use output
        id: use_output
        run: |
          echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV
          echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV
          echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV
          echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV

      - name: Executing remote command
        uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master
        with:
          host: ${{env.NEW_DROPLET_IP}}
          username: ${{ vars.DO_BACKEND_USERNAME }}
          key: ${{ secrets.DO_SSHKEY }}
          script: |

            # create env file for backend
            cat << EOF > /home/zubhub_backend/.env
            ENVIRONMENT=production
            DEFAULT_FRONTEND_DOMAIN=${{env.FRONTEND_DOMAIN}}.unstructured.studio
            DEFAULT_BACKEND_DOMAIN=${{env.API_DOMAIN}}.unstructured.studio
            DEFAULT_DISPLAY_NAME=ZubHub
            DEFAULT_FRONTEND_PROTOCOL=https
            DEFAULT_BACKEND_PROTOCOL=https
            SECRET_KEY=random string
            DEBUG=1
            STORE_MEDIA_LOCALLY=1
            MEDIA_SECRET=random string
            DEFAULT_MEDIA_SERVER_PROTOCOL=https
            DEFAULT_MEDIA_SERVER_DOMAIN=${{env.MEDIA_DOMAIN}}.unstructured.studio
            POSTGRES_NAME=postgres
            POSTGRES_USER=postgres
            POSTGRES_PASSWORD=postgres
            POSTGRES_HOST=db
            RABBITMQ_USERNAME=admin
            RABBITMQ_PASSWORD=admin
            RABBITMQ_MANAGEMENT_ALLOW_WEB_ACCESS=true
            CELERY_BROKER=amqp://admin:admin@rabbitmq:5672/
            CELERY_BACKEND=django-db
            PROXY_COUNT=0
            DETECT_MISCONFIG=0
            SUPERUSER_PASSWORD=dummy_password
            EOF

            # deploy project
            cd /home/zubhub
            # Use sed to prepend "NODE_ENV=development" to the frontend build command.
            # This is neccessary to avoid long build times on the test droplet.
            sed -i '/"build":/ s/node /NODE_ENV=development node /' \
            /home/zubhub/zubhub_frontend/zubhub/package.json
            sudo bash /home/zubhub/test_vm_deployment/deploy_test_fullstack.sh

      - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue_number: ${{ github.event.pull_request.number }}
          owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          message: |
            Test VM is ready ✅✅✅
            You can access it with the url:
            https://${{env.FRONTEND_DOMAIN}}.unstructured.studio
            Default username: dummy
            default password: dummy_password
######################################################################################

#################################################################################
# The following job is for destroying the test droplet and its DNS records
##################################################################################
  destroy_test_vm:
    if: |
      (
        github.event.action == 'closed' &&
        contains(github.event.pull_request.labels.*.name, 'create-test-vm')
      ) ||
      (
        github.event.action == 'unlabeled' &&
        github.event.label.name == 'create-test-vm'
      )
    runs-on: ubuntu-latest
    steps:
      - name: Get pr number and droplet ip
        id: get_pr_number_and_droplet_ip
        uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
        with:
          token: ${{ secrets.DO_ACCESS_TOKEN }}
          script: |

            # This script will be executed inside a composite action.
            # rename GITHUB_OUTPUT so that it's clear that it's a
            # composite output and should be handled differently from $GITHUB_OUTPUT
            COMPOSITE_OUTPUT=$GITHUB_OUTPUT
            # for events like unlabelled, github.event.pull_request.number is available
            PR_NUMBER=${{ github.event.pull_request.number }}
            if [[ -z "$PR_NUMBER" ]] ; then
              # github.event.pull_request.number is not available in closed event
              sudo apt-get install jq -y # TODO: remove this since jq. seems jq is installed by default
              PR_NUMBER=$(curl --silent --show-error -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
                https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/pulls | \
                jq -r '.[0].number')
            fi

            if [[ -z "$PR_NUMBER" ]] ; then
              echo "PR_NUMBER is not available. Exiting..."
              exit 1
            fi
            DROPLET_NAME=zubhub-test-$PR_NUMBER

            # Get droplet ip
            DROPLET_IP=$(doctl compute droplet get $DROPLET_NAME \
              --format PublicIPv4 --no-header 2>/dev/null || true)
            if [[ -z "$DROPLET_IP" ]] ; then
              echo "Droplet does not exist. Exiting..."
              exit 0
            fi
            echo "DROPLET_IP=$DROPLET_IP" >> $COMPOSITE_OUTPUT
            echo "DROPLET_NAME=$DROPLET_NAME" >> $COMPOSITE_OUTPUT

      - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
        env:
          DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
        if: ${{ env.DROPLET_IP != '' }}
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue_number: ${{ github.event.pull_request.number }}
          owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          message: |
            @${{ github.actor }} is deleting test VM ...

      - name: delete DNS records and test droplet
        uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master
        env:
          DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
          DROPLET_NAME: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_NAME }}
        with:
          token: ${{ secrets.DO_ACCESS_TOKEN }}
          script: |

            # Get the domain records
            records=$(doctl compute domain records list unstructured.studio --format ID,Data --no-header)
            # Loop through the records
            while IFS= read -r record; do
              # Split the record into ID and IP
              ID=$(echo $record | cut -d' ' -f1)
              IP=$(echo $record | cut -d' ' -f2)

              # Delete the record if it points to the droplet IP
              if [[ -n "$ID" && "$IP" == "${{env.DROPLET_IP}}" ]]; then
                doctl compute domain records delete unstructured.studio $ID --force
              fi
            done <<< "$records"
            # delete test droplet
            if [[ -n "${{ env.DROPLET_NAME }}" ]] ; then
              doctl compute droplet  delete ${{ env.DROPLET_NAME }} --force
            fi

      - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master
        env:
          DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }}
        if: ${{ env.DROPLET_IP != '' }}
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue_number: ${{ github.event.pull_request.number }}
          owner: ${{ github.repository_owner }}
          repo: ${{ github.event.repository.name }}
          message: |
            Test VM deleted ✅✅✅
######################################################################################