setup pre-commit #350
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create/Destroy Test VM | |
| on: | |
| pull_request_target: | |
| types: [opened, reopened, synchronize, labeled, unlabeled, closed] | |
| jobs: | |
| ################################################################################## | |
| # The following job is for creating a new DO droplet, creating DNS records for it | |
| # and copying the project files to it. | |
| ################################################################################## | |
| create_test_vm: | |
| if: | | |
| ( | |
| ( | |
| github.event.action == 'opened' || | |
| github.event.action == 'reopened' || | |
| github.event.action == 'synchronize' | |
| ) && contains(github.event.pull_request.labels.*.name, 'create-test-vm') | |
| ) || | |
| ( | |
| github.event.action == 'labeled' && | |
| github.event.label.name == 'create-test-vm' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue_number: ${{ github.event.pull_request.number }} | |
| owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| message: | | |
| @${{ github.actor }} is creating a test VM for this PR 🚀🚀🚀 | |
| This may take a few minutes so relax and grab a cup of coffee ☕ | |
| We will notify you when the VM is ready. | |
| - name: Get checkout commit ref and repo | |
| run: | | |
| # for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250 | |
| merge_commit_sha=${{ github.event.pull_request.merge_commit_sha }} | |
| head_ref=${{ github.event.pull_request.head.ref }} | |
| target_repo=${{ github.repository }} | |
| pr_repo=${{ github.event.pull_request.head.repo.full_name }} | |
| CHECKOUT_COMMIT_REF=$([ "$merge_commit_sha" != "" ] && \ | |
| echo "$merge_commit_sha" || echo "$head_ref") | |
| CHECKOUT_REPO=$([ "$merge_commit_sha" != "" ] && \ | |
| echo "$target_repo" || echo "$pr_repo") | |
| echo "CHECKOUT_COMMIT_REF=$CHECKOUT_COMMIT_REF" >> $GITHUB_ENV | |
| echo "CHECKOUT_REPO=$CHECKOUT_REPO" >> $GITHUB_ENV | |
| # careful with this as you are checking out the source branch of the PR | |
| # which is a potential security risk. Only do this when you know what you are doing | |
| # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques | |
| # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git | |
| # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. | |
| - uses: unstructuredstudio/zubhub/.github/actions/checkout@master | |
| with: | |
| ref: ${{ env.CHECKOUT_COMMIT_REF }} | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| - name: Create new droplet and DNS records | |
| id: create_droplet | |
| uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master | |
| with: | |
| token: ${{ secrets.DO_ACCESS_TOKEN }} | |
| script: | | |
| # This script will be executed inside a composite action. | |
| # rename GITHUB_OUTPUT so that it's clear that it's a | |
| # composite output and should be handled differently from $GITHUB_OUTPUT | |
| COMPOSITE_OUTPUT=$GITHUB_OUTPUT | |
| NEW_DROPLET_NAME=zubhub-test-${{ github.event.pull_request.number }} | |
| FRONTEND_DOMAIN=${NEW_DROPLET_NAME} | |
| API_DOMAIN=api.${FRONTEND_DOMAIN} | |
| MEDIA_DOMAIN=media.${FRONTEND_DOMAIN} | |
| echo "NEW_DROPLET_NAME=$NEW_DROPLET_NAME" >> $COMPOSITE_OUTPUT | |
| echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $COMPOSITE_OUTPUT | |
| echo "API_DOMAIN=$API_DOMAIN" >> $COMPOSITE_OUTPUT | |
| echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $COMPOSITE_OUTPUT | |
| # check if droplet already exists and exit script if it does | |
| NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \ | |
| --format PublicIPv4 --no-header 2>/dev/null || true) | |
| if [[ -n "$NEW_DROPLET_IP" ]] ; then | |
| echo "Droplet already exists. Save droplet IP to env variable and exit..." | |
| echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT | |
| exit 0 | |
| fi | |
| # create new droplet | |
| doctl compute droplet create $NEW_DROPLET_NAME --image \ | |
| ${{ secrets.ZUBHUB_TEST_SNAPSHOT_ID }} --tag-name zubhub-test --size s-1vcpu-1gb \ | |
| --region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait | |
| sleep 30s | |
| NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \ | |
| --format PublicIPv4 --no-header) | |
| echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $COMPOSITE_OUTPUT | |
| # we only need records for frontend, media server and api server | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $FRONTEND_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $API_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $MEDIA_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| - uses: unstructuredstudio/zubhub/.github/actions/scp_action@master | |
| env: | |
| NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }} | |
| with: | |
| host: ${{env.NEW_DROPLET_IP}} | |
| username: ${{ vars.DO_BACKEND_USERNAME }} | |
| key: ${{ secrets.DO_SSHKEY }} | |
| source: "." | |
| target: "/home/zubhub" | |
| - name: Set output | |
| id: set_output | |
| env: | |
| NEW_DROPLET_IP: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).NEW_DROPLET_IP }} | |
| FRONTEND_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).FRONTEND_DOMAIN }} | |
| API_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).API_DOMAIN }} | |
| MEDIA_DOMAIN: ${{ fromJson(steps.create_droplet.outputs.JSON_STRING).MEDIA_DOMAIN }} | |
| run: | | |
| echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $GITHUB_OUTPUT | |
| echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $GITHUB_OUTPUT | |
| echo "API_DOMAIN=$API_DOMAIN" >> $GITHUB_OUTPUT | |
| echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $GITHUB_OUTPUT | |
| echo "CHECKOUT_COMMIT_REF=${{ env.CHECKOUT_COMMIT_REF }}" >> $GITHUB_OUTPUT | |
| echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT | |
| outputs: | |
| NEW_DROPLET_IP: ${{ steps.set_output.outputs.NEW_DROPLET_IP }} | |
| FRONTEND_DOMAIN: ${{ steps.set_output.outputs.FRONTEND_DOMAIN }} | |
| API_DOMAIN: ${{ steps.set_output.outputs.API_DOMAIN }} | |
| MEDIA_DOMAIN: ${{ steps.set_output.outputs.MEDIA_DOMAIN }} | |
| CHECKOUT_COMMIT_REF: ${{ steps.set_output.outputs.CHECKOUT_COMMIT_REF }} | |
| CHECKOUT_REPO: ${{ steps.set_output.outputs.CHECKOUT_REPO }} | |
| ################################################################################# | |
| ################################################################################# | |
| # The following job is for building docker images and pushing them to dockerhub | |
| ################################################################################## | |
| build_and_push: | |
| needs: create_test_vm | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| service: ['frontend', 'web', 'celery', 'media'] | |
| steps: | |
| - name: Use output | |
| id: use_output | |
| run: | | |
| echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV | |
| echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV | |
| echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV | |
| echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_COMMIT_REF=${{ needs.create_test_vm.outputs.CHECKOUT_COMMIT_REF }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REPO=${{ needs.create_test_vm.outputs.CHECKOUT_REPO }}" >> $GITHUB_ENV | |
| # careful with this as you are checking out the source branch of the PR | |
| # which is a potential security risk. Only do this when you know what you are doing | |
| # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques | |
| # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git | |
| # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. | |
| - uses: unstructuredstudio/zubhub/.github/actions/checkout@master | |
| with: | |
| ref: ${{ env.CHECKOUT_COMMIT_REF }} | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| - name: create a .env file in the zubhub_frontend/zubhub/ directory | |
| run: | | |
| # create env file for frontend. | |
| # This vm is only for testing so we don't need to worry about secrets being exposed | |
| # (provided we don't put sensitive info in the env file) | |
| cat << EOF > ./zubhub_frontend/zubhub/.env | |
| GENERATE_SOURCEMAP=false | |
| DISABLE_ESLINT_PLUGIN=true | |
| REACT_APP_NODE_ENV=development | |
| REACT_APP_BACKEND_DEVELOPMENT_URL=https://${{env.API_DOMAIN}}.unstructured.studio | |
| REACT_APP_BACKEND_PRODUCTION_URL=https://${{env.API_DOMAIN}}.unstructured.studio | |
| REACT_APP_DOSPACE_ACCESS_KEY_ID= | |
| REACT_APP_DOSPACE_ACCESS_SECRET_KEY= | |
| REACT_APP_VIDEO_UPLOAD_URL= | |
| REACT_APP_VIDEO_FOLDER_NAME=videos | |
| REACT_APP_DEV_VIDEO_FOLDER_NAME=dev_videos | |
| REACT_APP_VIDEO_UPLOAD_PRESET_NAME=video_upload_preset | |
| REACT_APP_DEV_VIDEO_UPLOAD_PRESET_NAME=dev_video_upload_preset | |
| EOF | |
| - name: Build and push ${{ matrix.service }} | |
| uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master | |
| with: | |
| username: ${{ vars.DOCKERHUB_USERNAME }} | |
| token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # for frontend, context is ./zubhub_frontend/zubhub/ | |
| # for backend services, context is ./zubhub_backend/ | |
| context: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || 'backend' }}/ | |
| # for frontend, Dockerfile is ./zubhub_frontend/zubhub/Dockerfile.prod | |
| # for backend services, Dockerfile is ./zubhub_backend/compose/<service>/prod/Dockerfile | |
| file: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || format('backend/compose/{0}', matrix.service) }}/${{ matrix.service != 'frontend' && 'prod/' || '' }}Dockerfile${{ matrix.service == 'frontend' && '.prod' || '' }} | |
| push: true | |
| tags: unstructuredstudio/zubhub-test_${{ matrix.service }}:latest | |
| ################################################################################## | |
| ################################################################################# | |
| # The following job is for deploying the project on the test droplet | |
| ################################################################################## | |
| deploy: | |
| needs: | |
| - create_test_vm | |
| - build_and_push | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Use output | |
| id: use_output | |
| run: | | |
| echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV | |
| echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV | |
| echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV | |
| echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV | |
| - name: Executing remote command | |
| uses: unstructuredstudio/zubhub/.github/actions/ssh_action@master | |
| with: | |
| host: ${{env.NEW_DROPLET_IP}} | |
| username: ${{ vars.DO_BACKEND_USERNAME }} | |
| key: ${{ secrets.DO_SSHKEY }} | |
| script: | | |
| # create env file for backend | |
| cat << EOF > /home/zubhub_backend/.env | |
| ENVIRONMENT=production | |
| DEFAULT_FRONTEND_DOMAIN=${{env.FRONTEND_DOMAIN}}.unstructured.studio | |
| DEFAULT_BACKEND_DOMAIN=${{env.API_DOMAIN}}.unstructured.studio | |
| DEFAULT_DISPLAY_NAME=ZubHub | |
| DEFAULT_FRONTEND_PROTOCOL=https | |
| DEFAULT_BACKEND_PROTOCOL=https | |
| SECRET_KEY=random string | |
| DEBUG=1 | |
| STORE_MEDIA_LOCALLY=1 | |
| MEDIA_SECRET=random string | |
| DEFAULT_MEDIA_SERVER_PROTOCOL=https | |
| DEFAULT_MEDIA_SERVER_DOMAIN=${{env.MEDIA_DOMAIN}}.unstructured.studio | |
| POSTGRES_NAME=postgres | |
| POSTGRES_USER=postgres | |
| POSTGRES_PASSWORD=postgres | |
| POSTGRES_HOST=db | |
| RABBITMQ_USERNAME=admin | |
| RABBITMQ_PASSWORD=admin | |
| RABBITMQ_MANAGEMENT_ALLOW_WEB_ACCESS=true | |
| CELERY_BROKER=amqp://admin:admin@rabbitmq:5672/ | |
| CELERY_BACKEND=django-db | |
| PROXY_COUNT=0 | |
| DETECT_MISCONFIG=0 | |
| SUPERUSER_PASSWORD=dummy_password | |
| EOF | |
| # deploy project | |
| cd /home/zubhub | |
| # Use sed to prepend "NODE_ENV=development" to the frontend build command. | |
| # This is neccessary to avoid long build times on the test droplet. | |
| sed -i '/"build":/ s/node /NODE_ENV=development node /' \ | |
| /home/zubhub/zubhub_frontend/zubhub/package.json | |
| sudo bash /home/zubhub/test_vm_deployment/deploy_test_fullstack.sh | |
| - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue_number: ${{ github.event.pull_request.number }} | |
| owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| message: | | |
| Test VM is ready ✅✅✅ | |
| You can access it with the url: | |
| https://${{env.FRONTEND_DOMAIN}}.unstructured.studio | |
| ###################################################################################### | |
| ################################################################################# | |
| # The following job is for destroying the test droplet and its DNS records | |
| ################################################################################## | |
| destroy_test_vm: | |
| if: | | |
| ( | |
| github.event.action == 'closed' && | |
| contains(github.event.pull_request.labels.*.name, 'create-test-vm') | |
| ) || | |
| ( | |
| github.event.action == 'unlabeled' && | |
| github.event.label.name == 'create-test-vm' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get pr number and droplet ip | |
| id: get_pr_number_and_droplet_ip | |
| uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master | |
| with: | |
| token: ${{ secrets.DO_ACCESS_TOKEN }} | |
| script: | | |
| # This script will be executed inside a composite action. | |
| # rename GITHUB_OUTPUT so that it's clear that it's a | |
| # composite output and should be handled differently from $GITHUB_OUTPUT | |
| COMPOSITE_OUTPUT=$GITHUB_OUTPUT | |
| # for events like unlabelled, github.event.pull_request.number is available | |
| PR_NUMBER=${{ github.event.pull_request.number }} | |
| if [[ -z "$PR_NUMBER" ]] ; then | |
| # github.event.pull_request.number is not available in closed event | |
| sudo apt-get install jq -y # TODO: remove this since jq. seems jq is installed by default | |
| PR_NUMBER=$(curl --silent --show-error -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/pulls | \ | |
| jq -r '.[0].number') | |
| fi | |
| if [[ -z "$PR_NUMBER" ]] ; then | |
| echo "PR_NUMBER is not available. Exiting..." | |
| exit 1 | |
| fi | |
| DROPLET_NAME=zubhub-test-$PR_NUMBER | |
| # Get droplet ip | |
| DROPLET_IP=$(doctl compute droplet get $DROPLET_NAME \ | |
| --format PublicIPv4 --no-header 2>/dev/null || true) | |
| if [[ -z "$DROPLET_IP" ]] ; then | |
| echo "Droplet does not exist. Exiting..." | |
| exit 0 | |
| fi | |
| echo "DROPLET_IP=$DROPLET_IP" >> $COMPOSITE_OUTPUT | |
| echo "DROPLET_NAME=$DROPLET_NAME" >> $COMPOSITE_OUTPUT | |
| - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master | |
| env: | |
| DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }} | |
| if: ${{ env.DROPLET_IP != '' }} | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue_number: ${{ github.event.pull_request.number }} | |
| owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| message: | | |
| @${{ github.actor }} is deleting test VM ... | |
| - name: delete DNS records and test droplet | |
| uses: unstructuredstudio/zubhub/.github/actions/doctl_action@master | |
| env: | |
| DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }} | |
| DROPLET_NAME: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_NAME }} | |
| with: | |
| token: ${{ secrets.DO_ACCESS_TOKEN }} | |
| script: | | |
| # Get the domain records | |
| records=$(doctl compute domain records list unstructured.studio --format ID,Data --no-header) | |
| # Loop through the records | |
| while IFS= read -r record; do | |
| # Split the record into ID and IP | |
| ID=$(echo $record | cut -d' ' -f1) | |
| IP=$(echo $record | cut -d' ' -f2) | |
| # Delete the record if it points to the droplet IP | |
| if [[ -n "$ID" && "$IP" == "${{env.DROPLET_IP}}" ]]; then | |
| doctl compute domain records delete unstructured.studio $ID --force | |
| fi | |
| done <<< "$records" | |
| # delete test droplet | |
| if [[ -n "${{ env.DROPLET_NAME }}" ]] ; then | |
| doctl compute droplet delete ${{ env.DROPLET_NAME }} --force | |
| fi | |
| - uses: unstructuredstudio/zubhub/.github/actions/comment_action@master | |
| env: | |
| DROPLET_IP: ${{ fromJson(steps.get_pr_number_and_droplet_ip.outputs.JSON_STRING).DROPLET_IP }} | |
| if: ${{ env.DROPLET_IP != '' }} | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| issue_number: ${{ github.event.pull_request.number }} | |
| owner: ${{ github.repository_owner }} | |
| repo: ${{ github.event.repository.name }} | |
| message: | | |
| Test VM deleted ✅✅✅ | |
| ###################################################################################### |