fix: fixes the slash issue #295
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create/Destroy Test VM | |
| on: | |
| pull_request_target: | |
| types: [opened, reopened, synchronize, labeled, unlabeled, closed] | |
| jobs: | |
| ################################################################################## | |
| # The following job is for creating a new DO droplet, creating DNS records for it | |
| # and copying the project files to it. | |
| ################################################################################## | |
| create_test_vm: | |
| if: | | |
| ( | |
| ( | |
| github.event.action == 'opened' || | |
| github.event.action == 'reopened' || | |
| github.event.action == 'synchronize' | |
| ) && contains(github.event.pull_request.labels.*.name, 'create-test-vm') | |
| ) || | |
| ( | |
| github.event.action == 'labeled' && | |
| github.event.label.name == 'create-test-vm' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get checkout commit ref and repo | |
| run: | | |
| # for context read this https://github.com/Arelle/Arelle/pull/938/files#r1377926250 | |
| merge_commit_sha=${{ github.event.pull_request.merge_commit_sha }} | |
| head_ref=${{ github.event.pull_request.head.ref }} | |
| target_repo=${{ github.repository }} | |
| pr_repo=${{ github.event.pull_request.head.repo.full_name }} | |
| CHECKOUT_COMMIT_REF=$([ "$merge_commit_sha" != "" ] && \ | |
| echo "$merge_commit_sha" || echo "$head_ref") | |
| CHECKOUT_REPO=$([ "$merge_commit_sha" != "" ] && \ | |
| echo "$target_repo" || echo "$pr_repo") | |
| echo "CHECKOUT_COMMIT_REF=$CHECKOUT_COMMIT_REF" >> $GITHUB_ENV | |
| echo "CHECKOUT_REPO=$CHECKOUT_REPO" >> $GITHUB_ENV | |
| echo "********************************* merge_commit_sha=$merge_commit_sha" | |
| echo "********************************* head_ref=$head_ref" | |
| echo "********************************* target_repo=$target_repo" | |
| echo "********************************* pr_repo=$pr_repo" | |
| echo "********************************* CHECKOUT_COMMIT_REF=$CHECKOUT_COMMIT_REF" | |
| echo "********************************* CHECKOUT_REPO=$CHECKOUT_REPO" | |
| # careful with this as you are checking out the source branch of the PR | |
| # which is a potential security risk. Only do this when you know what you are doing | |
| # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques | |
| # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git | |
| # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. | |
| - name: Checkout source branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ env.CHECKOUT_COMMIT_REF }} | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DO_ACCESS_TOKEN }} | |
| - name: Create new droplet and DNS records | |
| run: | | |
| NEW_DROPLET_NAME=zubhub-test-${{ github.event.pull_request.number }} | |
| FRONTEND_DOMAIN=${NEW_DROPLET_NAME} | |
| API_DOMAIN=api.${FRONTEND_DOMAIN} | |
| MEDIA_DOMAIN=media.${FRONTEND_DOMAIN} | |
| echo "NEW_DROPLET_NAME=$NEW_DROPLET_NAME" >> $GITHUB_ENV | |
| echo "FRONTEND_DOMAIN=$FRONTEND_DOMAIN" >> $GITHUB_ENV | |
| echo "API_DOMAIN=$API_DOMAIN" >> $GITHUB_ENV | |
| echo "MEDIA_DOMAIN=$MEDIA_DOMAIN" >> $GITHUB_ENV | |
| # check if droplet already exists and exit script if it does | |
| NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \ | |
| --format PublicIPv4 --no-header 2>/dev/null || true) | |
| if [[ -n "$NEW_DROPLET_IP" ]] ; then | |
| echo "Droplet already exists. Save droplet IP to env variable and exit..." | |
| echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| # create new droplet | |
| doctl compute droplet create $NEW_DROPLET_NAME --image \ | |
| ${{ secrets.ZUBHUB_TEST_SNAPSHOT_ID }} --tag-name zubhub-test --size s-1vcpu-1gb \ | |
| --region nyc1 --enable-monitoring --ssh-keys ${{ secrets.DO_PUBLIC_SSHKEY_FP }} --wait | |
| sleep 30s | |
| NEW_DROPLET_IP=$(doctl compute droplet get $NEW_DROPLET_NAME \ | |
| --format PublicIPv4 --no-header) | |
| echo "NEW_DROPLET_IP=$NEW_DROPLET_IP" >> $GITHUB_ENV | |
| # we only need records for frontend, media server and api server | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $FRONTEND_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $API_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| doctl compute domain records create unstructured.studio --record-type A --record-name \ | |
| $MEDIA_DOMAIN --record-data $NEW_DROPLET_IP --record-ttl 600 | |
| - name: Copy file via scp | |
| uses: appleboy/scp-action@master | |
| with: | |
| host: ${{env.NEW_DROPLET_IP}} | |
| username: ${{ secrets.DO_BACKEND_USERNAME }} | |
| key: ${{ secrets.DO_SSHKEY }} | |
| source: "." | |
| target: "/home/zubhub" | |
| - name: Set output | |
| id: set_output | |
| run: | | |
| echo "::set-output name=NEW_DROPLET_IP::$NEW_DROPLET_IP" | |
| echo "::set-output name=FRONTEND_DOMAIN::$FRONTEND_DOMAIN" | |
| echo "::set-output name=API_DOMAIN::$API_DOMAIN" | |
| echo "::set-output name=MEDIA_DOMAIN::$MEDIA_DOMAIN" | |
| echo "::set-output name=CHECKOUT_COMMIT_REF::${{ env.CHECKOUT_COMMIT_REF }}" | |
| echo "::set-output name=CHECKOUT_REPO::${{ env.CHECKOUT_REPO }}" | |
| outputs: | |
| NEW_DROPLET_IP: ${{ steps.set_output.outputs.NEW_DROPLET_IP }} | |
| FRONTEND_DOMAIN: ${{ steps.set_output.outputs.FRONTEND_DOMAIN }} | |
| API_DOMAIN: ${{ steps.set_output.outputs.API_DOMAIN }} | |
| MEDIA_DOMAIN: ${{ steps.set_output.outputs.MEDIA_DOMAIN }} | |
| CHECKOUT_COMMIT_REF: ${{ steps.set_output.outputs.CHECKOUT_COMMIT_REF }} | |
| CHECKOUT_REPO: ${{ steps.set_output.outputs.CHECKOUT_REPO }} | |
| ################################################################################# | |
| ################################################################################# | |
| # The following job is for building docker images and pushing them to dockerhub | |
| ################################################################################## | |
| build_and_push: | |
| needs: create_test_vm | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| service: ['frontend', 'web', 'celery', 'media'] | |
| steps: | |
| - name: Use output | |
| id: use_output | |
| run: | | |
| echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV | |
| echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV | |
| echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV | |
| echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_COMMIT_REF=${{ needs.create_test_vm.outputs.CHECKOUT_COMMIT_REF }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REPO=${{ needs.create_test_vm.outputs.CHECKOUT_REPO }}" >> $GITHUB_ENV | |
| # careful with this as you are checking out the source branch of the PR | |
| # which is a potential security risk. Only do this when you know what you are doing | |
| # See https://stackoverflow.com/questions/75873833/how-to-protect-github-secrets-in-pull-request-actions-from-malicious-pull-reques | |
| # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git | |
| # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. | |
| - name: Checkout source branch | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ env.CHECKOUT_COMMIT_REF }} | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| - name: create a .env file in the zubhub_frontend/zubhub/ directory | |
| run: | | |
| # create env file for frontend. | |
| # This vm is only for testing so we don't need to worry about secrets being exposed | |
| # (provided we don't put sensitive info in the env file) | |
| cat << EOF > ./zubhub_frontend/zubhub/.env | |
| GENERATE_SOURCEMAP=false | |
| DISABLE_ESLINT_PLUGIN=true | |
| REACT_APP_NODE_ENV=development | |
| REACT_APP_BACKEND_DEVELOPMENT_URL=https://${{env.API_DOMAIN}}.unstructured.studio | |
| REACT_APP_BACKEND_PRODUCTION_URL=https://${{env.API_DOMAIN}}.unstructured.studio | |
| REACT_APP_DOSPACE_ACCESS_KEY_ID= | |
| REACT_APP_DOSPACE_ACCESS_SECRET_KEY= | |
| REACT_APP_VIDEO_UPLOAD_URL= | |
| REACT_APP_VIDEO_FOLDER_NAME=videos | |
| REACT_APP_DEV_VIDEO_FOLDER_NAME=dev_videos | |
| REACT_APP_VIDEO_UPLOAD_PRESET_NAME=video_upload_preset | |
| REACT_APP_DEV_VIDEO_UPLOAD_PRESET_NAME=dev_video_upload_preset | |
| EOF | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push ${{ matrix.service }} | |
| id: docker_build | |
| uses: docker/build-push-action@v2 | |
| with: | |
| # for frontend, context is ./zubhub_frontend/zubhub/ | |
| # for backend services, context is ./zubhub_backend/ | |
| context: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || 'backend' }}/ | |
| # for frontend, Dockerfile is ./zubhub_frontend/zubhub/Dockerfile.prod | |
| # for backend services, Dockerfile is ./zubhub_backend/compose/<service>/prod/Dockerfile | |
| file: ./zubhub_${{ matrix.service == 'frontend' && 'frontend/zubhub' || format('backend/compose/{0}', matrix.service) }}/${{ matrix.service != 'frontend' && 'prod/' || '' }}Dockerfile${{ matrix.service == 'frontend' && '.prod' || '' }} | |
| push: true | |
| tags: unstructuredstudio/zubhub-test_${{ matrix.service }}:latest | |
| - name: Image digest | |
| run: echo ${{ steps.docker_build.outputs.digest }} | |
| ################################################################################## | |
| ################################################################################# | |
| # The following job is for deploying the project on the test droplet | |
| ################################################################################## | |
| deploy: | |
| needs: | |
| - create_test_vm | |
| - build_and_push | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Use output | |
| id: use_output | |
| run: | | |
| echo "NEW_DROPLET_IP=${{ needs.create_test_vm.outputs.NEW_DROPLET_IP }}" >> $GITHUB_ENV | |
| echo "FRONTEND_DOMAIN=${{ needs.create_test_vm.outputs.FRONTEND_DOMAIN }}" >> $GITHUB_ENV | |
| echo "API_DOMAIN=${{ needs.create_test_vm.outputs.API_DOMAIN }}" >> $GITHUB_ENV | |
| echo "MEDIA_DOMAIN=${{ needs.create_test_vm.outputs.MEDIA_DOMAIN }}" >> $GITHUB_ENV | |
| - name: Executing remote command | |
| uses: appleboy/ssh-action@master | |
| with: | |
| host: ${{env.NEW_DROPLET_IP}} | |
| username: ${{ secrets.DO_BACKEND_USERNAME }} | |
| key: ${{ secrets.DO_SSHKEY }} | |
| script: | | |
| # create env file for backend | |
| cat << EOF > /home/zubhub_backend/.env | |
| ENVIRONMENT=production | |
| DEFAULT_FRONTEND_DOMAIN=${{env.FRONTEND_DOMAIN}}.unstructured.studio | |
| DEFAULT_BACKEND_DOMAIN=${{env.API_DOMAIN}}.unstructured.studio | |
| DEFAULT_DISPLAY_NAME=ZubHub | |
| DEFAULT_FRONTEND_PROTOCOL=https | |
| DEFAULT_BACKEND_PROTOCOL=https | |
| SECRET_KEY=random string | |
| DEBUG=1 | |
| STORE_MEDIA_LOCALLY=1 | |
| MEDIA_SECRET=random string | |
| DEFAULT_MEDIA_SERVER_PROTOCOL=https | |
| DEFAULT_MEDIA_SERVER_DOMAIN=${{env.MEDIA_DOMAIN}}.unstructured.studio | |
| POSTGRES_NAME=postgres | |
| POSTGRES_USER=postgres | |
| POSTGRES_PASSWORD=postgres | |
| POSTGRES_HOST=db | |
| GF_ADMIN_USER=admin | |
| GF_ADMIN_PASSWORD=admin | |
| RABBITMQ_DEFAULT_USER=admin | |
| RABBITMQ_DEFAULT_PASS=admin | |
| CELERY_BROKER=amqp://admin:admin@rabbitmq:5672/ | |
| CELERY_BACKEND=django-db | |
| CELERY_FLOWER_USER=admin | |
| CELERY_FLOWER_PASSWORD=admin | |
| PROXY_COUNT=0 | |
| DETECT_MISCONFIG=0 | |
| SUPERUSER_PASSWORD=dummy_password | |
| EOF | |
| # deploy project | |
| cd /home/zubhub | |
| # Use sed to prepend "NODE_ENV=development" to the frontend build command. | |
| # This is neccessary to avoid long build times on the test droplet. | |
| sed -i '/"build":/ s/node /NODE_ENV=development node /' \ | |
| /home/zubhub/zubhub_frontend/zubhub/package.json | |
| sudo bash /home/zubhub/test_vm_deployment/deploy_test_fullstack.sh | |
| ###################################################################################### | |
| ################################################################################# | |
| # The following job is for destroying the test droplet and its DNS records | |
| ################################################################################## | |
| destroy_test_vm: | |
| if: | | |
| github.event.action == 'closed' || | |
| (github.event.action == 'unlabeled' && github.event.label.name == 'create-test-vm') || | |
| ( | |
| github.event.action == 'reopened' || | |
| github.event.action == 'synchronize' | |
| ) && contains(github.event.pull_request.labels.*.name, 'create-test-vm') != true | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DO_ACCESS_TOKEN }} | |
| - name: Get pr number and droplet ip | |
| run: | | |
| # for events like unlabelled, github.event.pull_request.number is available | |
| PR_NUMBER=${{ github.event.pull_request.number }} | |
| if [[ -z "$PR_NUMBER" ]] ; then | |
| # github.event.pull_request.number is not available in closed event | |
| sudo apt-get install jq -y | |
| PR_NUMBER=$(curl --silent --show-error -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/pulls | \ | |
| jq -r '.[0].number') | |
| fi | |
| if [[ -z "$PR_NUMBER" ]] ; then | |
| echo "PR_NUMBER is not available. Exiting..." | |
| exit 1 | |
| fi | |
| DROPLET_NAME=zubhub-test-$PR_NUMBER | |
| # Get droplet ip | |
| DROPLET_IP=$(doctl compute droplet get $DROPLET_NAME \ | |
| --format PublicIPv4 --no-header 2>/dev/null || true) | |
| if [[ -z "$DROPLET_IP" ]] ; then | |
| echo "Droplet does not exist. Exiting..." | |
| exit 0 | |
| fi | |
| echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV | |
| echo "DROPLET_IP=$DROPLET_IP" >> $GITHUB_ENV | |
| echo "DROPLET_NAME=$DROPLET_NAME" >> $GITHUB_ENV | |
| - name: delete DNS records | |
| run: | | |
| # Get the domain records | |
| records=$(doctl compute domain records list unstructured.studio --format ID,Data --no-header) | |
| # Loop through the records | |
| while IFS= read -r record; do | |
| # Split the record into ID and IP | |
| ID=$(echo $record | cut -d' ' -f1) | |
| IP=$(echo $record | cut -d' ' -f2) | |
| # Delete the record if it points to the droplet IP | |
| if [[ -n "$ID" && "$IP" == "${{env.DROPLET_IP}}" ]]; then | |
| doctl compute domain records delete unstructured.studio $ID --force | |
| fi | |
| done <<< "$records" | |
| - name: Delete test droplet | |
| run: | | |
| if [[ -n "${{ env.DROPLET_NAME }}" ]] ; then | |
| doctl compute droplet delete ${{ env.DROPLET_NAME }} --force | |
| fi | |
| ###################################################################################### |