refactor github actions

* use matrix strategy for similar jobs to save time * update version of third-party actions * reuse workflows Issue: #1030 Signed-off-by: Ndibe Raymond Olisaemeka <[email protected]>
unstructuredstudio · Dec 19, 2023 · ade434f · ade434f
1 parent 5c506c2
commit ade434f
Show file tree

Hide file tree

Showing 10 changed files with 155 additions and 87 deletions.
diff --git a/.github/workflows/.checkout.yml b/.github/workflows/.checkout.yml
@@ -0,0 +1,20 @@
+name: Checkout files workflow
+on:
+  workflow_call:
+    inputs:
+      ref:
+        required: false
+        type: string
+      repository:
+        required: false
+        type: string
+
+jobs:
+  checkout:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout files
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          repository: ${{ inputs.repository }}
diff --git a/.github/workflows/.docker_build_and_push.yml b/.github/workflows/.docker_build_and_push.yml
@@ -0,0 +1,44 @@
+name: Docker build and push workflow
+on:
+  workflow_call:
+    inputs:
+      context:
+        required: true
+        type: string
+      file:
+        required: true
+        type: string
+      tags:
+        required: true
+        type: string
+      push:
+        required: true
+        type: boolean
+
+jobs:
+  docker_build_and_push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.context }}}
+          file: ${{ inputs.file }}
+          push: ${{ inputs.push }}
+          tags: ${{ inputs.tags }}
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/workflows/.scp_action.yml b/.github/workflows/.scp_action.yml
@@ -0,0 +1,23 @@
+name: Copy files workflow
+on:
+  workflow_call:
+    inputs:
+      source:
+        required: true
+        type: string
+      target:
+        required: true
+        type: string
+
+jobs:
+  copy_files:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Copy file via scp
+        uses: appleboy/scp-action@master
+        with:
+          host: ${{ secrets.DO_BACKEND_HOST }}
+          username: ${{ secrets.DO_BACKEND_USERNAME }}
+          key: ${{ secrets.DO_SSHKEY }}
+          source: ${{ inputs.source }}
+          target: ${{ inputs.target }}
diff --git a/.github/workflows/.ssh_action.yml b/.github/workflows/.ssh_action.yml
@@ -0,0 +1,19 @@
+name: Execute remote command workflow
+on:
+  workflow_call:
+    inputs:
+      script:
+        required: true
+        type: string
+
+jobs:
+  execute_command:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Executing remote command
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.DO_BACKEND_HOST }}
+          username: ${{ secrets.DO_BACKEND_USERNAME }}
+          key: ${{ secrets.DO_SSHKEY }}
+          script: ${{ inputs.script }}
diff --git a/.github/workflows/build_deploy_backend.yml b/.github/workflows/build_deploy_backend.yml
@@ -13,81 +13,43 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  build_and_push:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        service: ['web', 'celery', 'media']
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Checkout files
-        uses: actions/checkout@v2
-
-      - name: Build and push django api
-        id: docker_build_web
-        uses: docker/build-push-action@v2
-        with:
-          context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/web/prod/Dockerfile
-          push: true
-          tags: unstructuredstudio/zubhub-services_web:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Build and push celery worker
-        id: docker_build_celery
-        uses: docker/build-push-action@v2
-        with:
-          context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/celery/prod/Dockerfile
-          push: true
-          tags: unstructuredstudio/zubhub-services_celery:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+      - uses: unstructuredstudio/zubhub/.github/workflows/.checkout.yml@master
 
-      - name: Build and push media service
-        id: docker_build_media
-        uses: docker/build-push-action@v2
+      - name: Build and push ${{ matrix.service }}
+        uses: unstructuredstudio/zubhub/.github/workflows/.docker_build_and_push.yml@master
         with:
           context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/media/prod/Dockerfile
+          file: ./zubhub_backend/compose/${{ matrix.service }}/prod/Dockerfile
           push: true
-          tags: unstructuredstudio/zubhub-services_media:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+          tags: unstructuredstudio/zubhub-services_${{ matrix.service }}:latest
+        secrets: inherit
 
   deploy:
-    needs: build
+    needs: build_and_push
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: unstructuredstudio/zubhub/.github/workflows/.checkout.yml@master
 
-      - name: Copy file via scp
-        uses: appleboy/scp-action@master
+      - uses: unstructuredstudio/zubhub/.github/workflows/.scp_action.yml@master
         with:
-          host: ${{ secrets.DO_BACKEND_HOST }}
-          username: ${{ secrets.DO_BACKEND_USERNAME }}
-          key: ${{ secrets.DO_SSHKEY }}
           source: "."
           target: "/home/zubhub-services/zubhub"
+        secrets: inherit
 
-      - name: Executing remote  command
-        uses: appleboy/ssh-action@master
+      - uses: unsctructuredstudio/zubhub/.github/workflows/.ssh_action.yml@master
         with:
-          host: ${{ secrets.DO_BACKEND_HOST }}
-          username: ${{ secrets.DO_BACKEND_USERNAME }}
-          key: ${{ secrets.DO_SSHKEY }}
           script: |
             cp /home/zubhub-services/zubhub/zubhub_backend/compose/deploy_backend.sh /home/zubhub-services/
             sudo bash /home/zubhub-services/deploy_backend.sh
             doctl compute droplet list 'zubhub-services*' > droplets.txt
             droplets_count=`wc -l < droplets.txt`
             rm droplets.txt
             docker service scale zubhub-services_web=$(($droplets_count - 1))
+        secrets: inherit
diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml
@@ -11,8 +11,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - run: |

diff --git a/.github/workflows/build_deploy_frontend.yml b/.github/workflows/build_deploy_frontend.yml
@@ -17,21 +17,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Checkout files
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./zubhub_frontend/zubhub/
           file: ./zubhub_frontend/zubhub/Dockerfile.prod
@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
       - name: Copy file via scp
         uses: appleboy/scp-action@master

diff --git a/.github/workflows/build_locust.yml b/.github/workflows/build_locust.yml
@@ -23,21 +23,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Checkout files
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build and push locust
         id: docker_build_locust
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./locust/
           file: ./locust/Dockerfile

diff --git a/.github/workflows/create_destroy_test_vm.yaml b/.github/workflows/create_destroy_test_vm.yaml
@@ -47,7 +47,7 @@ jobs:
 # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
 # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
       - name: Checkout source branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.CHECKOUT_COMMIT_REF }}
           repository: ${{ env.CHECKOUT_REPO }}
@@ -148,7 +148,7 @@ jobs:
 # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
 # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
       - name: Checkout source branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.CHECKOUT_COMMIT_REF }}
           repository: ${{ env.CHECKOUT_REPO }}
@@ -175,20 +175,20 @@ jobs:
           EOF
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push ${{ matrix.service }}
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           # for frontend, context is ./zubhub_frontend/zubhub/
           # for backend services, context is ./zubhub_backend/

diff --git a/zubhub_backend/docker-compose.prod.yml b/zubhub_backend/docker-compose.prod.yml
@@ -95,20 +95,20 @@ services:
     depends_on:
       - rabbitmq
 
-  prometheus:
-    image: prom/prometheus
-    command:
-      - --config.file=/etc/prometheus/prometheus.yml
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-      placement:
-        max_replicas_per_node: 1
-        constraints:
-          - "node.role==manager"
-    volumes:
-      - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+  # prometheus:
+  #   image: prom/prometheus
+  #   command:
+  #     - --config.file=/etc/prometheus/prometheus.yml
+  #   deploy:
+  #     replicas: 1
+  #     restart_policy:
+  #       condition: on-failure
+  #     placement:
+  #       max_replicas_per_node: 1
+  #       constraints:
+  #         - "node.role==manager"
+  #   volumes:
+  #     - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro
 
 secrets:
   zubhub_services_secrets: