refactor github actions (#1041)

* use matrix strategy for similar jobs to save time * update version of third-party actions * use composite actions Issue: #1030 Signed-off-by: Ndibe Raymond Olisaemeka <[email protected]> Co-authored-by: Ndibe Raymond Olisaemeka <[email protected]>
unstructuredstudio · Dec 20, 2023 · 9de2230 · 9de2230
1 parent b6a3b28
commit 9de2230
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 81 deletions.
diff --git a/.github/actions/checkout/action.yml b/.github/actions/checkout/action.yml
@@ -0,0 +1,17 @@
+name: Checkout files action
+inputs:
+  ref:
+    required: false
+    type: string
+  repository:
+    required: false
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout files
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        repository: ${{ inputs.repository }}
diff --git a/.github/actions/docker_build_and_push/action.yml b/.github/actions/docker_build_and_push/action.yml
@@ -0,0 +1,47 @@
+name: Docker build and push action
+inputs:
+  dockerhub_username:
+    required: true
+    type: string
+  dockerhub_token:
+    required: true
+    type: string
+  context:
+    required: true
+    type: string
+  file:
+    required: true
+    type: string
+  tags:
+    required: true
+    type: string
+  push:
+    required: true
+    type: boolean
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ inputs.dockerhub_username }}
+        password: ${{ inputs.dockerhub_token }}
+
+    - name: Build and push
+      id: docker_build
+      uses: docker/build-push-action@v5
+      with:
+        context: ${{ inputs.context }}}
+        file: ${{ inputs.file }}
+        push: ${{ inputs.push }}
+        tags: ${{ inputs.tags }}
+
+    - name: Image digest
+      run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.github/actions/scp_action/action.yml b/.github/actions/scp_action/action.yml
@@ -0,0 +1,29 @@
+name: Copy files action
+inputs:
+  host:
+    required: true
+    type: string
+  username:
+    required: true
+    type: string
+  key:
+    required: true
+    type: string
+  source:
+    required: true
+    type: string
+  target:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Copy file via scp
+      uses: appleboy/scp-action@master
+      with:
+        host: ${{ inputs.host }}
+        username: ${{ inputs.username }}
+        key: ${{ inputs.key }}
+        source: ${{ inputs.source }}
+        target: ${{ inputs.target }}
diff --git a/.github/actions/ssh_action/action.yml b/.github/actions/ssh_action/action.yml
@@ -0,0 +1,25 @@
+name: Execute remote command action
+inputs:
+  host:
+    required: true
+    type: string
+  username:
+    required: true
+    type: string
+  key:
+    required: true
+    type: string
+  script:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Executing remote command
+      uses: appleboy/ssh-action@master
+      with:
+        host: ${{ inputs.host }}
+        username: ${{ inputs.username }}
+        key: ${{ inputs.key }}
+        script: ${{ inputs.script }}
diff --git a/.github/workflows/build_deploy_backend.yml b/.github/workflows/build_deploy_backend.yml
@@ -13,73 +13,40 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  build_and_push:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        service: ['web', 'celery', 'media']
     steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Checkout files
-        uses: actions/checkout@v2
-
-      - name: Build and push django api
-        id: docker_build_web
-        uses: docker/build-push-action@v2
-        with:
-          context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/web/prod/Dockerfile
-          push: true
-          tags: unstructuredstudio/zubhub-services_web:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Build and push celery worker
-        id: docker_build_celery
-        uses: docker/build-push-action@v2
-        with:
-          context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/celery/prod/Dockerfile
-          push: true
-          tags: unstructuredstudio/zubhub-services_celery:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+      - uses: unstructuredstudio/zubhub/.github/actions/checkout@master
 
-      - name: Build and push media service
-        id: docker_build_media
-        uses: docker/build-push-action@v2
+      - name: Build and push ${{ matrix.service }}
+        uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master
         with:
+          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
           context: ./zubhub_backend/
-          file: ./zubhub_backend/compose/media/prod/Dockerfile
+          file: ./zubhub_backend/compose/${{ matrix.service }}/prod/Dockerfile
           push: true
-          tags: unstructuredstudio/zubhub-services_media:latest
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+          tags: unstructuredstudio/zubhub-services_${{ matrix.service }}:latest
 
   deploy:
-    needs: build
+    needs: build_and_push
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: unstructuredstudio/zubhub/.github/actions/checkout@master
 
-      - name: Copy file via scp
-        uses: appleboy/scp-action@master
+      - uses: unstructuredstudio/zubhub/.github/actions/scp_action@master
         with:
           host: ${{ secrets.DO_BACKEND_HOST }}
           username: ${{ secrets.DO_BACKEND_USERNAME }}
           key: ${{ secrets.DO_SSHKEY }}
           source: "."
           target: "/home/zubhub-services/zubhub"
 
-      - name: Executing remote  command
-        uses: appleboy/ssh-action@master
+      - uses: unsctructuredstudio/zubhub/.github/actions/ssh_action@master
         with:
           host: ${{ secrets.DO_BACKEND_HOST }}
           username: ${{ secrets.DO_BACKEND_USERNAME }}

diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml
@@ -11,8 +11,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - run: |

diff --git a/.github/workflows/build_deploy_frontend.yml b/.github/workflows/build_deploy_frontend.yml
@@ -17,21 +17,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Checkout files
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./zubhub_frontend/zubhub/
           file: ./zubhub_frontend/zubhub/Dockerfile.prod
@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
 
       - name: Copy file via scp
         uses: appleboy/scp-action@master

diff --git a/.github/workflows/build_locust.yml b/.github/workflows/build_locust.yml
@@ -23,21 +23,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Checkout files
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build and push locust
         id: docker_build_locust
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: ./locust/
           file: ./locust/Dockerfile

diff --git a/.github/workflows/create_destroy_test_vm.yaml b/.github/workflows/create_destroy_test_vm.yaml
@@ -47,7 +47,7 @@ jobs:
 # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
 # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
       - name: Checkout source branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.CHECKOUT_COMMIT_REF }}
           repository: ${{ env.CHECKOUT_REPO }}
@@ -148,7 +148,7 @@ jobs:
 # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git
 # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch.
       - name: Checkout source branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.CHECKOUT_COMMIT_REF }}
           repository: ${{ env.CHECKOUT_REPO }}
@@ -175,20 +175,20 @@ jobs:
           EOF
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push ${{ matrix.service }}
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           # for frontend, context is ./zubhub_frontend/zubhub/
           # for backend services, context is ./zubhub_backend/

diff --git a/zubhub_backend/docker-compose.prod.yml b/zubhub_backend/docker-compose.prod.yml
@@ -95,20 +95,20 @@ services:
     depends_on:
       - rabbitmq
 
-  prometheus:
-    image: prom/prometheus
-    command:
-      - --config.file=/etc/prometheus/prometheus.yml
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-      placement:
-        max_replicas_per_node: 1
-        constraints:
-          - "node.role==manager"
-    volumes:
-      - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+  # prometheus:
+  #   image: prom/prometheus
+  #   command:
+  #     - --config.file=/etc/prometheus/prometheus.yml
+  #   deploy:
+  #     replicas: 1
+  #     restart_policy:
+  #       condition: on-failure
+  #     placement:
+  #       max_replicas_per_node: 1
+  #       constraints:
+  #         - "node.role==manager"
+  #   volumes:
+  #     - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro
 
 secrets:
   zubhub_services_secrets: