Enhance CI/CD image build (opea-project#961)

Signed-off-by: chensuyue <[email protected]> Signed-off-by: Arthur Leung <[email protected]>
arcyleung · Nov 4, 2024 · 409c3b0 · 409c3b0
1 parent f051a41
commit 409c3b0
Show file tree

Hide file tree

Showing 15 changed files with 114 additions and 161 deletions.
diff --git a/.github/workflows/_example-workflow.yml b/.github/workflows/_example-workflow.yml
@@ -12,6 +12,10 @@ on:
       example:
         required: true
         type: string
+      services:
+        default: ""
+        required: false
+        type: string
       tag:
         default: "latest"
         required: false
@@ -77,6 +81,7 @@ jobs:
         with:
           work_dir: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build
           docker_compose_path: ${{ github.workspace }}/${{ inputs.example }}/docker_image_build/build.yaml
+          service_list: ${{ inputs.services }}
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
 
@@ -105,7 +110,6 @@ jobs:
       example: ${{ inputs.example }}
       hardware: ${{ inputs.node }}
       tag: ${{ inputs.tag }}
-      context: "CD"
     secrets: inherit
 
 ####################################################################################################

diff --git a/.github/workflows/_manifest-e2e.yml b/.github/workflows/_manifest-e2e.yml
@@ -20,11 +20,6 @@ on:
         description: "Tag to apply to images, default is latest"
         required: false
         type: string
-      context:
-        default: "CI"
-        description: "CI or CD"
-        required: false
-        type: string
 
 jobs:
   manifest-test:
@@ -51,7 +46,7 @@ jobs:
 
       - name: Set variables
         run: |
-          echo "IMAGE_REPO=$OPEA_IMAGE_REPO" >> $GITHUB_ENV
+          echo "IMAGE_REPO=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
           echo "IMAGE_TAG=${{ inputs.tag }}" >> $GITHUB_ENV
           lower_example=$(echo "${{ inputs.example }}" | tr '[:upper:]' '[:lower:]')
           echo "NAMESPACE=$lower_example-$(tr -dc a-z0-9 </dev/urandom | head -c 16)" >> $GITHUB_ENV
@@ -60,7 +55,6 @@ jobs:
           echo "continue_test=true" >> $GITHUB_ENV
           echo "should_cleanup=false" >> $GITHUB_ENV
           echo "skip_validate=true" >> $GITHUB_ENV
-          echo "CONTEXT=${{ inputs.context }}" >> $GITHUB_ENV
           echo "NAMESPACE=$NAMESPACE"
 
       - name: Kubectl install

diff --git a/.github/workflows/manual-image-build.yml b/.github/workflows/manual-image-build.yml
@@ -0,0 +1,57 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build specific images on manual event
+on:
+  workflow_dispatch:
+    inputs:
+      nodes:
+        default: "gaudi,xeon"
+        description: "Hardware to run test"
+        required: true
+        type: string
+      example:
+        default: "ChatQnA"
+        description: 'Build images belong to which example?'
+        required: true
+        type: string
+      services:
+        default: "chatqna,chatqna-without-rerank"
+        description: 'Service list to build'
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      opea_branch:
+        default: "main"
+        description: 'OPEA branch for image build'
+        required: false
+        type: string
+jobs:
+  get-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      nodes: ${{ steps.get-matrix.outputs.nodes }}
+    steps:
+      - name: Create Matrix
+        id: get-matrix
+        run: |
+          nodes=($(echo ${{ inputs.nodes }} | tr ',' ' '))
+          nodes_json=$(printf '%s\n' "${nodes[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "nodes=$nodes_json" >> $GITHUB_OUTPUT
+
+  image-build:
+    needs: job1
+    strategy:
+      matrix:
+        node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
+      fail-fast: false
+    uses: ./.github/workflows/_example-workflow.yml
+    with:
+      node: ${{ matrix.node }}
+      example: ${{ inputs.example }}
+      services: ${{ inputs.services }}
+    secrets: inherit
diff --git a/.github/workflows/pr-bum_list_check.yml b/.github/workflows/pr-bum_list_check.yml
diff --git a/.github/workflows/push-image-build.yml b/.github/workflows/push-image-build.yml
@@ -23,12 +23,10 @@ jobs:
   image-build:
     needs: job1
     strategy:
-      matrix:
-        example: ${{ fromJSON(needs.job1.outputs.run_matrix).include.*.example }}
-        node: ["gaudi","xeon"]
+      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
       fail-fast: false
     uses: ./.github/workflows/_example-workflow.yml
     with:
-      node: ${{ matrix.node }}
+      node: ${{ matrix.hardware }}
       example: ${{ matrix.example }}
     secrets: inherit
diff --git a/ChatQnA/tests/test_manifest_on_gaudi.sh b/ChatQnA/tests/test_manifest_on_gaudi.sh
@@ -6,7 +6,7 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 ROLLOUT_TIMEOUT_SECONDS="1800s"
@@ -15,15 +15,10 @@ KUBECTL_TIMEOUT_SECONDS="60s"
 function init_chatqna() {
     # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/chatqna:latest#image: \"opea/chatqna:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/ChatQnA/tests/test_manifest_on_xeon.sh b/ChatQnA/tests/test_manifest_on_xeon.sh
@@ -6,7 +6,7 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 ROLLOUT_TIMEOUT_SECONDS="1800s"
@@ -15,15 +15,10 @@ KUBECTL_TIMEOUT_SECONDS="60s"
 function init_chatqna() {
     # replace the mount dir "path: /mnt/opea-models" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/chatqna:latest#image: \"opea/chatqna:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/CodeGen/tests/test_manifest_on_gaudi.sh b/CodeGen/tests/test_manifest_on_gaudi.sh
@@ -6,22 +6,17 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 function init_codegen() {
     # executed under path manifest/codegen/xeon
     # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/codegen:latest#image: \"opea/codegen:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/CodeGen/tests/test_manifest_on_xeon.sh b/CodeGen/tests/test_manifest_on_xeon.sh
@@ -6,22 +6,17 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 function init_codegen() {
     # executed under path manifest/codegen/xeon
     # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/codegen:latest#image: \"opea/codegen:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/CodeTrans/tests/test_manifest_on_gaudi.sh b/CodeTrans/tests/test_manifest_on_gaudi.sh
@@ -6,22 +6,17 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 function init_codetrans() {
     # executed under path manifest/codetrans/xeon
     # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/codetrans:latest#image: \"opea/codetrans:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/CodeTrans/tests/test_manifest_on_xeon.sh b/CodeTrans/tests/test_manifest_on_xeon.sh
@@ -6,22 +6,17 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 function init_codetrans() {
     # executed under path manifest/codetrans/xeon
     # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/codetrans:latest#image: \"opea/codetrans:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }

diff --git a/DocSum/tests/test_manifest_on_gaudi.sh b/DocSum/tests/test_manifest_on_gaudi.sh
@@ -6,22 +6,17 @@ set -xe
 USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 MOUNT_DIR=/home/$USER_ID/.cache/huggingface/hub
-IMAGE_REPO=${IMAGE_REPO:-}
+IMAGE_REPO=${IMAGE_REPO:-opea}
 IMAGE_TAG=${IMAGE_TAG:-latest}
 
 function init_docsum() {
     # executed under path manifest/docsum/xeon
     # replace the mount dir "path: /mnt/model" with "path: $CHART_MOUNT"
     find . -name '*.yaml' -type f -exec sed -i "s#path: /mnt/opea-models#path: $MOUNT_DIR#g" {} \;
-    if [ $CONTEXT == "CI" ]; then
-        # replace megaservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/docsum:latest#image: \"opea/docsum:${IMAGE_TAG}#g" {} \;
-    else
-        # replace microservice image tag
-        find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
-    fi
-    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/opea/"
-    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}opea/#g" {} \;
+    # replace microservice image tag
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/\(.*\):latest#image: \"opea/\1:${IMAGE_TAG}#g" {} \;
+    # replace the repository "image: opea/*" with "image: $IMAGE_REPO/"
+    find . -name '*.yaml' -type f -exec sed -i "s#image: \"opea/*#image: \"${IMAGE_REPO}/#g" {} \;
     # set huggingface token
     find . -name '*.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#$(cat /home/$USER_ID/.cache/huggingface/token)#g" {} \;
 }