refactor build image build and push workflow (#385)

* refactor build image build and push workflow

Signed-off-by: chensuyue <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/docker/compose/asr-compose.yaml

@@ -0,0 +1,17 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  asr:
+    build:
+      dockerfile: comps/asr/Dockerfile
+    image: ${REGISTRY}opea/asr:${TAG:-latest}
+  whisper:
+    build:
+      dockerfile: comps/asr/whisper/Dockerfile
+    image: ${REGISTRY}opea/whisper:${TAG:-latest}
+  whisper-gaudi:
+    build:
+      dockerfile: comps/asr/whisper/Dockerfile_hpu
+    image: ${REGISTRY}opea/whisper-gaudi:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -0,0 +1,12 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  dataprep-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis:${TAG:-latest}
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  embedding-tei:
+    build:
+      dockerfile: comps/embeddings/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-tei:${TAG:-latest}

.github/workflows/docker/compose/llms-compose.yaml

@@ -0,0 +1,17 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  llm-tgi:
+    build:
+      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-tgi:${TAG:-latest}
+  llm-docsum-tgi:
+    build:
+      dockerfile: comps/llms/summarization/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-docsum-tgi:${TAG:-latest}
+  llm-faqgen-tgi:
+    build:
+      dockerfile: comps/llms/faq-generation/tgi/Dockerfile
+    image: ${REGISTRY}opea/llm-faqgen-tgi:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  reranking-tei:
+    build:
+      dockerfile: comps/reranks/tei/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-tei:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  retriever-redis:
+    build:
+      dockerfile: comps/retrievers/langchain/redis/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-redis:${TAG:-latest}

.github/workflows/docker/compose/tts-compose.yaml

@@ -0,0 +1,17 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  asr:
+    build:
+      dockerfile: comps/tts/Dockerfile
+    image: ${REGISTRY}opea/asr:${TAG:-latest}
+  speecht5:
+    build:
+      dockerfile: comps/tts/speecht5/Dockerfile
+    image: ${REGISTRY}opea/speecht5:${TAG:-latest}
+  speecht5-gaudi:
+    build:
+      dockerfile: comps/tts/speecht5/Dockerfile_hpu
+    image: ${REGISTRY}opea/speecht5-gaudi:${TAG:-latest}

.github/workflows/docker/compose/web_retrievers-compose.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  web-retriever-chroma:
+    build:
+      dockerfile: comps/web_retrievers/langchain/chroma/docker/Dockerfile
+    image: ${REGISTRY}opea/web-retriever-chroma:${TAG:-latest}

.github/workflows/image-build-on-manual.yml

@@ -0,0 +1,60 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build latest images on manual event
+
+on:
+  workflow_dispatch:
+    inputs:
+      services:
+        default: "asr,dataprep,retrievers"
+        description: "List of services to build"
+        required: true
+        type: string
+      tag:
+        default: "latest"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      node:
+        default: "docker-build-gaudi"
+        description: "Node to run the build on [docker-build-xeon, docker-build-gaudi]"
+        required: true
+        type: string
+
+jobs:
+  get-build-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.get-services.outputs.services }}
+    steps:
+      - name: Get test Services
+        id: get-services
+        run: |
+          services=$(echo ${{ github.event.inputs.services }} | jq -R 'split(",") | map(gsub(" "; ""))')
+          echo "services=$services" >> $GITHUB_OUTPUT
+
+  image-build:
+    needs: get-build-matrix
+    strategy:
+      matrix:
+        service: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
+    runs-on: ${{ github.event.inputs.node }}
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Build image
+        env:
+          service: ${{ matrix.service }}
+        uses: opea-project/validation/actions/image-build/action.yml@main
+        with:
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}
+        secrets: inherit

.github/workflows/image-build-on-push.yml

@@ -17,13 +17,59 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  job1:
-    uses: ./.github/workflows/reuse-get-test-matrix.yml
+  get-build-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.get-services.outputs.services }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get Test Services
+        id: get-services
+        run: |
+          base_commit=$(git rev-parse HEAD~1)
+          merged_commit=$(git log -1 --format='%H')
+          changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true
+          services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u | jq -R '.' | jq -sc '.') || true
+          echo "services=$services"
+          echo "services=$services" >> $GITHUB_OUTPUT
 
   image-build:
-    needs: job1
+    needs: get-build-matrix
     strategy:
-      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
-    uses: ./.github/workflows/reuse-image-build.yml
-    with:
-      micro_service: "${{ matrix.service }}"
+      matrix:
+        service: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
+        node: [docker-build-xeon, docker-build-gaudi]
+    runs-on: ${{ matrix.node }}
+    continue-on-error: true
+    steps:
+      - name: Clean up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Check Docker Compose File Exists
+        env:
+          service: ${{ matrix.service }}
+        run: |
+          docker_compose_path="${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml"
+          if [ -e $docker_compose_path ]; then
+            echo "file_exists=true" >> $GITHUB_ENV
+            echo "docker_compose_path=${docker_compose_path}" >> $GITHUB_ENV
+          else
+            echo "file_exists=false" >> $GITHUB_ENV
+            echo "docker_compose_path=${docker_compose_path} for this service does not exist, so skipping image build for this service!!!"
+          fi
+
+      - name: Build Image
+        if: env.file_exists == 'true'
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ env.docker_compose_path }}
+          registry: ${OPEA_IMAGE_REPO}

.github/workflows/scripts/docker_images_build_push.sh

@@ -42,7 +42,7 @@ case ${micro_service} in
     "embeddings/langchain")
         IMAGE_NAME="opea/embedding-tei"
         ;;
-    "retrievers/langchain")
+    "retrievers/langchain/redis")
         IMAGE_NAME="opea/retriever-redis"
         ;;
     "reranks/tei")