enhance docker image build (#409)

Signed-off-by: chensuyue <[email protected]>

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -10,3 +10,7 @@ services:
     build:
       dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
     image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}

.github/workflows/image-build-on-manual.yml

@@ -7,18 +7,18 @@ on:
   workflow_dispatch:
     inputs:
       services:
-        default: "asr,dataprep,retrievers"
-        description: "List of services to build"
+        default: "asr"
+        description: "List of services to build including [asr,dataprep,embeddings,llms,reranks,retrievers,tts,web_retrievers]"
         required: true
         type: string
       tag:
         default: "latest"
         description: "Tag to apply to images"
         required: true
         type: string
-      node:
-        default: "docker-build-gaudi"
-        description: "Node to run the build on [docker-build-xeon, docker-build-gaudi]"
+      nodes:
+        default: "docker-build-xeon,docker-build-gaudi"
+        description: "List of nodes to run the build including [docker-build-xeon,docker-build-gaudi]"
         required: true
         type: string
 
@@ -27,19 +27,26 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       services: ${{ steps.get-services.outputs.services }}
+      nodes: ${{ steps.get-services.outputs.nodes }}
     steps:
       - name: Get test Services
         id: get-services
         run: |
-          services=$(echo ${{ github.event.inputs.services }} | jq -R 'split(",") | map(gsub(" "; ""))')
+          set -x
+          service_list=($(echo ${{ github.event.inputs.services }} | tr ',' ' '))
+          services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
           echo "services=$services" >> $GITHUB_OUTPUT
+          node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
+          nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
+          echo "nodes=$nodes" >> $GITHUB_OUTPUT
 
   image-build:
     needs: get-build-matrix
     strategy:
       matrix:
         service: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
-    runs-on: ${{ github.event.inputs.node }}
+        node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
+    runs-on: ${{ matrix.node }}
     continue-on-error: true
     steps:
       - name: Clean Up Working Directory
@@ -52,9 +59,9 @@ jobs:
       - name: Build image
         env:
           service: ${{ matrix.service }}
-        uses: opea-project/validation/actions/image-build/action.yml@main
+        uses: opea-project/validation/actions/image-build@main
         with:
           work_dir: ${{ github.workspace }}
           docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
           registry: ${OPEA_IMAGE_REPO}
-        secrets: inherit
+          tag: ${{ github.event.inputs.tag }}

.github/workflows/image-build-on-schedule.yml

@@ -0,0 +1,62 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build latest images on schedule event
+
+on:
+  schedule:
+    - cron: "0 17 * * *" # 5:00 PM UTC every day, 1:00 AM CST every day
+
+env:
+  tag: "latest"
+
+jobs:
+  check-build:
+    runs-on: ubuntu-latest
+    outputs:
+      run_build: ${{ steps.get-changes.outputs.run_build }}
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check if build is needed
+        id: get-changes
+        run: |
+          set -x
+          changed_files=$(git log --since="24 hours ago" --name-only --pretty=format: | grep comps/cores | grep -vE '*.md') || true
+          if [ -z "$changed_files" ]; then
+            echo "No changes in the last 24 hours for the 'comps/cores' directory"
+            echo "run_build=false" >> $GITHUB_OUTPUT
+          else
+            echo "Changes detected in the last 24 hours for the 'comps/cores' directory"
+            echo "run_build=true" >> $GITHUB_OUTPUT
+          fi
+
+  image-build:
+    needs: check-build
+    if: ${{ needs.check-build.outputs.run_build == 'true' }}
+    strategy:
+      matrix:
+        service: ["asr", "dataprep", "embeddings", "llms", "reranks", "retrievers", "tts", "web_retrievers"]
+        node: ["docker-build-xeon", "docker-build-gaudi"]
+    runs-on: ${{ matrix.node }}
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Build image
+        env:
+          service: ${{ matrix.service }}
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}
+          tag: ${tag}