update image build yaml (#529)

Signed-off-by: chensuyue <[email protected]>
Signed-off-by: zepan <[email protected]>

.github/workflows/_comps-workflow.yml

@@ -20,10 +20,17 @@ on:
         default: true
         required: false
         type: boolean
-      # scan:
-      #   default: true
-      #   required: false
-      #   type: boolean
+      test:
+        default: true
+        description: "Test comps with docker compose"
+        required: false
+        type: boolean
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
+
 jobs:
   ####################################################################################################
   # Image Build
@@ -39,82 +46,38 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Clone required Repo
+        id: get-yaml-path
         run: |
           cd ${{ github.workspace }}/.github/workflows/docker/compose
           # service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
-          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
-          echo ${docker_compose_yml}
+          if [[ "${{ inputs.mode }}" == "CD" ]]; then
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
+          else
+            docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          fi
+          echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
           if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
           fi
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
           fi
-          # echo "service=$service" >> $GITHUB_ENV
 
       - name: Build Image
         if: ${{ fromJSON(inputs.build) }}
         uses: opea-project/validation/actions/image-build@main
         with:
-          work_dir: ${{ github.workspace }}/
-          docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          work_dir: ${{ github.workspace }}
+          docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
           registry: ${OPEA_IMAGE_REPO}opea
           tag: ${{ inputs.tag }}
-  # ####################################################################################################
-  # # Trivy Scan
-  # ####################################################################################################
-  #   get-image-list:
-  #     needs: [build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
-  #     runs-on: ubuntu-latest
-  #     outputs:
-  #       matrix: ${{ steps.scan-matrix.outputs.matrix }}
-  #     steps:
-  #       - name: Checkout out Repo
-  #         uses: actions/checkout@v4
-
-  #       - name: Set Matrix
-  #         id: scan-matrix
-  #         run: |
-  #           pip install yq
-  #           compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
-  #           echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
-
-  #   scan-images:
-  #     needs: [get-image-list, build-images]
-  #     if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
-  #     runs-on: "docker-build-${{ inputs.node }}"
-  #     strategy:
-  #       matrix:
-  #         image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
-  #       fail-fast: false
-  #     steps:
-  #       - name: Pull Image
-  #         run: |
-  #           docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
-
-  #       - name: Scan Container
-  #         uses: opea-project/validation/actions/trivy-scan@main
-  #         with:
-  #           image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
-  #           output: ${{ matrix.image }}-scan.txt
-
-  #       - name: Cleanup
-  #         if: always()
-  #         run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
-
-  #       - uses: actions/[email protected]
-  #         with:
-  #           name: ${{ matrix.image }}-scan
-  #           path: ${{ matrix.image }}-scan.txt
-  #           overwrite: true
 
   ####################################################################################################
   # Docker Compose Test
   ####################################################################################################
   test-service-compose:
     needs: [build-images]
+    if: ${{ fromJSON(inputs.test) }}
     uses: ./.github/workflows/_run-docker-compose.yml
     with:
       tag: ${{ inputs.tag }}

.github/workflows/docker/compose/agent-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  comps-agent-langchain:
+    build:
+      dockerfile: comps/agent/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest}

.github/workflows/docker/compose/chathistory-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  chathistory-mongo-server:
+    build:
+      dockerfile: comps/chathistory/mongo/docker/Dockerfile
+    image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose-cd.yaml

@@ -0,0 +1,25 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# others: dataprep-redis-llama-index,dataprep-on-ray-redis
+services:
+  dataprep-redis-llama-index:
+    build:
+      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
+  dataprep-on-ray-redis:
+    build:
+      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
+  dataprep-milvus:
+    build:
+      dockerfile: comps/dataprep/milvus/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest}
+  dataprep-pgvector:
+    build:
+      dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest}
+  dataprep-pinecone:
+    build:
+      dockerfile: comps/dataprep/pinecone/docker/Dockerfile
+    image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -13,11 +13,3 @@ services:
     build:
       dockerfile: comps/dataprep/qdrant/docker/Dockerfile
     image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
-  dataprep-redis-llama-index:
-    build:
-      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
-  dataprep-on-ray-redis:
-    build:
-      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
-    image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose-cd.yaml

@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest}
+  embedding-langchain-mosec:
+    build:
+      dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest}
+  embedding-tei-llama-index:
+    build:
+      dockerfile: comps/embeddings/llama_index/docker/Dockerfile
+    image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  guardrails-pii-detection:
+    build:
+      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose.yaml

@@ -9,7 +9,3 @@ services:
     build:
       dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
     image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest}
-  guardrails-pii-detection:
-    build:
-      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
-    image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/llms-compose-cd.yaml

@@ -0,0 +1,8 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  llm-native:
+    build:
+      dockerfile: comps/llms/text-generation/native/docker/Dockerfile
+    image: ${REGISTRY}opea/llm-native:${TAG:-latest}

.github/workflows/docker/compose/lvms-compose-cd.yaml

@@ -0,0 +1,23 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  lvm:
+    build:
+      dockerfile: comps/lvms/Dockerfile
+    image: ${REGISTRY}opea/lvm:${TAG:-latest}
+  # Xeon CPU
+  llava:
+    build:
+      dockerfile: comps/lvms/llava/Dockerfile
+    image: ${REGISTRY}opea/llava:${TAG:-latest}
+  # Gaudi2 HPU
+  llava_hpu:
+    build:
+      dockerfile: comps/lvms/llava/Dockerfile_hpu
+    image: ${REGISTRY}opea/llava_hpu:${TAG:-latest}
+  lvm-tgi:
+    build:
+      dockerfile: comps/lvms/Dockerfile_tgi
+    image: ${REGISTRY}opea/lvm-tgi:${TAG:-latest}

.github/workflows/docker/compose/nginx-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  nginx:
+    build:
+      dockerfile: comps/nginx/docker/Dockerfile
+    image: ${REGISTRY}opea/nginx:${TAG:-latest}

.github/workflows/docker/compose/prompt_registry-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  promptregistry-mongo-server:
+    build:
+      dockerfile: comps/prompt_registry/mongo/docker/Dockerfile
+    image: ${REGISTRY}opea/promptregistry-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose-cd.yaml

@@ -0,0 +1,16 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  reranking-fastrag:
+    build:
+      dockerfile: comps/reranks/fastrag/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-fastrag:${TAG:-latest}
+  reranking-langchain-mosec-endpoint:
+    build:
+      dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-langchain-mosec-endpoint:${TAG:-latest}
+  reranking-langchain-mosec:
+    build:
+      dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile
+    image: ${REGISTRY}opea/reranking-langchain-mosec:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -11,3 +11,19 @@ services:
     build:
       dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile
     image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
+  retriever-pgvector:
+    build:
+      dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest}
+  retriever-pinecone:
+    build:
+      dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-pinecone:${TAG:-latest}
+  retriever-milvus:
+    build:
+      dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-milvus:${TAG:-latest}
+  retriever-redis-llamaindex:
+    build:
+      dockerfile: comps/retrievers/llamaindex/docker/Dockerfile
+    image: ${REGISTRY}opea/retriever-redis-llamaindex:${TAG:-latest}

.github/workflows/manual-comps-test.yml

@@ -7,14 +7,29 @@ on:
     inputs:
       services:
         default: "asr"
-        description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
+        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,llms,lvms,nginx,prompt_registry,reranks,retrievers,tts,web_retrievers]"
         required: true
         type: string
       build:
         default: true
         description: "Build test required images for Comps"
         required: false
         type: boolean
+      test:
+        default: true
+        description: "Test comps with docker compose"
+        required: false
+        type: boolean
+      tag:
+        default: "comps"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      mode:
+        default: "CD"
+        description: "Whether the test range is CI or CD"
+        required: false
+        type: string
 
 permissions: read-all
 
@@ -40,6 +55,8 @@ jobs:
     uses: ./.github/workflows/_comps-workflow.yml
     with:
       service: ${{ matrix.service }}
-      tag: "comps"
+      tag: ${{ inputs.tag }}
       node: gaudi
+      mode: ${{ inputs.mode }}
+      test: ${{ inputs.test }}
     secrets: inherit

comps/nginx/docker/Dockerfile

@@ -6,15 +6,15 @@ FROM nginx:alpine
 
 RUN apk add --no-cache gettext
 
-COPY nginx.conf.template /etc/nginx/nginx.conf.template
+COPY comps/nginx/docker/nginx.conf.template /etc/nginx/nginx.conf.template
 
 ENV FRONTEND_SERVICE_IP=localhost
 ENV FRONTEND_SERVICE_PORT=5173
 ENV BACKEND_SERVICE_NAME=chatqna
 ENV BACKEND_SERVICE_IP=localhost
 ENV BACKEND_SERVICE_PORT=8888
 
-COPY start-nginx.sh /usr/local/bin/start-nginx.sh
+COPY comps/nginx/docker/start-nginx.sh /usr/local/bin/start-nginx.sh
 RUN chmod +x /usr/local/bin/start-nginx.sh
 
 CMD ["/usr/local/bin/start-nginx.sh"]

comps/retrievers/llamaindex/docker/Dockerfile

@@ -4,6 +4,7 @@
 FROM ubuntu:22.04
 
 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
+    python3-pip \
     libgl1-mesa-glx \
     libjemalloc-dev