Merge branch 'main' into arc_vllm

.github/CODEOWNERS

@@ -1,3 +1,6 @@
+# Code owners will review PRs within their respective folders. 
+# Typically, ownership is organized at the second-level subdirectory under the homepage
+/*/ [email protected]
 /comps/agent/ [email protected]
 /comps/finetuning/ [email protected]
 /comps/guardrails/ [email protected]

.github/workflows/_comps-workflow.yml

@@ -66,9 +66,6 @@ jobs:
           fi
 
           cd ${{ github.workspace }}
-          if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
-              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 && cd ../
-          fi
           if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
               git clone https://github.com/vllm-project/vllm.git vllm-openvino
           fi

.github/workflows/docker/compose/lvms-compose-cd.yaml

@@ -13,16 +13,11 @@ services:
       dockerfile: comps/lvms/llava/Dockerfile
     image: ${REGISTRY:-opea}/llava:${TAG:-latest}
   # Gaudi2 HPU
-  llava_hpu:
+  llava-hpu:
     build:
       dockerfile: comps/lvms/llava/Dockerfile_hpu
     image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest}
   lvm-tgi:
     build:
       dockerfile: comps/lvms/Dockerfile_tgi
     image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest}
-  llava-tgi:
-    build:
-      context: tgi-gaudi
-      dockerfile: Dockerfile
-    image: ${REGISTRY:-opea}/llava-tgi:${TAG:-latest}

.github/workflows/docker/compose/vectorstores-compose-cd.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# this file should be run in the root of the repo
+services:
+  vectorstore-pathway:
+    build:
+      dockerfile: comps/vectorstores/langchain/pathway
+    image: ${REGISTRY:-opea}/vectorstore-pathway:${TAG:-latest}

.github/workflows/manual-docker-publish.yml

@@ -6,8 +6,8 @@ on:
   workflow_dispatch:
     inputs:
       services:
-        default: "asr"
-        description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
+        default: ""
+        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,knowledgegraphs,llms,lvms,nginx,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
         required: false
         type: string
       images:

.github/workflows/mix-megaservice-test.yml

@@ -68,7 +68,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /GenAIComps && pip install -r requirements.txt && python setup.py install"
+          docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /GenAIComps && pip install --no-cache-dir -r requirements.txt && python setup.py install"
 
       - name: Run UT
         run: |

.github/workflows/pr-dockerfile-path-scan.yaml

@@ -118,3 +118,34 @@ jobs:
             echo "Please modify the corresponding README in GenAIExamples repo and ask [email protected] for final confirmation."
             exit 1
           fi
+
+  Dockerfile-addition-detection-in-GenAIComps:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check if the Dockerfile has been added
+        run: |
+          set -e
+          shopt -s globstar
+          cd ${{github.workspace}}
+          is_use="FALSE"
+          used_files=""
+          merged_commit=$(git log -1 --format='%H')
+          changed_files="$(git diff --name-status --diff-filter=A ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)"
+          if [ -n "$changed_files" ]; then
+            for file in $changed_files; do
+              if find "${{github.workspace}}/.github/workflows/docker/compose/" -name "*$(echo "$file" | awk -F '/' '{print $2}')*" |grep -q .; then
+                echo "Please check if the added $file is included in the yaml under path ${{github.workspace}}/.github/workflows/docker/compose/."
+              else
+                echo "Please create a new compose file named service_name-compose-cd.yaml in ${{github.workspace}}/.github/workflows/docker/compose/ for $file and fill it in."
+              fi
+            done
+            exit 1
+          fi

.github/workflows/scripts/codeScan/bandit.sh

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 source /GenAIComps/.github/workflows/scripts/change_color
-pip install bandit==1.7.8
+pip install --no-cache-dir bandit==1.7.8
 log_dir=/GenAIComps/.github/workflows/scripts/codeScan
 python -m bandit -r -lll -iii /GenAIComps > ${log_dir}/bandit.log
 exit_code=$?

.github/workflows/scripts/freeze_requirements.sh

@@ -55,7 +55,7 @@ function check_branch_name() {
 
 function main() {
     check_branch_name
-    echo "::group::pip install pip-tools" && pip install pip-tools --upgrade && echo "::endgroup::"
+    echo "::group::pip install --no-cache-dir pip-tools" && pip install --no-cache-dir pip-tools --upgrade && echo "::endgroup::"
     export -f freeze
     find . -name "requirements.txt" | xargs -n 1 -I {} bash -c 'freeze "$@"' _ {}
 }

comps/__init__.py

@@ -13,6 +13,7 @@
     LLMParamsDoc,
     SearchedDoc,
     SearchedMultimodalDoc,
+    LVMSearchedMultimodalDoc,
     RerankedDoc,
     TextDoc,
     RAGASParams,
@@ -45,6 +46,7 @@
     RetrievalToolGateway,
     FaqGenGateway,
     VisualQnAGateway,
+    MultimodalRAGWithVideosGateway,
 )
 
 # Telemetry

comps/agent/langchain/README.md

@@ -33,34 +33,34 @@ The tools are registered with a yaml file. We support the following types of too
 
 Currently we have implemented OpenAI chat completion compatible API for agents. We are working to support OpenAI assistants APIs.
 
-# 🚀2. Start Agent Microservice
+## 🚀2. Start Agent Microservice
 
-## 2.1 Option 1: with Python
+### 2.1 Option 1: with Python
 
-### 2.1.1 Install Requirements
+#### 2.1.1 Install Requirements
 
 ```bash
 cd comps/agent/langchain/
 pip install -r requirements.txt
 ```
 
-### 2.1.2 Start Microservice with Python Script
+#### 2.1.2 Start Microservice with Python Script
 
 ```bash
 cd comps/agent/langchain/
 python agent.py
 ```
 
-## 2.2 Option 2. Start Microservice with Docker
+### 2.2 Option 2. Start Microservice with Docker
 
-### 2.2.1 Build Microservices
+#### 2.2.1 Build Microservices
 
 ```bash
 cd GenAIComps/ # back to GenAIComps/ folder
 docker build -t opea/comps-agent-langchain:latest -f comps/agent/langchain/docker/Dockerfile .
 ```
 
-### 2.2.2 Start microservices
+#### 2.2.2 Start microservices
 
 ```bash
 export ip_address=$(hostname -I | awk '{print $1}')
@@ -87,7 +87,7 @@ docker logs comps-langchain-agent-endpoint
 > docker run --rm --runtime=runc --name="comps-langchain-agent-endpoint" -v ./comps/agent/langchain/:/home/user/comps/agent/langchain/ -p 9090:9090 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e model=${model} -e ip_address=${ip_address} -e strategy=react -e llm_endpoint_url=http://${ip_address}:8080 -e llm_engine=tgi -e recursion_limit=5 -e require_human_feedback=false -e tools=/home/user/comps/agent/langchain/tools/custom_tools.yaml opea/comps-agent-langchain:latest
 > ```
 
-# 🚀 3. Validate Microservice
+## 🚀 3. Validate Microservice
 
 Once microservice starts, user can use below script to invoke.
 
@@ -104,7 +104,7 @@ data: [DONE]
 
 ```
 
-# 🚀 4. Provide your own tools
+## 🚀 4. Provide your own tools
 
 - Define tools
 
@@ -180,7 +180,7 @@ data: 'The weather information in Austin is not available from the Open Platform
 data: [DONE]
 ```
 
-# 5. Customize agent strategy
+## 5. Customize agent strategy
 
 For advanced developers who want to implement their own agent strategies, you can add a separate folder in `src\strategy`, implement your agent by inherit the `BaseAgent` class, and add your strategy into the `src\agent.py`. The architecture of this agent microservice is shown in the diagram below as a reference.
 ![Architecture Overview](agent_arch.jpg)

comps/asr/Dockerfile

@@ -15,7 +15,7 @@ COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
     if [ "${ARCH}" = "cpu" ]; then \
-        pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu ; \
+        pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu ; \
         pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r /home/user/comps/asr/requirements.txt ; \
     else \
         pip install --no-cache-dir -r /home/user/comps/asr/requirements.txt ; \
@@ -25,4 +25,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user
 
 WORKDIR /home/user/comps/asr
 
-ENTRYPOINT ["python", "asr.py"]
+ENTRYPOINT ["python", "asr.py"]

comps/asr/whisper/Dockerfile

@@ -23,7 +23,7 @@ USER user
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r /home/user/comps/asr/requirements.txt && \
     if [ "${ARCH}" = "cpu" ]; then \
-        pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu ; \
+        pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu ; \
         pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r /home/user/comps/asr/requirements.txt ; \
     else \
         pip install --no-cache-dir -r /home/user/comps/asr/requirements.txt ; \

comps/asr/whisper/Dockerfile_hpu

@@ -25,7 +25,7 @@ USER user
 # Install requirements and optimum habana
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r /home/user/comps/asr/requirements.txt && \
-    pip install optimum[habana]
+    pip install --no-cache-dir optimum[habana]
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 

comps/cores/mega/constants.py

@@ -42,6 +42,7 @@ class MegaServiceEndpoint(Enum):
     CODE_TRANS = "/v1/codetrans"
     DOC_SUMMARY = "/v1/docsum"
     SEARCH_QNA = "/v1/searchqna"
+    MULTIMODAL_RAG_WITH_VIDEOS = "/v1/mmragvideoqna"
     TRANSLATION = "/v1/translation"
     RETRIEVALTOOL = "/v1/retrievaltool"
     FAQ_GEN = "/v1/faqgen"