GenAIComps microservices refactor

.github/workflows/docker/compose/3rd_parties-compose.yaml

@@ -0,0 +1,12 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+services:
+  embedding-mosec:
+    build:
+      dockerfile: comps/3rd_parties/mosec/deployment/docker/Dockerfile
+    image: ${REGISTRY:-opea}/embedding-mosec:${TAG:-latest}
+  nginx:
+    build:
+      dockerfile: comps/3rd_parties/nginx/deployment/docker/Dockerfile
+    image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -3,6 +3,10 @@
 
 # this file should be run in the root of the repo
 services:
+  dataprep:
+    build:
+      dockerfile: comps/dataprep/src/Dockerfile
+    image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}
   dataprep-redis:
     build:
       dockerfile: comps/dataprep/redis/langchain/Dockerfile

.github/workflows/docker/compose/embeddings-compose.yaml

@@ -5,41 +5,29 @@
 services:
   embedding-tei:
     build:
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
   embedding-multimodal-clip:
     build:
-      dockerfile: comps/embeddings/multimodal_clip/Dockerfile
+      dockerfile: comps/embeddings/src/integrations/dependency/clip/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal-clip:${TAG:-latest}
   embedding-multimodal-bridgetower:
     build:
-      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile
+      dockerfile: comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower:${TAG:-latest}
   embedding-multimodal:
     build:
-      dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest}
-  embedding-langchain-mosec-endpoint:
-    build:
-      dockerfile: comps/embeddings/mosec/langchain/dependency/Dockerfile
-    image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest}
   embedding-langchain-mosec:
     build:
-      dockerfile: comps/embeddings/mosec/langchain/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest}
-  embedding-tei-llama-index:
-    build:
-      dockerfile: comps/embeddings/tei/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
   embedding-multimodal-bridgetower-gaudi:
     build:
-      dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu
+      dockerfile: comps/embeddings/src/integrations/dependency/bridgetower/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower-gaudi:${TAG:-latest}
   embedding-predictionguard:
     build:
-      dockerfile: comps/embeddings/predictionguard/Dockerfile
+      dockerfile: comps/embeddings/src/Dockerfile
     image: ${REGISTRY:-opea}/embedding-predictionguard:${TAG:-latest}
-  embedding-reranking-local:
-    build:
-      dockerfile: comps/embeddings/tei/langchain/Dockerfile.dynamic_batching
-    image: ${REGISTRY:-opea}/embedding-reranking-local:${TAG:-latest}

.github/workflows/docker/compose/image2image-compose.yaml

@@ -5,9 +5,9 @@
 services:
   image2image:
     build:
-      dockerfile: comps/image2image/Dockerfile
+      dockerfile: comps/image2image/src/Dockerfile
     image: ${REGISTRY:-opea}/image2image:${TAG:-latest}
   image2image-gaudi:
     build:
-      dockerfile: comps/image2image/Dockerfile.intel_hpu
+      dockerfile: comps/image2image/src/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/image2image-gaudi:${TAG:-latest}

.github/workflows/docker/compose/image2video-compose.yaml

@@ -5,13 +5,13 @@
 services:
   image2video:
     build:
-      dockerfile: comps/image2video/Dockerfile
+      dockerfile: comps/image2video/src/Dockerfile
     image: ${REGISTRY:-opea}/image2video:${TAG:-latest}
   svd:
     build:
-      dockerfile: comps/image2video/dependency/Dockerfile
+      dockerfile: comps/image2video/src/dependency/Dockerfile
     image: ${REGISTRY:-opea}/svd:${TAG:-latest}
   svd-gaudi:
     build:
-      dockerfile: comps/image2video/dependency/Dockerfile.intel_hpu
+      dockerfile: comps/image2video/src/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/svd-gaudi:${TAG:-latest}

.github/workflows/docker/compose/llms-compose.yaml

@@ -5,7 +5,7 @@
 services:
   llm-tgi:
     build:
-      dockerfile: comps/llms/text-generation/tgi/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   llm-ollama:
     build:
@@ -21,7 +21,7 @@ services:
     image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest}
   llm-vllm:
     build:
-      dockerfile: comps/llms/text-generation/vllm/langchain/Dockerfile
+      dockerfile: comps/llms/src/text-generation/Dockerfile
     image: ${REGISTRY:-opea}/llm-vllm:${TAG:-latest}
   llm-native:
     build:
@@ -50,10 +50,6 @@ services:
     build:
       dockerfile: comps/llms/utils/lm-eval/Dockerfile
     image: ${REGISTRY:-opea}/llm-eval:${TAG:-latest}
-  llm-vllm-llamaindex:
-    build:
-      dockerfile: comps/llms/text-generation/vllm/llama_index/Dockerfile
-    image: ${REGISTRY:-opea}/llm-vllm-llamaindex:${TAG:-latest}
   llm-textgen-predictionguard:
     build:
       dockerfile: comps/llms/text-generation/predictionguard/Dockerfile

.github/workflows/docker/compose/nginx-compose.yaml

@@ -5,5 +5,5 @@
 services:
   nginx:
     build:
-      dockerfile: comps/nginx/Dockerfile
+      dockerfile: comps/3rd_parties/nginx/deployment/docker/Dockerfile
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

.github/workflows/docker/compose/ragas-compose.yaml

@@ -3,5 +3,5 @@
 services:
   ragas:
     build:
-      dockerfile: comps/ragas/tgi/langchain/Dockerfile
+      dockerfile: comps/ragas/src/tgi/langchain/Dockerfile
     image: ${REGISTRY:-opea}/ragas:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose.yaml

@@ -5,21 +5,21 @@
 services:
   reranking-tei:
     build:
-      dockerfile: comps/reranks/tei/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}
   reranking-videoqna:
     build:
-      dockerfile: comps/reranks/videoqna/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile # TODO. need to update
     image: ${REGISTRY:-opea}/reranking-videoqna:${TAG:-latest}
   reranking-fastrag:
     build:
-      dockerfile: comps/reranks/fastrag/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile # TODO. need to update
     image: ${REGISTRY:-opea}/reranking-fastrag:${TAG:-latest}
   reranking-langchain-mosec-endpoint:
     build:
-      dockerfile: comps/reranks/mosec/langchain/dependency/Dockerfile
+      dockerfile: comps/reranks/src//Dockerfile # TODO. need to update
     image: ${REGISTRY:-opea}/reranking-langchain-mosec-endpoint:${TAG:-latest}
   reranking-langchain-mosec:
     build:
-      dockerfile: comps/reranks/mosec/langchain/Dockerfile
+      dockerfile: comps/reranks/src/Dockerfile
     image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -3,6 +3,10 @@
 
 # this file should be run in the root of the repo
 services:
+  retriever:
+    build:
+      dockerfile: comps/retrievers/src/Dockerfile
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
   retriever-redis:
     build:
       dockerfile: comps/retrievers/redis/langchain/Dockerfile

.github/workflows/pr-examples-test.yml

@@ -4,17 +4,14 @@
 name: Example-test
 
 on:
-  pull_request_target:
+  pull_request:
     branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
       - .github/workflows/pr-examples-test.yml
       - comps/cores/**
-      - comps/embeddings/langchain/**
-      - comps/retrievers/langchain/redis/**
-      - comps/reranks/tei/**
-      - comps/llms/text-generation/tgi/**
-      - comps/dataprep/redis/langchain/**
+      - comps/dataprep/src/redis/langchain/**
+      - comps/retrievers/src/redis/langchain/**
       - requirements.txt
       - "!**.md"
 

.github/workflows/pr-microservice-test.yml

@@ -4,8 +4,8 @@
 name: MicroService-test
 
 on:
-  pull_request_target:
-    branches: ["main", "*rc"]
+  pull_request:
+    branches: ["main", "*rc", "refactor_comps"]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
     paths:
       - comps/**
@@ -52,9 +52,8 @@ jobs:
           hardware: ${{ matrix.hardware }}
         run: |
           cd tests
-          echo "log_name=${service}_${hardware}" >> $GITHUB_ENV
-          if [ ${hardware} = "intel_cpu" ]; then on_hw=""; else on_hw="_on_${hardware}"; fi
-          timeout 60m bash $(find . -type f -name test_${service}${on_hw}.sh)
+          echo "log_name=${service}" >> $GITHUB_ENV
+          timeout 60m bash $(find . -type f -name test_${service}.sh)
 
       - name: Clean up container
         if: cancelled() || failure()

.github/workflows/scripts/get_test_matrix.sh

@@ -5,79 +5,127 @@
 # service: service path name, like 'agent_langchain', 'asr_whisper'
 # hardware: 'intel_cpu', 'intel_hpu', ...
 
-set -xe
+set -e
 cd $WORKSPACE
 changed_files_full=$changed_files_full
 run_matrix="{\"include\":["
 
 # add test services when comps code change
 function find_test_1() {
-    local pre_service=$1
+    local pre_service_path=$1
     local n=$2
     local all_service=$3
 
-    common_file_change=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service} | cut -d'/' -f$n | grep -E '*.py' | grep -vE '__init__.py|version.py' | sort -u) || true
+    common_file_change=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service_path} | cut -d'/' -f$n | grep -E '*.py' | grep -vE '__init__.py|version.py' | sort -u) || true
     if [ "$common_file_change" ] || [ "$all_service" = "true" ]; then
         # if common files changed, run all services
-        services=$(ls ${pre_service} | cut -d'/' -f$n | grep -vE '*.md|*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true
+        services=$(ls ${pre_service_path} | cut -d'/' -f$n | grep -vE '*.md|*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true
         all_service="true"
     else
         # if specific service files changed, only run the specific service
-        services=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service} | cut -d'/' -f$n | grep -vE '*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true
+        services=$(printf '%s\n' "${changed_files[@]}"| grep ${pre_service_path} | cut -d'/' -f$n | grep -vE '*.py|*.sh|*.yaml|*.yml|*.pdf' | sort -u) || true
     fi
 
     for service in ${services}; do
-        service=$pre_service/$service
-        if [[ $(ls ${service} | grep -E "Dockerfile*") ]]; then
-            service_name=$(echo $service | tr '/' '_' | cut -c7-) # comps/dataprep/redis/langchain -> dataprep_redis_langchain
-            default_service_script_path=$(find ./tests -type f -name test_${service_name}.sh) || true
-            if [ "$default_service_script_path" ]; then
-                run_matrix="${run_matrix}{\"service\":\"${service_name}\",\"hardware\":\"intel_cpu\"},"
+        service_path=$pre_service_path/$service
+        if [[ $(ls ${service_path} | grep -E "Dockerfile*") ]]; then
+            if [[ $(ls ${service_path} | grep "integrations") ]]; then
+                # new org with `src` and `integrations` folder
+                set -x # Print commands and their arguments as they are executed
+                run_all_interation="false"
+                service_name=$(echo $service_path | sed 's:/src::' | tr '/' '_' | cut -c7-) # comps/retrievers/src/redis/langchain -> retrievers_redis_langchain
+                common_file_change_insight=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -vE 'integrations' | sort -u) || true
+                if [ "$common_file_change_insight" ]; then
+                    # if common file changed, run all integrations
+                    run_all_interation="true"
+                fi
+                if [ "$run_all_interation" = "false" ]; then
+                    changed_integrations=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -E 'integrations' | grep -E '*.py' | cut -d'/' -f$((n+2)) | cut -d'.' -f1 | sort -u)  || true
+                    for integration in ${changed_integrations}; do
+                        find_test=$(find ./tests -type f -name test_${service_name}_${integration}*.sh) || true
+                        if [ "$find_test" ]; then
+                            fill_in_matrix "$find_test"
+                        else
+                            run_all_interation="true"
+                            break
+                        fi
+                    done
+                fi
+                if [ "$run_all_interation" = "true" ]; then
+                    find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true
+                    if [ "$find_test" ]; then
+                        fill_in_matrix "$find_test"
+                    fi
+                fi
+                set +x
+            else
+                # old org without 'src' folder
+                service_name=$(echo $service_path | tr '/' '_' | cut -c7-) # comps/retrievers/redis/langchain -> retrievers_redis_langchain
+                find_test=$(find ./tests -type f -name test_${service_name}*.sh) || true
+                if [ "$find_test" ]; then
+                    fill_in_matrix "$find_test"
+                fi
             fi
-            other_service_script_path=$(find ./tests -type f -name test_${service_name}_on_*.sh) || true
-            for script in ${other_service_script_path}; do
-                _service=$(echo $script | cut -d'/' -f4 | cut -d'.' -f1 | cut -c6-)
-                hardware=${_service#*_on_}
-                run_matrix="${run_matrix}{\"service\":\"${service_name}\",\"hardware\":\"${hardware}\"},"
-            done
         else
-            find_test_1 $service $((n+1)) $all_service
+            find_test_1 $service_path $((n+1)) $all_service
         fi
     done
 }
 
+function fill_in_matrix() {
+    find_test=$1
+    for test in ${find_test}; do
+        _service=$(echo $test | cut -d'/' -f4 | cut -d'.' -f1 | cut -c6-)
+        _fill_in_matrix $_service
+    done
+}
+
+function _fill_in_matrix() {
+    _service=$1
+    if [ $(echo ${_service} | grep -c "_on_") == 0 ]; then
+        service=${_service}
+        hardware="intel_cpu"
+    else
+        hardware=${_service#*_on_}
+    fi
+    echo "service=${_service}, hardware=${hardware}"
+    if [[ $(echo ${run_matrix} | grep -c "{\"service\":\"${_service}\",\"hardware\":\"${hardware}\"},") == 0 ]]; then
+        run_matrix="${run_matrix}{\"service\":\"${_service}\",\"hardware\":\"${hardware}\"},"
+        echo "------------------ add one service ------------------"
+    fi
+    sleep 1s
+}
+
 # add test case when test scripts code change
 function find_test_2() {
     test_files=$(printf '%s\n' "${changed_files[@]}" | grep -E "*.sh") || true
     for test_file in ${test_files}; do
-        _service=$(echo $test_file | cut -d'/' -f3 | cut -d'.' -f1 | cut -c6-)
-        if [ $(echo ${_service} | grep -c "_on_") == 0 ]; then
-            service=${_service}
-            hardware="intel_cpu"
-        else
-            service=${_service%_on_*}
-            hardware=${_service#*_on_}
-        fi
-        if [[ $(echo ${run_matrix} | grep -c "{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},") == 0 ]]; then
-            run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
+        if [ -f $test_file ]; then
+            _service=$(echo $test_file | cut -d'/' -f3 | cut -d'.' -f1 | cut -c6-)
+            _fill_in_matrix $_service
         fi
     done
 }
 
 function main() {
 
-    changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'comps/' | grep -vE '*.md|comps/cores') || true
+    changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'comps/' | grep -vE '*.md|comps/cores|comps/3rd_parties|deployment|*.yaml') || true
+    echo "===========start find_test_1============"
+    echo "changed_files=${changed_files}"
     find_test_1 "comps" 2 false
     sleep 1s
+    echo "run_matrix=${run_matrix}"
     echo "===========finish find_test_1============"
 
     changed_files=$(printf '%s\n' "${changed_files_full[@]}" | grep 'tests/' | grep -vE '*.md|*.txt|tests/cores') || true
+    echo "===========start find_test_2============"
+    echo "changed_files=${changed_files}"
     find_test_2
     sleep 1s
+    echo "run_matrix=${run_matrix}"
     echo "===========finish find_test_2============"
 
     run_matrix=$run_matrix"]}"
-    echo "run_matrix=${run_matrix}"
     echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
 }