[Reorg] Reorg Folder to Support Different Vendors (#625)

Signed-off-by: letonghan <[email protected]>
Signed-off-by: chensuyue <[email protected]>

.github/workflows/_get-test-matrix.yml

@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
+    if: ${{ !github.event.pull_request.draft }}
     steps:
       - name: Get checkout ref
         run: |
@@ -35,7 +36,6 @@ jobs:
       - name: Get test matrix
         id: get-test-matrix
         run: |
-          set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
             LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
             "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
@@ -46,52 +46,8 @@ jobs:
           fi
           merged_commit=$(git log -1 --format='%H')
 
-          changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
-          grep 'comps/' | grep -vE '*.md|comps/cores')" || true
-          services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u) || true
-          run_matrix="{\"include\":["
-          for service in ${services}; do
-              hardware="gaudi" # default hardware, set based on the changed files
-              if [[ $(ls comps/${service} | grep -E "Dockerfile*|docker*") ]]; then
-                  run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
-              fi
-              vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.sh|docker|*.yaml|*.yml' | sort -u)
-              for vendor in ${vendors}; do
-                  if [[ $(ls comps/${service}/${vendor} | grep -E "Dockerfile*|docker*") ]]; then
-                      run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}\",\"hardware\":\"${hardware}\"},"
-                  fi
-                  sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py|Dockerfile|*.sh|docker|*.yaml|*.yml' | sort -u)
-                  for sub_vendor in ${sub_vendors}; do
-                      if [[ $(ls comps/${service}/${vendor}/${sub_vendor} | grep -E "Dockerfile*|docker*") ]]; then
-                          run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}/${sub_vendor}\",\"hardware\":\"${hardware}\"},"
-                      fi
-                  done
-              done
-          done
+          changed_files="$(git diff --name-only ${base_commit} ${merged_commit})" || true
+          export changed_files_full=$changed_files
 
-          # add test for comps/dataprep/utils.py
-          if [[ "${changed_files[@]}" =~ "comps/dataprep/utils.py" ]]; then
-              service_list=("dataprep/qdrant" "dataprep/redis/langchain")
-              hardware="gaudi" # default hardware, set based on the changed files
-              for service in ${service_list[@]}; do
-                  if [ $(echo ${run_matrix} | grep -c ${service}) == 0 ]; then
-                      run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
-                  fi
-              done
-          fi
-
-          # add test for test scripts update
-          changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \
-          grep 'tests/' | grep -vE '*.md|*.txt|tests/cores')" || true
-          run_matrix_match=$(echo $run_matrix | tr '/' '_')
-          test_files=$(printf '%s\n' "${changed_files[@]}" | grep -E "tests/test_*" | grep -E "*.sh") || true
-          for test_file in ${test_files}; do
-              service=$(echo $test_file | cut -d'/' -f2 | cut -d'.' -f1 | cut -c6-)
-              hardware="gaudi" # default hardware, set based on the changed files
-              if [ $(echo ${run_matrix_match} | grep -c ${service}) == 0 ]; then
-                  run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"},"
-              fi
-          done
-
-          run_matrix=$run_matrix"]}"
-          echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
+          export WORKSPACE=${{ github.workspace }}
+          bash .github/workflows/scripts/get_test_matrix.sh

.github/workflows/docker/compose/agent-compose-cd.yaml

@@ -5,5 +5,5 @@
 services:
   comps-agent-langchain:
     build:
-      dockerfile: comps/agent/langchain/docker/Dockerfile
+      dockerfile: comps/agent/langchain/Dockerfile
     image: ${REGISTRY:-opea}/comps-agent-langchain:${TAG:-latest}

.github/workflows/docker/compose/asr-compose.yaml

@@ -6,13 +6,13 @@
 services:
   asr:
     build:
-      dockerfile: comps/asr/Dockerfile
+      dockerfile: comps/asr/whisper/Dockerfile
     image: ${REGISTRY:-opea}/asr:${TAG:-latest}
   whisper:
     build:
-      dockerfile: comps/asr/whisper/Dockerfile
+      dockerfile: comps/asr/whisper/dependency/Dockerfile
     image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
   whisper-gaudi:
     build:
-      dockerfile: comps/asr/whisper/Dockerfile_hpu
+      dockerfile: comps/asr/whisper/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest}

.github/workflows/docker/compose/chathistory-compose-cd.yaml

@@ -5,5 +5,5 @@
 services:
   chathistory-mongo-server:
     build:
-      dockerfile: comps/chathistory/mongo/docker/Dockerfile
+      dockerfile: comps/chathistory/mongo/Dockerfile
     image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose-cd.yaml

@@ -5,21 +5,21 @@
 services:
   dataprep-redis-llama-index:
     build:
-      dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
+      dockerfile: comps/dataprep/redis/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
   dataprep-on-ray-redis:
     build:
-      dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
+      dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
   dataprep-milvus:
     build:
-      dockerfile: comps/dataprep/milvus/docker/Dockerfile
+      dockerfile: comps/dataprep/milvus/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
   dataprep-pgvector:
     build:
-      dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
+      dockerfile: comps/dataprep/pgvector/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
   dataprep-pinecone:
     build:
-      dockerfile: comps/dataprep/pinecone/docker/Dockerfile
+      dockerfile: comps/dataprep/pinecone/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}

.github/workflows/docker/compose/dataprep-compose.yaml

@@ -7,9 +7,9 @@
 services:
   dataprep-redis:
     build:
-      dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile
+      dockerfile: comps/dataprep/redis/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
   dataprep-qdrant:
     build:
-      dockerfile: comps/dataprep/qdrant/docker/Dockerfile
+      dockerfile: comps/dataprep/qdrant/langchain/Dockerfile
     image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose-cd.yaml

@@ -4,13 +4,13 @@
 services:
   embedding-langchain-mosec-endpoint:
     build:
-      dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
+      dockerfile: comps/embeddings/mosec/langchain/dependency/Dockerfile
     image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest}
   embedding-langchain-mosec:
     build:
-      dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
+      dockerfile: comps/embeddings/mosec/langchain/Dockerfile
     image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest}
   embedding-tei-llama-index:
     build:
-      dockerfile: comps/embeddings/llama_index/docker/Dockerfile
+      dockerfile: comps/embeddings/tei/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}

.github/workflows/docker/compose/embeddings-compose.yaml

@@ -6,5 +6,5 @@
 services:
   embedding-tei:
     build:
-      dockerfile: comps/embeddings/langchain/docker/Dockerfile
+      dockerfile: comps/embeddings/tei/langchain/Dockerfile
     image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose-cd.yaml

@@ -4,5 +4,5 @@
 services:
   guardrails-pii-detection:
     build:
-      dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
+      dockerfile: comps/guardrails/pii_detection/Dockerfile
     image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest}

.github/workflows/docker/compose/guardrails-compose.yaml

@@ -7,5 +7,5 @@
 services:
   guardrails-tgi:
     build:
-      dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
+      dockerfile: comps/guardrails/llama_guard/langchain/Dockerfile
     image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}

.github/workflows/docker/compose/llms-compose-cd.yaml

@@ -4,7 +4,7 @@
 services:
   llm-native:
     build:
-      dockerfile: comps/llms/text-generation/native/docker/Dockerfile
+      dockerfile: comps/llms/text-generation/native/Dockerfile
     image: ${REGISTRY:-opea}/llm-native:${TAG:-latest}
   vllm-openvino:
     build:

.github/workflows/docker/compose/llms-compose.yaml

@@ -10,29 +10,29 @@ services:
     image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
   llm-ollama:
     build:
-      dockerfile: comps/llms/text-generation/ollama/Dockerfile
+      dockerfile: comps/llms/text-generation/ollama/langchain/Dockerfile
     image: ${REGISTRY:-opea}/llm-ollama:${TAG:-latest}
   llm-docsum-tgi:
     build:
-      dockerfile: comps/llms/summarization/tgi/Dockerfile
+      dockerfile: comps/llms/summarization/tgi/langchain/Dockerfile
     image: ${REGISTRY:-opea}/llm-docsum-tgi:${TAG:-latest}
   llm-faqgen-tgi:
     build:
-      dockerfile: comps/llms/faq-generation/tgi/Dockerfile
+      dockerfile: comps/llms/faq-generation/tgi/langchain/Dockerfile
     image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest}
   llm-vllm:
     build:
-      dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.microservice
+      dockerfile: comps/llms/text-generation/vllm/langchain/Dockerfile
     image: ${REGISTRY:-opea}/llm-vllm:${TAG:-latest}
   llm-vllm-hpu:
     build:
-      dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.hpu
+      dockerfile: comps/llms/text-generation/vllm/langchain/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/llm-vllm-hpu:${TAG:-latest}
   llm-vllm-ray:
     build:
-      dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.microservice
+      dockerfile: comps/llms/text-generation/vllm/ray/Dockerfile
     image: ${REGISTRY:-opea}/llm-vllm-ray:${TAG:-latest}
   llm-vllm-ray-hpu:
     build:
-      dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.vllmray
+      dockerfile: comps/llms/text-generation/vllm/ray/dependency/Dockerfile
     image: ${REGISTRY:-opea}/llm-vllm-ray-hpu:${TAG:-latest}

.github/workflows/docker/compose/lvms-compose-cd.yaml

@@ -5,19 +5,19 @@
 services:
   lvm:
     build:
-      dockerfile: comps/lvms/Dockerfile
+      dockerfile: comps/lvms/llava/Dockerfile
     image: ${REGISTRY:-opea}/lvm:${TAG:-latest}
   # Xeon CPU
   llava:
     build:
-      dockerfile: comps/lvms/llava/Dockerfile
+      dockerfile: comps/lvms/llava/dependency/Dockerfile
     image: ${REGISTRY:-opea}/llava:${TAG:-latest}
   # Gaudi2 HPU
   llava-hpu:
     build:
-      dockerfile: comps/lvms/llava/Dockerfile_hpu
+      dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest}
   lvm-tgi:
     build:
-      dockerfile: comps/lvms/Dockerfile_tgi
+      dockerfile: comps/lvms/tgi-llava/Dockerfile
     image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest}

.github/workflows/docker/compose/nginx-compose-cd.yaml

@@ -5,5 +5,5 @@
 services:
   nginx:
     build:
-      dockerfile: comps/nginx/docker/Dockerfile
+      dockerfile: comps/nginx/Dockerfile
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}

.github/workflows/docker/compose/prompt_registry-compose-cd.yaml

@@ -5,5 +5,5 @@
 services:
   promptregistry-mongo-server:
     build:
-      dockerfile: comps/prompt_registry/mongo/docker/Dockerfile
+      dockerfile: comps/prompt_registry/mongo/Dockerfile
     image: ${REGISTRY:-opea}/promptregistry-mongo-server:${TAG:-latest}

.github/workflows/docker/compose/reranks-compose-cd.yaml

@@ -4,15 +4,15 @@
 services:
   reranking-fastrag:
     build:
-      dockerfile: comps/reranks/fastrag/docker/Dockerfile
+      dockerfile: comps/reranks/fastrag/Dockerfile
     image: ${REGISTRY:-opea}/reranking-fastrag:${TAG:-latest}
   reranking-langchain-mosec-endpoint:
     build:
-      dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile
+      dockerfile: comps/reranks/mosec/langchain/dependency/Dockerfile
     image: ${REGISTRY:-opea}/reranking-langchain-mosec-endpoint:${TAG:-latest}
   reranking-langchain-mosec:
     build:
-      dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile
+      dockerfile: comps/reranks/mosec/langchain/Dockerfile
     image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest}
   reranking-mosec-neural-speed:
     build:

.github/workflows/docker/compose/reranks-compose.yaml

@@ -6,5 +6,5 @@
 services:
   reranking-tei:
     build:
-      dockerfile: comps/reranks/tei/docker/Dockerfile
+      dockerfile: comps/reranks/tei/Dockerfile
     image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose-cd.yaml

@@ -5,17 +5,17 @@
 services:
   retriever-pgvector:
     build:
-      dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile
+      dockerfile: comps/retrievers/pgvector/langchain/Dockerfile
     image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest}
   retriever-pinecone:
     build:
-      dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile
+      dockerfile: comps/retrievers/pinecone/langchain/Dockerfile
     image: ${REGISTRY:-opea}/retriever-pinecone:${TAG:-latest}
   retriever-milvus:
     build:
-      dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile
+      dockerfile: comps/retrievers/milvus/langchain/Dockerfile
     image: ${REGISTRY:-opea}/retriever-milvus:${TAG:-latest}
   retriever-redis-llamaindex:
     build:
-      dockerfile: comps/retrievers/llamaindex/docker/Dockerfile
+      dockerfile: comps/retrievers/redis/llama_index/Dockerfile
     image: ${REGISTRY:-opea}/retriever-redis-llamaindex:${TAG:-latest}

.github/workflows/docker/compose/retrievers-compose.yaml

@@ -5,9 +5,9 @@
 services:
   retriever-redis:
     build:
-      dockerfile: comps/retrievers/langchain/redis/docker/Dockerfile
+      dockerfile: comps/retrievers/redis/langchain/Dockerfile
     image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
   retriever-qdrant:
     build:
-      dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile
+      dockerfile: comps/retrievers/qdrant/haystack/Dockerfile
     image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest}

.github/workflows/docker/compose/tts-compose.yaml

@@ -6,13 +6,13 @@
 services:
   tts:
     build:
-      dockerfile: comps/tts/Dockerfile
+      dockerfile: comps/tts/speecht5/Dockerfile
     image: ${REGISTRY:-opea}/tts:${TAG:-latest}
   speecht5:
     build:
-      dockerfile: comps/tts/speecht5/Dockerfile
+      dockerfile: comps/tts/speecht5/dependency/Dockerfile
     image: ${REGISTRY:-opea}/speecht5:${TAG:-latest}
   speecht5-gaudi:
     build:
-      dockerfile: comps/tts/speecht5/Dockerfile_hpu
+      dockerfile: comps/tts/speecht5/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/speecht5-gaudi:${TAG:-latest}

.github/workflows/docker/compose/vectorstores-compose-cd.yaml

@@ -5,5 +5,5 @@
 services:
   vectorstore-pathway:
     build:
-      dockerfile: comps/vectorstores/langchain/pathway
+      dockerfile: comps/vectorstores/pathway
     image: ${REGISTRY:-opea}/vectorstore-pathway:${TAG:-latest}

.github/workflows/docker/compose/web_retrievers-compose.yaml

@@ -6,5 +6,5 @@
 services:
   web-retriever-chroma:
     build:
-      dockerfile: comps/web_retrievers/langchain/chroma/docker/Dockerfile
+      dockerfile: comps/web_retrievers/chroma/langchain/Dockerfile
     image: ${REGISTRY:-opea}/web-retriever-chroma:${TAG:-latest}

.github/workflows/pr-dockerfile-path-scan.yaml

@@ -72,7 +72,7 @@ jobs:
           if [[ "$is_use" == "TRUE" ]]; then
             echo "Warning: Changed Dockerfile paths:"
             echo "$used_files"
-            echo "Please modify the corresponding README in GenAIComps and ask [email protected] for final confirmation."
+            echo "Please modify the corresponding README in GenAIComps."
             exit 1
           fi