diff --git a/.github/workflows/docker/compose/chathistory-compose-cd.yaml b/.github/workflows/docker/compose/chathistory-compose.yaml similarity index 100% rename from .github/workflows/docker/compose/chathistory-compose-cd.yaml rename to .github/workflows/docker/compose/chathistory-compose.yaml diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml index 80f46ab7e..c7c2089a3 100644 --- a/.github/workflows/docker/compose/dataprep-compose-cd.yaml +++ b/.github/workflows/docker/compose/dataprep-compose-cd.yaml @@ -7,10 +7,6 @@ services: build: dockerfile: comps/dataprep/redis/llama_index/Dockerfile image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest} - dataprep-on-ray-redis: - build: - dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile - image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest} dataprep-milvus: build: dockerfile: comps/dataprep/milvus/langchain/Dockerfile @@ -23,7 +19,15 @@ services: build: dockerfile: comps/dataprep/pinecone/langchain/Dockerfile image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest} + dataprep-redis: #need to rename + build: + dockerfile: comps/dataprep/multimodal/redis/langchain/Dockerfile + image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest} dataprep-vdms: build: - dockerfile: comps/dataprep/vdms/multimodal_langchain/docker/Dockerfile + dockerfile: comps/dataprep/vdms/multimodal_langchain/Dockerfile + image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} + dataprep-vdmsxx: #need to rename + build: + dockerfile: comps/dataprep/vdms/langchain/Dockerfile image: ${REGISTRY:-opea}/dataprep-vdms:${TAG:-latest} diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml index 0f8ca4e83..b697af133 100644 --- a/.github/workflows/docker/compose/dataprep-compose.yaml +++ b/.github/workflows/docker/compose/dataprep-compose.yaml @@ -13,3 +13,7 @@ services: build: dockerfile: comps/dataprep/qdrant/langchain/Dockerfile image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest} + dataprep-on-ray-redis: + build: + dockerfile: comps/dataprep/redis/langchain_ray/Dockerfile + image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest} diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml index 9d0afadc3..8abb89aa3 100644 --- a/.github/workflows/docker/compose/embeddings-compose-cd.yaml +++ b/.github/workflows/docker/compose/embeddings-compose-cd.yaml @@ -14,3 +14,19 @@ services: build: dockerfile: comps/embeddings/tei/llama_index/Dockerfile image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest} + embedding-multimodal: + build: + dockerfile: comps/embeddings/multimodal_clip/Dockerfile + image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} + bridgetower-embedder: + build: + dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile + image: ${REGISTRY:-opea}/bridgetower-embedder:${TAG:-latest} + bridgetower-embedder-gaudi: + build: + dockerfile: comps/embeddings/multimodal/bridgetower/Dockerfile.intel_hpu + image: ${REGISTRY:-opea}/bridgetower-embedder-gaudi:${TAG:-latest} + embedding-multimodalxx: #need to rename + build: + dockerfile: comps/embeddings/multimodal/multimodal_langchain/Dockerfile + image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} diff --git a/.github/workflows/docker/compose/feedback_management-compose-cd.yaml b/.github/workflows/docker/compose/feedback_management-compose-cd.yaml new file mode 100644 index 000000000..0a3cfce66 --- /dev/null +++ b/.github/workflows/docker/compose/feedback_management-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + feedbackmanagement: + build: + dockerfile: comps/feedback_management/mongo/Dockerfile + image: ${REGISTRY:-opea}/feedbackmanagement:${TAG:-latest} diff --git a/.github/workflows/docker/compose/finetuning-compose-cd.yaml b/.github/workflows/docker/compose/finetuning-compose-cd.yaml new file mode 100644 index 000000000..94e20941c --- /dev/null +++ b/.github/workflows/docker/compose/finetuning-compose-cd.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + finetuning: + build: + dockerfile: comps/finetuning/Dockerfile + image: ${REGISTRY:-opea}/finetuning:${TAG:-latest} + finetuning-gaudi: + build: + dockerfile: comps/finetuning/Dockerfile.intel_hpu + image: ${REGISTRY:-opea}/finetuning-gaudi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml index 2c1df0639..55d6f346e 100644 --- a/.github/workflows/docker/compose/guardrails-compose-cd.yaml +++ b/.github/workflows/docker/compose/guardrails-compose-cd.yaml @@ -6,3 +6,7 @@ services: build: dockerfile: comps/guardrails/pii_detection/Dockerfile image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest} + guardrails-toxicity-detection: + build: + dockerfile: comps/guardrails/toxicity_detection/Dockerfile + image: ${REGISTRY:-opea}/guardrails-toxicity-detection:${TAG:-latest} diff --git a/.github/workflows/docker/compose/intent_detection-compose-cd.yaml b/.github/workflows/docker/compose/intent_detection-compose-cd.yaml new file mode 100644 index 000000000..ed4c9a052 --- /dev/null +++ b/.github/workflows/docker/compose/intent_detection-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + llm-tgi: + build: + dockerfile: comps/intent_detection/langchain/Dockerfile + image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/knowledgegraphs-compose-cd.yaml b/.github/workflows/docker/compose/knowledgegraphs-compose-cd.yaml new file mode 100644 index 000000000..bfd47fda0 --- /dev/null +++ b/.github/workflows/docker/compose/knowledgegraphs-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + knowledge_graphs: + build: + dockerfile: comps/knowledgegraphs/langchain/Dockerfile + image: ${REGISTRY:-opea}/knowledge_graphs:${TAG:-latest} diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml index 604dbc85f..aa34c2d77 100644 --- a/.github/workflows/docker/compose/llms-compose-cd.yaml +++ b/.github/workflows/docker/compose/llms-compose-cd.yaml @@ -11,6 +11,10 @@ services: context: vllm-openvino dockerfile: Dockerfile.openvino image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest} + llm-nativexx: #need to rename + build: + dockerfile: comps/llms/utils/lm-eval/Dockerfile + image: ${REGISTRY:-opea}/llm-native:${TAG:-latest} llm-vllm-llamaindex: build: dockerfile: comps/llms/text-generation/vllm/llama_index/Dockerfile diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml index 97b1a8a89..c5a72df21 100644 --- a/.github/workflows/docker/compose/lvms-compose-cd.yaml +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -17,7 +17,11 @@ services: build: dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest} - lvm-tgi: + lvm-video-llama: build: - dockerfile: comps/lvms/tgi-llava/Dockerfile - image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} + dockerfile: comps/lvms/video-llama/Dockerfile + image: ${REGISTRY:-opea}/lvm-video-llama:${TAG:-latest} + video-llama-lvm-server: + build: + dockerfile: comps/lvms/video-llama/dependency/Dockerfile + image: ${REGISTRY:-opea}/video-llama-lvm-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose.yaml b/.github/workflows/docker/compose/lvms-compose.yaml new file mode 100644 index 000000000..5df2c5182 --- /dev/null +++ b/.github/workflows/docker/compose/lvms-compose.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + lvm-tgi: + build: + dockerfile: comps/lvms/tgi-llava/Dockerfile + image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/nginx-compose-cd.yaml b/.github/workflows/docker/compose/nginx-compose.yaml similarity index 100% rename from .github/workflows/docker/compose/nginx-compose-cd.yaml rename to .github/workflows/docker/compose/nginx-compose.yaml diff --git a/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml b/.github/workflows/docker/compose/prompt_registry-compose.yaml similarity index 100% rename from .github/workflows/docker/compose/prompt_registry-compose-cd.yaml rename to .github/workflows/docker/compose/prompt_registry-compose.yaml diff --git a/.github/workflows/docker/compose/ragas-compose-cd.yaml b/.github/workflows/docker/compose/ragas-compose-cd.yaml new file mode 100644 index 000000000..aedad1b89 --- /dev/null +++ b/.github/workflows/docker/compose/ragas-compose-cd.yaml @@ -0,0 +1,7 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +services: + ragas: + build: + dockerfile: comps/ragas/tgi/langchain/Dockerfile + image: ${REGISTRY:-opea}/ragas:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose-cd.yaml b/.github/workflows/docker/compose/reranks-compose-cd.yaml index 3e5e7caab..f6180e0ef 100644 --- a/.github/workflows/docker/compose/reranks-compose-cd.yaml +++ b/.github/workflows/docker/compose/reranks-compose-cd.yaml @@ -14,3 +14,7 @@ services: build: dockerfile: comps/reranks/mosec/langchain/Dockerfile image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest} + reranking-videoragqna: + build: + dockerfile: comps/reranks/video-rag-qna/Dockerfile + image: ${REGISTRY:-opea}/reranking-videoragqna:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose-cd.yaml b/.github/workflows/docker/compose/retrievers-compose-cd.yaml index a8aefeda5..fdf128b1e 100644 --- a/.github/workflows/docker/compose/retrievers-compose-cd.yaml +++ b/.github/workflows/docker/compose/retrievers-compose-cd.yaml @@ -19,3 +19,15 @@ services: build: dockerfile: comps/retrievers/redis/llama_index/Dockerfile image: ${REGISTRY:-opea}/retriever-redis-llamaindex:${TAG:-latest} + retriever-pathway: + build: + dockerfile: comps/retrievers/pathway/langchain/Dockerfile + image: ${REGISTRY:-opea}/retriever-pathway:${TAG:-latest} + multimodal-retriever-redis: + build: + dockerfile: comps/retrievers/multimodal/redis/langchain/Dockerfile + image: ${REGISTRY:-opea}/multimodal-retriever-redis:${TAG:-latest} + retriever-vdms: + build: + dockerfile: comps/retrievers/vdms/langchain/Dockerfile + image: ${REGISTRY:-opea}/retriever-vdms:${TAG:-latest} diff --git a/.github/workflows/docker/compose/tts-compose-cd.yaml b/.github/workflows/docker/compose/tts-compose-cd.yaml new file mode 100644 index 000000000..8adad51e0 --- /dev/null +++ b/.github/workflows/docker/compose/tts-compose-cd.yaml @@ -0,0 +1,10 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +# images used by GenAIExamples: reranking-tei +services: + tts: #need to rename + build: + dockerfile: comps/tts/Dockerfile + image: ${REGISTRY:-opea}/tts:${TAG:-latest} diff --git a/.github/workflows/docker/compose/vectorstores-compose-cd.yaml b/.github/workflows/docker/compose/vectorstores-compose-cd.yaml index a3940b13d..584927e95 100644 --- a/.github/workflows/docker/compose/vectorstores-compose-cd.yaml +++ b/.github/workflows/docker/compose/vectorstores-compose-cd.yaml @@ -5,5 +5,5 @@ services: vectorstore-pathway: build: - dockerfile: comps/vectorstores/pathway + dockerfile: comps/vectorstores/pathway/Dockerfile image: ${REGISTRY:-opea}/vectorstore-pathway:${TAG:-latest} diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml index d24718375..254adb9bb 100644 --- a/.github/workflows/pr-dockerfile-path-scan.yaml +++ b/.github/workflows/pr-dockerfile-path-scan.yaml @@ -20,12 +20,18 @@ jobs: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* - - name: Checkout code + - name: Checkout Code uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check for changed Dockerfile paths in yaml + - name: Clone GenAIExamples + run: | + cd .. + git clone https://github.com/opea-project/GenAIExamples + + - name: Check Dockerfile Paths in Readme + if: always() run: | set -e shopt -s globstar @@ -36,7 +42,7 @@ jobs: changed_files="$(git diff --name-status --diff-filter=DR ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)" if [ -n "$changed_files" ]; then for file in $changed_files; do - if grep -q "$file" .github/workflows/docker/compose/*.yaml; then + if grep -q "$file" ./**/*.md; then is_use="TRUE" used_files+="$file " fi @@ -46,64 +52,68 @@ jobs: if [[ "$is_use" == "TRUE" ]]; then echo "Warning: Changed Dockerfile paths:" echo "$used_files" - echo "Please modify the corresponding yaml in GenAIComps/.github/workflows/docker/compose and ask suyue.chen@intel.com for final confirmation." + echo "Please modify the corresponding README in GenAIComps." exit 1 fi - - name: Check for changed Dockerfile paths in readme + - name: Check all Dockerfile mode and if it exists in yaml if: always() run: | set -e shopt -s globstar + no_add="FALSE" cd ${{github.workspace}} - is_use="FALSE" - used_files="" - merged_commit=$(git log -1 --format='%H') - changed_files="$(git diff --name-status --diff-filter=DR ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)" - if [ -n "$changed_files" ]; then - for file in $changed_files; do - if grep -q "$file" ./**/*.md; then - is_use="TRUE" - used_files+="$file " + Dockerfiles=$(find ./comps -name '*Dockerfile*'|sed 's/^\.\///') + if [ -n "$Dockerfiles" ]; then + for Dockerfile in $Dockerfiles; do + service=$(echo "$Dockerfile" | awk -F '/' '{print $2}') + if grep -q "$Dockerfile" ../GenAIExamples/**/*build.yaml*; then + mode="" #CI + else + mode="-cd" #CD + fi + + yaml_file=${{github.workspace}}/.github/workflows/docker/compose/$service'-compose'$mode'.yaml' + if [ -e "$yaml_file" ]; then + # check service yaml if exist + if ! grep -q "$Dockerfile" "$yaml_file"; then + echo "AR: Update $Dockerfile to .github/workflows/docker/compose/"$service"-compose"$mode".yaml." + no_add="TRUE" + fi + else + echo "AR: Create a new compose file .github/workflows/docker/compose/"$service"-compose"$mode".yaml for $Dockerfile and fill it in." + no_add="TRUE" fi done fi - if [[ "$is_use" == "TRUE" ]]; then - echo "Warning: Changed Dockerfile paths:" - echo "$used_files" - echo "Please modify the corresponding README in GenAIComps." + if [[ "$no_add" == "TRUE" ]]; then exit 1 fi - - name: Check new Dockerfile in compose yaml + - name: Check for the exist of Dockerfile in all yamls. if: always() run: | - set -xe shopt -s globstar + no_exist="FALSE" cd ${{github.workspace}} - no_add="FALSE" - merged_commit=$(git log -1 --format='%H') - changed_files="$(git diff --name-status --diff-filter=A ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/Dockerfile**' | cut -f2)" - changed_yamls="$(git diff --name-status --diff-filter=AM ${{ github.event.pull_request.base.sha }} ${merged_commit} -- '**/*.yaml**' | cut -f2)" - if [ -n "$changed_files" ]; then - for file in $changed_files; do - service=$(echo "$file" | awk -F '/' '{print $2}') - if find "${{github.workspace}}/.github/workflows/docker/compose/" -name "*$service*" |grep -q .; then - if [ -n "$changed_files" ] && grep -q $service'-compose-cd.yaml' <<< "$changed_yamls"; then - echo "The $file has been added to the ${{github.workspace}}/.github/workflows/docker/compose/"$service"-compose-cd.yaml." - else - echo "Please check if the added $file is included in the yaml under path ${{github.workspace}}/.github/workflows/docker/compose/"$service"-compose-cd.yaml." - no_add="TRUE" + yamls=$(find .github/workflows/docker/compose/ -name '*.yaml') + if [ -n "$yamls" ]; then + for yaml in $yamls; do + dockerfiles=$(grep 'dockerfile:' "$yaml"|sed 's/dockerfile: //') + for dockerfile in $dockerfiles; do + if [[ "$dockerfile" == *"comps/"* ]]; then + if ! [ -e "$dockerfile" ]; then + # cat "$yaml" + echo "AR: The dockerfile path "$dockerfile" in "$yaml" does not exist, remove or update it." + no_exist="TRUE" + fi fi - else - echo "Please create a new compose file named "$service"-compose-cd.yaml in ${{github.workspace}}/.github/workflows/docker/compose/ for $file and fill it in." - no_add="TRUE" - fi + done done fi - if [[ "$no_add" == "TRUE" ]]; then + if [[ "$no_exist" == "TRUE" ]]; then exit 1 fi