From ed44b445c90ab36e76dc3197d4be85fb0f9dbc7d Mon Sep 17 00:00:00 2001 From: Huiling Bao Date: Fri, 30 Aug 2024 17:40:28 +0800 Subject: [PATCH 01/16] fix SearchedMultimodalDoc in docarray (#583) Signed-off-by: BaoHuiling --- comps/cores/proto/docarray.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index e9716209d..1a29aa329 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -1,11 +1,11 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import numpy as np from docarray import BaseDoc, DocList -from docarray.documents import AudioDoc, VideoDoc +from docarray.documents import AudioDoc from docarray.typing import AudioUrl, ImageUrl from pydantic import Field, conint, conlist, field_validator @@ -43,14 +43,6 @@ class TextImageDoc(BaseDoc): ] -class ImageDoc(BaseDoc): - image_path: str - - -class TextImageDoc(BaseDoc): - doc: Tuple[Union[TextDoc, ImageDoc]] - - class Base64ByteStrDoc(BaseDoc): byte_str: str @@ -110,14 +102,8 @@ class Config: json_encoders = {np.ndarray: lambda x: x.tolist()} -class SearchedMultimodalDoc(BaseDoc): - retrieved_docs: List[TextImageDoc] - initial_query: str - top_n: int = 1 - metadata: Optional[List[Dict]] = None - - class Config: - json_encoders = {np.ndarray: lambda x: x.tolist()} +class SearchedMultimodalDoc(SearchedDoc): + metadata: List[Dict[str, Any]] class GeneratedDoc(BaseDoc): From b541fd8bdb277b523fa13b247a274509c7dfebe3 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Fri, 30 Aug 2024 17:44:08 +0800 Subject: [PATCH 02/16] update image build yaml (#529) Signed-off-by: chensuyue Signed-off-by: zepan --- .github/workflows/_comps-workflow.yml | 79 +++++-------------- .../docker/compose/agent-compose-cd.yaml | 9 +++ .../compose/chathistory-compose-cd.yaml | 9 +++ .../docker/compose/dataprep-compose-cd.yaml | 25 ++++++ .../docker/compose/dataprep-compose.yaml | 8 -- .../docker/compose/embeddings-compose-cd.yaml | 16 ++++ .../docker/compose/guardrails-compose-cd.yaml | 8 ++ .../docker/compose/guardrails-compose.yaml | 4 - .../docker/compose/llms-compose-cd.yaml | 8 ++ .../docker/compose/lvms-compose-cd.yaml | 23 ++++++ .../docker/compose/nginx-compose-cd.yaml | 9 +++ .../compose/prompt_registry-compose-cd.yaml | 9 +++ .../docker/compose/reranks-compose-cd.yaml | 16 ++++ .../docker/compose/retrievers-compose.yaml | 16 ++++ .github/workflows/manual-comps-test.yml | 21 ++++- comps/nginx/docker/Dockerfile | 4 +- comps/retrievers/llamaindex/docker/Dockerfile | 1 + tests/test_embeddings_llama_index.sh | 16 ++-- tests/test_nginx.sh | 4 +- 19 files changed, 201 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/docker/compose/agent-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/chathistory-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/dataprep-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/embeddings-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/guardrails-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/llms-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/lvms-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/nginx-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/prompt_registry-compose-cd.yaml create mode 100644 .github/workflows/docker/compose/reranks-compose-cd.yaml diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml index 58cb02dba..c2a68b0ca 100644 --- a/.github/workflows/_comps-workflow.yml +++ b/.github/workflows/_comps-workflow.yml @@ -20,10 +20,17 @@ on: default: true required: false type: boolean - # scan: - # default: true - # required: false - # type: boolean + test: + default: true + description: "Test comps with docker compose" + required: false + type: boolean + mode: + default: "CD" + description: "Whether the test range is CI or CD" + required: false + type: string + jobs: #################################################################################################### # Image Build @@ -39,82 +46,38 @@ jobs: uses: actions/checkout@v4 - name: Clone required Repo + id: get-yaml-path run: | cd ${{ github.workspace }}/.github/workflows/docker/compose # service=$(echo ${{ inputs.service }} | cut -d'_' -f1) - docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml - echo ${docker_compose_yml} + if [[ "${{ inputs.mode }}" == "CD" ]]; then + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml + else + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml + fi + echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 fi if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/vllm-project/vllm.git vllm-openvino fi - # echo "service=$service" >> $GITHUB_ENV - name: Build Image if: ${{ fromJSON(inputs.build) }} uses: opea-project/validation/actions/image-build@main with: - work_dir: ${{ github.workspace }}/ - docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml + work_dir: ${{ github.workspace }} + docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }} registry: ${OPEA_IMAGE_REPO}opea tag: ${{ inputs.tag }} - # #################################################################################################### - # # Trivy Scan - # #################################################################################################### - # get-image-list: - # needs: [build-images] - # if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }} - # runs-on: ubuntu-latest - # outputs: - # matrix: ${{ steps.scan-matrix.outputs.matrix }} - # steps: - # - name: Checkout out Repo - # uses: actions/checkout@v4 - - # - name: Set Matrix - # id: scan-matrix - # run: | - # pip install yq - # compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml - # echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT - - # scan-images: - # needs: [get-image-list, build-images] - # if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}} - # runs-on: "docker-build-${{ inputs.node }}" - # strategy: - # matrix: - # image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }} - # fail-fast: false - # steps: - # - name: Pull Image - # run: | - # docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} - # echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV - - # - name: Scan Container - # uses: opea-project/validation/actions/trivy-scan@main - # with: - # image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }} - # output: ${{ matrix.image }}-scan.txt - - # - name: Cleanup - # if: always() - # run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} - - # - uses: actions/upload-artifact@v4.3.4 - # with: - # name: ${{ matrix.image }}-scan - # path: ${{ matrix.image }}-scan.txt - # overwrite: true #################################################################################################### # Docker Compose Test #################################################################################################### test-service-compose: needs: [build-images] + if: ${{ fromJSON(inputs.test) }} uses: ./.github/workflows/_run-docker-compose.yml with: tag: ${{ inputs.tag }} diff --git a/.github/workflows/docker/compose/agent-compose-cd.yaml b/.github/workflows/docker/compose/agent-compose-cd.yaml new file mode 100644 index 000000000..a285ecc34 --- /dev/null +++ b/.github/workflows/docker/compose/agent-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + comps-agent-langchain: + build: + dockerfile: comps/agent/langchain/docker/Dockerfile + image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest} diff --git a/.github/workflows/docker/compose/chathistory-compose-cd.yaml b/.github/workflows/docker/compose/chathistory-compose-cd.yaml new file mode 100644 index 000000000..f8930cde8 --- /dev/null +++ b/.github/workflows/docker/compose/chathistory-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + chathistory-mongo-server: + build: + dockerfile: comps/chathistory/mongo/docker/Dockerfile + image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml new file mode 100644 index 000000000..19f4c063d --- /dev/null +++ b/.github/workflows/docker/compose/dataprep-compose-cd.yaml @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# others: dataprep-redis-llama-index,dataprep-on-ray-redis +services: + dataprep-redis-llama-index: + build: + dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile + image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest} + dataprep-on-ray-redis: + build: + dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile + image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest} + dataprep-milvus: + build: + dockerfile: comps/dataprep/milvus/docker/Dockerfile + image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest} + dataprep-pgvector: + build: + dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile + image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest} + dataprep-pinecone: + build: + dockerfile: comps/dataprep/pinecone/docker/Dockerfile + image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest} diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml index 5cca84cb4..1671235f4 100644 --- a/.github/workflows/docker/compose/dataprep-compose.yaml +++ b/.github/workflows/docker/compose/dataprep-compose.yaml @@ -13,11 +13,3 @@ services: build: dockerfile: comps/dataprep/qdrant/docker/Dockerfile image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest} - dataprep-redis-llama-index: - build: - dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest} - dataprep-on-ray-redis: - build: - dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest} diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml new file mode 100644 index 000000000..3d08a1b53 --- /dev/null +++ b/.github/workflows/docker/compose/embeddings-compose-cd.yaml @@ -0,0 +1,16 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + embedding-langchain-mosec-endpoint: + build: + dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile + image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest} + embedding-langchain-mosec: + build: + dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile + image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest} + embedding-tei-llama-index: + build: + dockerfile: comps/embeddings/llama_index/docker/Dockerfile + image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest} diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml new file mode 100644 index 000000000..e6365a99d --- /dev/null +++ b/.github/workflows/docker/compose/guardrails-compose-cd.yaml @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + guardrails-pii-detection: + build: + dockerfile: comps/guardrails/pii_detection/docker/Dockerfile + image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest} diff --git a/.github/workflows/docker/compose/guardrails-compose.yaml b/.github/workflows/docker/compose/guardrails-compose.yaml index 30592aecb..eec433c25 100644 --- a/.github/workflows/docker/compose/guardrails-compose.yaml +++ b/.github/workflows/docker/compose/guardrails-compose.yaml @@ -9,7 +9,3 @@ services: build: dockerfile: comps/guardrails/llama_guard/docker/Dockerfile image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest} - guardrails-pii-detection: - build: - dockerfile: comps/guardrails/pii_detection/docker/Dockerfile - image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest} diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml new file mode 100644 index 000000000..f60e0e921 --- /dev/null +++ b/.github/workflows/docker/compose/llms-compose-cd.yaml @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + llm-native: + build: + dockerfile: comps/llms/text-generation/native/docker/Dockerfile + image: ${REGISTRY}opea/llm-native:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml new file mode 100644 index 000000000..fbdad3011 --- /dev/null +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -0,0 +1,23 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + lvm: + build: + dockerfile: comps/lvms/Dockerfile + image: ${REGISTRY}opea/lvm:${TAG:-latest} + # Xeon CPU + llava: + build: + dockerfile: comps/lvms/llava/Dockerfile + image: ${REGISTRY}opea/llava:${TAG:-latest} + # Gaudi2 HPU + llava_hpu: + build: + dockerfile: comps/lvms/llava/Dockerfile_hpu + image: ${REGISTRY}opea/llava_hpu:${TAG:-latest} + lvm-tgi: + build: + dockerfile: comps/lvms/Dockerfile_tgi + image: ${REGISTRY}opea/lvm-tgi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/nginx-compose-cd.yaml b/.github/workflows/docker/compose/nginx-compose-cd.yaml new file mode 100644 index 000000000..e6cf05aa4 --- /dev/null +++ b/.github/workflows/docker/compose/nginx-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + nginx: + build: + dockerfile: comps/nginx/docker/Dockerfile + image: ${REGISTRY}opea/nginx:${TAG:-latest} diff --git a/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml b/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml new file mode 100644 index 000000000..52923a2f1 --- /dev/null +++ b/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml @@ -0,0 +1,9 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + promptregistry-mongo-server: + build: + dockerfile: comps/prompt_registry/mongo/docker/Dockerfile + image: ${REGISTRY}opea/promptregistry-mongo-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose-cd.yaml b/.github/workflows/docker/compose/reranks-compose-cd.yaml new file mode 100644 index 000000000..85339c8b8 --- /dev/null +++ b/.github/workflows/docker/compose/reranks-compose-cd.yaml @@ -0,0 +1,16 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +services: + reranking-fastrag: + build: + dockerfile: comps/reranks/fastrag/docker/Dockerfile + image: ${REGISTRY}opea/reranking-fastrag:${TAG:-latest} + reranking-langchain-mosec-endpoint: + build: + dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile + image: ${REGISTRY}opea/reranking-langchain-mosec-endpoint:${TAG:-latest} + reranking-langchain-mosec: + build: + dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile + image: ${REGISTRY}opea/reranking-langchain-mosec:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose.yaml b/.github/workflows/docker/compose/retrievers-compose.yaml index d9de4b27f..289871ed5 100644 --- a/.github/workflows/docker/compose/retrievers-compose.yaml +++ b/.github/workflows/docker/compose/retrievers-compose.yaml @@ -11,3 +11,19 @@ services: build: dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest} + retriever-pgvector: + build: + dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile + image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest} + retriever-pinecone: + build: + dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile + image: ${REGISTRY}opea/retriever-pinecone:${TAG:-latest} + retriever-milvus: + build: + dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile + image: ${REGISTRY}opea/retriever-milvus:${TAG:-latest} + retriever-redis-llamaindex: + build: + dockerfile: comps/retrievers/llamaindex/docker/Dockerfile + image: ${REGISTRY}opea/retriever-redis-llamaindex:${TAG:-latest} diff --git a/.github/workflows/manual-comps-test.yml b/.github/workflows/manual-comps-test.yml index 010cd0d7a..bde3bf9fa 100644 --- a/.github/workflows/manual-comps-test.yml +++ b/.github/workflows/manual-comps-test.yml @@ -7,7 +7,7 @@ on: inputs: services: default: "asr" - description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]" + description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,llms,lvms,nginx,prompt_registry,reranks,retrievers,tts,web_retrievers]" required: true type: string build: @@ -15,6 +15,21 @@ on: description: "Build test required images for Comps" required: false type: boolean + test: + default: true + description: "Test comps with docker compose" + required: false + type: boolean + tag: + default: "comps" + description: "Tag to apply to images" + required: true + type: string + mode: + default: "CD" + description: "Whether the test range is CI or CD" + required: false + type: string permissions: read-all @@ -40,6 +55,8 @@ jobs: uses: ./.github/workflows/_comps-workflow.yml with: service: ${{ matrix.service }} - tag: "comps" + tag: ${{ inputs.tag }} node: gaudi + mode: ${{ inputs.mode }} + test: ${{ inputs.test }} secrets: inherit diff --git a/comps/nginx/docker/Dockerfile b/comps/nginx/docker/Dockerfile index 6816fb0a5..447d3946a 100644 --- a/comps/nginx/docker/Dockerfile +++ b/comps/nginx/docker/Dockerfile @@ -6,7 +6,7 @@ FROM nginx:alpine RUN apk add --no-cache gettext -COPY nginx.conf.template /etc/nginx/nginx.conf.template +COPY comps/nginx/docker/nginx.conf.template /etc/nginx/nginx.conf.template ENV FRONTEND_SERVICE_IP=localhost ENV FRONTEND_SERVICE_PORT=5173 @@ -14,7 +14,7 @@ ENV BACKEND_SERVICE_NAME=chatqna ENV BACKEND_SERVICE_IP=localhost ENV BACKEND_SERVICE_PORT=8888 -COPY start-nginx.sh /usr/local/bin/start-nginx.sh +COPY comps/nginx/docker/start-nginx.sh /usr/local/bin/start-nginx.sh RUN chmod +x /usr/local/bin/start-nginx.sh CMD ["/usr/local/bin/start-nginx.sh"] diff --git a/comps/retrievers/llamaindex/docker/Dockerfile b/comps/retrievers/llamaindex/docker/Dockerfile index fec155b05..4b022718a 100644 --- a/comps/retrievers/llamaindex/docker/Dockerfile +++ b/comps/retrievers/llamaindex/docker/Dockerfile @@ -4,6 +4,7 @@ FROM ubuntu:22.04 RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + python3-pip \ libgl1-mesa-glx \ libjemalloc-dev diff --git a/tests/test_embeddings_llama_index.sh b/tests/test_embeddings_llama_index.sh index 048726044..81eac442b 100644 --- a/tests/test_embeddings_llama_index.sh +++ b/tests/test_embeddings_llama_index.sh @@ -11,12 +11,12 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH echo $(pwd) - docker build --no-cache -t opea/embedding-tei-llamaindex:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/llama_index/docker/Dockerfile . + docker build --no-cache -t opea/embedding-tei-llama-index:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/llama_index/docker/Dockerfile . if [ $? -ne 0 ]; then - echo "opea/embedding-tei-llamaindex built fail" + echo "opea/embedding-tei-llama-index built fail" exit 1 else - echo "opea/embedding-tei-llamaindex built successful" + echo "opea/embedding-tei-llama-index built successful" fi } @@ -24,17 +24,17 @@ function start_service() { tei_endpoint=5001 model="BAAI/bge-large-en-v1.5" revision="refs/pr/5" - docker run -d --name="test-comps-embedding-tei-llamaindex-endpoint" -p $tei_endpoint:80 -v ./data:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision + docker run -d --name="test-comps-embedding-tei-llama-index-endpoint" -p $tei_endpoint:80 -v ./data:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 --model-id $model --revision $revision export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}" tei_service_port=5010 - docker run -d --name="test-comps-embedding-tei-llamaindex-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p ${tei_service_port}:6000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei-llamaindex:comps + docker run -d --name="test-comps-embedding-tei-llama-index-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p ${tei_service_port}:6000 --ipc=host -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/embedding-tei-llama-index:comps sleep 3m } function validate_microservice() { tei_service_port=5010 URL="http://${ip_address}:$tei_service_port/v1/embeddings" - docker logs test-comps-embedding-tei-llamaindex-server >> ${LOG_PATH}/embedding.log + docker logs test-comps-embedding-tei-llama-index-server >> ${LOG_PATH}/embedding.log HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d '{"text":"What is Deep Learning?"}' -H 'Content-Type: application/json' "$URL") if [ "$HTTP_STATUS" -eq 200 ]; then echo "[ embedding - llama_index ] HTTP status is 200. Checking content..." @@ -44,12 +44,12 @@ function validate_microservice() { echo "[ embedding - llama_index ] Content is as expected." else echo "[ embedding - llama_index ] Content does not match the expected result: $CONTENT" - docker logs test-comps-embedding-tei-llamaindex-server >> ${LOG_PATH}/embedding.log + docker logs test-comps-embedding-tei-llama-index-server >> ${LOG_PATH}/embedding.log exit 1 fi else echo "[ embedding - llama_index ] HTTP status is not 200. Received status was $HTTP_STATUS" - docker logs test-comps-embedding-tei-llamaindex-server >> ${LOG_PATH}/embedding.log + docker logs test-comps-embedding-tei-llama-index-server >> ${LOG_PATH}/embedding.log exit 1 fi } diff --git a/tests/test_nginx.sh b/tests/test_nginx.sh index 3675a5753..626c6974a 100644 --- a/tests/test_nginx.sh +++ b/tests/test_nginx.sh @@ -9,8 +9,8 @@ LOG_PATH="$WORKPATH/tests" ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { - cd $WORKPATH/comps/nginx/docker - docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/nginx:comps -f ./Dockerfile . + cd $WORKPATH + docker build --no-cache --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -t opea/nginx:comps -f comps/nginx/docker/Dockerfile . if [ $? -ne 0 ]; then echo "opea/nginx built fail" exit 1 From 84a7e573b88eed6d263b3ba164361ea06fccc46d Mon Sep 17 00:00:00 2001 From: Liangyx2 Date: Sun, 1 Sep 2024 22:19:25 +0800 Subject: [PATCH 03/16] add microservice for intent detection (#131) * add microservice for intent detection Signed-off-by: Liangyx2 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update license copyright Signed-off-by: Liangyx2 * add ut Signed-off-by: Liangyx2 * refine Signed-off-by: Liangyx2 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update folder Signed-off-by: Liangyx2 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix test Signed-off-by: Liangyx2 --------- Signed-off-by: Liangyx2 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- comps/intent_detection/README.md | 99 +++++++++++++++++++ comps/intent_detection/langchain/Dockerfile | 25 +++++ .../langchain/docker_compose_intent.yaml | 32 ++++++ .../langchain/intent_detection.py | 47 +++++++++ .../langchain/requirements.txt | 9 ++ comps/intent_detection/langchain/template.py | 8 ++ tests/test_intent_detection_langchain.sh | 57 +++++++++++ 7 files changed, 277 insertions(+) create mode 100644 comps/intent_detection/README.md create mode 100644 comps/intent_detection/langchain/Dockerfile create mode 100644 comps/intent_detection/langchain/docker_compose_intent.yaml create mode 100644 comps/intent_detection/langchain/intent_detection.py create mode 100644 comps/intent_detection/langchain/requirements.txt create mode 100644 comps/intent_detection/langchain/template.py create mode 100644 tests/test_intent_detection_langchain.sh diff --git a/comps/intent_detection/README.md b/comps/intent_detection/README.md new file mode 100644 index 000000000..fa9062bb6 --- /dev/null +++ b/comps/intent_detection/README.md @@ -0,0 +1,99 @@ +# Intent Detection Microservice by TGI + +# πŸš€1. Start Microservice with Python（Option 1οΌ‰ + +## 1.1 Install Requirements + +```bash +pip install -r requirements.txt +``` + +## 1.2 Start TGI Service + +```bash +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms" +docker run -p 8008:80 -v ./data:/data --name tgi_service --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model} +``` + +## 1.3 Verify the TGI Service + +```bash +curl http://${your_ip}:8008/generate \ + -X POST \ + -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \ + -H 'Content-Type: application/json' +``` + +## 1.4 Setup Environment Variables + +```bash +export TGI_LLM_ENDPOINT="http://${your_ip}:8008" +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/intent" +``` + +## 1.5 Start Intent Detection Microservice with Python Script + +Start intent detection microservice with below command. + +```bash +cd /your_project_path/GenAIComps/ +cp comps/intent_detection/langchain/intent_detection.py . +python intent_detection.py +``` + +# πŸš€2. Start Microservice with Docker (Option 2) + +## 2.1 Start TGI Service + +Please refer to 1.2. + +## 2.2 Setup Environment Variables + +```bash +export TGI_LLM_ENDPOINT="http://${your_ip}:8008" +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/intent" +``` + +## 2.3 Build Docker Image + +```bash +cd /your_project_path/GenAIComps +docker build --no-cache -t opea/llm-tgi:latest -f comps/intent_detection/langchain/Dockerfile . +``` + +## 2.4 Run Docker with CLI (Option A) + +```bash +docker run -it --name="intent-tgi-server" --net=host --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:latest +``` + +## 2.5 Run with Docker Compose (Option B) + +```bash +cd /your_project_path/GenAIComps/comps/intent_detection/langchain +export LLM_MODEL_ID=${your_hf_llm_model} +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export TGI_LLM_ENDPOINT="http://tgi-service:80" +export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export LANGCHAIN_API_KEY=${your_langchain_api_key} +docker compose -f docker_compose_intent.yaml up -d +``` + +# πŸš€3. Consume Microservice + +Once intent detection microservice is started, user can use below command to invoke the microservice. + +```bash +curl http://${your_ip}:9000/v1/chat/intent\ + -X POST \ + -d '{"query":"What is Deep Learning?","max_new_tokens":10,"top_k":1,"temperature":0.001,"streaming":false}' \ + -H 'Content-Type: application/json' +``` diff --git a/comps/intent_detection/langchain/Dockerfile b/comps/intent_detection/langchain/Dockerfile new file mode 100644 index 000000000..297b1b88f --- /dev/null +++ b/comps/intent_detection/langchain/Dockerfile @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM langchain/langchain:latest + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/user/comps/intent_detection/langchain/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/intent_detection/langchain +ENTRYPOINT ["python", "intent_detection.py"] diff --git a/comps/intent_detection/langchain/docker_compose_intent.yaml b/comps/intent_detection/langchain/docker_compose_intent.yaml new file mode 100644 index 000000000..2a1524273 --- /dev/null +++ b/comps/intent_detection/langchain/docker_compose_intent.yaml @@ -0,0 +1,32 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + tgi_service: + image: ghcr.io/huggingface/text-generation-inference:1.4 + container_name: tgi-service + ports: + - "8008:80" + volumes: + - "./data:/data" + shm_size: 1g + command: --model-id ${LLM_MODEL_ID} + llm: + image: opea/llm-tgi:latest + container_name: intent-tgi-server + ports: + - "9000:9000" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/intent_detection/langchain/intent_detection.py b/comps/intent_detection/langchain/intent_detection.py new file mode 100644 index 000000000..bf2e430c6 --- /dev/null +++ b/comps/intent_detection/langchain/intent_detection.py @@ -0,0 +1,47 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +from langchain import LLMChain, PromptTemplate +from langchain_community.llms import HuggingFaceEndpoint +from langsmith import traceable + +from comps import GeneratedDoc, LLMParamsDoc, ServiceType, opea_microservices, register_microservice +from comps.intent_detection.langchain.template import IntentTemplate + + +@register_microservice( + name="opea_service@llm_intent", + service_type=ServiceType.LLM, + endpoint="/v1/chat/intent", + host="0.0.0.0", + port=9000, +) +@traceable(run_type="llm") +def llm_generate(input: LLMParamsDoc): + llm_endpoint = os.getenv("TGI_LLM_ENDPOINT", "http://localhost:8080") + llm = HuggingFaceEndpoint( + endpoint_url=llm_endpoint, + max_new_tokens=input.max_new_tokens, + top_k=input.top_k, + top_p=input.top_p, + typical_p=input.typical_p, + temperature=input.temperature, + repetition_penalty=input.repetition_penalty, + streaming=input.streaming, + timeout=600, + ) + + prompt = PromptTemplate(template=IntentTemplate.generate_intent_template, input_variables=["query"]) + + llm_chain = LLMChain(prompt=prompt, llm=llm) + + response = llm_chain.invoke(input.query) + response = response["text"] + print("response", response) + return GeneratedDoc(text=response, prompt=input.query) + + +if __name__ == "__main__": + opea_microservices["opea_service@llm_intent"].start() diff --git a/comps/intent_detection/langchain/requirements.txt b/comps/intent_detection/langchain/requirements.txt new file mode 100644 index 000000000..55cf47ae7 --- /dev/null +++ b/comps/intent_detection/langchain/requirements.txt @@ -0,0 +1,9 @@ +docarray[full] +fastapi +huggingface_hub +langchain==0.1.16 +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +shortuuid diff --git a/comps/intent_detection/langchain/template.py b/comps/intent_detection/langchain/template.py new file mode 100644 index 000000000..1a425ce43 --- /dev/null +++ b/comps/intent_detection/langchain/template.py @@ -0,0 +1,8 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +class IntentTemplate: + def generate_intent_template(query): + return f"""Please identify the intent of the user query. You may only respond with "chitchat" or "QA" without explanations or engaging in conversation. +### User Query: {query}, ### Response: """ diff --git a/tests/test_intent_detection_langchain.sh b/tests/test_intent_detection_langchain.sh new file mode 100644 index 000000000..45910ca7f --- /dev/null +++ b/tests/test_intent_detection_langchain.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/llm-tgi:latest -f comps/intent_detection/langchain/Dockerfile . +} + +function start_service() { + tgi_endpoint=5004 + # Remember to set HF_TOKEN before invoking this test! + export HUGGINGFACEHUB_API_TOKEN=${HF_TOKEN} + model=Intel/neural-chat-7b-v3-3 + docker run -d --name="test-comps-intent-tgi-endpoint" -p $tgi_endpoint:80 -v ./data:/data --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id $model + + export TGI_LLM_ENDPOINT="http://${ip_address}:${tgi_endpoint}" + tei_service_port=5005 + unset http_proxy + docker run -d --name="test-comps-intent-tei-server" -p ${tei_service_port}:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:latest + sleep 3m +} + +function validate_microservice() { + tei_service_port=5005 + http_proxy="" curl http://${ip_address}:${tei_service_port}/v1/chat/intent\ + -X POST \ + -d '{"query":"What is Deep Learning?","max_new_tokens":10,"top_k":1,"temperature":0.001,"streaming":false}' \ + -H 'Content-Type: application/json' + docker logs test-comps-intent-tei-server + docker logs test-comps-intent-tgi-endpoint +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-intent*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main From ae71eeee27796c3f39a85f16f387600ff7f460b9 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Mon, 2 Sep 2024 14:48:48 +0800 Subject: [PATCH 04/16] Make the scanning method optional. (#580) Signed-off-by: zepan --- .github/workflows/manual-bom-scan.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-bom-scan.yml b/.github/workflows/manual-bom-scan.yml index 9275406ba..4a781b254 100644 --- a/.github/workflows/manual-bom-scan.yml +++ b/.github/workflows/manual-bom-scan.yml @@ -15,6 +15,16 @@ on: description: "Tag to apply to images" required: true type: string + sbom-scan: + default: true + description: "Enable sbom-scan" + required: false + type: boolean + trivy-scan: + default: true + description: "Enable trivy-scan" + required: false + type: boolean permissions: read-all jobs: @@ -55,7 +65,7 @@ jobs: - name: SBOM Scan Container uses: anchore/sbom-action@v0.17.1 - if: always() + if: ${{ fromJSON(inputs.sbom-scan) }} with: image: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }} output-file: ${{ matrix.image }}-sbom-scan.txt @@ -63,7 +73,7 @@ jobs: - name: Security Scan Container uses: aquasecurity/trivy-action@0.24.0 - if: always() + if: ${{ fromJSON(inputs.trivy-scan) }} with: image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }} output: ${{ matrix.image }}-trivy-scan.txt From 0379aebd28f4a1a7361dca8d5941503b2a99e824 Mon Sep 17 00:00:00 2001 From: kevinintel Date: Tue, 3 Sep 2024 06:53:55 +0800 Subject: [PATCH 05/16] add code owners (#586) --- .github/CODEOWNERS | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 972b2a181..2636246ae 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,21 @@ +/comps/agent/ xuhui.ren@intel.com +/comps/finetuning/ xinyu.ye@intel.com +/comps/guardrails/ liang1.lv@intel.com /comps/asr/ sihan.chen@intel.com +/comps/intent_detection/ liang1.lv@intel.com +/comps/knowledgegraphs/ xuhui.ren@intel.com /comps/cores/ liang1.lv@intel.com /comps/dataprep/ xinyu.ye@intel.com /comps/embeddings/ xuhui.ren@intel.com /comps/guardrails/ letong.han@intel.com /comps/llms/ liang1.lv@intel.com +/comps/lvms/ sihan.chen@intel.com +/comps/nginx/ letong.han@intel.com +/comps/prompt_registry/ hoong.tee.yeoh@intel.com +/comps/chathistory/ yogesh.pandey@intel.com /comps/reranks/ xuhui.ren@intel.com /comps/retrievers/ xuhui.ren@intel.com /comps/tts/ sihan.chen@intel.com +/comps/ragas/ xuhui.ren@intel.com +/comps/vectorstores/ xinyu.ye@intel.com +/comps/web_retrievers/ sihan.chen@intel.com From d60907163e85f2399fbb2fb712e2adefc11fd678 Mon Sep 17 00:00:00 2001 From: Letong Han <106566639+letonghan@users.noreply.github.com> Date: Tue, 3 Sep 2024 10:27:40 +0800 Subject: [PATCH 06/16] remove revision for tei (#584) Signed-off-by: letonghan --- comps/embeddings/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/comps/embeddings/README.md b/comps/embeddings/README.md index 407644ada..8ac6dfe0c 100644 --- a/comps/embeddings/README.md +++ b/comps/embeddings/README.md @@ -44,8 +44,7 @@ First, you need to start a TEI service. ```bash your_port=8090 model="BAAI/bge-large-en-v1.5" -revision="refs/pr/5" -docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 --model-id $model --revision $revision +docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 --model-id $model ``` Then you need to test your TEI service using the following commands: @@ -88,8 +87,7 @@ First, you need to start a TEI service. ```bash your_port=8090 model="BAAI/bge-large-en-v1.5" -revision="refs/pr/5" -docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 --model-id $model --revision $revision +docker run -p $your_port:80 -v ./data:/data --name tei_server -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 --model-id $model ``` Then you need to test your TEI service using the following commands: From 12fd97aec172499f21294071aad91e30d8fc9cfa Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Tue, 3 Sep 2024 12:00:54 +0800 Subject: [PATCH 07/16] Bug fix (#591) * Check if the document exists. Signed-off-by: zepan * Add flag output. Signed-off-by: zepan * Modify nginx readme. Signed-off-by: zepan * Modify document detection logic Signed-off-by: zepan --------- Signed-off-by: zepan --- .github/workflows/_comps-workflow.yml | 23 +++++++++++++++++++---- comps/nginx/README.md | 4 ++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml index c2a68b0ca..3a7591de0 100644 --- a/.github/workflows/_comps-workflow.yml +++ b/.github/workflows/_comps-workflow.yml @@ -38,6 +38,8 @@ jobs: build-images: runs-on: "docker-build-${{ inputs.node }}" continue-on-error: true + outputs: + file_exists: ${{ steps.get-yaml-path.outputs.file_exists }} steps: - name: Clean Up Working Directory run: sudo rm -rf ${{github.workspace}}/* @@ -51,10 +53,23 @@ jobs: cd ${{ github.workspace }}/.github/workflows/docker/compose # service=$(echo ${{ inputs.service }} | cut -d'_' -f1) if [[ "${{ inputs.mode }}" == "CD" ]]; then - docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml + if [ -f "${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml" ]; then + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml + echo "file_exists=true" >> $GITHUB_OUTPUT + else + echo "There is no CD part of ${{ inputs.service }} that needs to be executed." + echo "file_exists=false" >> $GITHUB_OUTPUT + fi else - docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml + if [ -f "${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml" ]; then + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml + echo "file_exists=true" >> $GITHUB_OUTPUT + else + echo "There is no CI part of ${{ inputs.service }} that needs to be executed." + echo "file_exists=false" >> $GITHUB_OUTPUT + fi fi + echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 @@ -64,7 +79,7 @@ jobs: fi - name: Build Image - if: ${{ fromJSON(inputs.build) }} + if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }} uses: opea-project/validation/actions/image-build@main with: work_dir: ${{ github.workspace }} @@ -77,7 +92,7 @@ jobs: #################################################################################################### test-service-compose: needs: [build-images] - if: ${{ fromJSON(inputs.test) }} + if: ${{ fromJSON(inputs.test) && needs.build-images.outputs.file_exists == 'true' }} uses: ./.github/workflows/_run-docker-compose.yml with: tag: ${{ inputs.tag }} diff --git a/comps/nginx/README.md b/comps/nginx/README.md index 416b9b2e1..cae340331 100644 --- a/comps/nginx/README.md +++ b/comps/nginx/README.md @@ -7,8 +7,8 @@ In GenAIComps, we utilize nginx to streamline our network services. We provide a ## πŸš€1. Build Docker Image ```bash -cd docker -docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./Dockerfile . +cd ../.. +docker build -t opea/nginx:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/nginx/docker/Dockerfile . ``` ## πŸš€2. Environment Settings From 8325d5d9ba47330e3145af41c0d3d80e16fc726d Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Tue, 3 Sep 2024 14:17:26 +0800 Subject: [PATCH 08/16] Remove duplicate code (#596) Signed-off-by: zepan --- .github/workflows/_comps-workflow.yml | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml index 3a7591de0..ca402c445 100644 --- a/.github/workflows/_comps-workflow.yml +++ b/.github/workflows/_comps-workflow.yml @@ -53,24 +53,18 @@ jobs: cd ${{ github.workspace }}/.github/workflows/docker/compose # service=$(echo ${{ inputs.service }} | cut -d'_' -f1) if [[ "${{ inputs.mode }}" == "CD" ]]; then - if [ -f "${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml" ]; then - docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml - echo "file_exists=true" >> $GITHUB_OUTPUT - else - echo "There is no CD part of ${{ inputs.service }} that needs to be executed." - echo "file_exists=false" >> $GITHUB_OUTPUT - fi + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml else - if [ -f "${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml" ]; then - docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml - echo "file_exists=true" >> $GITHUB_OUTPUT - else - echo "There is no CI part of ${{ inputs.service }} that needs to be executed." - echo "file_exists=false" >> $GITHUB_OUTPUT - fi + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml fi - echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT + if [ -f "$docker_compose_yml" ]; then + echo "file_exists=true" >> $GITHUB_OUTPUT + else + echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed." + echo "file_exists=false" >> $GITHUB_OUTPUT + fi + if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 fi From 2a53e2547dbc95b5f8ef442ec300be23d0471d1a Mon Sep 17 00:00:00 2001 From: sri-intel <108247623+srinarayan-srikanthan@users.noreply.github.com> Date: Tue, 3 Sep 2024 20:55:33 -0400 Subject: [PATCH 09/16] adding embedding support for CLIP based models for VideoRAGQnA example for v0.9 (#538) * clip embedding support Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * test script for embedding Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * fix freeze workflow (#522) Signed-off-by: Sun, Xuehao Signed-off-by: srinarayan-srikanthan * Fix Dataprep Potential Error in get_file (#540) * fix get file error & refine logs Signed-off-by: letonghan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: letonghan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Support SearchedDoc input type in LLM for No Rerank Pipeline (#541) Signed-off-by: letonghan Signed-off-by: srinarayan-srikanthan * Add dependency for pdf2image and OCR processing (#421) Signed-off-by: srinarayan-srikanthan * Add local_embedding return 768 length to align with chatqna example (#313) Signed-off-by: Chendi.Xue Signed-off-by: srinarayan-srikanthan * add telemetry doc (#536) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Add video-llama LVM microservice under lvms (#495) Signed-off-by: BaoHuiling Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Fix the data load issue for structured files (#505) Signed-off-by: XuhuiRen Signed-off-by: srinarayan-srikanthan * Add finetuning component (#502) Signed-off-by: Xinyu Ye Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: test Co-authored-by: root Co-authored-by: Letong Han <106566639+letonghan@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * add torchvision into requirements (#546) Signed-off-by: chensuyue Signed-off-by: srinarayan-srikanthan * Use Gaudi base images from Dockerhub (#526) * Use Gaudi base images from Dockerhub Signed-off-by: Abolfazl Shahbazi * Fixing the malformed tag Signed-off-by: Abolfazl Shahbazi * fix another malformed tag Signed-off-by: Abolfazl Shahbazi --------- Signed-off-by: Abolfazl Shahbazi Signed-off-by: srinarayan-srikanthan * Add toxicity detection microservice (#338) * Add toxicity detection microservice Signed-off-by: Qun Gao * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Modification to toxicity plugin PR (#432) * changed microservice to use Service.GUARDRAILS and input/output to TextDoc Signed-off-by: Tyler Wilbers * simplify dockerfile to use langchain Signed-off-by: Tyler Wilbers * sort requirements Signed-off-by: Tyler Wilbers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Tyler Wilbers Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Minor SPDX header update (#434) Signed-off-by: Abolfazl Shahbazi * Remove 'langsmith' per code review (#534) Signed-off-by: Abolfazl Shahbazi * Add toxicity detection microservices with E2E testing Signed-off-by: Qun Gao --------- Signed-off-by: Qun Gao Signed-off-by: Tyler Wilbers Signed-off-by: Abolfazl Shahbazi Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Abolfazl Shahbazi Co-authored-by: Tyler W Signed-off-by: srinarayan-srikanthan * rename script and use 5xxx Signed-off-by: BaoHuiling Signed-off-by: srinarayan-srikanthan * add proxy for build Signed-off-by: BaoHuiling Signed-off-by: srinarayan-srikanthan * fixed commit issues Signed-off-by: srinarayan-srikanthan * Fix docarray constraint Signed-off-by: srinarayan-srikanthan * updated docarray Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * rm telemetry which cause error in mega Signed-off-by: BaoHuiling * renamed dirs Signed-off-by: srinarayan-srikanthan * renamed test Signed-off-by: srinarayan-srikanthan --------- Signed-off-by: srinarayan-srikanthan Signed-off-by: Sun, Xuehao Signed-off-by: letonghan Signed-off-by: Chendi.Xue Signed-off-by: BaoHuiling Signed-off-by: XuhuiRen Signed-off-by: Xinyu Ye Signed-off-by: chensuyue Signed-off-by: Abolfazl Shahbazi Signed-off-by: Qun Gao Signed-off-by: Tyler Wilbers Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sun, Xuehao Co-authored-by: Letong Han <106566639+letonghan@users.noreply.github.com> Co-authored-by: Zaili Wang <109502517+ZailiWang@users.noreply.github.com> Co-authored-by: Chendi.Xue Co-authored-by: Sihan Chen <39623753+Spycsh@users.noreply.github.com> Co-authored-by: Huiling Bao Co-authored-by: XuhuiRen <44249229+XuhuiRen@users.noreply.github.com> Co-authored-by: XinyuYe-Intel Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: test Co-authored-by: root Co-authored-by: chen, suyue Co-authored-by: Abolfazl Shahbazi Co-authored-by: qgao007 <108324932+qgao007@users.noreply.github.com> Co-authored-by: Tyler W --- comps/cores/proto/docarray.py | 1 + comps/dataprep/milvus/prepare_doc_milvus.py | 1 + comps/embeddings/multimodal_clip/README.md | 52 +++++++++++ comps/embeddings/multimodal_clip/__init__.py | 2 + .../multimodal_clip/docker/Dockerfile | 29 +++++++ .../docker/docker_compose_embedding.yaml | 22 +++++ .../multimodal_clip/embedding_multimodal.py | 86 +++++++++++++++++++ .../multimodal_clip/embeddings_clip.py | 50 +++++++++++ .../multimodal_clip/requirements.txt | 14 +++ comps/finetuning/README.md | 6 ++ comps/finetuning/datasets/.gitkeep | 0 comps/finetuning/handlers.py | 7 ++ comps/finetuning/jobs/.gitkeep | 0 comps/finetuning/lanuch.sh | 12 +++ comps/finetuning/llm_on_ray/common/logging.py | 56 ++++++++++++ .../llm_on_ray/finetune/finetune.py | 1 + comps/guardrails/toxicity_detection/README.md | 1 + .../toxicity_detection/docker/Dockerfile | 1 + .../lvms/video-llama/server/requirements.txt | 1 + comps/lvms/video-llama/server/server.py | 4 + tests/test_embeddings_clip_multimodal.sh | 61 +++++++++++++ tests/test_lvms_video-llama.sh | 2 + 22 files changed, 409 insertions(+) create mode 100644 comps/embeddings/multimodal_clip/README.md create mode 100644 comps/embeddings/multimodal_clip/__init__.py create mode 100644 comps/embeddings/multimodal_clip/docker/Dockerfile create mode 100644 comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml create mode 100644 comps/embeddings/multimodal_clip/embedding_multimodal.py create mode 100644 comps/embeddings/multimodal_clip/embeddings_clip.py create mode 100644 comps/embeddings/multimodal_clip/requirements.txt create mode 100644 comps/finetuning/datasets/.gitkeep create mode 100644 comps/finetuning/jobs/.gitkeep create mode 100644 comps/finetuning/lanuch.sh create mode 100644 comps/finetuning/llm_on_ray/common/logging.py create mode 100755 tests/test_embeddings_clip_multimodal.sh diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 1a29aa329..587278ec1 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -64,6 +64,7 @@ class EmbedDoc(BaseDoc): fetch_k: int = 20 lambda_mult: float = 0.5 score_threshold: float = 0.2 + constraints: Optional[Union[Dict[str, Any], None]] = None class EmbedMultimodalDoc(EmbedDoc): diff --git a/comps/dataprep/milvus/prepare_doc_milvus.py b/comps/dataprep/milvus/prepare_doc_milvus.py index 38ad4ef42..6fa363603 100644 --- a/comps/dataprep/milvus/prepare_doc_milvus.py +++ b/comps/dataprep/milvus/prepare_doc_milvus.py @@ -133,6 +133,7 @@ def ingest_data_to_milvus(doc_path: DocPath, embedder): ) content = document_loader(path) + if logflag: logger.info("[ ingest data ] file content loaded") diff --git a/comps/embeddings/multimodal_clip/README.md b/comps/embeddings/multimodal_clip/README.md new file mode 100644 index 000000000..aca6611d0 --- /dev/null +++ b/comps/embeddings/multimodal_clip/README.md @@ -0,0 +1,52 @@ +# Multimodal CLIP Embeddings Microservice + +The Multimodal CLIP Embedding Microservice is designed to efficiently convert textual strings and images into vectorized embeddings, facilitating seamless integration into various machine learning and data processing workflows. This service utilizes advanced algorithms to generate high-quality embeddings that capture the semantic essence of the input text and images, making it ideal for applications in multi-modal data processing, information retrieval, and similar fields. + +Key Features: + +**High Performance**: Optimized for quick and reliable conversion of textual data and image inputs into vector embeddings. + +**Scalability**: Built to handle high volumes of requests simultaneously, ensuring robust performance even under heavy loads. + +**Ease of Integration**: Provides a simple and intuitive API, allowing for straightforward integration into existing systems and workflows. + +**Customizable**: Supports configuration and customization to meet specific use case requirements, including different embedding models and preprocessing techniques. + +Users are albe to configure and build embedding-related services according to their actual needs. + +## πŸš€1. Start Microservice with Docker + +### 1.1 Build Docker Image + +#### Build Langchain Docker + +```bash +cd ../../.. +docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/docker/Dockerfile . +``` + +### 1.2 Run Docker with Docker Compose + +```bash +cd comps/embeddings/multimodal_clip/docker +docker compose -f docker_compose_embedding.yaml up -d +``` + +## πŸš€2. Consume Embedding Service + +### 2.1 Check Service Status + +```bash +curl http://localhost:6000/v1/health_check\ + -X GET \ + -H 'Content-Type: application/json' +``` + +### 2.2 Consume Embedding Service + +```bash +curl http://localhost:6000/v1/embeddings \ + -X POST -d '{"text":"Sample text"}' \ + -H 'Content-Type: application/json' + +``` diff --git a/comps/embeddings/multimodal_clip/__init__.py b/comps/embeddings/multimodal_clip/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/embeddings/multimodal_clip/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/embeddings/multimodal_clip/docker/Dockerfile b/comps/embeddings/multimodal_clip/docker/Dockerfile new file mode 100644 index 000000000..aefba8aad --- /dev/null +++ b/comps/embeddings/multimodal_clip/docker/Dockerfile @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM langchain/langchain:latest + +ARG ARCH="cpu" + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ + pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal_clip/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/embeddings/multimodal_clip + +ENTRYPOINT ["python", "embedding_multimodal.py"] diff --git a/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml b/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml new file mode 100644 index 000000000..7d314b93a --- /dev/null +++ b/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + embedding: + image: opea/embedding-multimodal:latest + container_name: embedding-multimodal-server + ports: + - "6000:6000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/embeddings/multimodal_clip/embedding_multimodal.py b/comps/embeddings/multimodal_clip/embedding_multimodal.py new file mode 100644 index 000000000..9320a827b --- /dev/null +++ b/comps/embeddings/multimodal_clip/embedding_multimodal.py @@ -0,0 +1,86 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import datetime +import os +import time +from typing import Union + +from dateparser.search import search_dates +from embeddings_clip import vCLIP + +from comps import ( + EmbedDoc, + ServiceType, + TextDoc, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) + + +def filtler_dates(prompt): + + base_date = datetime.datetime.today() + today_date = base_date.date() + dates_found = search_dates(prompt, settings={"PREFER_DATES_FROM": "past", "RELATIVE_BASE": base_date}) + + if dates_found is not None: + for date_tuple in dates_found: + date_string, parsed_date = date_tuple + date_out = str(parsed_date.date()) + time_out = str(parsed_date.time()) + hours, minutes, seconds = map(float, time_out.split(":")) + year, month, day_out = map(int, date_out.split("-")) + + rounded_seconds = min(round(parsed_date.second + 0.5), 59) + parsed_date = parsed_date.replace(second=rounded_seconds, microsecond=0) + + iso_date_time = parsed_date.isoformat() + iso_date_time = str(iso_date_time) + + if date_string == "today": + constraints = {"date": ["==", date_out]} + elif date_out != str(today_date) and time_out == "00:00:00": ## exact day (example last friday) + constraints = {"date": ["==", date_out]} + elif ( + date_out == str(today_date) and time_out == "00:00:00" + ): ## when search_date interprates words as dates output is todays date + time 00:00:00 + constraints = {} + else: ## Interval of time:last 48 hours, last 2 days,.. + constraints = {"date_time": [">=", {"_date": iso_date_time}]} + return constraints + + else: + return {} + + +@register_microservice( + name="opea_service@embedding_multimodal", + service_type=ServiceType.EMBEDDING, + endpoint="/v1/embeddings", + host="0.0.0.0", + port=6000, + input_datatype=TextDoc, + output_datatype=EmbedDoc, +) +@register_statistics(names=["opea_service@embedding_multimodal"]) +def embedding(input: TextDoc) -> EmbedDoc: + start = time.time() + + if isinstance(input, TextDoc): + # Handle text input + embed_vector = embeddings.embed_query(input.text).tolist()[0] + res = EmbedDoc(text=input.text, embedding=embed_vector, constraints=filtler_dates(input.text)) + + else: + raise ValueError("Invalid input type") + + statistics_dict["opea_service@embedding_multimodal"].append_latency(time.time() - start, None) + return res + + +if __name__ == "__main__": + embeddings = vCLIP({"model_name": "openai/clip-vit-base-patch32", "num_frm": 4}) + opea_microservices["opea_service@embedding_multimodal"].start() diff --git a/comps/embeddings/multimodal_clip/embeddings_clip.py b/comps/embeddings/multimodal_clip/embeddings_clip.py new file mode 100644 index 000000000..39db85b6e --- /dev/null +++ b/comps/embeddings/multimodal_clip/embeddings_clip.py @@ -0,0 +1,50 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import torch +import torch.nn as nn +from einops import rearrange +from transformers import AutoProcessor, AutoTokenizer, CLIPModel + +model_name = "openai/clip-vit-base-patch32" + +clip = CLIPModel.from_pretrained(model_name) +processor = AutoProcessor.from_pretrained(model_name) +tokenizer = AutoTokenizer.from_pretrained(model_name) + + +class vCLIP(nn.Module): + def __init__(self, cfg): + super().__init__() + + self.num_frm = cfg["num_frm"] + self.model_name = cfg["model_name"] + + def embed_query(self, texts): + """Input is list of texts.""" + text_inputs = tokenizer(texts, padding=True, return_tensors="pt") + text_features = clip.get_text_features(**text_inputs) + return text_features + + def get_embedding_length(self): + return len(self.embed_query("sample_text")) + + def get_image_embeddings(self, images): + """Input is list of images.""" + image_inputs = processor(images=images, return_tensors="pt") + image_features = clip.get_image_features(**image_inputs) + return image_features + + def get_video_embeddings(self, frames_batch): + """Input is list of list of frames in video.""" + self.batch_size = len(frames_batch) + vid_embs = [] + for frames in frames_batch: + frame_embeddings = self.get_image_embeddings(frames) + frame_embeddings = rearrange(frame_embeddings, "(b n) d -> b n d", b=len(frames_batch)) + # Normalize, mean aggregate and return normalized video_embeddings + frame_embeddings = frame_embeddings / frame_embeddings.norm(dim=-1, keepdim=True) + video_embeddings = frame_embeddings.mean(dim=1) + video_embeddings = video_embeddings / video_embeddings.norm(dim=-1, keepdim=True) + vid_embs.append(video_embeddings) + return torch.cat(vid_embs, dim=0) diff --git a/comps/embeddings/multimodal_clip/requirements.txt b/comps/embeddings/multimodal_clip/requirements.txt new file mode 100644 index 000000000..c914a0d52 --- /dev/null +++ b/comps/embeddings/multimodal_clip/requirements.txt @@ -0,0 +1,14 @@ +dateparser +docarray[full] +einops +fastapi +huggingface_hub +langchain +open_clip_torch +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +sentence_transformers +shortuuid +uvicorn diff --git a/comps/finetuning/README.md b/comps/finetuning/README.md index 44ee3d10c..33da63c29 100644 --- a/comps/finetuning/README.md +++ b/comps/finetuning/README.md @@ -92,10 +92,12 @@ Assuming a training file `alpaca_data.json` is uploaded, it can be downloaded in ```bash # upload a training file + curl http://${your_ip}:8015/v1/finetune/upload_training_files -X POST -H "Content-Type: multipart/form-data" -F "files=@./alpaca_data.json" # create a finetuning job curl http://${your_ip}:8015/v1/fine_tuning/jobs \ + -X POST \ -H "Content-Type: application/json" \ -d '{ @@ -104,18 +106,22 @@ curl http://${your_ip}:8015/v1/fine_tuning/jobs \ }' # list finetuning jobs + curl http://${your_ip}:8015/v1/fine_tuning/jobs -X GET # retrieve one finetuning job curl http://localhost:8015/v1/fine_tuning/jobs/retrieve -X POST -H "Content-Type: application/json" -d '{ + "fine_tuning_job_id": ${fine_tuning_job_id}}' # cancel one finetuning job + curl http://localhost:8015/v1/fine_tuning/jobs/cancel -X POST -H "Content-Type: application/json" -d '{ "fine_tuning_job_id": ${fine_tuning_job_id}}' # list checkpoints of a finetuning job curl http://${your_ip}:8015/v1/finetune/list_checkpoints -X POST -H "Content-Type: application/json" -d '{"fine_tuning_job_id": ${fine_tuning_job_id}}' + ``` diff --git a/comps/finetuning/datasets/.gitkeep b/comps/finetuning/datasets/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/comps/finetuning/handlers.py b/comps/finetuning/handlers.py index 6aa7e5d3e..e0d04bf24 100644 --- a/comps/finetuning/handlers.py +++ b/comps/finetuning/handlers.py @@ -53,7 +53,9 @@ def update_job_status(job_id: FineTuningJobID): status = str(job_status).lower() # Ray status "stopped" is OpenAI status "cancelled" status = "cancelled" if status == "stopped" else status + logger.info(f"Status of job {job_id} is '{status}'") + running_finetuning_jobs[job_id].status = status if status == "finished" or status == "cancelled" or status == "failed": break @@ -105,7 +107,9 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas ) finetune_config.General.output_dir = os.path.join(JOBS_PATH, job.id) if os.getenv("DEVICE", ""): + logger.info(f"specific device: {os.getenv('DEVICE')}") + finetune_config.Training.device = os.getenv("DEVICE") finetune_config_file = f"{JOBS_PATH}/{job.id}.yaml" @@ -120,6 +124,7 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas # Path to the local directory that contains the script.py file runtime_env={"working_dir": "./"}, ) + logger.info(f"Submitted Ray job: {ray_job_id} ...") running_finetuning_jobs[job.id] = job @@ -172,7 +177,9 @@ async def save_content_to_local_disk(save_path: str, content): content = await content.read() fout.write(content) except Exception as e: + logger.info(f"Write file failed. Exception: {e}") + raise Exception(status_code=500, detail=f"Write file {save_path} failed. Exception: {e}") diff --git a/comps/finetuning/jobs/.gitkeep b/comps/finetuning/jobs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/comps/finetuning/lanuch.sh b/comps/finetuning/lanuch.sh new file mode 100644 index 000000000..a7e249b6f --- /dev/null +++ b/comps/finetuning/lanuch.sh @@ -0,0 +1,12 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +if [[ -n "$RAY_PORT" ]];then + export RAY_ADDRESS=http://127.0.0.1:$RAY_PORT + ray start --head --port $RAY_PORT +else + export RAY_ADDRESS=http://127.0.0.1:8265 + ray start --head +fi + +python finetuning_service.py diff --git a/comps/finetuning/llm_on_ray/common/logging.py b/comps/finetuning/llm_on_ray/common/logging.py new file mode 100644 index 000000000..e2aec567a --- /dev/null +++ b/comps/finetuning/llm_on_ray/common/logging.py @@ -0,0 +1,56 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright 2023 The LLM-on-Ray Authors. + +import functools +import logging +import logging.config +import traceback + +__all__ = ["logger", "get_logger"] + +use_accelerate_log = False +logger_name = "common" + +logging_config = { + "version": 1, + "loggers": { + "root": {"level": "INFO", "handlers": ["consoleHandler"]}, + "common": { + "level": "INFO", + "handlers": ["consoleHandler"], + "qualname": "common", + "propagate": 0, + }, + }, + "handlers": { + "consoleHandler": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "standardFormatter", + }, + }, + "formatters": { + "standardFormatter": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + "datefmt": "", + } + }, +} + +if logging_config is not None: + try: + logging.config.dictConfig(logging_config) + except Exception: + traceback.print_exc() + exit(1) + +if use_accelerate_log: + import accelerate + + get_logger = functools.partial(accelerate.logging.get_logger, name=logger_name) +else: + get_logger = functools.partial(logging.getLogger, name=logger_name) + +logger = get_logger() diff --git a/comps/finetuning/llm_on_ray/finetune/finetune.py b/comps/finetuning/llm_on_ray/finetune/finetune.py index 03b8adfaa..868dc889e 100644 --- a/comps/finetuning/llm_on_ray/finetune/finetune.py +++ b/comps/finetuning/llm_on_ray/finetune/finetune.py @@ -405,6 +405,7 @@ def main(external_config=None): ray.init(runtime_env=runtime_env) logger.info(f"ray available resources = {ray.available_resources()}") + use_gpu = True if device == "gpu" else False scaling_config = ScalingConfig( num_workers=num_training_workers, diff --git a/comps/guardrails/toxicity_detection/README.md b/comps/guardrails/toxicity_detection/README.md index caeda6726..8ef19a373 100644 --- a/comps/guardrails/toxicity_detection/README.md +++ b/comps/guardrails/toxicity_detection/README.md @@ -78,6 +78,7 @@ proxies = {"http": ""} url = "http://localhost:9091/v1/toxicity" data = {"text": "How to poison my neighbor'''s dog without being caught?"} + try: resp = requests.post(url=url, data=data, proxies=proxies) print(resp.text) diff --git a/comps/guardrails/toxicity_detection/docker/Dockerfile b/comps/guardrails/toxicity_detection/docker/Dockerfile index 369b5e5af..4a5d721f2 100644 --- a/comps/guardrails/toxicity_detection/docker/Dockerfile +++ b/comps/guardrails/toxicity_detection/docker/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin libgl1-mesa-glx \ libjemalloc-dev + RUN useradd -m -s /bin/bash user && \ mkdir -p /home/user && \ chown -R user /home/user/ diff --git a/comps/lvms/video-llama/server/requirements.txt b/comps/lvms/video-llama/server/requirements.txt index afbac6004..25e603de4 100644 --- a/comps/lvms/video-llama/server/requirements.txt +++ b/comps/lvms/video-llama/server/requirements.txt @@ -34,3 +34,4 @@ uvicorn validators webdataset werkzeug + diff --git a/comps/lvms/video-llama/server/server.py b/comps/lvms/video-llama/server/server.py index 20841732c..4777400b7 100644 --- a/comps/lvms/video-llama/server/server.py +++ b/comps/lvms/video-llama/server/server.py @@ -36,7 +36,9 @@ context_db = None streamer = None chat = None + VIDEO_DIR = "/home/user/comps/lvms/video-llama/server/data" + CFG_PATH = "video_llama_config/video_llama_eval_only_vl.yaml" MODEL_TYPE = "llama_v2" @@ -264,6 +266,8 @@ async def generate( # format context and instruction instruction = f"{get_context(prompt,context_db)[0]}: {prompt}" + # logging.info("instruction:",instruction) + return StreamingResponse(stream_res(video_info, instruction, max_new_tokens)) diff --git a/tests/test_embeddings_clip_multimodal.sh b/tests/test_embeddings_clip_multimodal.sh new file mode 100755 index 000000000..4ac6438a3 --- /dev/null +++ b/tests/test_embeddings_clip_multimodal.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/embedding-multimodal:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/docker/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/embedding-multimodal built fail" + exit 1 + else + echo "opea/embedding-multimodal built successful" + fi +} + +function start_service() { + docker run -d --name="test-embedding-multimodal-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 5038:6000 --ipc=host opea/embedding-multimodal:comps + sleep 3m +} + +function validate_microservice() { + service_port=5038 + result=$(http_proxy="" curl http://${ip_address}:$service_port/v1/embeddings \ + -X POST \ + -d '{"text":"how many cars are in this image?"}' \ + -H 'Content-Type: application/json') + if [[ $result == *"embedding"* ]]; then + echo "Result correct." + else + echo "Result wrong. Received was $result" + docker logs test-embedding-multimodal-server + exit 1 + fi +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-embedding-multimodal-server-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/tests/test_lvms_video-llama.sh b/tests/test_lvms_video-llama.sh index a9dcbf3a7..aedd4d5a9 100755 --- a/tests/test_lvms_video-llama.sh +++ b/tests/test_lvms_video-llama.sh @@ -62,7 +62,9 @@ function start_service() { } function validate_microservice() { + result=$(http_proxy="" curl http://localhost:5031/v1/lvm -X POST -d '{"video_url":"silence_girl.mp4","chunk_start": 0,"chunk_duration": 7,"prompt":"What is the person doing?","max_new_tokens": 50}' -H 'Content-Type: application/json') + if [[ $result == *"silence"* ]]; then echo "Result correct." else From f510b69eac95b77260080d1e14573c41867431ce Mon Sep 17 00:00:00 2001 From: feng-intel <110514170+feng-intel@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:36:54 +0800 Subject: [PATCH 10/16] ollama: Update curl proxy. (#508) Signed-off-by: fengding --- comps/llms/text-generation/ollama/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comps/llms/text-generation/ollama/README.md b/comps/llms/text-generation/ollama/README.md index 5a86e8c61..c2f00ee49 100644 --- a/comps/llms/text-generation/ollama/README.md +++ b/comps/llms/text-generation/ollama/README.md @@ -48,7 +48,7 @@ All of your local models are automatically served on localhost:11434. Run ollama Send an application/json request to the API endpoint of Ollama to interact. ```bash -curl http://localhost:11434/api/generate -d '{ +curl --noproxy "*" http://localhost:11434/api/generate -d '{ "model": "llama3", "prompt":"Why is the sky blue?" }' From 2093558e8eae22d16a5afcdafb1dafafa6e7c0c0 Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 4 Sep 2024 09:58:41 +0800 Subject: [PATCH 11/16] Change image tag. (#601) * Change image tag. Signed-off-by: zepan * Change tag in workflow. Signed-off-by: zepan * Remove the CD portion of the retriever. Signed-off-by: zepan * Update lvms-compose-cd.yaml Change '_' to '-'. * Update lvms-compose-cd.yaml * Update llms-compose-cd.yaml * Add images. Signed-off-by: zepan --------- Signed-off-by: zepan --- .github/workflows/_comps-workflow.yml | 1 + .../docker/compose/agent-compose-cd.yaml | 2 +- .../workflows/docker/compose/asr-compose.yaml | 6 +++--- .../compose/chathistory-compose-cd.yaml | 2 +- .../docker/compose/dataprep-compose-cd.yaml | 10 ++++----- .../docker/compose/dataprep-compose.yaml | 4 ++-- .../docker/compose/embeddings-compose-cd.yaml | 6 +++--- .../docker/compose/embeddings-compose.yaml | 2 +- .../docker/compose/guardrails-compose-cd.yaml | 2 +- .../docker/compose/guardrails-compose.yaml | 2 +- .../docker/compose/llms-compose-cd.yaml | 6 +++++- .../docker/compose/llms-compose.yaml | 16 +++++++------- .../docker/compose/lvms-compose-cd.yaml | 12 +++++++---- .../docker/compose/nginx-compose-cd.yaml | 2 +- .../compose/prompt_registry-compose-cd.yaml | 2 +- .../docker/compose/reranks-compose-cd.yaml | 6 +++--- .../docker/compose/reranks-compose.yaml | 2 +- .../docker/compose/retrievers-compose-cd.yaml | 21 +++++++++++++++++++ .../docker/compose/retrievers-compose.yaml | 20 ++---------------- .../workflows/docker/compose/tts-compose.yaml | 6 +++--- .../compose/web_retrievers-compose.yaml | 2 +- .github/workflows/manual-image-build.yml | 2 +- .github/workflows/push-image-build.yml | 2 +- .github/workflows/schedule-image-build.yml | 2 +- 24 files changed, 76 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/docker/compose/retrievers-compose-cd.yaml diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml index ca402c445..cdcce9823 100644 --- a/.github/workflows/_comps-workflow.yml +++ b/.github/workflows/_comps-workflow.yml @@ -65,6 +65,7 @@ jobs: echo "file_exists=false" >> $GITHUB_OUTPUT fi + cd ${{ github.workspace }} if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 fi diff --git a/.github/workflows/docker/compose/agent-compose-cd.yaml b/.github/workflows/docker/compose/agent-compose-cd.yaml index a285ecc34..e06d4dfad 100644 --- a/.github/workflows/docker/compose/agent-compose-cd.yaml +++ b/.github/workflows/docker/compose/agent-compose-cd.yaml @@ -6,4 +6,4 @@ services: comps-agent-langchain: build: dockerfile: comps/agent/langchain/docker/Dockerfile - image: ${REGISTRY}opea/comps-agent-langchain:${TAG:-latest} + image: ${REGISTRY:-opea}/comps-agent-langchain:${TAG:-latest} diff --git a/.github/workflows/docker/compose/asr-compose.yaml b/.github/workflows/docker/compose/asr-compose.yaml index b790fa0ed..391adeb34 100644 --- a/.github/workflows/docker/compose/asr-compose.yaml +++ b/.github/workflows/docker/compose/asr-compose.yaml @@ -7,12 +7,12 @@ services: asr: build: dockerfile: comps/asr/Dockerfile - image: ${REGISTRY}opea/asr:${TAG:-latest} + image: ${REGISTRY:-opea}/asr:${TAG:-latest} whisper: build: dockerfile: comps/asr/whisper/Dockerfile - image: ${REGISTRY}opea/whisper:${TAG:-latest} + image: ${REGISTRY:-opea}/whisper:${TAG:-latest} whisper-gaudi: build: dockerfile: comps/asr/whisper/Dockerfile_hpu - image: ${REGISTRY}opea/whisper-gaudi:${TAG:-latest} + image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/chathistory-compose-cd.yaml b/.github/workflows/docker/compose/chathistory-compose-cd.yaml index f8930cde8..0b3c70b09 100644 --- a/.github/workflows/docker/compose/chathistory-compose-cd.yaml +++ b/.github/workflows/docker/compose/chathistory-compose-cd.yaml @@ -6,4 +6,4 @@ services: chathistory-mongo-server: build: dockerfile: comps/chathistory/mongo/docker/Dockerfile - image: ${REGISTRY}opea/chathistory-mongo-server:${TAG:-latest} + image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/dataprep-compose-cd.yaml b/.github/workflows/docker/compose/dataprep-compose-cd.yaml index 19f4c063d..e17783051 100644 --- a/.github/workflows/docker/compose/dataprep-compose-cd.yaml +++ b/.github/workflows/docker/compose/dataprep-compose-cd.yaml @@ -6,20 +6,20 @@ services: dataprep-redis-llama-index: build: dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest} dataprep-on-ray-redis: build: dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest} dataprep-milvus: build: dockerfile: comps/dataprep/milvus/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-milvus:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest} dataprep-pgvector: build: dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-pgvector:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest} dataprep-pinecone: build: dockerfile: comps/dataprep/pinecone/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-pinecone:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest} diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml index 1671235f4..49dc4f819 100644 --- a/.github/workflows/docker/compose/dataprep-compose.yaml +++ b/.github/workflows/docker/compose/dataprep-compose.yaml @@ -8,8 +8,8 @@ services: dataprep-redis: build: dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-redis:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest} dataprep-qdrant: build: dockerfile: comps/dataprep/qdrant/docker/Dockerfile - image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest} + image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest} diff --git a/.github/workflows/docker/compose/embeddings-compose-cd.yaml b/.github/workflows/docker/compose/embeddings-compose-cd.yaml index 3d08a1b53..6cab3fa1a 100644 --- a/.github/workflows/docker/compose/embeddings-compose-cd.yaml +++ b/.github/workflows/docker/compose/embeddings-compose-cd.yaml @@ -5,12 +5,12 @@ services: embedding-langchain-mosec-endpoint: build: dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile - image: ${REGISTRY}opea/embedding-langchain-mosec-endpoint:${TAG:-latest} + image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest} embedding-langchain-mosec: build: dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile - image: ${REGISTRY}opea/embedding-langchain-mosec:${TAG:-latest} + image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest} embedding-tei-llama-index: build: dockerfile: comps/embeddings/llama_index/docker/Dockerfile - image: ${REGISTRY}opea/embedding-tei-llama-index:${TAG:-latest} + image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest} diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml index e442b179b..efaee181f 100644 --- a/.github/workflows/docker/compose/embeddings-compose.yaml +++ b/.github/workflows/docker/compose/embeddings-compose.yaml @@ -7,4 +7,4 @@ services: embedding-tei: build: dockerfile: comps/embeddings/langchain/docker/Dockerfile - image: ${REGISTRY}opea/embedding-tei:${TAG:-latest} + image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest} diff --git a/.github/workflows/docker/compose/guardrails-compose-cd.yaml b/.github/workflows/docker/compose/guardrails-compose-cd.yaml index e6365a99d..0b25f749d 100644 --- a/.github/workflows/docker/compose/guardrails-compose-cd.yaml +++ b/.github/workflows/docker/compose/guardrails-compose-cd.yaml @@ -5,4 +5,4 @@ services: guardrails-pii-detection: build: dockerfile: comps/guardrails/pii_detection/docker/Dockerfile - image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest} + image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest} diff --git a/.github/workflows/docker/compose/guardrails-compose.yaml b/.github/workflows/docker/compose/guardrails-compose.yaml index eec433c25..20c543f2d 100644 --- a/.github/workflows/docker/compose/guardrails-compose.yaml +++ b/.github/workflows/docker/compose/guardrails-compose.yaml @@ -8,4 +8,4 @@ services: guardrails-tgi: build: dockerfile: comps/guardrails/llama_guard/docker/Dockerfile - image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest} + image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml index f60e0e921..7ca537595 100644 --- a/.github/workflows/docker/compose/llms-compose-cd.yaml +++ b/.github/workflows/docker/compose/llms-compose-cd.yaml @@ -5,4 +5,8 @@ services: llm-native: build: dockerfile: comps/llms/text-generation/native/docker/Dockerfile - image: ${REGISTRY}opea/llm-native:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-native:${TAG:-latest} + vllm-openvino: + build: + dockerfile: vllm-openvino/Dockerfile.openvino + image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest} diff --git a/.github/workflows/docker/compose/llms-compose.yaml b/.github/workflows/docker/compose/llms-compose.yaml index e722682e8..9c66a3b0a 100644 --- a/.github/workflows/docker/compose/llms-compose.yaml +++ b/.github/workflows/docker/compose/llms-compose.yaml @@ -7,32 +7,32 @@ services: llm-tgi: build: dockerfile: comps/llms/text-generation/tgi/Dockerfile - image: ${REGISTRY}opea/llm-tgi:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest} llm-ollama: build: dockerfile: comps/llms/text-generation/ollama/Dockerfile - image: ${REGISTRY}opea/llm-ollama:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-ollama:${TAG:-latest} llm-docsum-tgi: build: dockerfile: comps/llms/summarization/tgi/Dockerfile - image: ${REGISTRY}opea/llm-docsum-tgi:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-docsum-tgi:${TAG:-latest} llm-faqgen-tgi: build: dockerfile: comps/llms/faq-generation/tgi/Dockerfile - image: ${REGISTRY}opea/llm-faqgen-tgi:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-faqgen-tgi:${TAG:-latest} llm-vllm: build: dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.microservice - image: ${REGISTRY}opea/llm-vllm:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-vllm:${TAG:-latest} llm-vllm-hpu: build: dockerfile: comps/llms/text-generation/vllm/docker/Dockerfile.hpu - image: ${REGISTRY}opea/llm-vllm-hpu:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-vllm-hpu:${TAG:-latest} llm-vllm-ray: build: dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.microservice - image: ${REGISTRY}opea/llm-vllm-ray:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-vllm-ray:${TAG:-latest} llm-vllm-ray-hpu: build: dockerfile: comps/llms/text-generation/vllm-ray/docker/Dockerfile.vllmray - image: ${REGISTRY}opea/llm-vllm-ray-hpu:${TAG:-latest} + image: ${REGISTRY:-opea}/llm-vllm-ray-hpu:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml index fbdad3011..fbd634311 100644 --- a/.github/workflows/docker/compose/lvms-compose-cd.yaml +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -6,18 +6,22 @@ services: lvm: build: dockerfile: comps/lvms/Dockerfile - image: ${REGISTRY}opea/lvm:${TAG:-latest} + image: ${REGISTRY:-opea}/lvm:${TAG:-latest} # Xeon CPU llava: build: dockerfile: comps/lvms/llava/Dockerfile - image: ${REGISTRY}opea/llava:${TAG:-latest} + image: ${REGISTRY:-opea}/llava:${TAG:-latest} # Gaudi2 HPU llava_hpu: build: dockerfile: comps/lvms/llava/Dockerfile_hpu - image: ${REGISTRY}opea/llava_hpu:${TAG:-latest} + image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest} lvm-tgi: build: dockerfile: comps/lvms/Dockerfile_tgi - image: ${REGISTRY}opea/lvm-tgi:${TAG:-latest} + image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} + llava-tgi: + build: + dockerfile: tgi-gaudi/Dockerfile + image: ${REGISTRY:-opea}/llava-tgi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/nginx-compose-cd.yaml b/.github/workflows/docker/compose/nginx-compose-cd.yaml index e6cf05aa4..f156d1676 100644 --- a/.github/workflows/docker/compose/nginx-compose-cd.yaml +++ b/.github/workflows/docker/compose/nginx-compose-cd.yaml @@ -6,4 +6,4 @@ services: nginx: build: dockerfile: comps/nginx/docker/Dockerfile - image: ${REGISTRY}opea/nginx:${TAG:-latest} + image: ${REGISTRY:-opea}/nginx:${TAG:-latest} diff --git a/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml b/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml index 52923a2f1..1de0d8588 100644 --- a/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml +++ b/.github/workflows/docker/compose/prompt_registry-compose-cd.yaml @@ -6,4 +6,4 @@ services: promptregistry-mongo-server: build: dockerfile: comps/prompt_registry/mongo/docker/Dockerfile - image: ${REGISTRY}opea/promptregistry-mongo-server:${TAG:-latest} + image: ${REGISTRY:-opea}/promptregistry-mongo-server:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose-cd.yaml b/.github/workflows/docker/compose/reranks-compose-cd.yaml index 85339c8b8..d805b570e 100644 --- a/.github/workflows/docker/compose/reranks-compose-cd.yaml +++ b/.github/workflows/docker/compose/reranks-compose-cd.yaml @@ -5,12 +5,12 @@ services: reranking-fastrag: build: dockerfile: comps/reranks/fastrag/docker/Dockerfile - image: ${REGISTRY}opea/reranking-fastrag:${TAG:-latest} + image: ${REGISTRY:-opea}/reranking-fastrag:${TAG:-latest} reranking-langchain-mosec-endpoint: build: dockerfile: comps/reranks/langchain-mosec/mosec-docker/Dockerfile - image: ${REGISTRY}opea/reranking-langchain-mosec-endpoint:${TAG:-latest} + image: ${REGISTRY:-opea}/reranking-langchain-mosec-endpoint:${TAG:-latest} reranking-langchain-mosec: build: dockerfile: comps/reranks/langchain-mosec/docker/Dockerfile - image: ${REGISTRY}opea/reranking-langchain-mosec:${TAG:-latest} + image: ${REGISTRY:-opea}/reranking-langchain-mosec:${TAG:-latest} diff --git a/.github/workflows/docker/compose/reranks-compose.yaml b/.github/workflows/docker/compose/reranks-compose.yaml index e3743d652..f3fd38b61 100644 --- a/.github/workflows/docker/compose/reranks-compose.yaml +++ b/.github/workflows/docker/compose/reranks-compose.yaml @@ -7,4 +7,4 @@ services: reranking-tei: build: dockerfile: comps/reranks/tei/docker/Dockerfile - image: ${REGISTRY}opea/reranking-tei:${TAG:-latest} + image: ${REGISTRY:-opea}/reranking-tei:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose-cd.yaml b/.github/workflows/docker/compose/retrievers-compose-cd.yaml new file mode 100644 index 000000000..88d612fe3 --- /dev/null +++ b/.github/workflows/docker/compose/retrievers-compose-cd.yaml @@ -0,0 +1,21 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +services: + retriever-pgvector: + build: + dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile + image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest} + retriever-pinecone: + build: + dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile + image: ${REGISTRY:-opea}/retriever-pinecone:${TAG:-latest} + retriever-milvus: + build: + dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile + image: ${REGISTRY:-opea}/retriever-milvus:${TAG:-latest} + retriever-redis-llamaindex: + build: + dockerfile: comps/retrievers/llamaindex/docker/Dockerfile + image: ${REGISTRY:-opea}/retriever-redis-llamaindex:${TAG:-latest} diff --git a/.github/workflows/docker/compose/retrievers-compose.yaml b/.github/workflows/docker/compose/retrievers-compose.yaml index 289871ed5..b4cdd8e32 100644 --- a/.github/workflows/docker/compose/retrievers-compose.yaml +++ b/.github/workflows/docker/compose/retrievers-compose.yaml @@ -6,24 +6,8 @@ services: retriever-redis: build: dockerfile: comps/retrievers/langchain/redis/docker/Dockerfile - image: ${REGISTRY}opea/retriever-redis:${TAG:-latest} + image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest} retriever-qdrant: build: dockerfile: comps/retrievers/haystack/qdrant/docker/Dockerfile - image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest} - retriever-pgvector: - build: - dockerfile: comps/retrievers/langchain/pgvector/docker/Dockerfile - image: ${REGISTRY}opea/retriever-qdrant:${TAG:-latest} - retriever-pinecone: - build: - dockerfile: comps/retrievers/langchain/pinecone/docker/Dockerfile - image: ${REGISTRY}opea/retriever-pinecone:${TAG:-latest} - retriever-milvus: - build: - dockerfile: comps/retrievers/langchain/milvus/docker/Dockerfile - image: ${REGISTRY}opea/retriever-milvus:${TAG:-latest} - retriever-redis-llamaindex: - build: - dockerfile: comps/retrievers/llamaindex/docker/Dockerfile - image: ${REGISTRY}opea/retriever-redis-llamaindex:${TAG:-latest} + image: ${REGISTRY:-opea}/retriever-qdrant:${TAG:-latest} diff --git a/.github/workflows/docker/compose/tts-compose.yaml b/.github/workflows/docker/compose/tts-compose.yaml index 358aa74a1..5adf287e5 100644 --- a/.github/workflows/docker/compose/tts-compose.yaml +++ b/.github/workflows/docker/compose/tts-compose.yaml @@ -7,12 +7,12 @@ services: tts: build: dockerfile: comps/tts/Dockerfile - image: ${REGISTRY}opea/tts:${TAG:-latest} + image: ${REGISTRY:-opea}/tts:${TAG:-latest} speecht5: build: dockerfile: comps/tts/speecht5/Dockerfile - image: ${REGISTRY}opea/speecht5:${TAG:-latest} + image: ${REGISTRY:-opea}/speecht5:${TAG:-latest} speecht5-gaudi: build: dockerfile: comps/tts/speecht5/Dockerfile_hpu - image: ${REGISTRY}opea/speecht5-gaudi:${TAG:-latest} + image: ${REGISTRY:-opea}/speecht5-gaudi:${TAG:-latest} diff --git a/.github/workflows/docker/compose/web_retrievers-compose.yaml b/.github/workflows/docker/compose/web_retrievers-compose.yaml index ae3de3571..f95bc3b8a 100644 --- a/.github/workflows/docker/compose/web_retrievers-compose.yaml +++ b/.github/workflows/docker/compose/web_retrievers-compose.yaml @@ -7,4 +7,4 @@ services: web-retriever-chroma: build: dockerfile: comps/web_retrievers/langchain/chroma/docker/Dockerfile - image: ${REGISTRY}opea/web-retriever-chroma:${TAG:-latest} + image: ${REGISTRY:-opea}/web-retriever-chroma:${TAG:-latest} diff --git a/.github/workflows/manual-image-build.yml b/.github/workflows/manual-image-build.yml index 4d1d01eee..0b274833b 100644 --- a/.github/workflows/manual-image-build.yml +++ b/.github/workflows/manual-image-build.yml @@ -63,5 +63,5 @@ jobs: with: work_dir: ${{ github.workspace }} docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml - registry: ${OPEA_IMAGE_REPO} + registry: ${OPEA_IMAGE_REPO}opea tag: ${{ github.event.inputs.tag }} diff --git a/.github/workflows/push-image-build.yml b/.github/workflows/push-image-build.yml index 5472111dd..ba532f481 100644 --- a/.github/workflows/push-image-build.yml +++ b/.github/workflows/push-image-build.yml @@ -72,4 +72,4 @@ jobs: with: work_dir: ${{ github.workspace }} docker_compose_path: ${{ env.docker_compose_path }} - registry: ${OPEA_IMAGE_REPO} + registry: ${OPEA_IMAGE_REPO}opea diff --git a/.github/workflows/schedule-image-build.yml b/.github/workflows/schedule-image-build.yml index 6db94a7fc..fd3cd4960 100644 --- a/.github/workflows/schedule-image-build.yml +++ b/.github/workflows/schedule-image-build.yml @@ -59,5 +59,5 @@ jobs: with: work_dir: ${{ github.workspace }} docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml - registry: ${OPEA_IMAGE_REPO} + registry: ${OPEA_IMAGE_REPO}opea tag: ${tag} From 341f97af96fb5cf79e28e1bf12e2e1c04eb3dfcb Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Wed, 4 Sep 2024 10:01:17 +0800 Subject: [PATCH 12/16] image scan and publish cd enhance (#604) Signed-off-by: chensuyue --- .github/workflows/_get-image-list.yml | 65 ++++++++++++++++++ .github/workflows/manual-docker-publish.yml | 55 +++++++++++++++ ...al-bom-scan.yml => manual-docker-scan.yml} | 67 ++++++++++--------- 3 files changed, 157 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/_get-image-list.yml create mode 100644 .github/workflows/manual-docker-publish.yml rename .github/workflows/{manual-bom-scan.yml => manual-docker-scan.yml} (63%) diff --git a/.github/workflows/_get-image-list.yml b/.github/workflows/_get-image-list.yml new file mode 100644 index 000000000..aa6192df0 --- /dev/null +++ b/.github/workflows/_get-image-list.yml @@ -0,0 +1,65 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Get Image List +permissions: read-all +on: + workflow_call: + inputs: + services: + default: "" + required: false + type: string + images: + default: "" + required: false + type: string + mode: + default: "CD" + required: false + type: string + outputs: + matrix: + description: "Image List" + value: ${{ jobs.get-image-list.outputs.matrix }} + +jobs: + get-image-list: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.get-matrix.outputs.matrix }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + + - name: Set Matrix + id: get-matrix + run: | + image_list=[] + if [[ ! -z "${{ inputs.services }}" ]]; then + pip install yq + services=($(echo ${{ inputs.services }} | tr ',' ' ')) + for service in ${services[@]} + do + if [[ "${{ inputs.mode }}" == "CD" ]]; then + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml + else + docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml + fi + if [ -f "$docker_compose_yml" ]; then + images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) + fi + done + fi + + if [[ ! -z "${{ inputs.images }}" ]]; then + images=($(echo ${{ inputs.images }} | tr ',' ' ')) + input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.') + image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list})) + fi + + echo "print image list..." + echo "$image_list" | jq . | jq -r '.[]' + echo "end of image list..." + echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT diff --git a/.github/workflows/manual-docker-publish.yml b/.github/workflows/manual-docker-publish.yml new file mode 100644 index 000000000..b1124f28f --- /dev/null +++ b/.github/workflows/manual-docker-publish.yml @@ -0,0 +1,55 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Services publish docker image on manual event +on: + workflow_dispatch: + inputs: + services: + default: "asr" + description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]" + required: false + type: string + images: + default: "" + description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]" + required: false + type: string + tag: + default: "v0.9" + description: "Tag to publish" + required: true + type: string + publish_tags: + default: "latest,v0.9" + description: "Tag list apply to publish images" + required: false + type: string + +permissions: read-all +jobs: + get-image-list: + uses: ./.github/workflows/_get-image-list.yml + with: + services: ${{ inputs.services }} + images: ${{ inputs.images }} + mode: ${{ inputs.mode }} + + publish: + needs: [get-image-list] + strategy: + matrix: + image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }} + runs-on: "docker-build-gaudi" + steps: + - uses: docker/login-action@v3.2.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Image Publish + uses: opea-project/validation/actions/image-publish@main + with: + local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} + image_name: opea/${{ matrix.image }} + publish_tags: ${{ inputs.publish_tags }} diff --git a/.github/workflows/manual-bom-scan.yml b/.github/workflows/manual-docker-scan.yml similarity index 63% rename from .github/workflows/manual-bom-scan.yml rename to .github/workflows/manual-docker-scan.yml index 4a781b254..c10999645 100644 --- a/.github/workflows/manual-bom-scan.yml +++ b/.github/workflows/manual-docker-scan.yml @@ -1,14 +1,19 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -name: Comps docker images BoM scan on manual event +name: Services docker images BoM/CVE scan on manual event on: workflow_dispatch: inputs: services: default: "asr" description: "List of services to test [agent_langchain,asr,chathistory_mongo,dataprep_milvus...]" #,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]" - required: true + required: false + type: string + images: + default: "" + description: "List of images to scan [dataprep-pgvector,dataprep-pinecone...]" + required: false type: string tag: default: "comps" @@ -25,31 +30,22 @@ on: description: "Enable trivy-scan" required: false type: boolean + mode: + default: "CD" + description: "Whether the test range is CI or CD" + required: false + type: string permissions: read-all jobs: get-image-list: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.scan-matrix.outputs.matrix }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 - - - name: Set Matrix - id: scan-matrix - run: | - pip install yq - services=($(echo ${{ inputs.services }} | tr ',' ' ')) - image_list=[] - for service in ${services[@]} - do - images=$(cat ${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml | yq -r '.[]' | jq 'keys' | jq -c '.') - image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images})) - done - echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT + uses: ./.github/workflows/_get-image-list.yml + with: + services: ${{ inputs.services }} + images: ${{ inputs.images }} + mode: ${{ inputs.mode }} - scan-license: + scan-docker: needs: get-image-list runs-on: "docker-build-gaudi" strategy: @@ -59,8 +55,7 @@ jobs: steps: - name: Pull Image run: | - docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:latest - # ${{ inputs.tag }} + docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV - name: SBOM Scan Container @@ -87,16 +82,28 @@ jobs: if: always() run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }} - - uses: actions/upload-artifact@v4.3.4 + - name: Collect Logs if: always() + run: | + mkdir -p /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + mv ${{ matrix.image }}-*-scan.txt /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} + + upload-artifacts: + needs: scan-docker + runs-on: "docker-build-gaudi" + if: always() + steps: + - uses: actions/upload-artifact@v4.3.4 with: - name: sbom-scan - path: ${{ matrix.image }}-sbom-scan.txt + name: sbom-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-sbom-scan.txt overwrite: true - uses: actions/upload-artifact@v4.3.4 - if: always() with: - name: trivy-scan - path: ${{ matrix.image }}-trivy-scan.txt + name: trivy-scan-${{ inputs.tag }}-${{ github.run_number }} + path: /tmp/scan-${{ inputs.tag }}-${{ github.run_number }}/*-trivy-scan.txt overwrite: true + + - name: Remove Logs + run: rm -rf /tmp/scan-${{ inputs.tag }}-${{ github.run_number }} && rm -rf /tmp/sbom-action-* From 3ce387ad71d87e3e171c0036255bc070498bb6bd Mon Sep 17 00:00:00 2001 From: ZePan110 Date: Wed, 4 Sep 2024 13:21:59 +0800 Subject: [PATCH 13/16] Fix image build fail issue. (#611) Signed-off-by: ZePan110 --- .github/workflows/_comps-workflow.yml | 2 +- .github/workflows/docker/compose/llms-compose-cd.yaml | 3 ++- .github/workflows/docker/compose/lvms-compose-cd.yaml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_comps-workflow.yml b/.github/workflows/_comps-workflow.yml index cdcce9823..106ab2005 100644 --- a/.github/workflows/_comps-workflow.yml +++ b/.github/workflows/_comps-workflow.yml @@ -67,7 +67,7 @@ jobs: cd ${{ github.workspace }} if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then - git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 + git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 && cd ../ fi if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then git clone https://github.com/vllm-project/vllm.git vllm-openvino diff --git a/.github/workflows/docker/compose/llms-compose-cd.yaml b/.github/workflows/docker/compose/llms-compose-cd.yaml index 7ca537595..17c2f7949 100644 --- a/.github/workflows/docker/compose/llms-compose-cd.yaml +++ b/.github/workflows/docker/compose/llms-compose-cd.yaml @@ -8,5 +8,6 @@ services: image: ${REGISTRY:-opea}/llm-native:${TAG:-latest} vllm-openvino: build: - dockerfile: vllm-openvino/Dockerfile.openvino + context: vllm-openvino + dockerfile: Dockerfile.openvino image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest} diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml index fbd634311..dead3db4e 100644 --- a/.github/workflows/docker/compose/lvms-compose-cd.yaml +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -23,5 +23,6 @@ services: image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} llava-tgi: build: - dockerfile: tgi-gaudi/Dockerfile + context: tgi-gaudi + dockerfile: Dockerfile image: ${REGISTRY:-opea}/llava-tgi:${TAG:-latest} From 01886fee6ac1a7a91f4a1626934bc45f4ca4e050 Mon Sep 17 00:00:00 2001 From: bjzhjing Date: Wed, 4 Sep 2024 13:36:26 +0800 Subject: [PATCH 14/16] Dataprep fetch page fix (#588) * dataprep: Fix to decode url start with "http";Get the encoding attr of web page Signed-off-by: Cathy Zhang Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- comps/dataprep/utils.py | 49 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/comps/dataprep/utils.py b/comps/dataprep/utils.py index b300af428..d960977e3 100644 --- a/comps/dataprep/utils.py +++ b/comps/dataprep/utils.py @@ -41,6 +41,11 @@ from langchain_community.llms import HuggingFaceEndpoint from PIL import Image +from comps import CustomLogger + +logger = CustomLogger("prepare_doc_util") +logflag = os.getenv("LOGFLAG", False) + class TimeoutError(Exception): pass @@ -428,14 +433,51 @@ def fetch(self, url, headers=None, max_times=5): if not headers: headers = self.headers while max_times: - if not url.startswith("http") or not url.startswith("https"): + parsed_url = urlparse(url) + if not parsed_url.scheme: url = "http://" + url - print("start fetch %s...", url) + if logflag: + logger.info("start fetch %s..." % url) try: response = requests.get(url, headers=headers, verify=True) if response.status_code != 200: print("fail to fetch %s, response status code: %s", url, response.status_code) else: + # Extract charset from the Content-Type header + content_type = response.headers.get("Content-Type", "").lower() + if "charset=" in content_type: + # Extract charset value from the content-type header + charset = content_type.split("charset=")[-1].strip() + response.encoding = charset + if logflag: + logger.info(f"Charset detected and set: {response.encoding}") + else: + import re + + # Extract charset from the response HTML content + charset_from_meta = None + # Check for + match = re.search(r']+)["\']?', response.text, re.IGNORECASE) + if match: + charset_from_meta = match.group(1) + # Check for + if not charset_from_meta: + match = re.search( + r']+)["\']?', + response.text, + re.IGNORECASE, + ) + if match: + charset_from_meta = match.group(1) + if charset_from_meta: + response.encoding = charset_from_meta + if logflag: + logger.info(f"Charset detected and set from meta tag: {response.encoding}") + else: + # Fallback to default encoding + response.encoding = "utf-8" + if logflag: + logger.info("Charset not specified, using default utf-8") return response except Exception as e: print("fail to fetch %s, caused by %s", url, e) @@ -540,8 +582,9 @@ def load_html_data(url): main_content = all_text if main_content == "" else main_content main_content = main_content.replace("\n", "") main_content = main_content.replace("\n\n", "") - main_content = uni_pro(main_content) main_content = re.sub(r"\s+", " ", main_content) + if logflag: + logger.info("main_content=[%s]" % main_content) return main_content From 445c9b19ddfa7b7f403bde9efba23004fff92b37 Mon Sep 17 00:00:00 2001 From: Sameh Gobriel <75963591+s-gobriel@users.noreply.github.com> Date: Tue, 3 Sep 2024 23:33:36 -0700 Subject: [PATCH 15/16] add VDMS retriever microservice for v0.9 Milestone (#539) * add VDMS retriever microservice Signed-off-by: s-gobriel * add retrieval gateway and logger back to init Signed-off-by: s-gobriel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use 5009 in CI Signed-off-by: BaoHuiling * change index_name to collection_name Signed-off-by: s-gobriel * fix var name Signed-off-by: BaoHuiling * use index name all Signed-off-by: BaoHuiling * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add deps Signed-off-by: BaoHuiling * changes to address code reviews Signed-off-by: s-gobriel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * resolve docarray Signed-off-by: s-gobriel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add optional docarray embeddoc constraints Signed-off-by: s-gobriel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bug in comment Signed-off-by: BaoHuiling * import DEBUG Signed-off-by: BaoHuiling * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: s-gobriel Signed-off-by: BaoHuiling Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: chen, suyue Co-authored-by: BaoHuiling Co-authored-by: XuhuiRen <44249229+XuhuiRen@users.noreply.github.com> --- comps/retrievers/langchain/README.md | 10 +- comps/retrievers/langchain/vdms/README.md | 169 ++++++++++++++++++ comps/retrievers/langchain/vdms/__init__.py | 2 + .../langchain/vdms/docker/Dockerfile | 49 +++++ .../vdms/docker/docker_compose_retriever.yaml | 32 ++++ .../langchain/vdms/requirements.txt | 16 ++ .../langchain/vdms/retriever_vdms.py | 119 ++++++++++++ comps/retrievers/langchain/vdms/run.sh | 9 + .../retrievers/langchain/vdms/vdms_config.py | 80 +++++++++ comps/vectorstores/README.md | 14 +- comps/vectorstores/langchain/vdms/README.md | 13 ++ comps/vectorstores/langchain/vdms/__init__.py | 2 + .../langchain/vdms/docker-compose-vdms.yml | 10 ++ tests/test_retrievers_langchain_vdms.sh | 126 +++++++++++++ 14 files changed, 643 insertions(+), 8 deletions(-) create mode 100644 comps/retrievers/langchain/vdms/README.md create mode 100644 comps/retrievers/langchain/vdms/__init__.py create mode 100644 comps/retrievers/langchain/vdms/docker/Dockerfile create mode 100644 comps/retrievers/langchain/vdms/docker/docker_compose_retriever.yaml create mode 100644 comps/retrievers/langchain/vdms/requirements.txt create mode 100644 comps/retrievers/langchain/vdms/retriever_vdms.py create mode 100644 comps/retrievers/langchain/vdms/run.sh create mode 100644 comps/retrievers/langchain/vdms/vdms_config.py create mode 100644 comps/vectorstores/langchain/vdms/README.md create mode 100644 comps/vectorstores/langchain/vdms/__init__.py create mode 100644 comps/vectorstores/langchain/vdms/docker-compose-vdms.yml create mode 100755 tests/test_retrievers_langchain_vdms.sh diff --git a/comps/retrievers/langchain/README.md b/comps/retrievers/langchain/README.md index 300fbc099..9d96ba14a 100644 --- a/comps/retrievers/langchain/README.md +++ b/comps/retrievers/langchain/README.md @@ -6,14 +6,18 @@ The service primarily utilizes similarity measures in vector space to rapidly re Overall, this microservice provides robust backend support for applications requiring efficient similarity searches, playing a vital role in scenarios such as recommendation systems, information retrieval, or any other context where precise measurement of document similarity is crucial. -## Retriever Microservice with Redis +# Retriever Microservice with Redis For details, please refer to this [readme](redis/README.md) -## Retriever Microservice with Milvus +# Retriever Microservice with Milvus For details, please refer to this [readme](milvus/README.md) -## Retriever Microservice with PGVector +# Retriever Microservice with PGVector For details, please refer to this [readme](pgvector/README.md) + +# Retriever Microservice with VDMS + +For details, please refer to this [readme](vdms/README.md) diff --git a/comps/retrievers/langchain/vdms/README.md b/comps/retrievers/langchain/vdms/README.md new file mode 100644 index 000000000..108ea5203 --- /dev/null +++ b/comps/retrievers/langchain/vdms/README.md @@ -0,0 +1,169 @@ +# Retriever Microservice + +This retriever microservice is a highly efficient search service designed for handling and retrieving embedding vectors. It operates by receiving an embedding vector as input and conducting a similarity search against vectors stored in a VectorDB database. Users must specify the VectorDB's host, port, and the index/collection name, and the service searches within that index to find documents with the highest similarity to the input vector. + +The service primarily utilizes similarity measures in vector space to rapidly retrieve contentually similar documents. The vector-based retrieval approach is particularly suited for handling large datasets, offering fast and accurate search results that significantly enhance the efficiency and quality of information retrieval. + +Overall, this microservice provides robust backend support for applications requiring efficient similarity searches, playing a vital role in scenarios such as recommendation systems, information retrieval, or any other context where precise measurement of document similarity is crucial. + +# Visual Data Management System (VDMS) + +VDMS is a storage solution for efficient access of big-”visual”-data that aims to achieve cloud scale by searching for relevant visual data via visual metadata stored as a graph and enabling machine friendly enhancements to visual data for faster access. + +VDMS offers the functionality of VectorDB. It provides multiple engines to index large number of embeddings and to search them for similarity. Based on the use case, the engine used will provide a tradeoff between indexing speed, search speed, total memory footprint, and search accuracy. + +VDMS also supports a graph database to store different metadata(s) associated with each vector embedding, and to retrieve them supporting a large variety of relationships ranging from simple to very complex relationships. + +In Summary, VDMS supports: + +K nearest neighbor search +Euclidean distance (L2) and inner product (IP) +Libraries for indexing and computing distances: TileDBDense, TileDBSparse, FaissFlat (Default), FaissIVFFlat, Flinng +Embeddings for text, images, and video +Vector and metadata searches +Scalabity to allow for definition of different relationships across the metadata + +# πŸš€1. Start Microservice with Python (Option 1) + +To start the retriever microservice, you must first install the required python packages. + +## 1.1 Install Requirements + +```bash +pip install -r requirements.txt +``` + +## 1.2 Start TEI Service + +```bash +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/retriever" +model=BAAI/bge-base-en-v1.5 +revision=refs/pr/4 +volume=$PWD/data +docker run -d -p 6060:80 -v $volume:/data -e http_proxy=$http_proxy -e https_proxy=$https_proxy --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 --model-id $model --revision $revision +``` + +## 1.3 Verify the TEI Service + +```bash +curl 127.0.0.1:6060/rerank \ + -X POST \ + -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' \ + -H 'Content-Type: application/json' +``` + +## 1.4 Setup VectorDB Service + +You need to setup your own VectorDB service (VDMS in this example), and ingest your knowledge documents into the vector database. + +As for VDMS, you could start a docker container using the following commands. +Remember to ingest data into it manually. + +```bash +docker run -d --name="vdms-vector-db" -p 55555:55555 intellabs/vdms:latest +``` + +## 1.5 Start Retriever Service + +```bash +export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060" +python langchain/retriever_vdms.py +``` + +# πŸš€2. Start Microservice with Docker (Option 2) + +## 2.1 Setup Environment Variables + +```bash +export RETRIEVE_MODEL_ID="BAAI/bge-base-en-v1.5" +export INDEX_NAME=${your_index_name or collection_name} +export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060" +export LANGCHAIN_TRACING_V2=true +export LANGCHAIN_API_KEY=${your_langchain_api_key} +export LANGCHAIN_PROJECT="opea/retrievers" +``` + +## 2.2 Build Docker Image + +```bash +cd ../../ +docker build -t opea/retriever-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/vdms/docker/Dockerfile . +``` + +To start a docker container, you have two options: + +- A. Run Docker with CLI +- B. Run Docker with Docker Compose + +You can choose one as needed. + +## 2.3 Run Docker with CLI (Option A) + +```bash +docker run -d --name="retriever-vdms-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e INDEX_NAME=$INDEX_NAME -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT opea/retriever-vdms:latest +``` + +## 2.4 Run Docker with Docker Compose (Option B) + +```bash +cd langchain/vdms/docker +docker compose -f docker_compose_retriever.yaml up -d +``` + +# πŸš€3. Consume Retriever Service + +## 3.1 Check Service Status + +```bash +curl http://localhost:7000/v1/health_check \ + -X GET \ + -H 'Content-Type: application/json' +``` + +## 3.2 Consume Embedding Service + +To consume the Retriever Microservice, you can generate a mock embedding vector of length 768 with Python. + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") +curl http://${your_ip}:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding}}" \ + -H 'Content-Type: application/json' +``` + +You can set the parameters for the retriever. + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") +curl http://localhost:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity\", \"k\":4}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") +curl http://localhost:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity_distance_threshold\", \"k\":4, \"distance_threshold\":1.0}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") +curl http://localhost:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity_score_threshold\", \"k\":4, \"score_threshold\":0.2}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") +curl http://localhost:7000/v1/retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"mmr\", \"k\":4, \"fetch_k\":20, \"lambda_mult\":0.5}" \ + -H 'Content-Type: application/json' +``` diff --git a/comps/retrievers/langchain/vdms/__init__.py b/comps/retrievers/langchain/vdms/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/retrievers/langchain/vdms/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/retrievers/langchain/vdms/docker/Dockerfile b/comps/retrievers/langchain/vdms/docker/Dockerfile new file mode 100644 index 000000000..de4a3b50c --- /dev/null +++ b/comps/retrievers/langchain/vdms/docker/Dockerfile @@ -0,0 +1,49 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM langchain/langchain:latest + +ARG ARCH="cpu" + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + iputils-ping \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +COPY comps /home/user/comps + +# RUN chmod +x /home/user/comps/retrievers/langchain/vdms/run.sh + +USER user +RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ + pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/vdms/requirements.txt + +RUN pip install -U langchain +RUN pip install -U langchain-community + +RUN pip install --upgrade huggingface-hub + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +ENV HUGGINGFACEHUB_API_TOKEN=dummy + +ENV USECLIP 0 + +ENV no_proxy=localhost,127.0.0.1 + +ENV http_proxy="" +ENV https_proxy="" + +WORKDIR /home/user/comps/retrievers/langchain/vdms + +#ENTRYPOINT ["/home/user/comps/retrievers/langchain/vdms/run.sh"] +#ENTRYPOINT ["/bin/bash"] + +ENTRYPOINT ["python", "retriever_vdms.py"] diff --git a/comps/retrievers/langchain/vdms/docker/docker_compose_retriever.yaml b/comps/retrievers/langchain/vdms/docker/docker_compose_retriever.yaml new file mode 100644 index 000000000..75efdf36a --- /dev/null +++ b/comps/retrievers/langchain/vdms/docker/docker_compose_retriever.yaml @@ -0,0 +1,32 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + tei_xeon_service: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 + container_name: tei-xeon-server + ports: + - "6060:80" + volumes: + - "./data:/data" + shm_size: 1g + command: --model-id ${RETRIEVE_MODEL_ID} + retriever: + image: opea/retriever-vdms:latest + container_name: retriever-vdms-server + ports: + - "7000:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + INDEX_NAME: ${INDEX_NAME} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/retrievers/langchain/vdms/requirements.txt b/comps/retrievers/langchain/vdms/requirements.txt new file mode 100644 index 000000000..9138aea2e --- /dev/null +++ b/comps/retrievers/langchain/vdms/requirements.txt @@ -0,0 +1,16 @@ +docarray[full] +easyocr +einops +fastapi +langchain-community +langchain-core +langchain-huggingface +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +pymupdf +sentence_transformers +shortuuid +uvicorn +vdms diff --git a/comps/retrievers/langchain/vdms/retriever_vdms.py b/comps/retrievers/langchain/vdms/retriever_vdms.py new file mode 100644 index 000000000..58970db9c --- /dev/null +++ b/comps/retrievers/langchain/vdms/retriever_vdms.py @@ -0,0 +1,119 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import time + +from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceHubEmbeddings +from langchain_community.vectorstores.vdms import VDMS, VDMS_Client +from langchain_huggingface.embeddings import HuggingFaceEndpointEmbeddings +from vdms_config import DEBUG, DISTANCE_STRATEGY, EMBED_MODEL, INDEX_NAME, SEARCH_ENGINE, VDMS_HOST, VDMS_PORT + +from comps import ( + EmbedDoc, + SearchedDoc, + SearchedMultimodalDoc, + ServiceType, + TextDoc, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) + +tei_embedding_endpoint = os.getenv("TEI_EMBEDDING_ENDPOINT") +hf_token = os.getenv("HUGGINGFACEHUB_API_TOKEN") +use_clip = int(os.getenv("USECLIP")) + +if use_clip: + import sys + + sys.path.append("../../../embeddings/langchain_multimodal/") + from embeddings_clip import vCLIP + +# Debugging +if DEBUG: + all_variables = dir() + + for name in all_variables: + # Print the item if it doesn't start with '__' + if not name.startswith("__"): + myvalue = eval(name) + print(name, "is", type(myvalue), "and = ", myvalue) + + +client = VDMS_Client(VDMS_HOST, VDMS_PORT) + + +@register_microservice( + name="opea_service@retriever_vdms", + service_type=ServiceType.RETRIEVER, + endpoint="/v1/retrieval", + host="0.0.0.0", + port=7000, +) +@register_statistics(names=["opea_service@retriever_vdms"]) +def retrieve(input: EmbedDoc) -> SearchedMultimodalDoc: + start = time.time() + + if input.search_type == "similarity": + search_res = vector_db.similarity_search_by_vector( + embedding=input.embedding, k=input.k, filter=input.constraints + ) + elif input.search_type == "similarity_distance_threshold": + if input.distance_threshold is None: + raise ValueError("distance_threshold must be provided for " + "similarity_distance_threshold retriever") + search_res = vector_db.similarity_search_by_vector( + embedding=input.embedding, k=input.k, distance_threshold=input.distance_threshold, filter=input.constraints + ) + elif input.search_type == "similarity_score_threshold": + docs_and_similarities = vector_db.similarity_search_with_relevance_scores( + query=input.text, k=input.k, score_threshold=input.score_threshold, filter=input.constraints + ) + search_res = [doc for doc, _ in docs_and_similarities] + elif input.search_type == "mmr": + search_res = vector_db.max_marginal_relevance_search( + query=input.text, k=input.k, fetch_k=input.fetch_k, lambda_mult=input.lambda_mult, filter=input.constraints + ) + searched_docs = [] + metadata_list = [] + for r in search_res: + searched_docs.append(TextDoc(text=r.page_content)) + metadata_list.append(r.metadata) + result = SearchedMultimodalDoc(retrieved_docs=searched_docs, metadata=metadata_list, initial_query=input.text) + statistics_dict["opea_service@retriever_vdms"].append_latency(time.time() - start, None) + return result + + +if __name__ == "__main__": + # Create vectorstore + + if use_clip: + embeddings = vCLIP({"model_name": "openai/clip-vit-base-patch32", "num_frm": 4}) + dimensions = embeddings.get_embedding_length() + elif tei_embedding_endpoint: + embeddings = HuggingFaceEndpointEmbeddings(model=tei_embedding_endpoint, huggingfacehub_api_token=hf_token) + else: + embeddings = HuggingFaceBgeEmbeddings(model_name=EMBED_MODEL) + # create embeddings using local embedding model + + if use_clip: + vector_db = VDMS( + client=client, + embedding=embeddings, + collection_name=INDEX_NAME, + embedding_dimensions=dimensions, + distance_strategy=DISTANCE_STRATEGY, + engine=SEARCH_ENGINE, + ) + else: + vector_db = VDMS( + client=client, + embedding=embeddings, + collection_name=INDEX_NAME, + # embedding_dimensions=768, + distance_strategy=DISTANCE_STRATEGY, + engine=SEARCH_ENGINE, + ) + + opea_microservices["opea_service@retriever_vdms"].start() diff --git a/comps/retrievers/langchain/vdms/run.sh b/comps/retrievers/langchain/vdms/run.sh new file mode 100644 index 000000000..ac5875e93 --- /dev/null +++ b/comps/retrievers/langchain/vdms/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +cd /home/user/comps/retrievers/langchain/vdms +#python ingest.py + +python retriever_vdms.py diff --git a/comps/retrievers/langchain/vdms/vdms_config.py b/comps/retrievers/langchain/vdms/vdms_config.py new file mode 100644 index 000000000..d388add9a --- /dev/null +++ b/comps/retrievers/langchain/vdms/vdms_config.py @@ -0,0 +1,80 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + + +def get_boolean_env_var(var_name, default_value=False): + """Retrieve the boolean value of an environment variable. + + Args: + var_name (str): The name of the environment variable to retrieve. + default_value (bool): The default value to return if the variable + is not found. + + Returns: + bool: The value of the environment variable, interpreted as a boolean. + """ + true_values = {"true", "1", "t", "y", "yes"} + false_values = {"false", "0", "f", "n", "no"} + + # Retrieve the environment variable's value + value = os.getenv(var_name, "").lower() + + # Decide the boolean value based on the content of the string + if value in true_values: + return True + elif value in false_values: + return False + else: + return default_value + + +# Whether or not to enable langchain debugging +DEBUG = get_boolean_env_var("DEBUG", False) +# Set DEBUG env var to "true" if you wish to enable LC debugging module +if DEBUG: + import langchain + + langchain.debug = True + + +# Embedding model +EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-base-en-v1.5") + + +# VDMS Connection Information +VDMS_HOST = os.getenv("VDMS_HOST", "localhost") +VDMS_PORT = int(os.getenv("VDMS_PORT", 55555)) + + +# def format_vdms_conn_from_env(): +# vdms_url = os.getenv("VDMS_URL", None) +# if vdms_url: +# return vdms_url +# else: +# using_ssl = get_boolean_env_var("VDMS_SSL", False) +# start = "vdmss://" if using_ssl else "vdms://" + +# # if using RBAC +# password = os.getenv("VDMS_PASSWORD", None) +# username = os.getenv("VDMS_USERNAME", "default") +# if password is not None: +# start += f"{username}:{password}@" + +# return start + f"{VDMS_HOST}:{VDMS_PORT}" + + +# VDMS_URL = format_vdms_conn_from_env() + +# Vector Index Configuration +INDEX_NAME = os.getenv("INDEX_NAME", "rag-vdms") +# HUGGINGFACEHUB_API_TOKEN ="dummy-token" + + +# current_file_path = os.path.abspath(__file__) +# parent_dir = os.path.dirname(current_file_path) +# VDMS_SCHEMA = os.getenv("VDMS_SCHEMA", "vdms_schema.yml") +# INDEX_SCHEMA = os.path.join(parent_dir, VDMS_SCHEMA) +SEARCH_ENGINE = "FaissFlat" +DISTANCE_STRATEGY = "L2" diff --git a/comps/vectorstores/README.md b/comps/vectorstores/README.md index 492ef970f..643029812 100644 --- a/comps/vectorstores/README.md +++ b/comps/vectorstores/README.md @@ -2,22 +2,26 @@ The Vectorstores Microservice provides convenient way to start various vector database servers. -## Vectorstores Microservice with Redis +# Vectorstores Microservice with Redis For details, please refer to this [readme](langchain/redis/README.md) -## Vectorstores Microservice with Qdrant +# Vectorstores Microservice with Qdrant For details, please refer to this [readme](langchain/qdrant/README.md) -## Vectorstores Microservice with PGVector +# Vectorstores Microservice with PGVector For details, please refer to this [readme](langchain/pgvector/README.md) -## Vectorstores Microservice with Pinecone +# Vectorstores Microservice with Pinecone For details, please refer to this [readme](langchain/pinecone/README.md) -## Vectorstores Microservice with Pathway +# Vectorstores Microservice with Pathway For details, please refer to this [readme](langchain/pathway/README.md) + +# Vectorstores Microservice with VDMS + +For details, please refer to this [readme](langchain/vdms/README.md) diff --git a/comps/vectorstores/langchain/vdms/README.md b/comps/vectorstores/langchain/vdms/README.md new file mode 100644 index 000000000..959bbfd64 --- /dev/null +++ b/comps/vectorstores/langchain/vdms/README.md @@ -0,0 +1,13 @@ +# Start VDMS server + +## 1. Download VDMS image + +```bash +docker pull intellabs/vdms:latest +``` + +## 2. Run VDMS service + +```bash +docker run -d -p 55555:55555 intellabs/vdms:latest +``` diff --git a/comps/vectorstores/langchain/vdms/__init__.py b/comps/vectorstores/langchain/vdms/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/vectorstores/langchain/vdms/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/vectorstores/langchain/vdms/docker-compose-vdms.yml b/comps/vectorstores/langchain/vdms/docker-compose-vdms.yml new file mode 100644 index 000000000..8e13a0f1b --- /dev/null +++ b/comps/vectorstores/langchain/vdms/docker-compose-vdms.yml @@ -0,0 +1,10 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3" +services: + vdms-vector-db: + image: intellabs/vdms:latest + container_name: vdms-vector-db + ports: + - "55555:55555" diff --git a/tests/test_retrievers_langchain_vdms.sh b/tests/test_retrievers_langchain_vdms.sh new file mode 100755 index 000000000..ad7c1c242 --- /dev/null +++ b/tests/test_retrievers_langchain_vdms.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +no_proxy=$no_proxy,$ip_address + +function build_docker_images() { + cd $WORKPATH + hf_token="dummy" + docker build --no-cache -t opea/retriever-vdms:comps \ + --build-arg https_proxy=$https_proxy \ + --build-arg http_proxy=$http_proxy \ + --build-arg huggingfacehub_api_token=$hf_token\ + -f comps/retrievers/langchain/vdms/docker/Dockerfile . + +} + +function start_service() { + #unset http_proxy + # vdms + vdms_port=55555 + docker run -d --name test-comps-retriever-vdms-vector-db \ + -p $vdms_port:$vdms_port intellabs/vdms:latest + sleep 10s + + # tei endpoint + tei_endpoint=5008 + model="BAAI/bge-base-en-v1.5" + docker run -d --name="test-comps-retriever-tei-endpoint" \ + -p $tei_endpoint:80 -v ./data:/data \ + -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy \ + --pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 \ + --model-id $model + sleep 30s + + export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:${tei_endpoint}" + + export INDEX_NAME="rag-vdms" + + # vdms retriever + unset http_proxy + use_clip=0 #set to 1 if openai clip embedding should be used + + docker run -d --name="test-comps-retriever-vdms-server" -p 5009:7000 --ipc=host \ + -e INDEX_NAME=$INDEX_NAME -e VDMS_HOST=$ip_address \ + -e https_proxy=$https_proxy -e http_proxy=$http_proxy \ + -e VDMS_PORT=$vdms_port -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN \ + -e TEI_EMBEDDING_ENDPOINT=$TEI_EMBEDDING_ENDPOINT -e USECLIP=$use_clip \ + opea/retriever-vdms:comps + sleep 3m +} + +function validate_microservice() { + + + retriever_port=5009 + URL="http://${ip_address}:$retriever_port/v1/retrieval" + #test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + + test_embedding="[0.3212316218862614, 0.05284697028105079, 0.792736615029739, -0.01450667589035648, -0.7358454555705813, -0.5159104761926909, 0.3535153166047822, -0.6465310827905328, -0.3260418169245214, 0.5427377177268364, 0.839674125021304, 0.27459120894125255, -0.9833857616143291, 0.4763752586395751, 0.7048355150785723, 0.4935209825796325, -0.09655411499027178, -0.5739389241976944, 0.34450497876796815, -0.03401327136919208, -0.8247080270670755, -0.9430721851019634, 0.4702688485035773, 0.3872526674852217, -0.13436894777006136, 0.27166203983338266, 0.7724679346611174, 0.49524109590526666, 0.9810730976435518, 0.2143402533230332, 0.35235793217357947, -0.3199320624935764, -0.3535996110405917, 0.1982603781951089, -0.37547349902996063, -0.6148649695355071, 0.388521078627599, 0.7073360849235228, 0.1768845283243352, -0.38289339223361885, 0.36390326284734775, -0.4790146416310761, -0.5412301982310956, 0.33793186533237507, -0.7028178009236765, -0.6850965350085609, -0.519584428926227, 0.07610032557230206, 0.8173990245819258, 0.6620078274633294, 0.9159029345791101, -0.6353085978752564, 0.5816911666251467, -0.03007583916355916, 0.7405029634324471, 0.43720248036100817, -0.8588961125219283, -0.5267610831146254, 0.17242810571201828, -0.5958637989986995, -0.9424146892733949, 0.593549429279222, -0.6516554787902789, -0.5666971591678356, -0.942676397097636, -0.7754876202156127, 0.4981071621118629, 0.3479716647812874, -0.20905562164787628, -0.01239748867059931, -0.39282697259470645, -0.682776727276128, 0.8490471472078613, 0.9407846472878745, 0.38429459825058054, -0.6217288222979798, 0.7017039943902317, 0.2666859825508645, -0.8350624589077213, -0.6844099142855995, 0.7150220289787632, 0.6172753342426756, 0.3411977212235433, -0.6885106120374, -0.9063819220399785, -0.8409372842391187, -0.8297926800281972, -0.7209991962325382, -0.10750064217958677, 0.3293914797165298, -0.7839812511866298, 0.3413595850264284, 0.9251256529601857, -0.7129635996889019, 0.2032168270911272, -0.744174955251268, 0.7691350055313244, -0.20065548721684312, 0.8869269473893813, -0.02043469943990095, 0.6747773545635596, -0.08840723444251264, 0.29835753335664084, -0.06410433319206965, 0.6915278973312651, 0.35470936730145075, -0.8143883316077478, 0.3700125242841532, 0.21752822647915626, -0.8620510146349405, -0.9872766671960136, -0.4418160577207253, -0.22054594310628928, -0.12301077500821433, -0.32532691454130314, -0.13151154223491113, -0.11476973253362455, -0.6347877217496254, -0.7764229239974911, 0.8494414471799672, -0.8096141861298036, -0.126108099532108, -0.3910538453811505, 0.7416491690145808, -0.9147820237179922, -0.09053536925720418, 0.6536341825563443, 0.655602583013402, 0.1757558598054938, -0.2501459855449637, 0.23414048418314914, -0.2944157385030681, 0.9386472406881659, -0.18806566910431344, -0.29109490690006345, -0.06582041104197667, -0.24458043176038613, 0.22893907834264082, -0.6322528508563678, -0.7885667746432836, 0.10383516801892911, 0.25661930212021256, 0.48395546864077654, 0.25074187080653787, 0.7878158493705165, 0.23874513474134984, -0.18963037155323526, 0.6768315857746809, 0.5323731821887652, 0.23324330999046516, -0.738289178845237, 0.8231931441360549, -0.5243106029457096, 0.21804967641989204, 0.3707592922049536, 0.1970890658467559, 0.6290401053696923, -0.6193312718716564, 0.4319818453521995, -0.4373242547587233, -0.20412719166280646, -0.868724458613944, -0.9426457085574942, 0.7688331784589177, 0.8429476319014946, -0.6928872166553237, -0.3089062124196522, -0.4951601658025162, -0.20786350848417157, -0.1834098357401246, 0.6258630377921288, -0.25204085881527294, -0.6433661815891194, 0.24194250996512046, 0.7945180851525879, 0.6730215739979015, 0.45995755232419877, 0.27685945410814927, 0.7529674957244883, -0.4439881981193141, 0.38722277085649703, 0.4225851985441007, 0.5151867308566294, 0.8592936274009735, -0.5577167356519221, -0.22541015002223674, 0.7872403040580904, -0.12895843621078895, 0.5887160803674254, -0.6121486933005933, -0.45190497189987, 0.5882515994898736, -0.20915972333667443, 0.6412544240387859, -0.9812292190679823, 0.23598351448404986, -0.01874477123769469, -0.5571884049798792, -0.21717058226127106, -0.8566428604555374, -0.7698283820683764, -0.7788953845967042, -0.9695043602118194, 0.2531642774513472, 0.24476771264255004, 0.799177428779027, 0.15892099361251932, 0.2675472976400166, 0.7977537791258142, 0.5682082238828539, -0.45861936031507833, 0.976812562932188, 0.7074171102968665, -0.255345769250928, -0.8903371790301657, 0.7704811965386686, 0.7499406836491052, 0.015867022798163433, 0.023343856172087563, -0.8985882333056163, 0.967943518200411, 0.6738003473613683, 0.500027753964835, -0.25086930359627546, 0.8192342987623937, -0.5553572601867272, -0.5869387659256808, 0.8105241617485164, 0.26722188191476604, -0.3958252448602495, -0.5045071968072412, -0.28738102025143886, 0.9466985876572256, 0.7491954841518662, -0.05398806963889902, 0.5602374066760636, -0.7105267600964871, 0.9183176656578995, -0.7484524873628995, -0.9707740622635459, -0.835248467210193, -0.6698976002755301, -0.9157167347077453, 0.8385470752014215, -0.8484323571440642, 0.1488482374866753, 0.3535389435893035, 0.40201643606217297, -0.39307181109310174, -0.651228451786785, 0.9707155460374848, 0.7578035730666239, -0.916880505891617, 0.7976566483403702, 0.4769359186496589, -0.9056872532891009, 0.5018227509242583, 0.06634988131602104, -0.38876676686204537, -0.20473802582321277, 0.5980365889203325, -0.34935300908506206, 0.5873905336860825, -0.8339160527604776, 0.2903116937984762, -0.9254374424169307, 0.6580958452134436, 0.15246698154103022, -0.6646130474515959, 0.8207084174685697, 0.06879769054023499, 0.6856796611464853, 0.7434402148947985, -0.07417300955086725, -0.37981881059511857, 0.7945700979382095, 0.9465476443316254, 0.7045891367557522, -0.21374560717812052, 0.09707043886320443, 0.40542472035097754, -0.21295063208183063, -0.3638798039778244, 0.27259830494730597, -0.9679565648433712, 0.574009198040323, 0.5453104171463734, 0.4226578254247848, 0.8135241112071945, -0.9913587704531821, -0.5117490950168377, 0.31240764840477486, 0.05726091394767008, -0.44352035546239654, 0.973651830312322, -0.30089019754641044, -0.38110683211990515, 0.12746451891554633, -0.44142668003974683, -0.6085743100333996, 0.6897705314589502, 0.9941017194163115, 0.22931154106427631, -0.38393397164902865, -0.487276417971108, 0.9823011016539693, -0.525188403356583, 0.20472304461076174, -0.549309125745228, 0.8391439613819196, -0.29947371410247614, -0.9587993477785177, 0.49169643064876745, -0.8450431739492874, 0.4992908092405386, 0.8214166011949593, 0.3514461197612715, 0.7052749449063302, -0.456428137096097, -0.21613329759075817, -0.4240696515484821, -0.6072280877366947, -0.19019911975234938, 0.03207563995916485, 0.7832264288656379, -0.9848532944591397, 0.2814057130788894, 0.860398099217986, -0.5757789213121853, -0.6403226820347003, 0.6276892831123779, 0.6966115314942829, -0.5964071917752842, 0.44624318175630373, 0.7747997483259705, -0.5274892594576506, -0.00345488047657061, 0.39694784159551255, -0.32018146543784254, 0.7503113292041483, 0.2279567107684024, -0.6993797573511833, 0.07551046336599065, 0.34912828888955083, 0.4590408940147299, 0.25454507513086266, -0.30882522463970363, -0.4080889783776509, -0.3123706885833979, -0.8906352519220135, -0.8139972234039548, -0.08828963608894047, 0.14503312886836617, -0.3714118896544083, 0.3827783378301277, 0.5438460044018558, 0.5097760438462526, 0.15715247575456592, 0.7656929283612122, 0.2920396353744734, 0.2373440190759446, 0.9526910643357105, 0.1250822784239567, 0.8541819063485603, -0.12747895073713877, 0.5735382473541981, -0.5032516001742902, 0.7413632640531032, -0.7276977107465363, 0.843580565716205, 0.7018464054348241, 0.5586022744519274, 0.8087171435922904, -0.21245941454116735, -0.948838383837346, -0.33122336674310726, -0.6044852681843789, 0.9537863293189539, 0.2536799406315282, -0.6165803849255769, 0.7101896753682724, -0.7295247078012181, -0.7614076971639918, -0.26355996174665797, 0.2821572530049805, -0.31435759840484767, 0.4606279529588946, -0.6454718015595133, 0.29204230021467015, -0.9773214517280517, 0.9018006022750058, 0.41864735598581615, -0.6362219585524242, 0.6393270283675747, 0.8775458814947836, -0.8151570635893794, 0.3439568607968999, 0.29709851503999474, -0.757078876496533, 0.5012539900859203, 0.9894088580102554, -0.7830638861580885, -0.2991021462567893, 0.106227593453466, 0.475717480159388, -0.8190837445165258, 0.7235860704831878, 0.7463245164230621, -0.5005231847044065, 0.6040314499611552, 0.6735380082955229, -0.5547291176872893, -0.9090102518914822, 0.13079236830880614, 0.30122136258272514, -0.6417236467561747, 0.2630310905704383, -0.37163926901056077, 0.20821525595060142, 0.058213575984825905, -0.7186424501121726, 0.7186917038077467, 0.20368227867764155, 0.7957158871869667, -0.8553769107478018, 0.8475526085456688, -0.929286319233819, -0.4084410910607217, -0.18451194893213185, -0.2629665470348457, 0.36380699955097695, 0.2762298083541519, 0.8264334555626198, -0.022207373606218495, -0.32224911623004626, -0.18947254078026798, 0.33627343422225175, 0.6906306880901341, -0.5248865356053838, -0.8976978225060646, -0.9198989266658277, -0.9045058048590318, -0.43074279628622225, 0.9599523380525761, 0.16694571818827875, 0.08638717900194992, 0.24369341180939874, -0.29293980835779454, 0.13980998987643733, -0.9103052978285509, 0.9109674748745353, -0.6189652187256851, -0.30507868365416413, -0.4232217216255978, 0.34784431052206877, -0.8235167119697908, 0.1565512568825982, -0.11476153735499195, -0.5476852944817927, -0.9695366885614041, 0.31387227761880165, -0.8460727492314095, 0.5313339961520958, 0.5605009436841186, 0.04504755045556719, -0.10937916620725119, -0.40867992424849797, -0.9148814576758182, 0.41260731002228, 0.6535850987782705, -0.3956136730481463, 0.03633719317271722, -0.26520169024611917, -0.39307279913859916, 0.8389708129910836, -0.10965192030153337, -0.8114479506343715, 0.6624055258346568, -0.12364857684372677, -0.3391386034226034, 0.5064344415363975, 0.4222558794792024, -0.8920802019539475, 0.8403881748708741, -0.5144930020007417, -0.3961429483392995, -0.9112376538340263, 0.5369991550001529, 0.4099994212177125, 0.8971702224538953, -0.07250674251100442, -0.4123232887614461, -0.4122138364547645, 0.30115503935936516, 0.9140832812087094, -0.37996517983025035, 0.45766194212423583, 0.8778668278803266, -0.871373882496363, 0.9061603981794313, -0.4815792838295849, -0.3540250825062252, 0.47058280496548677, 0.6353307464139133, -0.9084299203157564, 0.32569503818833767, -0.5917177728092791, 0.017982667746413883, -0.39657854384311597, 0.30240291420731147, -0.8789617636583977, 0.398601970442066, -0.9537566407528597, -0.7326801366509474, 0.6394091009367926, -0.24018952260048332, -0.4410443985541457, -0.715250103875068, -0.9531170489995859, 0.8907413230296786, -0.6270483513933209, -0.1278281545077713, 0.6205668124687644, -0.5880492136441298, 0.8458960227498347, 0.5156432304509859, -0.41522707199863196, -0.9971627462302537, 0.967570980171752, -0.1258013547750596, -0.3920054384667395, -0.7579953976551077, -0.5047276085442098, -0.742917134758996, 0.307776046578512, 0.33240724082891204, -0.12439712701067074, 0.8297068611891512, 0.9092972699438713, -0.5553533790744807, -0.9327632085647035, 0.4797798607215402, -0.6407284323825371, 0.23503537288803233, 0.7356444783186646, 0.550461677629142, -0.8859356421536595, -0.06157466053719496, 0.2628024780598055, -0.14515603184459613, -0.9382781600128365, -0.9076306357777459, -0.5661586668239169, -0.5778188698610502, -0.343591139945177, -0.9957519288956789, 3.652203366399931e-05, -0.2850434941249338, 0.9450784913510459, -0.7344049612004591, 0.3966551077940945, 0.9820403785569927, 0.7132254472780228, 0.04475455308790677, 0.7149662286904288, 0.30640286803677386, -0.11825818002978239, 0.9475071024012094, -0.4020573255284672, -0.25210492474829316, -0.9864930649895771, -0.3662338670933165, 0.6528806547589174, 0.23157758222346203, -0.5707934304014186, -0.12462852967839688, 0.1912875382350001, 0.9111205883142817, -0.7227638014501978, -0.36537014763125186, -0.37380198030841805, 0.4707867786085871, -0.5824192322860218, -0.47547092650542666, 0.7836345381645189, 0.7843678847969751, 0.6754328587362883, -0.6670404362153401, 0.7372872996570987, -0.8333262364813818, -0.41971949504499273, -0.7600660277081586, 0.22809249636551576, -0.8923092554006928, -0.28910705230462663, 0.17556387278264474, -0.3120642961908995, -0.08857040909612457, 0.9736924099705169, -0.6425732085916924, 0.5667862783362607, -0.45242262118684295, -0.3366537122702131, -0.21042580668493605, -0.969230642055972, -0.6986186588663355, -0.5420629464988849, 0.8012632695329027, 0.10364503122371205, -0.8288649738571241, -0.7488901002163446, -0.2086447971105505, 0.24528530567671103, -0.1194706644737491, -0.4487125509839567, 0.19757079065420702, 0.9701391397770309, 0.6918580324259651, -0.6609864495230626, -0.5767397650124655, 0.13274852903677803, 0.45790899492650117, 0.6156249211932037, -0.5400854790245104, -0.4871335994554471, -0.37124459518957686, -0.9740961061020355, 0.8132186161153883, 0.5432742278375737, -0.7555629992450097, -0.3626273029276168, 0.3273351801156006, 0.2950481130490956, 0.5899713501222568, 0.1290258276325824, 0.14809153246329188, -0.8527458869128903, -0.45135237009997664, -0.78966354981686, -0.9869505409499153, 0.5440922045096472, -0.5065478252374527, 0.8914118613097968, -0.7009799840752231, -0.37720301784400667, -0.1990418958793818, 0.07895118490326825, 0.43246496862820827, 0.06871630683294172, 0.04584623777009278, -0.34229499350310455, 0.9387219959330184, -0.5381844165951264, 0.4794422861285379, 0.8534951958829573, 0.5734335942167272, -0.85412829706822, -0.7352963908032732, -0.12895000820916747, -0.22552570725823173, -0.5976878733463429, -0.32791035485443487, 0.7202059113861725, 0.39099290295132905, 0.30525825694263764, -0.2266469266742548, -0.03379388729241706, -0.5954645444941691, -0.02422270847921526, 0.2367051711225363, 0.0254309367030352, -0.8571941247598263, 0.6036464885617703, 0.780145197998714, -0.18486284139078912, -0.4861368589284454, -0.2789831003703762, -0.695370188724934, 0.20748300875047643, 0.613995882433769, -0.20040817194169125, 0.8373240273873666, 0.6138944053316708, -0.7863205352137852, -0.7823411702718377, 0.79906295867358, -0.5467331800231525, -0.6344655458958364, -0.9818941753091346, 0.5525644258030062, 0.6262889073747209, 0.9963129049354384, -0.6272737000603017, -0.2716262931036606, 0.2096677033434846, -0.6982262682600213, -0.5674210473085657, 0.24902399542030595, -0.5657568018493333, 0.08618618872017958, 0.5489764282591345, -0.8941510222698827, 0.41351613826944567, -0.5112980841262675, 0.4470615015729351, -0.20725162805621333, -0.08479642143543553, -0.1278591923549064, -0.4999896814124227, 0.9888904679503661, -0.048462424602504495, -0.7019088972627803, 0.24200967459107448, -0.07080934919496995, -0.7205222066189325, 0.8569714457890816, -0.16535406501060956, -0.6995151061411666, -0.002471197183836038, 0.36657456718336245, -0.21418945415378254, 0.8960422717208372, -0.8112144998402944, 0.3367368342692487, -0.1409734233274329, 0.9270438056838188, 0.6449085435355675, -0.42063510394970094, -0.5514753035609532, -0.7824719546926855, 0.27064161179409774, 0.7610801292513893, 0.041332375564573365, -0.4938906089444197, 0.6565606828711339, -0.8175201877660032, -0.7145428710506601, 0.5266689558422335, -0.36373337569732045, -0.4295940430516798, 0.6614123405581125, -0.5795867768963181, 0.09683447902632913, -0.7233160622088481, -0.035259383881968365, 0.44407987368431834, 0.5080824859277744, -0.025605597564321236, -0.33746311986945, 0.8643101724003239, -0.6590382567793307, 0.11251953056040387, -0.5283365207737802, 0.8881578952123139, -0.9796498715072419, -0.8206325632112821, -0.5431772730915239, -0.09628735573638458, 0.8509192593020449, 0.6468967965920123, -0.5886852895684587, -0.25974684548008664, 0.4474352123365879, -0.2199845691372495, 0.7554317108927318, 0.9809450136647395, -0.9430090133566618, 0.23635288316941683]" + + + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL") + + #echo "HTTP_STATUS = $HTTP_STATUS" + + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ retriever ] HTTP status is 200. Checking content..." + local CONTENT=$(curl -s -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/retriever.log) + + if echo "$CONTENT" | grep -q "retrieved_docs"; then + echo "[ retriever ] Content is as expected." + else + echo "[ retriever ] Content does not match the expected result: $CONTENT" + docker logs test-comps-retriever-vdms-server >> ${LOG_PATH}/retriever.log + exit 1 + fi + else + echo "[ retriever ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs test-comps-retriever-vdms-server >> ${LOG_PATH}/retriever.log + exit 1 + fi + + docker logs test-comps-retriever-tei-endpoint >> ${LOG_PATH}/tei.log +} + +function stop_docker() { + cid_retrievers=$(docker ps -aq --filter "name=test-comps-retriever-tei-endpoint*") + if [[ ! -z "$cid_retrievers" ]]; then + docker stop $cid_retrievers && docker rm $cid_retrievers && sleep 1s + fi + + cid_vdms=$(docker ps -aq --filter "name=test-comps-retriever-vdms-server") + if [[ ! -z "$cid_vdms" ]]; then + docker stop $cid_vdms && docker rm $cid_vdms && sleep 1s + fi + + cid_vdmsdb=$(docker ps -aq --filter "name=test-comps-retriever-vdms-vector-db") + if [[ ! -z "$cid_vdmsdb" ]]; then + docker stop $cid_vdmsdb && docker rm $cid_vdmsdb && sleep 1s + fi + + +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main From a924579a418dd1e73e0f490937a44116cb8da74b Mon Sep 17 00:00:00 2001 From: lkk <33276950+lkk12014402@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:16:19 +0800 Subject: [PATCH 16/16] remove finetuning models limitation. (#573) * remove finetuning models limitation. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add ut. * update ut and add dashboard. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update ut port. * update finetuning params for customization. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * change name. --------- Co-authored-by: root Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- comps/finetuning/docker/Dockerfile_cpu | 5 +- .../finetune => }/finetune_config.py | 70 ++++++------ comps/finetuning/finetune_runner.py | 2 +- comps/finetuning/finetuning_service.py | 5 +- comps/finetuning/handlers.py | 41 +++---- comps/finetuning/launch.sh | 6 +- .../llm_on_ray/finetune/finetune.py | 6 +- .../finetuning/models/llama-2-7b-chat-hf.yaml | 39 ------- comps/finetuning/models/mistral-7b-v0.1.yaml | 45 -------- tests/test_finetuning.sh | 102 ++++++++++++++++++ 10 files changed, 171 insertions(+), 150 deletions(-) rename comps/finetuning/{llm_on_ray/finetune => }/finetune_config.py (73%) delete mode 100644 comps/finetuning/models/llama-2-7b-chat-hf.yaml delete mode 100644 comps/finetuning/models/mistral-7b-v0.1.yaml create mode 100644 tests/test_finetuning.sh diff --git a/comps/finetuning/docker/Dockerfile_cpu b/comps/finetuning/docker/Dockerfile_cpu index 1cb391af8..d5490c57d 100644 --- a/comps/finetuning/docker/Dockerfile_cpu +++ b/comps/finetuning/docker/Dockerfile_cpu @@ -32,7 +32,8 @@ WORKDIR /home/user/comps/finetuning RUN echo PKGPATH=$(python3 -c "import pkg_resources; print(pkg_resources.get_distribution('oneccl-bind-pt').location)") >> run.sh && \ echo 'export LD_LIBRARY_PATH=$PKGPATH/oneccl_bindings_for_pytorch/opt/mpi/lib/:$LD_LIBRARY_PATH' >> run.sh && \ echo 'source $PKGPATH/oneccl_bindings_for_pytorch/env/setvars.sh' >> run.sh && \ - echo ray start --head >> run.sh && \ + echo ray start --head --dashboard-host=0.0.0.0 >> run.sh && \ + echo export RAY_ADDRESS=http://localhost:8265 >> run.sh && \ echo python finetuning_service.py >> run.sh -CMD bash run.sh \ No newline at end of file +CMD bash run.sh diff --git a/comps/finetuning/llm_on_ray/finetune/finetune_config.py b/comps/finetuning/finetune_config.py similarity index 73% rename from comps/finetuning/llm_on_ray/finetune/finetune_config.py rename to comps/finetuning/finetune_config.py index 391c6e6c8..b070281ff 100644 --- a/comps/finetuning/llm_on_ray/finetune/finetune_config.py +++ b/comps/finetuning/finetune_config.py @@ -7,6 +7,8 @@ from pydantic import BaseModel, validator +from comps.cores.proto.api_protocol import FineTuningJobsRequest + PRECISION_BF16 = "bf16" PRECISION_FP16 = "fp16" PRECISION_NO = "no" @@ -20,30 +22,31 @@ ACCELERATE_STRATEGY_DEEPSPEED = "DEEPSPEED" -class GeneralConfig(BaseModel): - trust_remote_code: bool - use_auth_token: Optional[str] +class LoadConfig(BaseModel): + trust_remote_code: bool = False + # set Huggingface token to access dataset/model + token: Optional[str] = None class LoraConfig(BaseModel): - task_type: str - r: int - lora_alpha: int - lora_dropout: float + task_type: str = "CAUSAL_LM" + r: int = 8 + lora_alpha: int = 32 + lora_dropout: float = 0.1 target_modules: Optional[List[str]] = None -class General(BaseModel): - base_model: str +class GeneralConfig(BaseModel): + base_model: str = None tokenizer_name: Optional[str] = None gaudi_config_name: Optional[str] = None - gpt_base_model: bool - output_dir: str + gpt_base_model: bool = False + output_dir: str = "./tmp" report_to: str = "none" resume_from_checkpoint: Optional[str] = None save_strategy: str = "no" - config: GeneralConfig - lora_config: Optional[LoraConfig] = None + config: LoadConfig = LoadConfig() + lora_config: Optional[LoraConfig] = LoraConfig() enable_gradient_checkpointing: bool = False @validator("report_to") @@ -52,10 +55,10 @@ def check_report_to(cls, v: str): return v -class Dataset(BaseModel): - train_file: str - validation_file: Optional[str] - validation_split_percentage: int +class DatasetConfig(BaseModel): + train_file: str = None + validation_file: Optional[str] = None + validation_split_percentage: int = 5 max_length: int = 512 group: bool = True block_size: int = 512 @@ -74,23 +77,23 @@ class Dataset(BaseModel): class RayResourceConfig(BaseModel): - CPU: int + CPU: int = 32 GPU: int = 0 HPU: int = 0 -class Training(BaseModel): - optimizer: str - batch_size: int - epochs: int +class TrainingConfig(BaseModel): + optimizer: str = "adamw_torch" + batch_size: int = 2 + epochs: int = 1 max_train_steps: Optional[int] = None - learning_rate: float - lr_scheduler: str - weight_decay: float + learning_rate: float = 5.0e-5 + lr_scheduler: str = "linear" + weight_decay: float = 0.0 device: str = DEVICE_CPU hpu_execution_mode: str = "lazy" - num_training_workers: int - resources_per_worker: RayResourceConfig + num_training_workers: int = 1 + resources_per_worker: RayResourceConfig = RayResourceConfig() accelerate_mode: str = ACCELERATE_STRATEGY_DDP mixed_precision: str = PRECISION_NO gradient_accumulation_steps: int = 1 @@ -151,6 +154,13 @@ def check_logging_steps(cls, v: int): class FinetuneConfig(BaseModel): - General: General - Dataset: Dataset - Training: Training + General: GeneralConfig = GeneralConfig() + Dataset: DatasetConfig = DatasetConfig() + Training: TrainingConfig = TrainingConfig() + + +class FineTuningParams(FineTuningJobsRequest): + # priority use FineTuningJobsRequest params + General: GeneralConfig = GeneralConfig() + Dataset: DatasetConfig = DatasetConfig() + Training: TrainingConfig = TrainingConfig() diff --git a/comps/finetuning/finetune_runner.py b/comps/finetuning/finetune_runner.py index 1ddfc4642..0161cd1ca 100644 --- a/comps/finetuning/finetune_runner.py +++ b/comps/finetuning/finetune_runner.py @@ -6,7 +6,7 @@ from pydantic_yaml import parse_yaml_raw_as from transformers import TrainerCallback, TrainerControl, TrainerState, TrainingArguments -from comps.finetuning.llm_on_ray.finetune.finetune_config import FinetuneConfig +from comps.finetuning.finetune_config import FinetuneConfig class FineTuneCallback(TrainerCallback): diff --git a/comps/finetuning/finetuning_service.py b/comps/finetuning/finetuning_service.py index 031380a5d..bd1423d21 100644 --- a/comps/finetuning/finetuning_service.py +++ b/comps/finetuning/finetuning_service.py @@ -8,7 +8,8 @@ from fastapi import BackgroundTasks, File, UploadFile from comps import opea_microservices, register_microservice -from comps.cores.proto.api_protocol import FineTuningJobIDRequest, FineTuningJobsRequest +from comps.cores.proto.api_protocol import FineTuningJobIDRequest +from comps.finetuning.finetune_config import FineTuningParams from comps.finetuning.handlers import ( DATASET_BASE_PATH, handle_cancel_finetuning_job, @@ -21,7 +22,7 @@ @register_microservice(name="opea_service@finetuning", endpoint="/v1/fine_tuning/jobs", host="0.0.0.0", port=8015) -def create_finetuning_jobs(request: FineTuningJobsRequest, background_tasks: BackgroundTasks): +def create_finetuning_jobs(request: FineTuningParams, background_tasks: BackgroundTasks): return handle_create_finetuning_jobs(request, background_tasks) diff --git a/comps/finetuning/handlers.py b/comps/finetuning/handlers.py index e0d04bf24..406bcd245 100644 --- a/comps/finetuning/handlers.py +++ b/comps/finetuning/handlers.py @@ -13,28 +13,21 @@ from ray.job_submission import JobSubmissionClient from comps import CustomLogger -from comps.cores.proto.api_protocol import ( - FineTuningJob, - FineTuningJobIDRequest, - FineTuningJobList, - FineTuningJobsRequest, -) -from comps.finetuning.llm_on_ray.finetune.finetune_config import FinetuneConfig +from comps.cores.proto.api_protocol import FineTuningJob, FineTuningJobIDRequest, FineTuningJobList +from comps.finetuning.finetune_config import FinetuneConfig, FineTuningParams logger = CustomLogger("finetuning_handlers") -MODEL_CONFIG_FILE_MAP = { - "meta-llama/Llama-2-7b-chat-hf": "./models/llama-2-7b-chat-hf.yaml", - "mistralai/Mistral-7B-v0.1": "./models/mistral-7b-v0.1.yaml", -} - DATASET_BASE_PATH = "datasets" JOBS_PATH = "jobs" +OUTPUT_DIR = "output" + if not os.path.exists(DATASET_BASE_PATH): os.mkdir(DATASET_BASE_PATH) - if not os.path.exists(JOBS_PATH): os.mkdir(JOBS_PATH) +if not os.path.exists(OUTPUT_DIR): + os.mkdir(OUTPUT_DIR) FineTuningJobID = str CHECK_JOB_STATUS_INTERVAL = 5 # Check every 5 secs @@ -62,23 +55,17 @@ def update_job_status(job_id: FineTuningJobID): time.sleep(CHECK_JOB_STATUS_INTERVAL) -def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tasks: BackgroundTasks): +def handle_create_finetuning_jobs(request: FineTuningParams, background_tasks: BackgroundTasks): base_model = request.model train_file = request.training_file train_file_path = os.path.join(DATASET_BASE_PATH, train_file) - model_config_file = MODEL_CONFIG_FILE_MAP.get(base_model) - if not model_config_file: - raise HTTPException(status_code=404, detail=f"Base model '{base_model}' not supported!") - if not os.path.exists(train_file_path): raise HTTPException(status_code=404, detail=f"Training file '{train_file}' not found!") - with open(model_config_file) as f: - finetune_config = parse_yaml_raw_as(FinetuneConfig, f) - + finetune_config = FinetuneConfig(General=request.General, Dataset=request.Dataset, Training=request.Training) + finetune_config.General.base_model = base_model finetune_config.Dataset.train_file = train_file_path - if request.hyperparameters is not None: if request.hyperparameters.epochs != "auto": finetune_config.Training.epochs = request.hyperparameters.epochs @@ -90,7 +77,7 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas finetune_config.Training.learning_rate = request.hyperparameters.learning_rate_multiplier if os.getenv("HF_TOKEN", None): - finetune_config.General.config.use_auth_token = os.getenv("HF_TOKEN", None) + finetune_config.General.config.token = os.getenv("HF_TOKEN", None) job = FineTuningJob( id=f"ft-job-{uuid.uuid4()}", @@ -105,12 +92,16 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas status="running", seed=random.randint(0, 1000) if request.seed is None else request.seed, ) - finetune_config.General.output_dir = os.path.join(JOBS_PATH, job.id) + finetune_config.General.output_dir = os.path.join(OUTPUT_DIR, job.id) if os.getenv("DEVICE", ""): logger.info(f"specific device: {os.getenv('DEVICE')}") finetune_config.Training.device = os.getenv("DEVICE") + if finetune_config.Training.device == "hpu": + if finetune_config.Training.resources_per_worker.HPU == 0: + # set 1 + finetune_config.Training.resources_per_worker.HPU = 1 finetune_config_file = f"{JOBS_PATH}/{job.id}.yaml" to_yaml_file(finetune_config_file, finetune_config) @@ -122,7 +113,7 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas # Entrypoint shell command to execute entrypoint=f"python finetune_runner.py --config_file {finetune_config_file}", # Path to the local directory that contains the script.py file - runtime_env={"working_dir": "./"}, + runtime_env={"working_dir": "./", "excludes": [f"{OUTPUT_DIR}"]}, ) logger.info(f"Submitted Ray job: {ray_job_id} ...") diff --git a/comps/finetuning/launch.sh b/comps/finetuning/launch.sh index bb5042ac6..f439db2de 100644 --- a/comps/finetuning/launch.sh +++ b/comps/finetuning/launch.sh @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 if [[ -n "$RAY_PORT" ]];then - ray start --head --port $RAY_PORT + ray start --head --port $RAY_PORT --dashboard-host=0.0.0.0 else - ray start --head + ray start --head --dashboard-host=0.0.0.0 export RAY_PORT=8265 fi -export RAY_ADDRESS=http://127.0.0.1:$RAY_PORT +export RAY_ADDRESS=http://localhost:$RAY_PORT python finetuning_service.py diff --git a/comps/finetuning/llm_on_ray/finetune/finetune.py b/comps/finetuning/llm_on_ray/finetune/finetune.py index 868dc889e..2ccb20b73 100644 --- a/comps/finetuning/llm_on_ray/finetune/finetune.py +++ b/comps/finetuning/llm_on_ray/finetune/finetune.py @@ -24,9 +24,9 @@ from ray.train.torch import TorchTrainer from comps import CustomLogger +from comps.finetuning.finetune_config import FinetuneConfig from comps.finetuning.llm_on_ray import common from comps.finetuning.llm_on_ray.finetune.data_process import DataProcessor -from comps.finetuning.llm_on_ray.finetune.finetune_config import FinetuneConfig logger = CustomLogger("llm_on_ray/finetune") @@ -171,8 +171,8 @@ def local_load(name, **load_config): else: # try to download and load dataset from huggingface.co load_config = config["General"].get("config", {}) - use_auth_token = load_config.get("use_auth_token", None) - raw_dataset = datasets.load_dataset(dataset_file, use_auth_token=use_auth_token) + use_auth_token = load_config.get("token", None) + raw_dataset = datasets.load_dataset(dataset_file, token=use_auth_token) validation_split_percentage = config["Dataset"].get("validation_split_percentage", 0) if "validation" not in raw_dataset.keys() and ( diff --git a/comps/finetuning/models/llama-2-7b-chat-hf.yaml b/comps/finetuning/models/llama-2-7b-chat-hf.yaml deleted file mode 100644 index d6ae5f34d..000000000 --- a/comps/finetuning/models/llama-2-7b-chat-hf.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -General: - base_model: meta-llama/Llama-2-7b-chat-hf - output_dir: "./tmp" - gpt_base_model: false - save_strategy: no - config: - trust_remote_code: false - use_auth_token: null - lora_config: - task_type: CAUSAL_LM - r: 8 - lora_alpha: 32 - lora_dropout: 0.1 - target_modules: - - q_proj - - v_proj - enable_gradient_checkpointing: false -Dataset: - train_file: examples/data/sample_finetune_data_small.jsonl - group: false - validation_file: null - validation_split_percentage: 5 -Training: - optimizer: adamw_torch - batch_size: 2 - epochs: 3 - learning_rate: 1.0e-05 - lr_scheduler: linear - weight_decay: 0.0 - mixed_precision: bf16 - device: cpu - num_training_workers: 1 - resources_per_worker: - CPU: 32 - gradient_accumulation_steps: 1 - logging_steps: 10 diff --git a/comps/finetuning/models/mistral-7b-v0.1.yaml b/comps/finetuning/models/mistral-7b-v0.1.yaml deleted file mode 100644 index 4334fa37e..000000000 --- a/comps/finetuning/models/mistral-7b-v0.1.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -General: - base_model: mistralai/Mistral-7B-v0.1 - output_dir: "./tmp" - gpt_base_model: false - save_strategy: no - config: - trust_remote_code: false - use_auth_token: null - lora_config: - task_type: CAUSAL_LM - r: 8 - lora_alpha: 32 - lora_dropout: 0.1 - target_modules: - - q_proj - - k_proj - - v_proj - - o_proj - - gate_proj - - up_proj - - down_proj - - lm_head - enable_gradient_checkpointing: false -Dataset: - train_file: examples/data/sample_finetune_data_small.jsonl - validation_file: null - validation_split_percentage: 5 -Training: - optimizer: adamw_torch - batch_size: 2 - epochs: 3 - learning_rate: 1.0e-05 - lr_scheduler: linear - weight_decay: 0.0 - mixed_precision: bf16 - device: cpu - num_training_workers: 2 - resources_per_worker: - CPU: 32 - accelerate_mode: DDP - gradient_accumulation_steps: 1 - logging_steps: 10 diff --git a/tests/test_finetuning.sh b/tests/test_finetuning.sh new file mode 100644 index 000000000..3ddbdfcf0 --- /dev/null +++ b/tests/test_finetuning.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') +finetuning_service_port=8015 +ray_port=8265 + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build -t opea/finetuning:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg HF_TOKEN=$HF_TOKEN -f comps/finetuning/docker/Dockerfile_cpu . + if [ $? -ne 0 ]; then + echo "opea/finetuning built fail" + exit 1 + else + echo "opea/finetuning built successful" + fi +} + +function start_service() { + export no_proxy="localhost,127.0.0.1,"${ip_address} + docker run -d --name="finetuning-server" -p $finetuning_service_port:$finetuning_service_port -p $ray_port:$ray_port --runtime=runc --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy opea/finetuning:latest + sleep 1m +} + +function validate_microservice() { + cd $LOG_PATH + export no_proxy="localhost,127.0.0.1,"${ip_address} + + # test /v1/dataprep upload file + URL="http://${ip_address}:$finetuning_service_port/v1/finetune/upload_training_files" + echo '[{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."},{"instruction": "Give three tips for staying healthy.", "input": "", "output": "1.Eat a balanced diet and make sure to include plenty of fruits and vegetables. \n2. Exercise regularly to keep your body active and strong. \n3. Get enough sleep and maintain a consistent sleep schedule."}]' > $LOG_PATH/test_data.json + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F 'files=@./test_data.json' -H 'Content-Type: multipart/form-data' "$URL") + HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g') + SERVICE_NAME="finetuning-server - upload - file" + + if [ "$HTTP_STATUS" -ne "200" ]; then + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs finetuning-server >> ${LOG_PATH}/finetuning-server_upload_file.log + exit 1 + else + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + fi + if [[ "$RESPONSE_BODY" != *"Training files uploaded"* ]]; then + echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY" + docker logs finetuning-server >> ${LOG_PATH}/finetuning-server_upload_file.log + exit 1 + else + echo "[ $SERVICE_NAME ] Content is as expected." + fi + + # test /v1/fine_tuning/jobs + URL="http://${ip_address}:$finetuning_service_port/v1/fine_tuning/jobs" + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -H 'Content-Type: application/json' -d '{"training_file": "test_data.json","model": "facebook/opt-125m"}' "$URL") + HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g') + SERVICE_NAME="finetuning-server - create finetuning job" + + if [ "$HTTP_STATUS" -ne "200" ]; then + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs finetuning-server >> ${LOG_PATH}/finetuning-server_create.log + exit 1 + else + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + fi + if [[ "$RESPONSE_BODY" != *'{"id":"ft-job'* ]]; then + echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY" + docker logs finetuning-server >> ${LOG_PATH}/finetuning-server_create.log + exit 1 + else + echo "[ $SERVICE_NAME ] Content is as expected." + fi + + sleep 3m +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=finetuning-server*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main