From 151399895043dd2e5198592e74b7081bf55e0806 Mon Sep 17 00:00:00 2001 From: Tiep Le <97980157+tileintel@users.noreply.github.com> Date: Thu, 5 Sep 2024 22:46:10 -0700 Subject: [PATCH] Retriever and lvm update for multimodal rag on videos (#606) * updates Signed-off-by: Tiep Le * cosmetic Signed-off-by: siddhivelankar23 * update redis schema Signed-off-by: siddhivelankar23 * update multimodal config and docker compose retriever Signed-off-by: siddhivelankar23 * update requirements Signed-off-by: siddhivelankar23 * update retriever redis Signed-off-by: siddhivelankar23 * multimodal retriever implementation Signed-off-by: siddhivelankar23 * test for multimodal retriever Signed-off-by: siddhivelankar23 * include prompt preparation for multimodal rag on videos application Signed-off-by: sjagtap1803 * fix template Signed-off-by: sjagtap1803 * add test for llava for mm_rag_on_videos Signed-off-by: sjagtap1803 * update test Signed-off-by: sjagtap1803 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix index not found Signed-off-by: sjagtap1803 * add LVMSearchedMultimodalDoc Signed-off-by: sjagtap1803 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove INDEX_SCHEMA Signed-off-by: siddhivelankar23 * revise folder structure to comps/retrievers/langchain/redis_multimodal Signed-off-by: siddhivelankar23 * update test Signed-off-by: siddhivelankar23 * change port of redis to resolve CI test Signed-off-by: siddhivelankar23 * update test Signed-off-by: siddhivelankar23 * update lvms test Signed-off-by: siddhivelankar23 --------- Signed-off-by: Tiep Le Signed-off-by: siddhivelankar23 Signed-off-by: sjagtap1803 Co-authored-by: siddhivelankar23 Co-authored-by: sjagtap1803 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- comps/__init__.py | 1 + comps/cores/proto/docarray.py | 18 +++ comps/lvms/lvm.py | 39 +++++- comps/lvms/requirements.txt | 1 + comps/lvms/template.py | 10 ++ .../langchain/redis_multimodal/README.md | 123 ++++++++++++++++++ .../langchain/redis_multimodal/__init__.py | 2 + .../redis_multimodal/docker/Dockerfile | 29 +++++ .../docker/docker_compose_retriever.yaml | 23 ++++ .../redis_multimodal/multimodal_config.py | 77 +++++++++++ .../redis_multimodal/requirements.txt | 11 ++ .../redis_multimodal/retriever_redis.py | 93 +++++++++++++ ...est_dataprep_redis_multimodal_langchain.sh | 2 +- tests/test_lvms_llava.sh | 20 +++ ...t_retrievers_langchain_multimodal_redis.sh | 84 ++++++++++++ 15 files changed, 525 insertions(+), 8 deletions(-) create mode 100644 comps/lvms/template.py create mode 100644 comps/retrievers/langchain/redis_multimodal/README.md create mode 100644 comps/retrievers/langchain/redis_multimodal/__init__.py create mode 100644 comps/retrievers/langchain/redis_multimodal/docker/Dockerfile create mode 100644 comps/retrievers/langchain/redis_multimodal/docker/docker_compose_retriever.yaml create mode 100644 comps/retrievers/langchain/redis_multimodal/multimodal_config.py create mode 100644 comps/retrievers/langchain/redis_multimodal/requirements.txt create mode 100644 comps/retrievers/langchain/redis_multimodal/retriever_redis.py create mode 100644 tests/test_retrievers_langchain_multimodal_redis.sh diff --git a/comps/__init__.py b/comps/__init__.py index c58ae42fe..bc7c8fe80 100644 --- a/comps/__init__.py +++ b/comps/__init__.py @@ -13,6 +13,7 @@ LLMParamsDoc, SearchedDoc, SearchedMultimodalDoc, + LVMSearchedMultimodalDoc, RerankedDoc, TextDoc, RAGASParams, diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 132b172bc..3c97880a3 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -107,6 +107,24 @@ class SearchedMultimodalDoc(SearchedDoc): metadata: List[Dict[str, Any]] +class LVMSearchedMultimodalDoc(SearchedMultimodalDoc): + max_new_tokens: conint(ge=0, le=1024) = 512 + top_k: int = 10 + top_p: float = 0.95 + typical_p: float = 0.95 + temperature: float = 0.01 + streaming: bool = False + repetition_penalty: float = 1.03 + chat_template: Optional[str] = Field( + default=None, + description=( + "A template to use for this conversion. " + "If this is not passed, the model's default chat template will be " + "used instead. We recommend that the template contains {context} and {question} for multimodal-rag on videos." + ), + ) + + class GeneratedDoc(BaseDoc): text: str prompt: str diff --git a/comps/lvms/lvm.py b/comps/lvms/lvm.py index 4ae900aae..d84e36637 100644 --- a/comps/lvms/lvm.py +++ b/comps/lvms/lvm.py @@ -5,12 +5,16 @@ import json import os import time +from typing import Union import requests +from langchain_core.prompts import PromptTemplate +from template import ChatTemplate from comps import ( CustomLogger, LVMDoc, + LVMSearchedMultimodalDoc, ServiceType, TextDoc, opea_microservices, @@ -29,20 +33,41 @@ endpoint="/v1/lvm", host="0.0.0.0", port=9399, - input_datatype=LVMDoc, - output_datatype=TextDoc, ) @register_statistics(names=["opea_service@lvm"]) -async def lvm(request: LVMDoc): +async def lvm(request: Union[LVMDoc, LVMSearchedMultimodalDoc]) -> TextDoc: if logflag: logger.info(request) start = time.time() - img_b64_str = request.image - prompt = request.prompt - max_new_tokens = request.max_new_tokens + if isinstance(request, LVMSearchedMultimodalDoc): + if logflag: + logger.info("[LVMSearchedMultimodalDoc ] input from retriever microservice") + retrieved_metadatas = request.metadata + img_b64_str = retrieved_metadatas[0]["b64_img_str"] + initial_query = request.initial_query + context = retrieved_metadatas[0]["transcript_for_inference"] + prompt = initial_query + if request.chat_template is None: + prompt = ChatTemplate.generate_multimodal_rag_on_videos_prompt(initial_query, context) + else: + prompt_template = PromptTemplate.from_template(request.chat_template) + input_variables = prompt_template.input_variables + if sorted(input_variables) == ["context", "question"]: + prompt = prompt_template.format(question=initial_query, context=context) + else: + logger.info( + f"[ LVMSearchedMultimodalDoc ] {prompt_template} not used, we only support 2 input variables ['question', 'context']" + ) + max_new_tokens = request.max_new_tokens + if logflag: + logger.info(f"prompt generated for [LVMSearchedMultimodalDoc ] input from retriever microservice: {prompt}") - inputs = {"img_b64_str": img_b64_str, "prompt": prompt, "max_new_tokens": max_new_tokens} + else: + img_b64_str = request.image + prompt = request.prompt + max_new_tokens = request.max_new_tokens + inputs = {"img_b64_str": img_b64_str, "prompt": prompt, "max_new_tokens": max_new_tokens} # forward to the LLaVA server response = requests.post(url=f"{lvm_endpoint}/generate", data=json.dumps(inputs), proxies={"http": None}) diff --git a/comps/lvms/requirements.txt b/comps/lvms/requirements.txt index 556dfb0c1..be1c23a6d 100644 --- a/comps/lvms/requirements.txt +++ b/comps/lvms/requirements.txt @@ -2,6 +2,7 @@ datasets docarray[full] fastapi huggingface_hub +langchain-core opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk diff --git a/comps/lvms/template.py b/comps/lvms/template.py new file mode 100644 index 000000000..71c2b2667 --- /dev/null +++ b/comps/lvms/template.py @@ -0,0 +1,10 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +class ChatTemplate: + + @staticmethod + def generate_multimodal_rag_on_videos_prompt(question: str, context: str): + template = """The transcript associated with the image is '{context}'. {question}""" + return template.format(context=context, question=question) diff --git a/comps/retrievers/langchain/redis_multimodal/README.md b/comps/retrievers/langchain/redis_multimodal/README.md new file mode 100644 index 000000000..edc2523a3 --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/README.md @@ -0,0 +1,123 @@ +# Retriever Microservice + +This retriever microservice is a highly efficient search service designed for handling and retrieving embedding vectors from multimodal data. It operates by receiving an embedding vector as input and conducting a similarity search against vectors stored in a VectorDB database. Users must specify the VectorDB's URL and the index name, and the service searches within that index to find documents with the highest similarity to the input vector. + +The service primarily utilizes similarity measures in vector space to rapidly retrieve contentually similar documents. The vector-based retrieval approach is particularly suited for handling large datasets, offering fast and accurate search results that significantly enhance the efficiency and quality of information retrieval. + +Overall, this microservice provides robust backend support for applications requiring efficient similarity searches, playing a vital role in scenarios such as recommendation systems, information retrieval, or any other context where precise measurement of document similarity is crucial. + +## 🚀1. Start Microservice with Python (Option 1) + +To start the retriever microservice, you must first install the required python packages. + +### 1.1 Install Requirements + +```bash +pip install -r requirements.txt +``` + +### 1.2 Setup VectorDB Service + +You need to setup your own VectorDB service (Redis in this example), and ingest your knowledge documents into the vector database. + +As for Redis, you could start a docker container using the following commands. +Remember to ingest data into it manually. + +```bash +docker run -d --name="redis-vector-db" -p 6379:6379 -p 8001:8001 redis/redis-stack:7.2.0-v9 +``` + +### 1.3 Ingest images or video + +Upload a video or images using the dataprep microservice, instructions can be found [here](https://github.com/opea-project/GenAIComps/tree/main/comps/dataprep/redis/multimodal_langchain/README.md). + +### 1.4 Start Retriever Service + +```bash +python retriever_redis.py +``` + +## 🚀2. Start Microservice with Docker (Option 2) + +### 2.1 Setup Environment Variables + +```bash +export your_ip=$(hostname -I | awk '{print $1}') +export REDIS_URL="redis://${your_ip}:6379" +export INDEX_NAME=${your_index_name} +``` + +### 2.2 Build Docker Image + +```bash +cd ../../../../ +docker build -t opea/multimodal-retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/redis_multimodal/docker/Dockerfile . +``` + +To start a docker container, you have two options: + +- A. Run Docker with CLI +- B. Run Docker with Docker Compose + +You can choose one as needed. + +### 2.3 Run Docker with CLI (Option A) + +```bash +docker run -d --name="multimodal-retriever-redis-server" -p 7000:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/multimodal-retriever-redis:latest +``` + +### 2.4 Run Docker with Docker Compose (Option B) + +```bash +cd docker +docker compose -f docker_compose_retriever.yaml up -d +``` + +## 🚀3. Consume Retriever Service + +### 3.1 Consume Embedding Service + +To consume the Retriever Microservice, you can generate a mock embedding vector of length 512 with Python. + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://${your_ip}:7000/v1/multimodal_retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding}}" \ + -H 'Content-Type: application/json' +``` + +You can set the parameters for the retriever. + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://localhost:7000/v1/multimodal_retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity\", \"k\":4}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://localhost:7000/v1/multimodal_retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity_distance_threshold\", \"k\":4, \"distance_threshold\":1.0}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://localhost:7000/v1/multimodal_retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"similarity_score_threshold\", \"k\":4, \"score_threshold\":0.2}" \ + -H 'Content-Type: application/json' +``` + +```bash +your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://localhost:7000/v1/multimodal_retrieval \ + -X POST \ + -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"mmr\", \"k\":4, \"fetch_k\":20, \"lambda_mult\":0.5}" \ + -H 'Content-Type: application/json' +``` diff --git a/comps/retrievers/langchain/redis_multimodal/__init__.py b/comps/retrievers/langchain/redis_multimodal/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/retrievers/langchain/redis_multimodal/docker/Dockerfile b/comps/retrievers/langchain/redis_multimodal/docker/Dockerfile new file mode 100644 index 000000000..f211005b9 --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/docker/Dockerfile @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM langchain/langchain:latest + +ARG ARCH="cpu" + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +COPY comps /home/user/comps + +USER user + +RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ + pip install --no-cache-dir -r /home/user/comps/retrievers/langchain/redis_multimodal/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/retrievers/langchain/redis_multimodal + +ENTRYPOINT ["python", "retriever_redis.py"] diff --git a/comps/retrievers/langchain/redis_multimodal/docker/docker_compose_retriever.yaml b/comps/retrievers/langchain/redis_multimodal/docker/docker_compose_retriever.yaml new file mode 100644 index 000000000..efba29a4e --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/docker/docker_compose_retriever.yaml @@ -0,0 +1,23 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "1.0" + +services: + retriever: + image: opea/multimodal-retriever-redis:latest + container_name: multimodal-retriever-redis-server + ports: + - "7000:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + INDEX_NAME: ${INDEX_NAME} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/retrievers/langchain/redis_multimodal/multimodal_config.py b/comps/retrievers/langchain/redis_multimodal/multimodal_config.py new file mode 100644 index 000000000..211851b7e --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/multimodal_config.py @@ -0,0 +1,77 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +current_file_path = os.path.abspath(__file__) +parent_dir = os.path.dirname(current_file_path) + + +def get_boolean_env_var(var_name, default_value=False): + """Retrieve the boolean value of an environment variable. + + Args: + var_name (str): The name of the environment variable to retrieve. + default_value (bool): The default value to return if the variable + is not found. + Returns: + bool: The value of the environment variable, interpreted as a boolean. + """ + true_values = {"true", "1", "t", "y", "yes"} + false_values = {"false", "0", "f", "n", "no"} + + # Retrieve the environment variable's value + value = os.getenv(var_name, "").lower() + + # Decide the boolean value based on the content of the string + if value in true_values: + return True + elif value in false_values: + return False + else: + return default_value + + +# Check for openai API key +# if "OPENAI_API_KEY" not in os.environ: +# raise Exception("Must provide an OPENAI_API_KEY as an env var.") + + +# Whether or not to enable langchain debugging +DEBUG = get_boolean_env_var("DEBUG", False) +# Set DEBUG env var to "true" if you wish to enable LC debugging module +if DEBUG: + import langchain + + langchain.debug = True + + +# Embedding model +EMBED_MODEL = os.getenv("EMBED_MODEL", "BridgeTower/bridgetower-large-itm-mlm-itc") + +# Redis Connection Information +REDIS_HOST = os.getenv("REDIS_HOST", "localhost") +REDIS_PORT = int(os.getenv("REDIS_PORT", 6379)) + + +def format_redis_conn_from_env(): + redis_url = os.getenv("REDIS_URL", None) + if redis_url: + return redis_url + else: + using_ssl = get_boolean_env_var("REDIS_SSL", False) + start = "rediss://" if using_ssl else "redis://" + + # if using RBAC + password = os.getenv("REDIS_PASSWORD", None) + username = os.getenv("REDIS_USERNAME", "default") + if password is not None: + start += f"{username}:{password}@" + + return start + f"{REDIS_HOST}:{REDIS_PORT}" + + +REDIS_URL = format_redis_conn_from_env() + +# Vector Index Configuration +INDEX_NAME = os.getenv("INDEX_NAME", "test-index") diff --git a/comps/retrievers/langchain/redis_multimodal/requirements.txt b/comps/retrievers/langchain/redis_multimodal/requirements.txt new file mode 100644 index 000000000..b7cef2866 --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/requirements.txt @@ -0,0 +1,11 @@ +docarray[full] +fastapi +langchain_community +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +redis +shortuuid +transformers +uvicorn diff --git a/comps/retrievers/langchain/redis_multimodal/retriever_redis.py b/comps/retrievers/langchain/redis_multimodal/retriever_redis.py new file mode 100644 index 000000000..f998ee28e --- /dev/null +++ b/comps/retrievers/langchain/redis_multimodal/retriever_redis.py @@ -0,0 +1,93 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import time +from typing import Union + +from langchain_community.vectorstores import Redis +from multimodal_config import INDEX_NAME, REDIS_URL + +from comps import ( + EmbedMultimodalDoc, + SearchedMultimodalDoc, + ServiceType, + TextDoc, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) +from comps.cores.proto.api_protocol import ( + ChatCompletionRequest, + RetrievalRequest, + RetrievalResponse, + RetrievalResponseData, +) +from comps.embeddings.multimodal_embeddings.bridgetower import BridgeTowerEmbedding + + +@register_microservice( + name="opea_service@multimodal_retriever_redis", + service_type=ServiceType.RETRIEVER, + endpoint="/v1/multimodal_retrieval", + host="0.0.0.0", + port=7000, +) +@register_statistics(names=["opea_service@multimodal_retriever_redis"]) +def retrieve( + input: Union[EmbedMultimodalDoc, RetrievalRequest, ChatCompletionRequest] +) -> Union[SearchedMultimodalDoc, RetrievalResponse, ChatCompletionRequest]: + + start = time.time() + # check if the Redis index has data + if vector_db.client.keys() == []: + search_res = [] + else: + # if the Redis index has data, perform the search + if input.search_type == "similarity": + search_res = vector_db.similarity_search_by_vector(embedding=input.embedding, k=input.k) + elif input.search_type == "similarity_distance_threshold": + if input.distance_threshold is None: + raise ValueError("distance_threshold must be provided for " + "similarity_distance_threshold retriever") + search_res = vector_db.similarity_search_by_vector( + embedding=input.embedding, k=input.k, distance_threshold=input.distance_threshold + ) + elif input.search_type == "similarity_score_threshold": + docs_and_similarities = vector_db.similarity_search_with_relevance_scores( + query=input.text, k=input.k, score_threshold=input.score_threshold + ) + search_res = [doc for doc, _ in docs_and_similarities] + elif input.search_type == "mmr": + search_res = vector_db.max_marginal_relevance_search( + query=input.text, k=input.k, fetch_k=input.fetch_k, lambda_mult=input.lambda_mult + ) + else: + raise ValueError(f"{input.search_type} not valid") + + # return different response format + retrieved_docs = [] + if isinstance(input, EmbedMultimodalDoc): + metadata_list = [] + for r in search_res: + metadata_list.append(r.metadata) + retrieved_docs.append(TextDoc(text=r.page_content)) + result = SearchedMultimodalDoc(retrieved_docs=retrieved_docs, initial_query=input.text, metadata=metadata_list) + else: + for r in search_res: + retrieved_docs.append(RetrievalResponseData(text=r.page_content, metadata=r.metadata)) + if isinstance(input, RetrievalRequest): + result = RetrievalResponse(retrieved_docs=retrieved_docs) + elif isinstance(input, ChatCompletionRequest): + input.retrieved_docs = retrieved_docs + input.documents = [doc.text for doc in retrieved_docs] + result = input + + statistics_dict["opea_service@multimodal_retriever_redis"].append_latency(time.time() - start, None) + return result + + +if __name__ == "__main__": + + embeddings = BridgeTowerEmbedding() + vector_db = Redis(embedding=embeddings, index_name=INDEX_NAME, redis_url=REDIS_URL) + opea_microservices["opea_service@multimodal_retriever_redis"].start() diff --git a/tests/test_dataprep_redis_multimodal_langchain.sh b/tests/test_dataprep_redis_multimodal_langchain.sh index e5a75f860..17d6985a1 100644 --- a/tests/test_dataprep_redis_multimodal_langchain.sh +++ b/tests/test_dataprep_redis_multimodal_langchain.sh @@ -271,7 +271,7 @@ function main() { validate_microservice delete_data stop_docker - # echo y | docker system prune + echo y | docker system prune } diff --git a/tests/test_lvms_llava.sh b/tests/test_lvms_llava.sh index 08f138e2f..0f6ec2602 100644 --- a/tests/test_lvms_llava.sh +++ b/tests/test_lvms_llava.sh @@ -45,6 +45,26 @@ function validate_microservice() { exit 1 fi + result=$(http_proxy="" curl http://localhost:5028/v1/lvm -XPOST -d '{"retrieved_docs": [], "initial_query": "What is this?", "top_n": 1, "metadata": [{"b64_img_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "transcript_for_inference": "yellow image"}]}' -H 'Content-Type: application/json') + if [[ $result == *"yellow"* ]]; then + echo "Result correct." + else + echo "Result wrong." + docker logs test-comps-lvm-llava + docker logs test-comps-lvm + exit 1 + fi + + result=$(http_proxy="" curl http://localhost:5028/v1/lvm -XPOST -d '{"retrieved_docs": [], "initial_query": "What is this?", "top_n": 1, "metadata": [{"b64_img_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "transcript_for_inference": "yellow image"}], "chat_template":"The caption of the image is: '\''{context}'\''. {question}"}' -H 'Content-Type: application/json') + if [[ $result == *"yellow"* ]]; then + echo "Result correct." + else + echo "Result wrong." + docker logs test-comps-lvm-llava + docker logs test-comps-lvm + exit 1 + fi + } function stop_docker() { diff --git a/tests/test_retrievers_langchain_multimodal_redis.sh b/tests/test_retrievers_langchain_multimodal_redis.sh new file mode 100644 index 000000000..f2de9cc9e --- /dev/null +++ b/tests/test_retrievers_langchain_multimodal_redis.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + docker build --no-cache -t opea/multimodal-retriever-redis:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/langchain/redis_multimodal/docker/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/multimodal-retriever-redis built fail" + exit 1 + else + echo "opea/multimodal-retriever-redis built successful" + fi +} + +function start_service() { + # redis + docker run -d --name test-comps-multimodal-retriever-redis-vector-db -p 5689:6379 -p 5011:8001 -e HTTPS_PROXY=$https_proxy -e HTTP_PROXY=$https_proxy redis/redis-stack:7.2.0-v9 + sleep 10s + + # redis retriever + export REDIS_URL="redis://${ip_address}:5689" + export INDEX_NAME="rag-redis" + retriever_port=5009 + unset http_proxy + docker run -d --name="test-comps-multimodal-retriever-redis-server" -p ${retriever_port}:7000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -e INDEX_NAME=$INDEX_NAME opea/multimodal-retriever-redis:comps + + sleep 5m +} + +function validate_microservice() { + retriever_port=5009 + export PATH="${HOME}/miniforge3/bin:$PATH" + source activate + URL="http://${ip_address}:$retriever_port/v1/multimodal_retrieval" + test_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") + + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ retriever ] HTTP status is 200. Checking content..." + local CONTENT=$(curl -s -X POST -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/retriever.log) + + if echo "$CONTENT" | grep -q "retrieved_docs"; then + echo "[ retriever ] Content is as expected." + else + echo "[ retriever ] Content does not match the expected result: $CONTENT" + docker logs test-comps-multimodal-retriever-redis-server >> ${LOG_PATH}/retriever.log + exit 1 + fi + else + echo "[ retriever ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs test-comps-multimodal-retriever-redis-server >> ${LOG_PATH}/retriever.log + exit 1 + fi +} + +function stop_docker() { + cid_retrievers=$(docker ps -aq --filter "name=test-comps-multimodal-retriever*") + if [[ ! -z "$cid_retrievers" ]]; then + docker stop $cid_retrievers && docker rm $cid_retrievers && sleep 1s + fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main