From b709a87769460b2f7dfdc528ade0a2e15a9bea0a Mon Sep 17 00:00:00 2001 From: letonghan Date: Wed, 12 Jun 2024 15:54:40 +0800 Subject: [PATCH 1/5] move file to langchain folder Signed-off-by: letonghan --- comps/dataprep/redis/{ => langchain}/__init__.py | 0 comps/dataprep/redis/{ => langchain}/config.py | 0 comps/dataprep/redis/{ => langchain}/docker/Dockerfile | 0 .../{ => langchain}/docker/docker-compose-dataprep-redis.yaml | 0 comps/dataprep/redis/{ => langchain}/prepare_doc_redis.py | 0 comps/dataprep/redis/{ => langchain}/requirements.txt | 0 comps/dataprep/redis/{ => langchain}/schema.yml | 0 comps/dataprep/redis/{ => langchain}/schema_dim_1024.yml | 0 comps/dataprep/redis/{ => langchain}/schema_dim_768.yml | 0 comps/dataprep/redis/{ => langchain}/schema_lcdocs_dim_768.yml | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename comps/dataprep/redis/{ => langchain}/__init__.py (100%) rename comps/dataprep/redis/{ => langchain}/config.py (100%) rename comps/dataprep/redis/{ => langchain}/docker/Dockerfile (100%) rename comps/dataprep/redis/{ => langchain}/docker/docker-compose-dataprep-redis.yaml (100%) rename comps/dataprep/redis/{ => langchain}/prepare_doc_redis.py (100%) rename comps/dataprep/redis/{ => langchain}/requirements.txt (100%) rename comps/dataprep/redis/{ => langchain}/schema.yml (100%) rename comps/dataprep/redis/{ => langchain}/schema_dim_1024.yml (100%) rename comps/dataprep/redis/{ => langchain}/schema_dim_768.yml (100%) rename comps/dataprep/redis/{ => langchain}/schema_lcdocs_dim_768.yml (100%) diff --git a/comps/dataprep/redis/__init__.py b/comps/dataprep/redis/langchain/__init__.py similarity index 100% rename from comps/dataprep/redis/__init__.py rename to comps/dataprep/redis/langchain/__init__.py diff --git a/comps/dataprep/redis/config.py b/comps/dataprep/redis/langchain/config.py similarity index 100% rename from comps/dataprep/redis/config.py rename to comps/dataprep/redis/langchain/config.py diff --git a/comps/dataprep/redis/docker/Dockerfile b/comps/dataprep/redis/langchain/docker/Dockerfile similarity index 100% rename from comps/dataprep/redis/docker/Dockerfile rename to comps/dataprep/redis/langchain/docker/Dockerfile diff --git a/comps/dataprep/redis/docker/docker-compose-dataprep-redis.yaml b/comps/dataprep/redis/langchain/docker/docker-compose-dataprep-redis.yaml similarity index 100% rename from comps/dataprep/redis/docker/docker-compose-dataprep-redis.yaml rename to comps/dataprep/redis/langchain/docker/docker-compose-dataprep-redis.yaml diff --git a/comps/dataprep/redis/prepare_doc_redis.py b/comps/dataprep/redis/langchain/prepare_doc_redis.py similarity index 100% rename from comps/dataprep/redis/prepare_doc_redis.py rename to comps/dataprep/redis/langchain/prepare_doc_redis.py diff --git a/comps/dataprep/redis/requirements.txt b/comps/dataprep/redis/langchain/requirements.txt similarity index 100% rename from comps/dataprep/redis/requirements.txt rename to comps/dataprep/redis/langchain/requirements.txt diff --git a/comps/dataprep/redis/schema.yml b/comps/dataprep/redis/langchain/schema.yml similarity index 100% rename from comps/dataprep/redis/schema.yml rename to comps/dataprep/redis/langchain/schema.yml diff --git a/comps/dataprep/redis/schema_dim_1024.yml b/comps/dataprep/redis/langchain/schema_dim_1024.yml similarity index 100% rename from comps/dataprep/redis/schema_dim_1024.yml rename to comps/dataprep/redis/langchain/schema_dim_1024.yml diff --git a/comps/dataprep/redis/schema_dim_768.yml b/comps/dataprep/redis/langchain/schema_dim_768.yml similarity index 100% rename from comps/dataprep/redis/schema_dim_768.yml rename to comps/dataprep/redis/langchain/schema_dim_768.yml diff --git a/comps/dataprep/redis/schema_lcdocs_dim_768.yml b/comps/dataprep/redis/langchain/schema_lcdocs_dim_768.yml similarity index 100% rename from comps/dataprep/redis/schema_lcdocs_dim_768.yml rename to comps/dataprep/redis/langchain/schema_lcdocs_dim_768.yml From f274289b95ad8e1acb55ae1ffe9f5bbc113b745a Mon Sep 17 00:00:00 2001 From: letonghan Date: Wed, 12 Jun 2024 16:29:18 +0800 Subject: [PATCH 2/5] support dataprep with llama_index Signed-off-by: letonghan --- comps/dataprep/redis/README.md | 22 ++++- comps/dataprep/redis/llama_index/__init__.py | 2 + comps/dataprep/redis/llama_index/config.py | 57 ++++++++++++ .../redis/llama_index/docker/Dockerfile | 30 ++++++ .../docker/docker-compose-dataprep-redis.yaml | 28 ++++++ .../redis/llama_index/prepare_doc_redis.py | 93 +++++++++++++++++++ .../redis/llama_index/requirements.txt | 15 +++ 7 files changed, 244 insertions(+), 3 deletions(-) create mode 100644 comps/dataprep/redis/llama_index/__init__.py create mode 100644 comps/dataprep/redis/llama_index/config.py create mode 100644 comps/dataprep/redis/llama_index/docker/Dockerfile create mode 100644 comps/dataprep/redis/llama_index/docker/docker-compose-dataprep-redis.yaml create mode 100644 comps/dataprep/redis/llama_index/prepare_doc_redis.py create mode 100644 comps/dataprep/redis/llama_index/requirements.txt diff --git a/comps/dataprep/redis/README.md b/comps/dataprep/redis/README.md index 87564c584..6d54a9f8f 100644 --- a/comps/dataprep/redis/README.md +++ b/comps/dataprep/redis/README.md @@ -1,10 +1,18 @@ # Dataprep Microservice with Redis +For dataprep microservice, we provide two frameworks: `Langchain` and `LlamaIndex`. + +We organized these two folders in the same way, so you can use either framework for dataprep microservice with the following constructions. + # 🚀1. Start Microservice with Python（Option 1） ## 1.1 Install Requirements ```bash +# for langchain +cd langchain +# for llama_index +cd llama_index pip install -r requirements.txt ``` @@ -47,10 +55,15 @@ export LANGCHAIN_PROJECT="opea/dataprep" ``` ## 2.3 Build Docker Image - +- Build docker image with langchain +```bash +cd ../../../../ +docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/docker/Dockerfile . +``` +- Build docker image with llama_index ```bash cd ../../../../ -docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/docker/Dockerfile . +docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/llama_index/docker/Dockerfile . ``` ## 2.4 Run Docker with CLI (Option A) @@ -62,7 +75,10 @@ docker run -d --name="dataprep-redis-server" -p 6007:6007 --ipc=host -e http_pro ## 2.5 Run with Docker Compose (Option B) ```bash -cd comps/dataprep/redis/docker +# for langchain +cd comps/dataprep/redis/langchain/docker +# for llama_index +cd comps/dataprep/redis/llama_index/docker docker compose -f docker-compose-dataprep-redis.yaml up -d ``` diff --git a/comps/dataprep/redis/llama_index/__init__.py b/comps/dataprep/redis/llama_index/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/dataprep/redis/llama_index/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/dataprep/redis/llama_index/config.py b/comps/dataprep/redis/llama_index/config.py new file mode 100644 index 000000000..e427b443d --- /dev/null +++ b/comps/dataprep/redis/llama_index/config.py @@ -0,0 +1,57 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os + +EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-base-en-v1.5") + +# Redis Connection Information +REDIS_HOST = os.getenv("REDIS_HOST", "localhost") +REDIS_PORT = int(os.getenv("REDIS_PORT", 6379)) + + +def get_boolean_env_var(var_name, default_value=False): + """Retrieve the boolean value of an environment variable. + + Args: + var_name (str): The name of the environment variable to retrieve. + default_value (bool): The default value to return if the variable + is not found. + + Returns: + bool: The value of the environment variable, interpreted as a boolean. + """ + true_values = {"true", "1", "t", "y", "yes"} + false_values = {"false", "0", "f", "n", "no"} + + # Retrieve the environment variable's value + value = os.getenv(var_name, "").lower() + + # Decide the boolean value based on the content of the string + if value in true_values: + return True + elif value in false_values: + return False + else: + return default_value + + +def format_redis_conn_from_env(): + redis_url = os.getenv("REDIS_URL", None) + if redis_url: + return redis_url + else: + using_ssl = get_boolean_env_var("REDIS_SSL", False) + start = "rediss://" if using_ssl else "redis://" + + # if using RBAC + password = os.getenv("REDIS_PASSWORD", None) + username = os.getenv("REDIS_USERNAME", "default") + if password is not None: + start += f"{username}:{password}@" + + return start + f"{REDIS_HOST}:{REDIS_PORT}" + +INDEX_NAME = os.getenv("INDEX_NAME", "rag-redis") +REDIS_URL = format_redis_conn_from_env() + diff --git a/comps/dataprep/redis/llama_index/docker/Dockerfile b/comps/dataprep/redis/llama_index/docker/Dockerfile new file mode 100644 index 000000000..8fcb8bab1 --- /dev/null +++ b/comps/dataprep/redis/llama_index/docker/Dockerfile @@ -0,0 +1,30 @@ + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM ubuntu:22.04 + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim \ + python3 \ + python3-pip + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/user/comps/dataprep/redis/llama_index/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/dataprep/redis/llama_index + +ENTRYPOINT ["python3", "prepare_doc_redis.py"] + diff --git a/comps/dataprep/redis/llama_index/docker/docker-compose-dataprep-redis.yaml b/comps/dataprep/redis/llama_index/docker/docker-compose-dataprep-redis.yaml new file mode 100644 index 000000000..8135a242c --- /dev/null +++ b/comps/dataprep/redis/llama_index/docker/docker-compose-dataprep-redis.yaml @@ -0,0 +1,28 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3" +services: + redis-vector-db: + image: redis/redis-stack:7.2.0-v9 + container_name: redis-vector-db + ports: + - "6379:6379" + - "8001:8001" + dataprep-redis: + image: opea/dataprep-redis:latest + container_name: dataprep-redis-server + ports: + - "6007:6007" + ipc: host + environment: + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + INDEX_NAME: ${INDEX_NAME} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/dataprep/redis/llama_index/prepare_doc_redis.py b/comps/dataprep/redis/llama_index/prepare_doc_redis.py new file mode 100644 index 000000000..53e50f8a5 --- /dev/null +++ b/comps/dataprep/redis/llama_index/prepare_doc_redis.py @@ -0,0 +1,93 @@ + +import os +import json +from pathlib import Path +from langsmith import traceable +from typing import List, Optional, Union +from fastapi import File, Form, HTTPException, UploadFile +from llama_index.embeddings.huggingface import HuggingFaceEmbedding +from redis import Redis +from redisvl.schema import IndexSchema +from llama_index.core import StorageContext, VectorStoreIndex, SimpleDirectoryReader +from llama_index.core.settings import Settings +from llama_index.vector_stores.redis import RedisVectorStore + +from config import EMBED_MODEL, REDIS_URL, INDEX_NAME +from comps import DocPath, opea_microservices, register_microservice + + +async def save_file_to_local_disk(save_path: str, file): + save_path = Path(save_path) + with save_path.open("wb") as fout: + try: + content = await file.read() + fout.write(content) + except Exception as e: + print(f"Write file failed. Exception: {e}") + raise HTTPException(status_code=500, detail=f"Write file {save_path} failed. Exception: {e}") + + +async def ingest_data_to_redis(doc_path: DocPath): + embedder = HuggingFaceEmbedding(model_name=EMBED_MODEL) + print(f"embedder: {embedder}") + Settings.embed_model = embedder + doc_path = doc_path.path + content = SimpleDirectoryReader(input_files=[doc_path]).load_data() + redis_client = Redis.from_url(REDIS_URL) + schema = IndexSchema.from_dict({ + "index": {"name": INDEX_NAME, "prefix": f"doc:{INDEX_NAME}"}, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "doc_id", "type": "tag"}, + {"name": "text", "type": "text"}, + {"name": "content", "type": "text"}, + {"name": "source", "type": "text"}, + {"name":"start_index", "type":"numeric"}, + {"name": "vector", "type": "vector", "attrs": {"dims": 768, "algorithm": "HNSW", "date_type": "FLOAT32"}} + ] + }) + vector_store = RedisVectorStore( + redis_client=redis_client, + schema=schema + ) + storage_context = StorageContext.from_defaults(vector_store=vector_store) + _ = VectorStoreIndex.from_documents( + content, + storage_context=storage_context + ) + print("[ ingest data ] data ingested into Redis DB.") + return True + + +@register_microservice( + name="opea_service@prepare_doc_redis", + endpoint="/v1/dataprep", + host="0.0.0.0", + port=6007 +) +@traceable(run_type="tool") +# llama index only support upload files now +async def ingest_documents(files: Optional[Union[UploadFile, List[UploadFile]]] = File(None)): + print(f"files:{files}") + if not files: + raise HTTPException(status_code=400, detail="Please provide at least one file.") + + if not isinstance(files, list): + files = [files] + upload_folder = "./uploaded_files/" + if not os.path.exists(upload_folder): + Path(upload_folder).mkdir(parents=True, exist_ok=True) + try: + for file in files: + save_path = upload_folder + file.filename + await save_file_to_local_disk(save_path, file) + await ingest_data_to_redis(DocPath(path=save_path)) + print(f"Successfully saved file {save_path}") + return {"status": 200, "message": "Data preparation succeeded"} + except Exception as e: + print(f"Data preparation failed. Exception: {e}") + raise HTTPException(status_code=500, detail=f"Data preparation failed. Exception: {e}") + + +if __name__ == "__main__": + opea_microservices["opea_service@prepare_doc_redis"].start() \ No newline at end of file diff --git a/comps/dataprep/redis/llama_index/requirements.txt b/comps/dataprep/redis/llama_index/requirements.txt new file mode 100644 index 000000000..1bb9d9cbe --- /dev/null +++ b/comps/dataprep/redis/llama_index/requirements.txt @@ -0,0 +1,15 @@ +docarray[full] +fastapi +huggingface_hub +langsmith +numpy +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +redis +sentence_transformers +llama-index +llama-index-vector-stores-redis +llama-index-readers-file +llama-index-embeddings-huggingface==0.2.0 +shortuuid From bf52baad8c559b4cee25dad0ba68a2149efca637 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 08:31:08 +0000 Subject: [PATCH 3/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/dataprep/redis/README.md | 4 ++ comps/dataprep/redis/llama_index/config.py | 2 +- .../redis/llama_index/prepare_doc_redis.py | 62 +++++++++---------- .../redis/llama_index/requirements.txt | 8 +-- 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/comps/dataprep/redis/README.md b/comps/dataprep/redis/README.md index 6d54a9f8f..6aa080141 100644 --- a/comps/dataprep/redis/README.md +++ b/comps/dataprep/redis/README.md @@ -55,12 +55,16 @@ export LANGCHAIN_PROJECT="opea/dataprep" ``` ## 2.3 Build Docker Image + - Build docker image with langchain + ```bash cd ../../../../ docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/langchain/docker/Dockerfile . ``` + - Build docker image with llama_index + ```bash cd ../../../../ docker build -t opea/dataprep-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/llama_index/docker/Dockerfile . diff --git a/comps/dataprep/redis/llama_index/config.py b/comps/dataprep/redis/llama_index/config.py index e427b443d..0f99cc05e 100644 --- a/comps/dataprep/redis/llama_index/config.py +++ b/comps/dataprep/redis/llama_index/config.py @@ -52,6 +52,6 @@ def format_redis_conn_from_env(): return start + f"{REDIS_HOST}:{REDIS_PORT}" + INDEX_NAME = os.getenv("INDEX_NAME", "rag-redis") REDIS_URL = format_redis_conn_from_env() - diff --git a/comps/dataprep/redis/llama_index/prepare_doc_redis.py b/comps/dataprep/redis/llama_index/prepare_doc_redis.py index 53e50f8a5..ec0ddf0fa 100644 --- a/comps/dataprep/redis/llama_index/prepare_doc_redis.py +++ b/comps/dataprep/redis/llama_index/prepare_doc_redis.py @@ -1,18 +1,21 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 -import os import json +import os from pathlib import Path -from langsmith import traceable from typing import List, Optional, Union + +from config import EMBED_MODEL, INDEX_NAME, REDIS_URL from fastapi import File, Form, HTTPException, UploadFile +from langsmith import traceable +from llama_index.core import SimpleDirectoryReader, StorageContext, VectorStoreIndex +from llama_index.core.settings import Settings from llama_index.embeddings.huggingface import HuggingFaceEmbedding +from llama_index.vector_stores.redis import RedisVectorStore from redis import Redis from redisvl.schema import IndexSchema -from llama_index.core import StorageContext, VectorStoreIndex, SimpleDirectoryReader -from llama_index.core.settings import Settings -from llama_index.vector_stores.redis import RedisVectorStore -from config import EMBED_MODEL, REDIS_URL, INDEX_NAME from comps import DocPath, opea_microservices, register_microservice @@ -34,37 +37,32 @@ async def ingest_data_to_redis(doc_path: DocPath): doc_path = doc_path.path content = SimpleDirectoryReader(input_files=[doc_path]).load_data() redis_client = Redis.from_url(REDIS_URL) - schema = IndexSchema.from_dict({ - "index": {"name": INDEX_NAME, "prefix": f"doc:{INDEX_NAME}"}, - "fields": [ - {"name": "id", "type": "tag"}, - {"name": "doc_id", "type": "tag"}, - {"name": "text", "type": "text"}, - {"name": "content", "type": "text"}, - {"name": "source", "type": "text"}, - {"name":"start_index", "type":"numeric"}, - {"name": "vector", "type": "vector", "attrs": {"dims": 768, "algorithm": "HNSW", "date_type": "FLOAT32"}} - ] - }) - vector_store = RedisVectorStore( - redis_client=redis_client, - schema=schema + schema = IndexSchema.from_dict( + { + "index": {"name": INDEX_NAME, "prefix": f"doc:{INDEX_NAME}"}, + "fields": [ + {"name": "id", "type": "tag"}, + {"name": "doc_id", "type": "tag"}, + {"name": "text", "type": "text"}, + {"name": "content", "type": "text"}, + {"name": "source", "type": "text"}, + {"name": "start_index", "type": "numeric"}, + { + "name": "vector", + "type": "vector", + "attrs": {"dims": 768, "algorithm": "HNSW", "date_type": "FLOAT32"}, + }, + ], + } ) + vector_store = RedisVectorStore(redis_client=redis_client, schema=schema) storage_context = StorageContext.from_defaults(vector_store=vector_store) - _ = VectorStoreIndex.from_documents( - content, - storage_context=storage_context - ) + _ = VectorStoreIndex.from_documents(content, storage_context=storage_context) print("[ ingest data ] data ingested into Redis DB.") return True -@register_microservice( - name="opea_service@prepare_doc_redis", - endpoint="/v1/dataprep", - host="0.0.0.0", - port=6007 -) +@register_microservice(name="opea_service@prepare_doc_redis", endpoint="/v1/dataprep", host="0.0.0.0", port=6007) @traceable(run_type="tool") # llama index only support upload files now async def ingest_documents(files: Optional[Union[UploadFile, List[UploadFile]]] = File(None)): @@ -90,4 +88,4 @@ async def ingest_documents(files: Optional[Union[UploadFile, List[UploadFile]]] if __name__ == "__main__": - opea_microservices["opea_service@prepare_doc_redis"].start() \ No newline at end of file + opea_microservices["opea_service@prepare_doc_redis"].start() diff --git a/comps/dataprep/redis/llama_index/requirements.txt b/comps/dataprep/redis/llama_index/requirements.txt index 1bb9d9cbe..3eb05757f 100644 --- a/comps/dataprep/redis/llama_index/requirements.txt +++ b/comps/dataprep/redis/llama_index/requirements.txt @@ -2,14 +2,14 @@ docarray[full] fastapi huggingface_hub langsmith +llama-index +llama-index-embeddings-huggingface==0.2.0 +llama-index-readers-file +llama-index-vector-stores-redis numpy opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk redis sentence_transformers -llama-index -llama-index-vector-stores-redis -llama-index-readers-file -llama-index-embeddings-huggingface==0.2.0 shortuuid From d271403ed93da8137b0bc883683f4988cec731fc Mon Sep 17 00:00:00 2001 From: letonghan Date: Wed, 12 Jun 2024 17:41:30 +0800 Subject: [PATCH 4/5] add e2e test script Signed-off-by: letonghan --- tests/test_dataprep_llama_index.sh | 67 ++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/test_dataprep_llama_index.sh diff --git a/tests/test_dataprep_llama_index.sh b/tests/test_dataprep_llama_index.sh new file mode 100644 index 000000000..1889e4040 --- /dev/null +++ b/tests/test_dataprep_llama_index.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/dataprep-redis:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/redis/llama_index/docker/Dockerfile . +} + +function start_service() { + docker run -d --name="test-comps-dataprep-redis" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 6379:6379 -p 8001:8001 --ipc=host redis/redis-stack:7.2.0-v9 + dataprep_service_port=5011 + REDIS_URL="redis://${ip_address}:6379" + docker run -d --name="test-comps-dataprep-redis-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e REDIS_URL=$REDIS_URL -p ${dataprep_service_port}:6007 --ipc=host opea/dataprep-redis:comps + sleep 1m +} + +function validate_microservice() { + dataprep_service_port=5011 + URL="http://${ip_address}:$dataprep_service_port/v1/dataprep" + echo "Deep learning is a subset of machine learning that utilizes neural networks with multiple layers to analyze various levels of abstract data representations. It enables computers to identify patterns and make decisions with minimal human intervention by learning from large amounts of data." > ./dataprep_file.txt + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ dataprep ] HTTP status is 200. Checking content..." + local CONTENT=$(curl -s -X POST -F 'files=@./dataprep_file.txt' -H 'Content-Type: multipart/form-data' "$URL" | tee ${LOG_PATH}/dataprep.log) + + if echo 'Data preparation succeeded' | grep -q "$EXPECTED_RESULT"; then + echo "[ dataprep ] Content is as expected." + else + echo "[ dataprep ] Content does not match the expected result: $CONTENT" + docker logs test-comps-dataprep-redis-server >> ${LOG_PATH}/dataprep.log + exit 1 + fi + else + echo "[ dataprep ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs test-comps-dataprep-redis-server >> ${LOG_PATH}/dataprep.log + exit 1 + fi +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-dataprep-redis*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main From 3ecb1263f17fbc766d4add3c277b4493ddbf700e Mon Sep 17 00:00:00 2001 From: letonghan Date: Wed, 12 Jun 2024 17:44:56 +0800 Subject: [PATCH 5/5] update test script name Signed-off-by: letonghan --- ...dataprep_llama_index.sh => test_dataprep_redis_llama_index.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_dataprep_llama_index.sh => test_dataprep_redis_llama_index.sh} (100%) diff --git a/tests/test_dataprep_llama_index.sh b/tests/test_dataprep_redis_llama_index.sh similarity index 100% rename from tests/test_dataprep_llama_index.sh rename to tests/test_dataprep_redis_llama_index.sh