remove length limitation of embedding (opea-project#347)

* remove length limitation of embedding * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * rm files * update haystack code * fix * resolve dependency * revert for llama_index * limit version haystack * fix ver --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
predictionguard · Aug 6, 2024 · 4b0594c · 4b0594c
1 parent 81b6ffe
commit 4b0594c
Show file tree

Hide file tree

Showing 27 changed files with 44 additions and 150 deletions.
diff --git a/README.md b/README.md
@@ -175,7 +175,7 @@ A `Microservices` can be created by using the decorator `register_microservice`.
 from langchain_community.embeddings import HuggingFaceHubEmbeddings
 from langsmith import traceable
 
-from comps import register_microservice, EmbedDoc768, ServiceType, TextDoc
+from comps import register_microservice, EmbedDoc, ServiceType, TextDoc
 
 
 @register_microservice(
@@ -185,13 +185,12 @@ from comps import register_microservice, EmbedDoc768, ServiceType, TextDoc
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc768,
+    output_datatype=EmbedDoc,
 )
 @traceable(run_type="embedding")
-def embedding(input: TextDoc) -> EmbedDoc768:
+def embedding(input: TextDoc) -> EmbedDoc:
     embed_vector = embeddings.embed_query(input.text)
-    embed_vector = embed_vector[:768]  # Keep only the first 768 elements
-    res = EmbedDoc768(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     return res
 ```
 

diff --git a/comps/__init__.py b/comps/__init__.py
@@ -9,7 +9,6 @@
     Base64ByteStrDoc,
     DocPath,
     EmbedDoc768,
-    EmbedDoc512,
     EmbedDoc1024,
     GeneratedDoc,
     LLMParamsDoc,

diff --git a/comps/dataprep/redis/README.md b/comps/dataprep/redis/README.md
@@ -64,7 +64,7 @@ python prepare_doc_redis_on_ray.py
 
 ## 2.1 Start Redis Stack Server
 
-Please refer to this [readme](../../../vectorstores/langchain/redis/README.md).
+Please refer to this [readme](../../vectorstores/langchain/redis/README.md).
 
 ## 2.2 Setup Environment Variables
 

diff --git a/comps/dataprep/redis/langchain/config.py b/comps/dataprep/redis/langchain/config.py
@@ -5,7 +5,7 @@
 
 # Embedding model
 
-EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-base-en-v1.5")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "BAAI/bge-large-en-v1.5")
 
 # Redis Connection Information
 REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
@@ -61,9 +61,4 @@ def format_redis_conn_from_env():
 INDEX_NAME = os.getenv("INDEX_NAME", "rag-redis")
 KEY_INDEX_NAME = os.getenv("KEY_INDEX_NAME", "file-keys")
 
-current_file_path = os.path.abspath(__file__)
-parent_dir = os.path.dirname(current_file_path)
-REDIS_SCHEMA = os.getenv("REDIS_SCHEMA", "schema_dim_768.yml")
 TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", 600))
-schema_path = os.path.join(parent_dir, REDIS_SCHEMA)
-INDEX_SCHEMA = schema_path
diff --git a/comps/dataprep/redis/langchain/prepare_doc_redis.py b/comps/dataprep/redis/langchain/prepare_doc_redis.py
@@ -10,7 +10,7 @@
 
 # from pyspark import SparkConf, SparkContext
 import redis
-from config import EMBED_MODEL, INDEX_NAME, INDEX_SCHEMA, KEY_INDEX_NAME, REDIS_HOST, REDIS_PORT, REDIS_URL
+from config import EMBED_MODEL, INDEX_NAME, KEY_INDEX_NAME, REDIS_HOST, REDIS_PORT, REDIS_URL
 from fastapi import Body, File, Form, HTTPException, UploadFile
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceHubEmbeddings
@@ -151,7 +151,6 @@ def ingest_data_to_redis(doc_path: DocPath):
             texts=batch_texts,
             embedding=embedder,
             index_name=INDEX_NAME,
-            index_schema=INDEX_SCHEMA,
             redis_url=REDIS_URL,
         )
         print(f"keys: {keys}")
@@ -195,7 +194,6 @@ async def ingest_link_to_redis(link_list: List[str]):
             texts=content,
             embedding=embedder,
             index_name=INDEX_NAME,
-            index_schema=INDEX_SCHEMA,
             redis_url=REDIS_URL,
         )
         print(f"keys: {keys}")

diff --git a/comps/dataprep/redis/langchain/schema.yml b/comps/dataprep/redis/langchain/schema.yml
diff --git a/comps/dataprep/redis/langchain/schema_dim_1024.yml b/comps/dataprep/redis/langchain/schema_dim_1024.yml
diff --git a/comps/dataprep/redis/langchain/schema_dim_768.yml b/comps/dataprep/redis/langchain/schema_dim_768.yml
diff --git a/comps/dataprep/redis/langchain/schema_lcdocs_dim_768.yml b/comps/dataprep/redis/langchain/schema_lcdocs_dim_768.yml
diff --git a/comps/dataprep/redis/langchain_ray/config.py b/comps/dataprep/redis/langchain_ray/config.py
@@ -60,9 +60,4 @@ def format_redis_conn_from_env():
 # Vector Index Configuration
 INDEX_NAME = os.getenv("INDEX_NAME", "rag-redis")
 
-current_file_path = os.path.abspath(__file__)
-parent_dir = os.path.dirname(current_file_path)
-REDIS_SCHEMA = os.getenv("REDIS_SCHEMA", "schema_dim_768.yml")
 TIMEOUT_SECONDS = int(os.getenv("TIMEOUT_SECONDS", 600))
-schema_path = os.path.join(parent_dir, REDIS_SCHEMA)
-INDEX_SCHEMA = schema_path
diff --git a/comps/dataprep/redis/langchain_ray/prepare_doc_redis_on_ray.py b/comps/dataprep/redis/langchain_ray/prepare_doc_redis_on_ray.py
@@ -21,7 +21,7 @@
 from typing import Callable, List, Optional, Union
 
 import pandas as pd
-from config import EMBED_MODEL, INDEX_NAME, INDEX_SCHEMA, REDIS_URL, TIMEOUT_SECONDS
+from config import EMBED_MODEL, INDEX_NAME, REDIS_URL, TIMEOUT_SECONDS
 from fastapi import Body, File, Form, HTTPException, UploadFile
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceHubEmbeddings
@@ -195,7 +195,6 @@ def data_to_redis(data):
             texts=batch_texts,
             embedding=embedder,
             index_name=INDEX_NAME,
-            index_schema=INDEX_SCHEMA,
             redis_url=REDIS_URL,
         )
         # print(f"Processed batch {i//batch_size + 1}/{(num_chunks-1)//batch_size + 1}")

diff --git a/comps/embeddings/langchain-mosec/embedding_mosec.py b/comps/embeddings/langchain-mosec/embedding_mosec.py
@@ -9,7 +9,7 @@
 from langsmith import traceable
 
 from comps import (
-    EmbedDoc768,
+    EmbedDoc,
     ServiceType,
     TextDoc,
     opea_microservices,
@@ -51,15 +51,14 @@ def empty_embedding() -> List[float]:
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc768,
+    output_datatype=EmbedDoc,
 )
 @traceable(run_type="embedding")
 @register_statistics(names=["opea_service@embedding_mosec"])
-def embedding(input: TextDoc) -> EmbedDoc768:
+def embedding(input: TextDoc) -> EmbedDoc:
     start = time.time()
     embed_vector = embeddings.embed_query(input.text)
-    embed_vector = embed_vector[:768]  # Keep only the first 768 elements
-    res = EmbedDoc768(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     statistics_dict["opea_service@embedding_mosec"].append_latency(time.time() - start, None)
     return res
 

diff --git a/comps/embeddings/langchain/embedding_tei.py b/comps/embeddings/langchain/embedding_tei.py
@@ -8,7 +8,7 @@
 from langsmith import traceable
 
 from comps import (
-    EmbedDoc768,
+    EmbedDoc,
     ServiceType,
     TextDoc,
     opea_microservices,
@@ -25,15 +25,14 @@
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc768,
+    output_datatype=EmbedDoc,
 )
 @traceable(run_type="embedding")
 @register_statistics(names=["opea_service@embedding_tei_langchain"])
-def embedding(input: TextDoc) -> EmbedDoc768:
+def embedding(input: TextDoc) -> EmbedDoc:
     start = time.time()
     embed_vector = embeddings.embed_query(input.text)
-    embed_vector = embed_vector[:768]  # Keep only the first 768 elements
-    res = EmbedDoc768(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     statistics_dict["opea_service@embedding_tei_langchain"].append_latency(time.time() - start, None)
     return res
 

diff --git a/comps/embeddings/langchain/local_embedding.py b/comps/embeddings/langchain/local_embedding.py
@@ -3,7 +3,7 @@
 
 from langchain_community.embeddings import HuggingFaceBgeEmbeddings
 
-from comps import EmbedDoc1024, ServiceType, TextDoc, opea_microservices, opea_telemetry, register_microservice
+from comps import EmbedDoc, ServiceType, TextDoc, opea_microservices, opea_telemetry, register_microservice
 
 
 @register_microservice(
@@ -13,12 +13,12 @@
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc1024,
+    output_datatype=EmbedDoc,
 )
 @opea_telemetry
-def embedding(input: TextDoc) -> EmbedDoc1024:
+def embedding(input: TextDoc) -> EmbedDoc:
     embed_vector = embeddings.embed_query(input.text)
-    res = EmbedDoc1024(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     return res
 
 

diff --git a/comps/embeddings/llama_index/embedding_tei.py b/comps/embeddings/llama_index/embedding_tei.py
@@ -6,7 +6,7 @@
 from langsmith import traceable
 from llama_index.embeddings.text_embeddings_inference import TextEmbeddingsInference
 
-from comps import EmbedDoc768, ServiceType, TextDoc, opea_microservices, register_microservice
+from comps import EmbedDoc, ServiceType, TextDoc, opea_microservices, register_microservice
 
 
 @register_microservice(
@@ -16,13 +16,12 @@
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc768,
+    output_datatype=EmbedDoc,
 )
 @traceable(run_type="embedding")
-def embedding(input: TextDoc) -> EmbedDoc768:
+def embedding(input: TextDoc) -> EmbedDoc:
     embed_vector = embeddings._get_query_embedding(input.text)
-    embed_vector = embed_vector[:768]  # Keep only the first 768 elements
-    res = EmbedDoc768(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     return res
 
 

diff --git a/comps/embeddings/llama_index/local_embedding.py b/comps/embeddings/llama_index/local_embedding.py
@@ -4,7 +4,7 @@
 from langsmith import traceable
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
-from comps import EmbedDoc1024, ServiceType, TextDoc, opea_microservices, register_microservice
+from comps import EmbedDoc, ServiceType, TextDoc, opea_microservices, register_microservice
 
 
 @register_microservice(
@@ -14,12 +14,12 @@
     host="0.0.0.0",
     port=6000,
     input_datatype=TextDoc,
-    output_datatype=EmbedDoc1024,
+    output_datatype=EmbedDoc,
 )
 @traceable(run_type="embedding")
-def embedding(input: TextDoc) -> EmbedDoc1024:
+def embedding(input: TextDoc) -> EmbedDoc:
     embed_vector = embeddings.get_text_embedding(input.text)
-    res = EmbedDoc1024(text=input.text, embedding=embed_vector)
+    res = EmbedDoc(text=input.text, embedding=embed_vector)
     return res
 
 

diff --git a/comps/retrievers/haystack/qdrant/requirements.txt b/comps/retrievers/haystack/qdrant/requirements.txt
@@ -1,12 +1,13 @@
 docarray[full]
 easyocr
 fastapi
-haystack-ai
+haystack-ai==2.2.4
 langchain_community
 langsmith
 opentelemetry-api
 opentelemetry-exporter-otlp
 opentelemetry-sdk
+prometheus_fastapi_instrumentator
 pymupdf
 qdrant-haystack
 sentence_transformers

diff --git a/comps/retrievers/haystack/qdrant/retriever_qdrant.py b/comps/retrievers/haystack/qdrant/retriever_qdrant.py
@@ -7,7 +7,7 @@
 from langsmith import traceable
 from qdrant_config import EMBED_DIMENSION, EMBED_ENDPOINT, EMBED_MODEL, INDEX_NAME, QDRANT_HOST, QDRANT_PORT
 
-from comps import EmbedDoc768, SearchedDoc, ServiceType, TextDoc, opea_microservices, register_microservice
+from comps import EmbedDoc, SearchedDoc, ServiceType, TextDoc, opea_microservices, register_microservice
 
 
 # Create a pipeline for querying a Qdrant document store
@@ -29,7 +29,7 @@ def initialize_qdrant_retriever() -> QdrantEmbeddingRetriever:
     port=7000,
 )
 @traceable(run_type="retriever")
-def retrieve(input: EmbedDoc768) -> SearchedDoc:
+def retrieve(input: EmbedDoc) -> SearchedDoc:
     search_res = retriever.run(query_embedding=input.embedding)["documents"]
     searched_docs = [TextDoc(text=r.content) for r in search_res]
     result = SearchedDoc(retrieved_docs=searched_docs, initial_query=input.text)

diff --git a/comps/retrievers/langchain/milvus/retriever_milvus.py b/comps/retrievers/langchain/milvus/retriever_milvus.py
@@ -20,7 +20,7 @@
 from langsmith import traceable
 
 from comps import (
-    EmbedDoc768,
+    EmbedDoc,
     SearchedDoc,
     ServiceType,
     TextDoc,
@@ -65,7 +65,7 @@ def empty_embedding() -> List[float]:
 )
 @traceable(run_type="retriever")
 @register_statistics(names=["opea_service@retriever_milvus"])
-def retrieve(input: EmbedDoc768) -> SearchedDoc:
+def retrieve(input: EmbedDoc) -> SearchedDoc:
     vector_db = Milvus(
         embeddings,
         connection_args={"host": MILVUS_HOST, "port": MILVUS_PORT},

diff --git a/comps/retrievers/langchain/pgvector/retriever_pgvector.py b/comps/retrievers/langchain/pgvector/retriever_pgvector.py
@@ -10,7 +10,7 @@
 from langsmith import traceable
 
 from comps import (
-    EmbedDoc768,
+    EmbedDoc,
     SearchedDoc,
     ServiceType,
     TextDoc,
@@ -32,7 +32,7 @@
 )
 @traceable(run_type="retriever")
 @register_statistics(names=["opea_service@retriever_pgvector"])
-def retrieve(input: EmbedDoc768) -> SearchedDoc:
+def retrieve(input: EmbedDoc) -> SearchedDoc:
     start = time.time()
     search_res = vector_db.similarity_search_by_vector(embedding=input.embedding)
     searched_docs = []

diff --git a/comps/retrievers/langchain/redis/ingest.py b/comps/retrievers/langchain/redis/ingest.py
@@ -14,7 +14,7 @@
 from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceEmbeddings, HuggingFaceHubEmbeddings
 from langchain_community.vectorstores import Redis
 from PIL import Image
-from redis_config import EMBED_MODEL, INDEX_NAME, INDEX_SCHEMA, REDIS_URL
+from redis_config import EMBED_MODEL, INDEX_NAME, REDIS_URL
 
 tei_embedding_endpoint = os.getenv("TEI_EMBEDDING_ENDPOINT")
 
@@ -112,7 +112,6 @@ def ingest_documents():
             texts=batch_texts,
             embedding=embedder,
             index_name=INDEX_NAME,
-            index_schema=INDEX_SCHEMA,
             redis_url=REDIS_URL,
         )
         print(f"Processed batch {i//batch_size + 1}/{(num_chunks-1)//batch_size + 1}")