revert retrievers and vectorstores

Signed-off-by: letonghan <[email protected]>

comps/retrievers/src/elasticsearch/langchain/Dockerfile → comps/retrievers/elasticsearch/langchain/Dockerfile

@@ -19,10 +19,10 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/elasticsearch/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/elasticsearch/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/elasticsearch/langchain
+WORKDIR /home/user/comps/retrievers/elasticsearch/langchain
 
 ENTRYPOINT ["python", "retriever_elasticsearch.py"]

comps/retrievers/src/elasticsearch/langchain/README.md → comps/retrievers/elasticsearch/langchain/README.md

@@ -76,7 +76,7 @@ export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
 
 ```bash
 cd ../../../../../
-docker build -t opea/retriever-elasticsearch:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/elasticsearch/langchain/Dockerfile .
+docker build -t opea/retriever-elasticsearch:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/elasticsearch/langchain/Dockerfile .
 ```
 
 To start a docker container, you have two options:
@@ -95,7 +95,7 @@ docker run -d --name="retriever-elasticsearch" -p 7000:7000 --ipc=host -e http_p
 ### 2.4 Run Docker with Docker Compose (Option B)
 
 ```bash
-cd comps/retrievers/src/elasticsearch/langchain
+cd comps/retrievers/elasticsearch/langchain
 docker compose -f docker_compose_retriever.yaml up -d
 ```
 

comps/retrievers/src/milvus/langchain/Dockerfile → comps/retrievers/milvus/langchain/Dockerfile

@@ -22,10 +22,10 @@ COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/milvus/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/milvus/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/milvus/langchain
+WORKDIR /home/user/comps/retrievers/milvus/langchain
 
-ENTRYPOINT ["python", "retriever_milvus.py"]
+ENTRYPOINT ["python", "retriever_milvus.py"]

comps/retrievers/src/milvus/langchain/README.md → comps/retrievers/milvus/langchain/README.md

@@ -10,7 +10,7 @@ pip install -r requirements.txt
 
 ### Start Milvus Server
 
-Please refer to this [readme](../../../../vectorstores/src/milvus/README.md).
+Please refer to this [readme](../../../vectorstores/milvus/README.md).
 
 ### Setup Environment Variables
 
@@ -37,7 +37,7 @@ python retriever_redis.py
 
 ```bash
 cd ../../
-docker build -t opea/retriever-milvus:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/milvus/langchain/Dockerfile .
+docker build -t opea/retriever-milvus:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/milvus/langchain/Dockerfile .
 ```
 
 ### Run Docker with CLI
@@ -100,4 +100,4 @@ curl http://localhost:7000/v1/retrieval \
   -X POST \
   -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding},\"search_type\":\"mmr\", \"k\":4, \"fetch_k\":20, \"lambda_mult\":0.5}" \
   -H 'Content-Type: application/json'
-```
+```

comps/vectorstores/src/vdms/__init__.py → comps/retrievers/milvus/langchain/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (C) 2024 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
+# SPDX-License-Identifier: Apache-2.0

comps/retrievers/src/milvus/langchain/config.py → comps/retrievers/milvus/langchain/config.py

@@ -15,4 +15,4 @@
 MOSEC_EMBEDDING_MODEL = os.environ.get("MOSEC_EMBEDDING_MODEL", "/home/user/bce-embedding-base_v1")
 MOSEC_EMBEDDING_ENDPOINT = os.environ.get("MOSEC_EMBEDDING_ENDPOINT", "")
 os.environ["OPENAI_API_BASE"] = MOSEC_EMBEDDING_ENDPOINT
-os.environ["OPENAI_API_KEY"] = "Dummy key"
+os.environ["OPENAI_API_KEY"] = "Dummy key"

comps/retrievers/src/milvus/langchain/requirements.txt → comps/retrievers/milvus/langchain/requirements.txt

@@ -22,4 +22,4 @@ python-docx==0.8.11
 sentence_transformers
 shortuuid
 tiktoken
-uvicorn
+uvicorn

comps/retrievers/src/milvus/langchain/retriever_milvus.py → comps/retrievers/milvus/langchain/retriever_milvus.py

@@ -120,4 +120,4 @@ async def retrieve(input: EmbedDoc) -> SearchedDoc:
             logger.info(f"[ retriever_milvus ] LOCAL_EMBEDDING_MODEL:{LOCAL_EMBEDDING_MODEL}")
         embeddings = HuggingFaceBgeEmbeddings(model_name=LOCAL_EMBEDDING_MODEL)
 
-    opea_microservices["opea_service@retriever_milvus"].start()
+    opea_microservices["opea_service@retriever_milvus"].start()

comps/retrievers/src/neo4j/langchain/Dockerfile → comps/retrievers/neo4j/langchain/Dockerfile

@@ -24,10 +24,10 @@ COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/neo4j/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/neo4j/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/neo4j/langchain
+WORKDIR /home/user/comps/retrievers/neo4j/langchain
 
 ENTRYPOINT ["python", "retriever_neo4j.py"]

comps/retrievers/src/neo4j/langchain/README.md → comps/retrievers/neo4j/langchain/README.md

@@ -46,7 +46,7 @@ python retriever_neo4j.py
 
 ```bash
 cd ../../
-docker build -t opea/retriever-neo4j:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/neo4j/langchain/Dockerfile .
+docker build -t opea/retriever-neo4j:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/neo4j/langchain/Dockerfile .
 ```
 
 ### Run Docker with CLI

comps/retrievers/src/neo4j/llama_index/Dockerfile → comps/retrievers/neo4j/llama_index/Dockerfile

@@ -26,10 +26,10 @@ COPY comps /home/user/comps
 
 RUN pip install --no-cache-dir --upgrade pip && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/neo4j/llama_index/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/neo4j/llama_index/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/neo4j/llama_index
+WORKDIR /home/user/comps/retrievers/neo4j/llama_index
 
 ENTRYPOINT ["python", "retriever_community_answers_neo4j.py"]

comps/retrievers/src/neo4j/llama_index/README.md → comps/retrievers/neo4j/llama_index/README.md

@@ -14,7 +14,7 @@ Retrieval follows these steps:
 
 ```bash
 cd ../../
-docker build -t opea/retriever-community-answers-neo4j:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/neo4j/llama_index/Dockerfile .
+docker build -t opea/retriever-community-answers-neo4j:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/neo4j/llama_index/Dockerfile .
 ```
 
 ### 2. Setup Environment Variables
@@ -40,7 +40,7 @@ source ./set_env.sh
 Docker compose will start 5 microservices: retriever-neo4j-llamaindex, dataprep-neo4j-llamaindex, neo4j-apoc, tgi-gaudi-service and tei-embedding-service. Neo4j database supports embeddings natively so we do not need a separate vector store. Checkout the blog [Introducing the Property Graph Index: A Powerful New Way to Build Knowledge Graphs with LLMs](https://www.llamaindex.ai/blog/introducing-the-property-graph-index-a-powerful-new-way-to-build-knowledge-graphs-with-llms) for a better understanding of Property Graph Store and Index.
 
 ```bash
-cd comps/retrievers/src/neo4j/llama_index
+cd comps/retrievers/neo4j/llama_index
 docker compose -f compose.yaml up -d
 ```
 

comps/retrievers/src/neo4j/llama_index/retriever_community_answers_neo4j.py → comps/retrievers/neo4j/llama_index/retriever_community_answers_neo4j.py

@@ -46,7 +46,7 @@
     RetrievalResponse,
     RetrievalResponseData,
 )
-from comps.dataprep.src.neo4j.llama_index.extract_graph_neo4j import GraphRAGStore, get_attribute_from_tgi_endpoint
+from comps.dataprep.neo4j.llama_index.extract_graph_neo4j import GraphRAGStore, get_attribute_from_tgi_endpoint
 
 logger = CustomLogger("retriever_neo4j")
 logflag = os.getenv("LOGFLAG", False)

comps/retrievers/src/pathway/langchain/Dockerfile → comps/retrievers/pathway/langchain/Dockerfile

@@ -19,10 +19,10 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/pathway/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/pathway/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/pathway/langchain
+WORKDIR /home/user/comps/retrievers/pathway/langchain
 
 ENTRYPOINT ["bash", "entrypoint.sh"]

comps/retrievers/src/pathway/langchain/README.md → comps/retrievers/pathway/langchain/README.md

@@ -41,7 +41,7 @@ curl 127.0.0.1:6060/rerank \
 #### Start Retriever Service
 
 Retriever service queries the Pathway vector store on incoming requests.
-Make sure that Pathway vector store is already running, [see Pathway vector store here](../../../../vectorstores/src/pathway/README.md).
+Make sure that Pathway vector store is already running, [see Pathway vector store here](../../../../vectorstores/pathway/README.md).
 
 Retriever service expects the Pathway host and port variables to connect to the vector DB. Set the Pathway vector store environment variables.
 
@@ -52,7 +52,7 @@ export PATHWAY_PORT=8666
 
 ```bash
 # make sure you are in the root folder of the repo
-docker build -t opea/retriever-pathway:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/pathway/langchain/Dockerfile .
+docker build -t opea/retriever-pathway:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/pathway/langchain/Dockerfile .
 
 docker run -p 7000:7000 -e PATHWAY_HOST=${PATHWAY_HOST} -e PATHWAY_PORT=${PATHWAY_PORT} -e http_proxy=$http_proxy -e https_proxy=$https_proxy --network="host" opea/retriever-pathway:latest
 ```
@@ -78,7 +78,7 @@ Note that following docker compose sets the `network_mode: host` in retriever im
 This will start the both the embedding and retriever services:
 
 ```bash
-cd comps/retrievers/src/pathway/langchain
+cd comps/retrievers/pathway/langchain
 
 docker compose -f docker_compose_retriever.yaml build
 docker compose -f docker_compose_retriever.yaml up

comps/retrievers/src/pgvector/langchain/Dockerfile → comps/retrievers/pgvector/langchain/Dockerfile

@@ -19,10 +19,10 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/pgvector/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/pgvector/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/pgvector/langchain
+WORKDIR /home/user/comps/retrievers/pgvector/langchain
 
 ENTRYPOINT ["python", "retriever_pgvector.py"]

comps/retrievers/src/pgvector/langchain/README.md → comps/retrievers/pgvector/langchain/README.md

@@ -72,7 +72,7 @@ export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
 
 ```bash
 cd ../../../../
-docker build -t opea/retriever-pgvector:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/pgvector/langchain/Dockerfile .
+docker build -t opea/retriever-pgvector:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/pgvector/langchain/Dockerfile .
 ```
 
 To start a docker container, you have two options:
@@ -91,7 +91,7 @@ docker run -d --name="retriever-pgvector" -p 7000:7000 --ipc=host -e http_proxy=
 ### 2.4 Run Docker with Docker Compose (Option B)
 
 ```bash
-cd comps/retrievers/src/pgvector/langchain
+cd comps/retrievers/pgvector/langchain
 docker compose -f docker_compose_retriever.yaml up -d
 ```
 

comps/retrievers/src/pinecone/langchain/Dockerfile → comps/retrievers/pinecone/langchain/Dockerfile

@@ -19,10 +19,10 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir --no-warn-script-location -r /home/user/comps/retrievers/src/pinecone/langchain/requirements.txt
+    pip install --no-cache-dir --no-warn-script-location -r /home/user/comps/retrievers/pinecone/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/pinecone/langchain
+WORKDIR /home/user/comps/retrieverss/pinecone/langchain
 
 ENTRYPOINT ["python", "retriever_pinecone.py"]

comps/retrievers/src/qdrant/haystack/Dockerfile → comps/retrievers/qdrant/haystack/Dockerfile

@@ -16,10 +16,10 @@ USER user
 COPY comps /home/user/comps
 
 RUN python -m pip install --no-cache-dir --upgrade pip setuptools && \
-    python -m pip install --no-cache-dir -r /home/user/comps/retrievers/src/qdrant/haystack/requirements.txt
+    python -m pip install --no-cache-dir -r /home/user/comps/retrievers/qdrant/haystack/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/qdrant/haystack
+WORKDIR /home/user/comps/retrievers/qdrant/haystack
 
 ENTRYPOINT ["python", "retriever_qdrant.py"]

comps/retrievers/src/qdrant/haystack/README.md → comps/retrievers/qdrant/haystack/README.md

@@ -10,7 +10,7 @@ pip install -r requirements.txt
 
 ### 1.2 Start Qdrant Server
 
-Please refer to this [readme](../../../../vectorstores/src/qdrant/README.md).
+Please refer to this [readme](../../../../vectorstores/qdrant/README.md).
 
 ### 1.3 Setup Environment Variables
 
@@ -42,7 +42,7 @@ export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
 
 ```bash
 cd ../../../../
-docker build -t opea/retriever-qdrant:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/qdrant/haystack/Dockerfile .
+docker build -t opea/retriever-qdrant:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/qdrant/haystack/Dockerfile .
 ```
 
 ### 2.3 Run Docker with CLI

comps/retrievers/src/redis/langchain/Dockerfile → comps/retrievers/redis/langchain/Dockerfile

@@ -19,10 +19,10 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/redis/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/redis/langchain/requirements.txt
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/redis/langchain
+WORKDIR /home/user/comps/retrievers/redis/langchain
 
 ENTRYPOINT ["python", "retriever_redis.py"]

comps/retrievers/src/redis/langchain/README.md → comps/retrievers/redis/langchain/README.md

@@ -81,7 +81,7 @@ export BRIDGE_TOWER_EMBEDDING=true
 
 ```bash
 cd ../../../../
-docker build -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/redis/langchain/Dockerfile .
+docker build -t opea/retriever-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/langchain/Dockerfile .
 ```
 
 To start a docker container, you have two options:

comps/retrievers/src/redis/langchain/retriever_redis.py → comps/retrievers/redis/langchain/retriever_redis.py

@@ -30,7 +30,7 @@
     RetrievalResponse,
     RetrievalResponseData,
 )
-from comps.embeddings.multimodal.bridgetower import BridgeTowerEmbedding
+from comps.embeddings.src.integrations.dependency.bridgetower import BridgeTowerEmbedding
 
 logger = CustomLogger("retriever_redis")
 logflag = os.getenv("LOGFLAG", False)

comps/retrievers/src/redis/llama_index/Dockerfile → comps/retrievers/redis/llama_index/Dockerfile

@@ -23,13 +23,13 @@ USER user
 
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then \
-      pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r /home/user/comps/retrievers/src/redis/llama_index/requirements.txt; \
+      pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r /home/user/comps/retrievers/redis/llama_index/requirements.txt; \
     else \
-      pip install --no-cache-dir -r /home/user/comps/retrievers/src/redis/llama_index/requirements.txt; \
+      pip install --no-cache-dir -r /home/user/comps/retrievers/redis/llama_index/requirements.txt; \
     fi;
 
 ENV PYTHONPATH=$PYTHONPATH:/home/user
 
-WORKDIR /home/user/comps/retrievers/src/redis/llama_index
+WORKDIR /home/user/comps/retrievers/redis/llama_index
 
 ENTRYPOINT ["python", "retriever_redis.py"]

comps/retrievers/src/redis/llama_index/README.md → comps/retrievers/redis/llama_index/README.md

@@ -48,7 +48,7 @@ export INDEX_NAME=${your_index_name}
 
 ```bash
 cd ../../../..
-docker build -t opea/retriever-redis-llamaindex:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/redis/llama_index/Dockerfile .
+docker build -t opea/retriever-redis-llamaindex:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/redis/llama_index/Dockerfile .
 ```
 
 To start a docker container, you have two options:

comps/retrievers/src/vdms/langchain/Dockerfile → comps/retrievers/vdms/langchain/Dockerfile

@@ -19,7 +19,7 @@ COPY comps /home/user/comps
 USER user
 RUN pip install --no-cache-dir --upgrade pip setuptools && \
     if [ ${ARCH} = "cpu" ]; then pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \
-    pip install --no-cache-dir -r /home/user/comps/retrievers/src/vdms/langchain/requirements.txt
+    pip install --no-cache-dir -r /home/user/comps/retrievers/vdms/langchain/requirements.txt
 
 RUN pip install --no-cache-dir -U \
     huggingface-hub \
@@ -32,6 +32,6 @@ ENV HUGGINGFACEHUB_API_TOKEN=dummy
 
 ENV USECLIP 0
 
-WORKDIR /home/user/comps/retrievers/src/vdms/langchain
+WORKDIR /home/user/comps/retrievers/vdms/langchain
 
 ENTRYPOINT ["python", "retriever_vdms.py"]

comps/retrievers/src/vdms/langchain/README.md → comps/retrievers/vdms/langchain/README.md

@@ -84,7 +84,7 @@ export TEI_EMBEDDING_ENDPOINT="http://${your_ip}:6060"
 
 ```bash
 cd ../../../../
-docker build -t opea/retriever-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/src/vdms/langchain/Dockerfile .
+docker build -t opea/retriever-vdms:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/vdms/langchain/Dockerfile .
 ```
 
 To start a docker container, you have two options: