[Enhence] Increase mosec_embedding forward timeout to support high co…

…ncurrency cases (opea-project#413) * increate forward timeout to support high concurrency cases * modifed max_forward_timeout as the env variable and refined the doc --------- Co-authored-by: lvliang-intel <[email protected]>
wangkl2 · Aug 6, 2024 · b61f61b · b61f61b
1 parent 75d6bc9
commit b61f61b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/comps/embeddings/langchain-mosec/README.md b/comps/embeddings/langchain-mosec/README.md
@@ -19,7 +19,7 @@ docker run -d --name="embedding-langchain-mosec-endpoint" -p 6001:8000  langchai
 # launch embedding microservice docker container
 
 ```
-export MOSEC_EMBEDDING_ENDPOINT=http://127.0.0.1:6001
+export MOSEC_EMBEDDING_ENDPOINT=http://{mosec_embedding_host_ip}:6001
 docker run -d --name="embedding-langchain-mosec-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 6000:6000 --ipc=host -e MOSEC_EMBEDDING_ENDPOINT=$MOSEC_EMBEDDING_ENDPOINT opea/embedding-langchain-mosec:latest
 ```
 

diff --git a/comps/embeddings/langchain-mosec/mosec-docker/server-ipex.py b/comps/embeddings/langchain-mosec/mosec-docker/server-ipex.py
@@ -113,8 +113,9 @@ def forward(self, data: List[EmbeddingRequest]) -> List[EmbeddingResponse]:
 if __name__ == "__main__":
     MAX_BATCH_SIZE = int(os.environ.get("MAX_BATCH_SIZE", 128))
     MAX_WAIT_TIME = int(os.environ.get("MAX_WAIT_TIME", 10))
+    MAX_FORWARD_TIMEOUT = int(os.environ.get("FORWARD_TIMEOUT", 60))
     server = Server()
-    emb = Runtime(Embedding, max_batch_size=MAX_BATCH_SIZE, max_wait_time=MAX_WAIT_TIME)
+    emb = Runtime(Embedding, max_batch_size=MAX_BATCH_SIZE, max_wait_time=MAX_WAIT_TIME, timeout=MAX_FORWARD_TIMEOUT)
     server.register_runtime(
         {
             "/v1/embeddings": [emb],