From fe5f39452b7fbca7e512611cef8c1a90c08feae8 Mon Sep 17 00:00:00 2001 From: lvliang-intel Date: Fri, 21 Jun 2024 10:25:59 +0800 Subject: [PATCH] Allow vllm-xft microservice to be accessed externally (#226) * add preload env in run.sh Signed-off-by: lvliang-intel * Allow vllm-xft microservice to be accessed externally Signed-off-by: lvliang-intel --------- Signed-off-by: lvliang-intel --- comps/llms/text-generation/vllm-xft/README.md | 2 +- comps/llms/text-generation/vllm-xft/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/comps/llms/text-generation/vllm-xft/README.md b/comps/llms/text-generation/vllm-xft/README.md index abc3b1885..68931d4ca 100644 --- a/comps/llms/text-generation/vllm-xft/README.md +++ b/comps/llms/text-generation/vllm-xft/README.md @@ -13,7 +13,7 @@ docker build -t opea/llm-vllm-xft:latest --build-arg https_proxy=$https_proxy -- ## 2 Run Docker with CLI ```bash -docker run -it -p 9000:9000 -v /home/sdp/Qwen2-7B-Instruct/:/Qwen2-7B-Instruct/ -e vLLM_LLM_ENDPOINT="http://localhost:18688" -e HF_DATASET_DIR="/Qwen2-7B-Instruct/" -e OUTPUT_DIR="./output" -e TOKEN_PATH="/Qwen2-7B-Instruct/" -e https_proxy=$https_proxy -e http_proxy=$http_proxy --ipc=host opea/llm-vllm-xft:latest +docker run -it -p 9000:9000 -v /home/sdp/Qwen2-7B-Instruct/:/Qwen2-7B-Instruct/ -e vLLM_LLM_ENDPOINT="http://localhost:18688" -e HF_DATASET_DIR="/Qwen2-7B-Instruct/" -e OUTPUT_DIR="./output" -e TOKEN_PATH="/Qwen2-7B-Instruct/" -e https_proxy=$https_proxy -e http_proxy=$http_proxy -e no_proxy=$no_proxy --ipc=host opea/llm-vllm-xft:latest ``` # 🚀3. Consume LLM Service diff --git a/comps/llms/text-generation/vllm-xft/run.sh b/comps/llms/text-generation/vllm-xft/run.sh index cd06458fc..b729760ca 100644 --- a/comps/llms/text-generation/vllm-xft/run.sh +++ b/comps/llms/text-generation/vllm-xft/run.sh @@ -20,7 +20,7 @@ python -m vllm.entrypoints.openai.api_server \ --dtype bf16 \ --kv-cache-dtype fp16 \ --served-model-name xft \ - --host localhost \ + --host 0.0.0.0 \ --port 18688 \ --trust-remote-code &