From 9fec2262613e6db71d617cacf321f84d6971dcc6 Mon Sep 17 00:00:00 2001 From: XinyaoWa Date: Tue, 29 Oct 2024 16:44:52 +0800 Subject: [PATCH] Add huggingface token for native llm (#827) * add huggingface token for native llm Signed-off-by: Xinyao Wang * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bug Signed-off-by: Xinyao Wang * fix bug Signed-off-by: Xinyao Wang --------- Signed-off-by: Xinyao Wang Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- comps/llms/text-generation/native/langchain/README.md | 1 + .../text-generation/native/langchain/docker_compose_llm.yaml | 1 + comps/llms/text-generation/native/langchain/utils.py | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/comps/llms/text-generation/native/langchain/README.md b/comps/llms/text-generation/native/langchain/README.md index 3b66ca8ad..3ce428aa6 100644 --- a/comps/llms/text-generation/native/langchain/README.md +++ b/comps/llms/text-generation/native/langchain/README.md @@ -12,6 +12,7 @@ In order to start Native LLM service, you need to setup the following environmen ```bash export LLM_NATIVE_MODEL="Qwen/Qwen2-7B-Instruct" +export HUGGINGFACEHUB_API_TOKEN="your_huggingface_token" ``` ### 1.2 Build Docker Image diff --git a/comps/llms/text-generation/native/langchain/docker_compose_llm.yaml b/comps/llms/text-generation/native/langchain/docker_compose_llm.yaml index f3a36e5bb..241853efc 100644 --- a/comps/llms/text-generation/native/langchain/docker_compose_llm.yaml +++ b/comps/llms/text-generation/native/langchain/docker_compose_llm.yaml @@ -21,6 +21,7 @@ services: HABANA_VISIBLE_DEVICES: all OMPI_MCA_btl_vader_single_copy_mechanism: none TOKENIZERS_PARALLELISM: false + HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} restart: unless-stopped networks: diff --git a/comps/llms/text-generation/native/langchain/utils.py b/comps/llms/text-generation/native/langchain/utils.py index 04cebfbd4..57bd05956 100644 --- a/comps/llms/text-generation/native/langchain/utils.py +++ b/comps/llms/text-generation/native/langchain/utils.py @@ -26,6 +26,7 @@ from pathlib import Path import torch +from huggingface_hub import login from optimum.habana.checkpoint_utils import ( get_ds_injection_policy, get_repo_root, @@ -42,6 +43,10 @@ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer from transformers.utils import check_min_version +HUGGINGFACEHUB_API_TOKEN = os.getenv("HUGGINGFACEHUB_API_TOKEN", "") +if HUGGINGFACEHUB_API_TOKEN != "": + login(token=HUGGINGFACEHUB_API_TOKEN) + def adjust_batch(batch, size): curr_size = batch["input_ids"].shape[1]