From 0358d6cf97601b13a6453a7cb72f5cedbf3a1dd5 Mon Sep 17 00:00:00 2001 From: dorren002 <63645716+dorren002@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:47:38 +0800 Subject: [PATCH] support windows (#356) Co-authored-by: wenduren Co-authored-by: lwj-st Co-authored-by: wangzhihong --- .github/workflows/win_test.yml | 186 ++++++++++++++++++ examples/rag_online.py | 7 +- lazyllm/common/logger.py | 2 +- .../auto/configure/core/configuration.py | 4 +- .../finetune/alpaca-lora/finetune.py | 2 +- lazyllm/components/finetune/alpacalora.py | 13 +- lazyllm/components/finetune/collie.py | 11 +- lazyllm/components/finetune/llamafactory.py | 4 +- lazyllm/configs.py | 4 +- lazyllm/launcher.py | 6 +- lazyllm/module/module.py | 2 +- .../module/onlineChatModule/doubaoModule.py | 4 +- lazyllm/module/onlineChatModule/glmModule.py | 9 +- lazyllm/module/onlineChatModule/kimiModule.py | 4 +- .../onlineChatModule/onlineChatModuleBase.py | 5 +- .../module/onlineChatModule/openaiModule.py | 15 +- lazyllm/module/onlineChatModule/qwenModule.py | 24 ++- .../onlineChatModule/sensenovaModule.py | 15 +- .../onlineEmbedding/onlineEmbeddingModule.py | 42 ++-- lazyllm/module/onlineEmbedding/qwenEmbed.py | 34 ++++ lazyllm/tools/rag/rerank.py | 11 +- lazyllm/tools/rag/web.py | 8 +- lazyllm/tools/webpages/webmodule.py | 8 +- tests/basic_tests/pytest.ini | 4 +- tests/basic_tests/test_module.py | 1 - tests/basic_tests/test_store.py | 2 + tests/charge_tests/pytest.ini | 2 +- tests/charge_tests/test_example.py | 1 - 28 files changed, 337 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/win_test.yml diff --git a/.github/workflows/win_test.yml b/.github/workflows/win_test.yml new file mode 100644 index 00000000..9246d4b3 --- /dev/null +++ b/.github/workflows/win_test.yml @@ -0,0 +1,186 @@ +name: Test on Windows + +on: + push: + branches: + - main + pull_request_target: + branches: + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request_target' && github.event.pull_request.id || github.ref }} + cancel-in-progress: true + +env: + POETRY_VERSION: "1.8.3" + PYTHON_VERSION: "3.10.9" + +jobs: + win-basic_tests: + runs-on: windows-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - name: Fetch PR source branch + if: github.event_name == 'pull_request_target' + run: | + if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo ${{ github.event.pull_request.head.repo.clone_url }} + echo ${{ github.event.pull_request.head.ref }} + git remote add external_repo ${{ github.event.pull_request.head.repo.clone_url }} + git fetch external_repo ${{ github.event.pull_request.head.ref }}:pr_branch + else + echo ${{ github.event.pull_request.head.ref }} + git fetch origin ${{ github.event.pull_request.head.ref }}:pr_branch + fi + + - name: Merge PR branch into main + if: github.event_name == 'pull_request_target' + run: | + git checkout main + git merge --no-ff pr_branch + git submodule update --init + + - name: Copy poetry.lock to root directory + shell: bash + run: | + git branch + cd LazyLLM-Env && git branch + cd .. + cp LazyLLM-Env/poetry.lock . + ls + + - name: Set up python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download model + shell: bash + run: | + pip install modelscope pyyaml + mkdir C:/Users/runneradmin/.lazyllm + ls C:/Users/runneradmin/.lazyllm + mkdir -p C:/Users/runneradmin/.cache/modelscope/hub + ln -s C:/Users/runneradmin/.cache/modelscope/hub C:/Users/runneradmin/.lazyllm/model + modelscope download Shanghai_AI_Laboratory/internlm2-chat-7b & + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + + - name: Build project with Poetry + shell: bash + run: | + poetry build + + - name: List dist directory + shell: bash + run: ls dist + + - name: Install the built package + shell: bash + run: | + pip install dist/lazyllm*.whl + + - name: basic_tests + shell: bash + run: | + git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git D:/a/LazyLLM/data + pip install -r tests/requirements.txt + export LAZYLLM_DATA_PATH=D:/a/LazyLLM/data + python -m pytest -m "not skip_on_win" -v --reruns=2 tests/basic_tests + timeout-minutes: 30 + env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + + + win-charge_tests: + if: | + !contains(github.event.head_commit.message, '[skip ci]') + && !contains(github.event.pull_request.title, '[skip ci]') + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - name: Merge PR branch into main + if: github.event_name == 'pull_request_target' + run: | + if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo ${{ github.event.pull_request.head.repo.clone_url }} + echo ${{ github.event.pull_request.head.ref }} + git remote add external_repo ${{ github.event.pull_request.head.repo.clone_url }} + git fetch external_repo ${{ github.event.pull_request.head.ref }}:pr_branch + else + echo ${{ github.event.pull_request.head.ref }} + git fetch origin ${{ github.event.pull_request.head.ref }}:pr_branch + fi + + - name: Merge PR branch into main + if: github.event_name == 'pull_request_target' + shell: bash + run: | + git checkout main + git merge --no-ff pr_branch + git submodule update --init + + - name: Copy poetry.lock to root directory + shell: bash + run: | + git branch + cd LazyLLM-Env && git branch + cd .. + cp LazyLLM-Env/poetry.lock . + + - name: Set up python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + + - name: Build project with Poetry + shell: bash + run: | + poetry build + + - name: List dist directory + shell: bash + run: ls dist + + - name: Install the built package + shell: bash + run: | + pip install dist/lazyllm*.whl + + - name: charge_tests + shell: bash + run : | + git clone https://$GITHUB_TOKEN@github.com/LazyAGI/LazyLLM-Data.git D:/a/LazyLLM/data + pip install -r tests/requirements.txt + export LAZYLLM_DATA_PATH=D:/a/LazyLLM/data + python -m pytest -v --reruns=2 tests/charge_tests + env : + LAZYLLM_KIMI_API_KEY: ${{ secrets.LAZYLLM_KIMI_API_KEY }} + LAZYLLM_GLM_API_KEY: ${{ secrets.LAZYLLM_GLM_API_KEY }} + LAZYLLM_QWEN_API_KEY: ${{ secrets.LAZYLLM_QWEN_API_KEY }} + LAZYLLM_SENSENOVA_API_KEY: ${{ secrets.LAZYLLM_SENSENOVA_API_KEY }} + LAZYLLM_SENSENOVA_SECRET_KEY: ${{ secrets.LAZYLLM_SENSENOVA_SECRET_KEY }} + LAZYLLM_PostgreSQL_URL: ${{ secrets.LAZYLLM_PostgreSQL_URL }} + GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + timeout-minutes: 30 diff --git a/examples/rag_online.py b/examples/rag_online.py index 62e370c9..66eb984a 100644 --- a/examples/rag_online.py +++ b/examples/rag_online.py @@ -2,7 +2,7 @@ # flake8: noqa: F821 import lazyllm -from lazyllm import pipeline, parallel, bind, SentenceSplitter, Document, Retriever, Reranker +from lazyllm import pipeline, parallel, bind, OnlineEmbeddingModule, SentenceSplitter, Document, Retriever, Reranker # Before running, set the environment variable: # @@ -27,15 +27,14 @@ prompt = 'You will play the role of an AI Q&A assistant and complete a dialogue task. In this task, you need to provide your answer based on the given context and question.' -documents = Document(dataset_path="rag_master", embed=lazyllm.OnlineEmbeddingModule(), manager=False) +documents = Document(dataset_path="rag_master", embed=OnlineEmbeddingModule(), manager=False) documents.create_node_group(name="sentences", transform=SentenceSplitter, chunk_size=1024, chunk_overlap=100) with pipeline() as ppl: with parallel().sum as ppl.prl: prl.retriever1 = Retriever(documents, group_name="sentences", similarity="cosine", topk=3) prl.retriever2 = Retriever(documents, "CoarseChunk", "bm25_chinese", 0.003, topk=3) - - ppl.reranker = Reranker("ModuleReranker", model="bge-reranker-large", topk=1, output_format='content', join=True) | bind(query=ppl.input) + ppl.reranker = Reranker("ModuleReranker", model=OnlineEmbeddingModule(type="rerank"), topk=1, output_format='content', join=True) | bind(query=ppl.input) ppl.formatter = (lambda nodes, query: dict(context_str=nodes, query=query)) | bind(query=ppl.input) ppl.llm = lazyllm.OnlineChatModule(stream=False).prompt(lazyllm.ChatPrompter(prompt, extro_keys=["context_str"])) diff --git a/lazyllm/common/logger.py b/lazyllm/common/logger.py index f062380d..37ad4e47 100644 --- a/lazyllm/common/logger.py +++ b/lazyllm/common/logger.py @@ -24,7 +24,7 @@ "{level}: ({name}:{line}) {message}", "LOG_FORMAT", ) -lazyllm.config.add("log_dir", str, "~/.lazyllm", "LOG_DIR") +lazyllm.config.add("log_dir", str, os.path.join(os.path.expanduser('~'), '.lazyllm'), "LOG_DIR") lazyllm.config.add("log_file_level", str, "ERROR", "LOG_FILE_LEVEL") lazyllm.config.add("log_file_size", str, "4 MB", "LOG_FILE_SIZE") lazyllm.config.add("log_file_retention", str, "7 days", "LOG_FILE_RETENTION") diff --git a/lazyllm/components/auto/configure/core/configuration.py b/lazyllm/components/auto/configure/core/configuration.py index 1b388f6b..6666d1aa 100644 --- a/lazyllm/components/auto/configure/core/configuration.py +++ b/lazyllm/components/auto/configure/core/configuration.py @@ -84,6 +84,6 @@ def query_deploy(self, gpu_type: str, gpu_num: int, model_name: str, max_token_n def get_configer(): global configer if configer is None: - configer = AutoConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/finetune.csv'), - os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs/deploy.csv')) + configer = AutoConfig(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs', 'finetune.csv'), + os.path.join(os.path.dirname(os.path.abspath(__file__)), 'configs', 'deploy.csv')) return configer diff --git a/lazyllm/components/finetune/alpaca-lora/finetune.py b/lazyllm/components/finetune/alpaca-lora/finetune.py index 879a42e9..380f9db2 100755 --- a/lazyllm/components/finetune/alpaca-lora/finetune.py +++ b/lazyllm/components/finetune/alpaca-lora/finetune.py @@ -60,7 +60,7 @@ def train( # noqa C901 # model/data params base_model: str = "", # the only required argument data_path: str = "", - output_dir: str = "./output_dir", + output_dir: str = os.path.abspath("./output_dir"), # training hyperparams batch_size: int = 128, micro_batch_size: int = 4, diff --git a/lazyllm/components/finetune/alpacalora.py b/lazyllm/components/finetune/alpacalora.py index ad3bd4e0..c0571aa5 100644 --- a/lazyllm/components/finetune/alpacalora.py +++ b/lazyllm/components/finetune/alpacalora.py @@ -42,14 +42,15 @@ def __init__(self, if not merge_path: save_path = os.path.join(lazyllm.config['train_target_root'], target_path) target_path, merge_path = os.path.join(save_path, "lazyllm_lora"), os.path.join(save_path, "lazyllm_merge") - os.system(f'mkdir -p {target_path} {merge_path}') + os.makedirs(target_path, exist_ok=True) + os.makedirs(merge_path, exist_ok=True) super().__init__( base_model, target_path, launcher=launcher, ) self.folder_path = os.path.dirname(os.path.abspath(__file__)) - deepspeed_config_path = os.path.join(self.folder_path, 'alpaca-lora/ds.json') + deepspeed_config_path = os.path.join(self.folder_path, 'alpaca-lora', 'ds.json') self.kw = copy.deepcopy(self.defatult_kw) self.kw['deepspeed'] = deepspeed_config_path self.kw['nccl_port'] = random.randint(19000, 20500) @@ -67,23 +68,23 @@ def cmd(self, trainset, valset=None) -> str: if not self.kw['data_path']: self.kw['data_path'] = trainset - run_file_path = os.path.join(self.folder_path, 'alpaca-lora/finetune.py') + run_file_path = os.path.join(self.folder_path, 'alpaca-lora', 'finetune.py') cmd = (f'python {run_file_path} ' f'--base_model={self.base_model} ' f'--output_dir={self.target_path} ' ) cmd += self.kw.parse_kwargs() - cmd += f' 2>&1 | tee {self.target_path}/{self.model_name}_$(date +"%Y-%m-%d_%H-%M-%S").log' + cmd += f' 2>&1 | tee {os.path.join(self.target_path, self.model_name)}_$(date +"%Y-%m-%d_%H-%M-%S").log' if self.merge_path: - run_file_path = os.path.join(self.folder_path, 'alpaca-lora/utils/merge_weights.py') + run_file_path = os.path.join(self.folder_path, 'alpaca-lora', 'utils', 'merge_weights.py') cmd = [cmd, f'python {run_file_path} ' f'--base={self.base_model} ' f'--adapter={self.target_path} ' f'--save_path={self.merge_path} ', - f' cp {self.base_model}/{self.cp_files} {self.merge_path} ' + f' cp {os.path.join(self.base_model, self.cp_files)} {self.merge_path} ' ] # cmd = 'realpath .' diff --git a/lazyllm/components/finetune/collie.py b/lazyllm/components/finetune/collie.py index 15790d84..1659fae9 100644 --- a/lazyllm/components/finetune/collie.py +++ b/lazyllm/components/finetune/collie.py @@ -41,7 +41,8 @@ def __init__(self, if not merge_path: save_path = os.path.join(lazyllm.config['train_target_root'], target_path) target_path, merge_path = os.path.join(save_path, "lazyllm_lora"), os.path.join(save_path, "lazyllm_merge") - os.system(f'mkdir -p {target_path} {merge_path}') + os.makedirs(target_path, exist_ok=True) + os.makedirs(merge_path, exist_ok=True) super().__init__( base_model, target_path, @@ -63,23 +64,23 @@ def cmd(self, trainset, valset=None) -> str: if not self.kw['data_path']: self.kw['data_path'] = trainset - run_file_path = os.path.join(self.folder_path, 'collie/finetune.py') + run_file_path = os.path.join(self.folder_path, 'collie', 'finetune.py') cmd = (f'python {run_file_path} ' f'--base_model={self.base_model} ' f'--output_dir={self.target_path} ' ) cmd += self.kw.parse_kwargs() - cmd += f' 2>&1 | tee {self.target_path}/{self.model_name}_$(date +"%Y-%m-%d_%H-%M-%S").log' + cmd += f' 2>&1 | tee {os.path.join(self.target_path, self.model_name)}_$(date +"%Y-%m-%d_%H-%M-%S").log' if self.merge_path: - run_file_path = os.path.join(self.folder_path, 'alpaca-lora/utils/merge_weights.py') + run_file_path = os.path.join(self.folder_path, 'alpaca-lora', 'utils', 'merge_weights.py') cmd = [cmd, f'python {run_file_path} ' f'--base={self.base_model} ' f'--adapter={self.target_path} ' f'--save_path={self.merge_path} ', - f' cp {self.base_model}/{self.cp_files} {self.merge_path} ' + f' cp {os.path.join(self.base_model,self.cp_files)} {self.merge_path} ' ] return cmd diff --git a/lazyllm/components/finetune/llamafactory.py b/lazyllm/components/finetune/llamafactory.py index 088cb48e..7bed14a4 100644 --- a/lazyllm/components/finetune/llamafactory.py +++ b/lazyllm/components/finetune/llamafactory.py @@ -47,7 +47,7 @@ def __init__(self, self.export_config_path = export_config_path self.config_folder_path = os.path.dirname(os.path.abspath(__file__)) - default_config_path = os.path.join(self.config_folder_path, 'llamafactory/sft.yaml') + default_config_path = os.path.join(self.config_folder_path, 'llamafactory', 'sft.yaml') self.template_dict = ArgsDict(self.load_yaml(default_config_path)) if self.config_path: @@ -64,7 +64,7 @@ def __init__(self, self.template_dict['template'] = self.get_template_name(base_model) self.template_dict.check_and_update(kw) - default_export_config_path = os.path.join(self.config_folder_path, 'llamafactory/lora_export.yaml') + default_export_config_path = os.path.join(self.config_folder_path, 'llamafactory', 'lora_export.yaml') self.export_dict = ArgsDict(self.load_yaml(default_export_config_path)) if self.export_config_path: diff --git a/lazyllm/configs.py b/lazyllm/configs.py index d935bd95..d5d67275 100644 --- a/lazyllm/configs.py +++ b/lazyllm/configs.py @@ -11,13 +11,13 @@ class Mode(Enum): class Config(object): - def __init__(self, prefix='LAZYLLM', home='~/.lazyllm/'): + def __init__(self, prefix='LAZYLLM', home=os.path.join(os.path.expanduser('~'), '.lazyllm')): self._config_params = dict() self._env_map_name = dict() self.prefix = prefix self.impl, self.cfgs = dict(), dict() self.add('home', str, os.path.expanduser(home), 'HOME') - os.system(f'mkdir -p {home}') + os.makedirs(home, exist_ok=True) self.cgf_path = os.path.join(self['home'], 'config.json') if os.path.exists(self.cgf_path): with open(self.cgf_path, 'r+') as f: diff --git a/lazyllm/launcher.py b/lazyllm/launcher.py index c4fc9a2c..98f0e070 100644 --- a/lazyllm/launcher.py +++ b/lazyllm/launcher.py @@ -117,8 +117,8 @@ def _start(self, *, fixed): cmd = self.get_executable_cmd(fixed=fixed) LOG.info(f'Command: {cmd}') if lazyllm.config['mode'] == lazyllm.Mode.Display: return - self.ps = subprocess.Popen(cmd.cmd, shell=True, executable='/bin/bash', - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + self.ps = subprocess.Popen(cmd.cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) self._get_jobid() self._enqueue_subprocess_output(hooks=self.output_hooks) @@ -234,7 +234,7 @@ def _get_jobid(self): self.jobid = self.ps.pid if self.ps else None def get_jobip(self): - return '0.0.0.0' + return '127.0.0.1' def wait(self): if self.ps: diff --git a/lazyllm/module/module.py b/lazyllm/module/module.py index 3d5eada8..0f0a67e8 100644 --- a/lazyllm/module/module.py +++ b/lazyllm/module/module.py @@ -617,7 +617,7 @@ def _get_train_tasks_impl(self, mode: Optional[str] = None, **kw): self._trainset, str) else self._trainset target_path = self._generate_target_path() if not os.path.exists(target_path): - os.system(f'mkdir -p {target_path}') + os.makedirs(target_path, exist_ok=True) kw = kw or self._get_train_or_deploy_args(mode, disable=['base_model', 'target_path']) task = getattr(self, f'_{mode}')(base_model=self._base_model, target_path=target_path, **kw) diff --git a/lazyllm/module/onlineChatModule/doubaoModule.py b/lazyllm/module/onlineChatModule/doubaoModule.py index 8e96f0d5..d4bba9f7 100644 --- a/lazyllm/module/onlineChatModule/doubaoModule.py +++ b/lazyllm/module/onlineChatModule/doubaoModule.py @@ -1,5 +1,5 @@ -import os import lazyllm +from urllib.parse import urljoin from typing import Union, Dict, List from .onlineChatModuleBase import OnlineChatModuleBase @@ -25,7 +25,7 @@ def _get_system_prompt(self): and support to users' questions and requests." def _set_chat_url(self): - self._url = os.path.join(self._base_url, 'chat/completions') + self._url = urljoin(self._base_url, 'chat/completions') def forward(self, __input: Union[Dict, str] = None, llm_chat_history: List[List[str]] = None, **kw): raise NotImplementedError("Individual user support is not friendly and is not supported yet") diff --git a/lazyllm/module/onlineChatModule/glmModule.py b/lazyllm/module/onlineChatModule/glmModule.py index 5cedf617..5dc82bc9 100644 --- a/lazyllm/module/onlineChatModule/glmModule.py +++ b/lazyllm/module/onlineChatModule/glmModule.py @@ -2,7 +2,7 @@ import os import requests from typing import Tuple, List - +from urllib.parse import urljoin import lazyllm from .onlineChatModuleBase import OnlineChatModuleBase from .fileHandler import FileHandlerBase @@ -11,7 +11,7 @@ class GLMModule(OnlineChatModuleBase, FileHandlerBase): TRAINABLE_MODEL_LIST = ["chatglm3-6b", "chatglm_12b", "chatglm_32b", "chatglm_66b", "chatglm_130b"] def __init__(self, - base_url: str = "https://open.bigmodel.cn/api/paas/v4", + base_url: str = "https://open.bigmodel.cn/api/paas/v4/", model: str = "glm-4", api_key: str = None, stream: str = True, @@ -76,8 +76,7 @@ def _upload_train_file(self, train_file): "Authorization": "Bearer " + self._api_key } - url = os.path.join(self._base_url, "files") - + url = urljoin(self._base_url, "files") self.get_finetune_data(train_file) file_object = { @@ -106,7 +105,7 @@ def _update_kw(self, data, normal_config): return cur_data def _create_finetuning_job(self, train_model, train_file_id, **kw) -> Tuple[str, str]: - url = os.path.join(self._base_url, "fine_tuning/jobs") + url = urljoin(self._base_url, "fine_tuning/jobs") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", diff --git a/lazyllm/module/onlineChatModule/kimiModule.py b/lazyllm/module/onlineChatModule/kimiModule.py index 962b2d8e..55cee12a 100644 --- a/lazyllm/module/onlineChatModule/kimiModule.py +++ b/lazyllm/module/onlineChatModule/kimiModule.py @@ -1,5 +1,5 @@ -import os import lazyllm +from urllib.parse import urljoin from .onlineChatModuleBase import OnlineChatModuleBase class KimiModule(OnlineChatModuleBase): @@ -29,4 +29,4 @@ def _get_system_prompt(self): "into other languages.") def _set_chat_url(self): - self._url = os.path.join(self._base_url, 'v1/chat/completions') + self._url = urljoin(self._base_url, 'v1/chat/completions') diff --git a/lazyllm/module/onlineChatModule/onlineChatModuleBase.py b/lazyllm/module/onlineChatModule/onlineChatModuleBase.py index 69cef08a..96ac6266 100644 --- a/lazyllm/module/onlineChatModule/onlineChatModuleBase.py +++ b/lazyllm/module/onlineChatModule/onlineChatModuleBase.py @@ -5,6 +5,7 @@ import requests import re from typing import Tuple, List, Dict, Union, Any +from urllib.parse import urljoin import time import lazyllm @@ -88,10 +89,10 @@ def _set_headers(self): } def _set_chat_url(self): - self._url = os.path.join(self._base_url, 'chat/completions') + self._url = urljoin(self._base_url, 'chat/completions') def _get_models_list(self): - url = os.path.join(self._base_url, 'models') + url = urljoin(self._base_url, 'models') headers = {'Authorization': 'Bearer ' + self._api_key} with requests.get(url, headers=headers) as r: if r.status_code != 200: diff --git a/lazyllm/module/onlineChatModule/openaiModule.py b/lazyllm/module/onlineChatModule/openaiModule.py index 45c94357..3eaee884 100644 --- a/lazyllm/module/onlineChatModule/openaiModule.py +++ b/lazyllm/module/onlineChatModule/openaiModule.py @@ -2,6 +2,7 @@ import os import requests from typing import Tuple, List +from urllib.parse import urljoin import lazyllm from .onlineChatModuleBase import OnlineChatModuleBase from .fileHandler import FileHandlerBase @@ -12,7 +13,7 @@ class OpenAIModule(OnlineChatModuleBase, FileHandlerBase): "davinci-002", "gpt-4-0613"] def __init__(self, - base_url: str = "https://api.openai.com/v1", + base_url: str = "https://api.openai.com/v1/", model: str = "gpt-3.5-turbo", api_key: str = None, stream: bool = True, @@ -65,7 +66,7 @@ def _upload_train_file(self, train_file): "Authorization": "Bearer " + self._api_key } - url = os.path.join(self._base_url, "files") + url = urljoin(self._base_url, "files") self.get_finetune_data(train_file) @@ -94,7 +95,7 @@ def _update_kw(self, data, normal_config): return current_train_data def _create_finetuning_job(self, train_model, train_file_id, **kw) -> Tuple[str, str]: - url = os.path.join(self._base_url, "fine_tuning/jobs") + url = urljoin(self._base_url, "fine_tuning/jobs") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -122,7 +123,7 @@ def _cancel_finetuning_job(self, fine_tuning_job_id=None): if not fine_tuning_job_id and not self.fine_tuning_job_id: return 'Invalid' job_id = fine_tuning_job_id if fine_tuning_job_id else self.fine_tuning_job_id - fine_tune_url = os.path.join(self._base_url, f"fine_tuning/jobs/{job_id}/cancel") + fine_tune_url = urljoin(self._base_url, f"fine_tuning/jobs/{job_id}/cancel") headers = { "Authorization": f"Bearer {self._api_key}" } @@ -136,7 +137,7 @@ def _cancel_finetuning_job(self, fine_tuning_job_id=None): return f'JOB {job_id} status: {status}' def _query_finetuned_jobs(self): - fine_tune_url = os.path.join(self._base_url, "fine_tuning/jobs") + fine_tune_url = urljoin(self._base_url, "fine_tuning/jobs") headers = { "Authorization": f"Bearer {self._api_key}", } @@ -178,7 +179,7 @@ def _get_log(self, fine_tuning_job_id=None): raise RuntimeError("No job ID specified. Please ensure that a valid 'fine_tuning_job_id' is " "provided as an argument or started a training job.") job_id = fine_tuning_job_id if fine_tuning_job_id else self.fine_tuning_job_id - fine_tune_url = os.path.join(self._base_url, f"fine_tuning/jobs/{job_id}/events") + fine_tune_url = urljoin(self._base_url, f"fine_tuning/jobs/{job_id}/events") headers = { "Authorization": f"Bearer {self._api_key}" } @@ -194,7 +195,7 @@ def _get_curr_job_model_id(self): return self.fine_tuning_job_id, model_id def _query_finetuning_job_info(self, fine_tuning_job_id): - fine_tune_url = os.path.join(self._base_url, f"fine_tuning/jobs/{fine_tuning_job_id}") + fine_tune_url = urljoin(self._base_url, f"fine_tuning/jobs/{fine_tuning_job_id}") headers = { "Authorization": f"Bearer {self._api_key}" } diff --git a/lazyllm/module/onlineChatModule/qwenModule.py b/lazyllm/module/onlineChatModule/qwenModule.py index 03cb6a16..d67f1e21 100644 --- a/lazyllm/module/onlineChatModule/qwenModule.py +++ b/lazyllm/module/onlineChatModule/qwenModule.py @@ -2,6 +2,7 @@ import os import requests from typing import Tuple, List +from urllib.parse import urljoin import lazyllm from .onlineChatModuleBase import OnlineChatModuleBase from .fileHandler import FileHandlerBase @@ -15,7 +16,7 @@ class QwenModule(OnlineChatModuleBase, FileHandlerBase): TRAINABLE_MODEL_LIST = ["qwen-turbo", "qwen-7b-chat", "qwen-72b-chat"] def __init__(self, - base_url: str = "https://dashscope.aliyuncs.com", + base_url: str = "https://dashscope.aliyuncs.com/", model: str = "qwen-plus", api_key: str = None, stream: bool = True, @@ -60,10 +61,7 @@ def _get_system_prompt(self): "your name is Tongyi Qianwen, and you are a useful assistant.") def _set_chat_url(self): - self._url = os.path.join(self._base_url, 'compatible-mode/v1/chat/completions') - - # def _set_chat_sft_url(self): - # self._url = os.path.join(self._base_url, ) + self._url = urljoin(self._base_url, 'compatible-mode/v1/chat/completions') def _convert_file_format(self, filepath: str) -> None: with open(filepath, 'r', encoding='utf-8') as fr: @@ -87,7 +85,7 @@ def _upload_train_file(self, train_file): "Authorization": "Bearer " + self._api_key } - url = os.path.join(self._base_url, "api/v1/files") + url = urljoin(self._base_url, "api/v1/files") self.get_finetune_data(train_file) @@ -126,7 +124,7 @@ def _update_kw(self, data, normal_config): return current_train_data def _create_finetuning_job(self, train_model, train_file_id, **kw) -> Tuple[str, str]: - url = os.path.join(self._base_url, "api/v1/fine-tunes") + url = urljoin(self._base_url, "api/v1/fine-tunes") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -153,7 +151,7 @@ def _cancel_finetuning_job(self, fine_tuning_job_id=None): if not fine_tuning_job_id and not self.fine_tuning_job_id: return 'Invalid' job_id = fine_tuning_job_id if fine_tuning_job_id else self.fine_tuning_job_id - fine_tune_url = os.path.join(self._base_url, f"api/v1/fine-tunes/{job_id}/cancel") + fine_tune_url = urljoin(self._base_url, f"api/v1/fine-tunes/{job_id}/cancel") headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json" @@ -168,7 +166,7 @@ def _cancel_finetuning_job(self, fine_tuning_job_id=None): return f'JOB {job_id} status: {status}' def _query_finetuned_jobs(self): - fine_tune_url = os.path.join(self._base_url, "api/v1/fine-tunes") + fine_tune_url = urljoin(self._base_url, "api/v1/fine-tunes") headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json" @@ -217,7 +215,7 @@ def _get_log(self, fine_tuning_job_id=None): raise RuntimeError("No job ID specified. Please ensure that a valid 'fine_tuning_job_id' is " "provided as an argument or started a training job.") job_id = fine_tuning_job_id if fine_tuning_job_id else self.fine_tuning_job_id - fine_tune_url = os.path.join(self._base_url, f"api/v1/fine-tunes/{job_id}/logs") + fine_tune_url = urljoin(self._base_url, f"api/v1/fine-tunes/{job_id}/logs") headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json" @@ -234,7 +232,7 @@ def _get_curr_job_model_id(self): return self.fine_tuning_job_id, model_id def _query_finetuning_job_info(self, fine_tuning_job_id): - fine_tune_url = os.path.join(self._base_url, f"api/v1/fine-tunes/{fine_tuning_job_id}") + fine_tune_url = urljoin(self._base_url, f"api/v1/fine-tunes/{fine_tuning_job_id}") headers = { "Authorization": f"Bearer {self._api_key}", "Content-Type": "application/json" @@ -265,7 +263,7 @@ def set_deploy_parameters(self, **kw): self._deploy_paramters = kw def _create_deployment(self) -> Tuple[str, str]: - url = os.path.join(self._base_url, "api/v1/deployments") + url = urljoin(self._base_url, "api/v1/deployments") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -286,7 +284,7 @@ def _create_deployment(self) -> Tuple[str, str]: return (deployment_id, status) def _query_deployment(self, deployment_id) -> str: - fine_tune_url = os.path.join(self._base_url, f"api/v1/deployments/{deployment_id}") + fine_tune_url = urljoin(self._base_url, f"api/v1/deployments/{deployment_id}") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}" diff --git a/lazyllm/module/onlineChatModule/sensenovaModule.py b/lazyllm/module/onlineChatModule/sensenovaModule.py index 1596ebd8..3ef68816 100644 --- a/lazyllm/module/onlineChatModule/sensenovaModule.py +++ b/lazyllm/module/onlineChatModule/sensenovaModule.py @@ -2,6 +2,7 @@ import os import requests from typing import Tuple, Dict, Any +from urllib.parse import urljoin import uuid import lazyllm from .onlineChatModuleBase import OnlineChatModuleBase @@ -11,7 +12,7 @@ class SenseNovaModule(OnlineChatModuleBase, FileHandlerBase): TRAINABLE_MODEL_LIST = ["nova-ptc-s-v2"] def __init__(self, - base_url: str = "https://api.sensenova.cn/v1/llm", + base_url: str = "https://api.sensenova.cn/v1/llm/", model: str = "SenseChat-5", api_key: str = None, secret_key: str = None, @@ -56,7 +57,7 @@ def encode_jwt_token(ak: str, sk: str) -> str: return token def _set_chat_url(self): - self._url = os.path.join(self._base_url, 'chat-completions') + self._url = urljoin(self._base_url, 'chat-completions') def _convert_msg_format(self, msg: Dict[str, Any]): try: @@ -121,7 +122,7 @@ def _upload_train_file(self, train_file): lazyllm.LOG.info(f"train file id: {train_file_id}") def _create_finetuning_dataset(description, files): - url = os.path.join(self._base_url, "fine-tune/datasets") + url = urljoin(self._base_url, "fine-tune/datasets") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -155,7 +156,7 @@ def _create_finetuning_dataset(description, files): return _create_finetuning_dataset("fine-tuning dataset", [train_file_id]) def _create_finetuning_job(self, train_model, train_file_id, **kw) -> Tuple[str, str]: - url = os.path.join(self._base_url, "fine-tunes") + url = urljoin(self._base_url, "fine-tunes") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -177,7 +178,7 @@ def _create_finetuning_job(self, train_model, train_file_id, **kw) -> Tuple[str, return (fine_tuning_job_id, status) def _query_finetuning_job(self, fine_tuning_job_id) -> Tuple[str, str]: - fine_tune_url = os.path.join(self._base_url, f"fine-tunes/{fine_tuning_job_id}") + fine_tune_url = urljoin(self._base_url, f"fine-tunes/{fine_tuning_job_id}") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}" @@ -196,7 +197,7 @@ def set_deploy_parameters(self, **kw): self._deploy_paramters = kw def _create_deployment(self) -> Tuple[str, str]: - url = os.path.join(self._base_url, "fine-tune/servings") + url = urljoin(self._base_url, "fine-tune/servings") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}", @@ -219,7 +220,7 @@ def _create_deployment(self) -> Tuple[str, str]: return (fine_tuning_job_id, status) def _query_deployment(self, deployment_id) -> str: - fine_tune_url = os.path.join(self._base_url, f"fine-tune/servings/{deployment_id}") + fine_tune_url = urljoin(self._base_url, f"fine-tune/servings/{deployment_id}") headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self._api_key}" diff --git a/lazyllm/module/onlineEmbedding/onlineEmbeddingModule.py b/lazyllm/module/onlineEmbedding/onlineEmbeddingModule.py index c7242ee8..f19772c7 100644 --- a/lazyllm/module/onlineEmbedding/onlineEmbeddingModule.py +++ b/lazyllm/module/onlineEmbedding/onlineEmbeddingModule.py @@ -4,7 +4,7 @@ from .openaiEmbed import OpenAIEmbedding from .glmEmbed import GLMEmbedding from .sensenovaEmbed import SenseNovaEmbedding -from .qwenEmbed import QwenEmbedding +from .qwenEmbed import QwenEmbedding, QwenReranking from .onlineEmbeddingModuleBase import OnlineEmbeddingModuleBase class __EmbedModuleMeta(type): @@ -16,10 +16,11 @@ def __instancecheck__(self, __instance: Any) -> bool: class OnlineEmbeddingModule(metaclass=__EmbedModuleMeta): - MODELS = {'openai': OpenAIEmbedding, - 'sensenova': SenseNovaEmbedding, - 'glm': GLMEmbedding, - 'qwen': QwenEmbedding} + EMBED_MODELS = {'openai': OpenAIEmbedding, + 'sensenova': SenseNovaEmbedding, + 'glm': GLMEmbedding, + 'qwen': QwenEmbedding} + RERANK_MODELS = {'qwen': QwenReranking} @staticmethod def _encapsulate_parameters(embed_url: str, @@ -33,6 +34,16 @@ def _encapsulate_parameters(embed_url: str, params.update(kwargs) return params + @staticmethod + def _check_available_source(available_models): + for source in available_models.keys(): + if lazyllm.config[f'{source}_api_key']: break + else: + raise KeyError(f"No api_key is configured for any of the models {available_models.keys()}.") + + assert source in available_models.keys(), f"Unsupported source: {source}" + return source + def __new__(self, source: str = None, embed_url: str = None, @@ -40,13 +51,16 @@ def __new__(self, **kwargs): params = OnlineEmbeddingModule._encapsulate_parameters(embed_url, embed_model_name, **kwargs) - if source is None: - if "api_key" in kwargs and kwargs["api_key"]: - raise ValueError("No source is given but an api_key is provided.") - for source in OnlineEmbeddingModule.MODELS.keys(): - if lazyllm.config[f'{source}_api_key']: break - else: - raise KeyError(f"No api_key is configured for any of the models {OnlineEmbeddingModule.MODELS.keys()}.") + if source is None and "api_key" in kwargs and kwargs["api_key"]: + raise ValueError("No source is given but an api_key is provided.") - assert source in OnlineEmbeddingModule.MODELS.keys(), f"Unsupported source: {source}" - return OnlineEmbeddingModule.MODELS[source](**params) + if kwargs.get("type", "embed") == "embed": + if source is None: + source = OnlineEmbeddingModule._check_available_source(OnlineEmbeddingModule.EMBED_MODELS) + return OnlineEmbeddingModule.EMBED_MODELS[source](**params) + elif kwargs.get("type") == "rerank": + if source is None: + source = OnlineEmbeddingModule._check_available_source(OnlineEmbeddingModule.RERANK_MODELS) + return OnlineEmbeddingModule.RERANK_MODELS[source](**params) + else: + raise ValueError("Unknown type of online embedding module.") diff --git a/lazyllm/module/onlineEmbedding/qwenEmbed.py b/lazyllm/module/onlineEmbedding/qwenEmbed.py index 4134e7b6..a559221c 100644 --- a/lazyllm/module/onlineEmbedding/qwenEmbed.py +++ b/lazyllm/module/onlineEmbedding/qwenEmbed.py @@ -25,3 +25,37 @@ def _encapsulated_data(self, text: str, **kwargs) -> Dict[str, str]: def _parse_response(self, response: Dict[str, Any]) -> List[float]: return response['output']['embeddings'][0]['embedding'] + + +class QwenReranking(OnlineEmbeddingModuleBase): + + def __init__(self, + embed_url: str = ("https://dashscope.aliyuncs.com/api/v1/services/" + "rerank/text-rerank/text-rerank"), + embed_model_name: str = "gte-rerank", + api_key: str = None, **kwargs): + super().__init__("QWEN", embed_url, api_key or lazyllm.config['qwen_api_key'], embed_model_name) + + @property + def type(self): + return "ONLINE_RERANK" + + def _encapsulated_data(self, query: str, documents: List[str], top_n: int, **kwargs) -> Dict[str, str]: + json_data = { + "input": { + "query": query, + "documents": documents + }, + "parameters": { + "top_n": top_n, + }, + "model": self._embed_model_name + } + if len(kwargs) > 0: + json_data.update(kwargs) + + return json_data + + def _parse_response(self, response: Dict[str, Any]) -> List[float]: + results = response['output']['results'] + return [result["index"] for result in results] diff --git a/lazyllm/tools/rag/rerank.py b/lazyllm/tools/rag/rerank.py index 112d5968..f6490fb8 100644 --- a/lazyllm/tools/rag/rerank.py +++ b/lazyllm/tools/rag/rerank.py @@ -88,11 +88,14 @@ def KeywordFilter( @Reranker.register_reranker() class ModuleReranker(Reranker): - def __init__(self, name: str = "ModuleReranker", target: Optional[str] = None, + def __init__(self, name: str = "ModuleReranker", model: Union[Callable, str] = None, target: Optional[str] = None, output_format: Optional[str] = None, join: Union[bool, str] = False, **kwargs) -> None: super().__init__(name, target, output_format, join, **kwargs) - assert 'model' in self._kwargs - self._reranker = lazyllm.TrainableModule(self._kwargs['model']) + assert model is not None, "Reranker model must be specified as a model name or a callable." + if isinstance(model, str): + self._reranker = lazyllm.TrainableModule(model) + else: + self._reranker = model def forward(self, nodes: List[DocNode], query: str = "") -> List[DocNode]: if not nodes: @@ -100,7 +103,7 @@ def forward(self, nodes: List[DocNode], query: str = "") -> List[DocNode]: docs = [node.get_text(metadata_mode=MetadataMode.EMBED) for node in nodes] top_n = self._kwargs['topk'] if 'topk' in self._kwargs else len(docs) - if self._reranker._deploy_type == lazyllm.deploy.Infinity: + if self._reranker.type == "ONLINE_RERANK" or self._reranker._deploy_type == lazyllm.deploy.Infinity: sorted_indices = self._reranker(query, documents=docs, top_n=top_n) else: inps = {'query': query, 'documents': docs, 'top_n': top_n} diff --git a/lazyllm/tools/rag/web.py b/lazyllm/tools/rag/web.py index 14a72061..0f3abbdf 100644 --- a/lazyllm/tools/rag/web.py +++ b/lazyllm/tools/rag/web.py @@ -219,10 +219,12 @@ def _work(self): self.api_url = self.doc_server._url.rsplit("/", 1)[0] self.web_ui = WebUi(self.api_url) self.demo = self.web_ui.create_ui() - self.url = f'http://0.0.0.0:{port}' + self.url = f'http://127.0.0.1:{port}' + self.broadcast_url = f'http://0.0.0.0:{port}' self.demo.queue().launch(server_name="0.0.0.0", server_port=port, prevent_thread_lock=True) - LOG.success(f'LazyLLM docwebmodule launched successfully: Running on local URL: {self.url}', flush=True) + LOG.success('LazyLLM docwebmodule launched successfully: Running on: ' + f'{self.broadcast_url}, local URL: {self.url}', flush=True) def _get_deploy_tasks(self): return Pipeline(self._work) @@ -256,7 +258,7 @@ def _print_url(self): def _verify_port_access(self, port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - result = s.connect_ex(("localhost", port)) + result = s.connect_ex(("127.0.0.1", port)) return result != 0 def __repr__(self): diff --git a/lazyllm/tools/webpages/webmodule.py b/lazyllm/tools/webpages/webmodule.py index 424dbeca..ec9b3a30 100644 --- a/lazyllm/tools/webpages/webmodule.py +++ b/lazyllm/tools/webpages/webmodule.py @@ -361,10 +361,12 @@ def _work(self): port = self.port assert self._verify_port_access(port), f'port {port} is occupied' - self.url = f'http://0.0.0.0:{port}' + self.url = f'http://127.0.0.1:{port}' + self.broadcast_url = f'http://0.0.0.0:{port}' self.demo.queue().launch(server_name="0.0.0.0", server_port=port, prevent_thread_lock=True) - LOG.success(f'LazyLLM webmodule launched successfully: Running on local URL: {self.url}', flush=True) + LOG.success('LazyLLM webmodule launched successfully: Running on: ' + f'{self.broadcast_url}, local URL: {self.url}', flush=True) def _update(self, *, mode=None, recursive=True): super(__class__, self)._update(mode=mode, recursive=recursive) @@ -398,5 +400,5 @@ def _find_can_use_network_port(self): def _verify_port_access(self, port): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - result = s.connect_ex(('localhost', port)) + result = s.connect_ex(('127.0.0.1', port)) return result != 0 diff --git a/tests/basic_tests/pytest.ini b/tests/basic_tests/pytest.ini index e528c26c..33961bb0 100644 --- a/tests/basic_tests/pytest.ini +++ b/tests/basic_tests/pytest.ini @@ -1,2 +1,4 @@ [pytest] -cache_dir = .pytest-cache \ No newline at end of file +cache_dir = .pytest-cache +markers = + skip_on_win: mark tests to skip on Windows diff --git a/tests/basic_tests/test_module.py b/tests/basic_tests/test_module.py index fcefed7e..37e16658 100644 --- a/tests/basic_tests/test_module.py +++ b/tests/basic_tests/test_module.py @@ -1,4 +1,3 @@ - import time import requests import pytest diff --git a/tests/basic_tests/test_store.py b/tests/basic_tests/test_store.py index d2188286..aed15261 100644 --- a/tests/basic_tests/test_store.py +++ b/tests/basic_tests/test_store.py @@ -1,5 +1,6 @@ import os import shutil +import pytest import tempfile import unittest from unittest.mock import MagicMock @@ -182,6 +183,7 @@ def test_group_others(self): self.assertEqual(self.store.is_group_active("group1"), True) self.assertEqual(self.store.is_group_active("group2"), False) +@pytest.mark.skip_on_win class TestMilvusStore(unittest.TestCase): def setUp(self): self.mock_embed = { diff --git a/tests/charge_tests/pytest.ini b/tests/charge_tests/pytest.ini index e528c26c..4521d113 100644 --- a/tests/charge_tests/pytest.ini +++ b/tests/charge_tests/pytest.ini @@ -1,2 +1,2 @@ [pytest] -cache_dir = .pytest-cache \ No newline at end of file +cache_dir = .pytest-cache diff --git a/tests/charge_tests/test_example.py b/tests/charge_tests/test_example.py index 077ca09b..8033fd67 100644 --- a/tests/charge_tests/test_example.py +++ b/tests/charge_tests/test_example.py @@ -18,7 +18,6 @@ def setup_method(self): self.env_vars = [ 'LAZYLLM_OPENAI_API_KEY', 'LAZYLLM_KIMI_API_KEY', - 'LAZYLLM_QWEN_API_KEY', 'LAZYLLM_SENSENOVA_API_KEY', ] self.webs = []