From 1e50b161e9a9b1c201999c28f97fd2b2fcedc470 Mon Sep 17 00:00:00 2001 From: cpacker Date: Fri, 15 Dec 2023 12:18:09 -0800 Subject: [PATCH 1/2] migrate to using completions endpoint by default --- memgpt/cli/cli_config.py | 2 +- memgpt/local_llm/chat_completion_proxy.py | 4 +++- memgpt/local_llm/constants.py | 1 + memgpt/local_llm/lmstudio/api.py | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/memgpt/cli/cli_config.py b/memgpt/cli/cli_config.py index 787d6bb590..010326e416 100644 --- a/memgpt/cli/cli_config.py +++ b/memgpt/cli/cli_config.py @@ -60,7 +60,7 @@ def configure_llm_endpoint(config: MemGPTConfig): model_endpoint_type = "azure" model_endpoint = get_azure_credentials()["azure_endpoint"] else: # local models - backend_options = ["webui", "webui-legacy", "llamacpp", "koboldcpp", "ollama", "lmstudio", "vllm", "openai"] + backend_options = ["webui", "webui-legacy", "llamacpp", "koboldcpp", "ollama", "lmstudio", "lmstudio-legacy", "vllm", "openai"] default_model_endpoint_type = None if config.model_endpoint_type in backend_options: # set from previous config diff --git a/memgpt/local_llm/chat_completion_proxy.py b/memgpt/local_llm/chat_completion_proxy.py index dae3d9ca07..8a1954f18e 100644 --- a/memgpt/local_llm/chat_completion_proxy.py +++ b/memgpt/local_llm/chat_completion_proxy.py @@ -89,7 +89,9 @@ def get_chat_completion( elif endpoint_type == "webui-legacy": result, usage = get_webui_completion_legacy(endpoint, prompt, context_window, grammar=grammar_name) elif endpoint_type == "lmstudio": - result, usage = get_lmstudio_completion(endpoint, prompt, context_window) + result, usage = get_lmstudio_completion(endpoint, prompt, context_window, api="completions") + elif endpoint_type == "lmstudio-legacy": + result, usage = get_lmstudio_completion(endpoint, prompt, context_window, api="chat") elif endpoint_type == "llamacpp": result, usage = get_llamacpp_completion(endpoint, prompt, context_window, grammar=grammar_name) elif endpoint_type == "koboldcpp": diff --git a/memgpt/local_llm/constants.py b/memgpt/local_llm/constants.py index bcdf830e52..b53b6ead64 100644 --- a/memgpt/local_llm/constants.py +++ b/memgpt/local_llm/constants.py @@ -4,6 +4,7 @@ "koboldcpp": "http://localhost:5001", "llamacpp": "http://localhost:8080", "lmstudio": "http://localhost:1234", + "lmstudio-legacy": "http://localhost:1234", "ollama": "http://localhost:11434", "webui-legacy": "http://localhost:5000", "webui": "http://localhost:5000", diff --git a/memgpt/local_llm/lmstudio/api.py b/memgpt/local_llm/lmstudio/api.py index 183e440e9b..2c6af47e1b 100644 --- a/memgpt/local_llm/lmstudio/api.py +++ b/memgpt/local_llm/lmstudio/api.py @@ -10,7 +10,7 @@ # TODO move to "completions" by default, not "chat" -def get_lmstudio_completion(endpoint, prompt, context_window, settings=SIMPLE, api="chat"): +def get_lmstudio_completion(endpoint, prompt, context_window, settings=SIMPLE, api="completions"): """Based on the example for using LM Studio as a backend from https://github.com/lmstudio-ai/examples/tree/main/Hello%2C%20world%20-%20OpenAI%20python%20client""" from memgpt.utils import printd From 1280d2ffa92bdce8ade249840f65a4553d85995a Mon Sep 17 00:00:00 2001 From: cpacker Date: Fri, 15 Dec 2023 12:22:52 -0800 Subject: [PATCH 2/2] added note about version to docs --- docs/lmstudio.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/lmstudio.md b/docs/lmstudio.md index 2314aec3b2..80f0ba3af7 100644 --- a/docs/lmstudio.md +++ b/docs/lmstudio.md @@ -6,6 +6,12 @@ **Context Overflow Policy = Stop at limit**: If you see "Context Overflow Policy" inside LM Studio's "Tools" panel on the right side (below "Server Model Settings"), set it to **Stop at limit**. The default setting "Keep the system prompt ... truncate middle" will break MemGPT. +!!! note "Update your LM Studio" + + The current `lmstudio` backend will only work if your LM Studio is version 0.2.9 or newer. + + If you are on a version of LM Studio older than 0.2.9 (<= 0.2.8), select `lmstudio-legacy` as your backend type. + image 1. Download [LM Studio](https://lmstudio.ai/) and the model you want to test with