Merge pull request #31 from DevXT-LLC/vlm-progress

VLM progress

Pipes.py

@@ -95,9 +95,9 @@ async def get_response(self, data, completion_type="chat"):
                 else data["prompt"]
             )
             response_text = (
-                response["messages"][1]["content"]
-                if completion_type == "chat"
-                else response["choices"][0]["text"]
+                response["choices"][0]["text"]
+                if completion_type != "chat"
+                else response["choices"][0]["message"]["content"]
             )
             img_gen_prompt = f"Users message: {user_message} \nAssistant response: {response_text} \n\n**The assistant is acting as a decision maker for creating stable diffusion images and only responds with a concise YES or NO answer on if it would make sense to generate an image based on the users message. No other explanation is needed!**\nShould an image be created to accompany the assistant response?\nAssistant: "
             logging.info(f"[IMG] Decision maker prompt: {img_gen_prompt}")
@@ -107,7 +107,7 @@ async def get_response(self, data, completion_type="chat"):
                 temperature=data["temperature"],
                 top_p=data["top_p"],
             )
-            create_img = str(create_img["messages"][1]["content"]).lower()
+            create_img = str(create_img["choices"][0]["message"]["content"]).lower()
             logging.info(f"[IMG] Decision maker response: {create_img}")
             if "yes" in create_img or "es," in create_img:
 
@@ -123,11 +123,8 @@ async def get_response(self, data, completion_type="chat"):
                     temperature=data["temperature"],
                     top_p=data["top_p"],
                 )
-                logging.info(
-                    f"[IMG] Image generation prompt: {image_generation_prompt}"
-                )
                 image_generation_prompt = str(
-                    image_generation_prompt["messages"][1]["content"]
+                    image_generation_prompt["choices"][0]["message"]["content"]
                 )
                 logging.info(
                     f"[IMG] Image generation response: {image_generation_prompt}"
@@ -142,26 +139,27 @@ async def get_response(self, data, completion_type="chat"):
                 )
         audio_response = None
         if "voice" in data:
-            if completion_type == "chat":
-                text_response = response["messages"][1]["content"]
-            else:
-                text_response = response["choices"][0]["text"]
+            text_response = (
+                response["choices"][0]["text"]
+                if completion_type != "chat"
+                else response["choices"][0]["message"]["content"]
+            )
             language = data["language"] if "language" in data else "en"
             audio_response = await self.ctts.generate(
                 text=text_response,
                 voice=data["voice"],
                 language=language,
                 local_uri=self.local_uri,
             )
-            if completion_type == "chat":
-                response["messages"][1][
+            if completion_type != "chat":
+                response["choices"][0]["text"] = f"{text_response}\n{audio_response}"
+            else:
+                response["choices"][0]["message"][
                     "content"
                 ] = f"{text_response}\n{audio_response}"
-            else:
-                response["choices"][0]["text"] = f"{text_response}\n{audio_response}"
         if generated_image:
-            if completion_type == "chat":
-                response["messages"][1]["content"] += f"\n\n{generated_image}"
-            else:
+            if completion_type != "chat":
                 response["choices"][0]["text"] += f"\n\n{generated_image}"
+            else:
+                response["choices"][0]["message"]["content"] += f"\n\n{generated_image}"
         return response, audio_response

ezlocalai/LLM.py

@@ -16,12 +16,9 @@
 
 def get_vision_models():
     return [
-        {"mistral-vlm-7b": "JoshXT/mistral-vlm-7b"},
         {"bakllava-1-7b": "mys/ggml_bakllava-1"},
         {"llava-v1.5-7b": "mys/ggml_llava-v1.5-7b"},
         {"llava-v1.5-13b": "mys/ggml_llava-v1.5-13b"},
-        {"yi-vl-6b": "cmp-nct/Yi-VL-6B-GGUF"},
-        {"sharegpt4v-7b": "cmp-nct/ShareGPT4V-7B-quant-gguf"},
     ]
 
 
@@ -157,17 +154,6 @@ def download_llm(model_name="", models_dir="models"):
             clip_url = (
                 f"https://huggingface.co/{model_url}/resolve/main/mmproj-model-f16.gguf"
             )
-        elif model_url == "cmp-nct/Yi-VL-6B-GGUF":
-            url = (
-                f"https://huggingface.co/{model_url}/resolve/main/ggml-model-Q5_K.gguf"
-            )
-            clip_url = f"https://huggingface.co/{model_url}/resolve/main/mmproj-model-f16-q6_k.gguf"
-        elif model_url == "cmp-nct/ShareGPT4V-7B-quant-gguf":
-            url = f"https://huggingface.co/cmp-nct/ShareGPT4V-7B-quant-gguf/resolve/main/ggml-model-q5_k?"
-            clip_url = f"https://huggingface.co/cmp-nct/ShareGPT4V-7B-quant-gguf/resolve/main/mmproj-model-f16.gguf"
-        elif model_url == "JoshXT/mistral-vlm-7b":
-            url = f"https://huggingface.co/JoshXT/mistral-vlm-7b/resolve/main/mistral-vlm-7b.Q5_K_M.gguf"
-            clip_url = f"https://huggingface.co/JoshXT/mistral-vlm-7b/resolve/main/mmproj-model-f16.gguf"
         else:
             url = (
                 (
@@ -303,6 +289,7 @@ def __init__(
         )
         self.params = {}
         self.model_name = DEFAULT_MODEL
+        chat_handler = None
         if self.model_name != "":
             self.params["model_path"] = download_llm(
                 model_name=self.model_name, models_dir=models_dir
@@ -319,7 +306,7 @@ def __init__(
                     model_name=self.model_name, models_dir=models_dir
                 )
                 if clip_path != "":
-                    self.params["chat_handler"] = llama_chat_format.Llava15ChatHandler(
+                    chat_handler = llama_chat_format.Llava15ChatHandler(
                         clip_model_path=clip_path, verbose=True
                     )
         else:
@@ -372,7 +359,8 @@ def __init__(
         else:
             self.params["n_batch"] = 1024
         if self.model_name != "":
-            self.lcpp = Llama(**self.params, embedding=True)
+            logging.info(f"[LLM] Parameters: {self.params}")
+            self.lcpp = Llama(**self.params, embedding=True, chat_handler=chat_handler)
         else:
             self.lcpp = None
         self.model_list = get_models()
@@ -395,6 +383,28 @@ def generate(
         system_message=None,
         **kwargs,
     ):
+        messages = [
+            {
+                "role": "system",
+                "content": (
+                    self.system_message if system_message is None else system_message
+                ),
+            }
+        ]
+        if isinstance(prompt, list):
+            messages.append(
+                {
+                    "role": "user",
+                    "content": prompt,
+                },
+            )
+        else:
+            messages.append(
+                {
+                    "role": "user",
+                    "content": prompt,
+                },
+            )
         if format_prompt:
             formatted_prompt = custom_format_prompt(
                 prompt=prompt,
@@ -467,6 +477,7 @@ def chat(self, messages, **kwargs):
         message = clean(
             message=data["choices"][0]["text"], stop_tokens=self.params["stop"]
         )
+        data["choices"][0]["message"] = {"content": message}
         messages.append({"role": "assistant", "content": message})
         data["messages"] = messages
         return data

setup.py

@@ -10,7 +10,7 @@
 
 setup(
     name="ezlocalai",
-    version="0.1.4",
+    version="0.1.5",
     description="ezlocalai is an easy to set up local artificial intelligence server with OpenAI Style Endpoints.",
     long_description=long_description,
     long_description_content_type="text/markdown",