Updates

.gitignore

@@ -10,4 +10,5 @@ tests/*.wav
 .env.local
 test.ipynb
 output.md
-ezlocalai.yml
+ezlocalai.yml
+.env

cuda.Dockerfile

@@ -12,8 +12,9 @@ WORKDIR /app
 ENV HOST=0.0.0.0 \
     CUDA_DOCKER_ARCH=all \
     LLAMA_CUBLAS=1 \ 
-    GGML_CUDA=1
-RUN pip install llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124 --no-cache-dir
+    GGML_CUDA=1 \
+    CMAKE_ARGS="-DGGML_CUDA=on"
+RUN CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python==0.3.1 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124 --no-cache-dir
 RUN git clone https://github.com/Josh-XT/DeepSeek-VL deepseek
 RUN pip install torch==2.3.1+cu121 torchaudio==2.3.1+cu121 --index-url https://download.pytorch.org/whl/cu121
 COPY cuda-requirements.txt .

ezlocalai/VLM.py

@@ -27,7 +27,8 @@ def __init__(self, model="deepseek-ai/deepseek-vl-1.3b-chat"):
                 cache_dir=os.path.join(os.getcwd(), "models"),
             )
             self.vl_gpt = self.vl_gpt.to(torch.bfloat16).cuda().eval()
-        except:
+        except Exception as e:
+            print(f"[VLM] Error: {e}")
             self.vl_chat_processor = None
             self.tokenizer = None
             self.vl_gpt = None

requirements.txt

@@ -25,4 +25,4 @@ optimum
 onnx
 diffusers[torch]
 torchaudio==2.3.1
-llama-cpp-python==0.2.90
+llama-cpp-python==0.3.1