REGRESSION: ChatLiteLLM: ValidationError only when using cache #22389
Comments
dosubot
bot
added
🔌: openai
|
You are the right , this is a bug with |
|
@keenborder786 @eyurtsev Hi! Can I ask what's the status of the fix? I'm forced to repay a lot of queries and it's really a pain. |
|
In the end I coded a workaround cache. It's a class that inherits from BaseCache and acts like InMemoryCache but actually stored a dill.dumps (as json cannot dumps a ChatGeneratin object) of a zlib compressed ChatGeneration. """
source : https://api.python.langchain.com/en/latest/_modules/langchain_community/cache.html#InMemoryCache
This workaround is to solve this: https://github.com/langchain-ai/langchain/issues/22389
Create a caching class that looks like it's just in memory but actually saves to sql
"""
import zlib
import sqlite3
import dill
from pathlib import Path, PosixPath
from typing import Union, Any, Optional
from threading import Lock
from langchain_core.caches import RETURN_VAL_TYPE, BaseCache
class SQLiteCacheFixed(BaseCache):
"""Cache that stores things in memory."""
def __init__(
self,
database_path: Union[str, PosixPath],
) -> None:
self.lock = Lock()
self.database_path = Path(database_path)
if database_path.exists():
self.clear()
else:
conn = sqlite3.connect(self.database_path)
cursor = conn.cursor()
with self.lock:
cursor.execute('''CREATE TABLE IF NOT EXISTS saved_llm_calls
(id INTEGER PRIMARY KEY AUTOINCREMENT,
data TEXT)''')
conn.close()
self._cache = {}
def lookup(self, prompt: str, llm_string: str) -> Optional[RETURN_VAL_TYPE]:
"""Look up based on prompt and llm_string."""
key = (prompt, llm_string)
if key in self._cache:
return self._cache[key]
def update(self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE) -> None:
"""Update cache based on prompt and llm_string."""
key = (prompt, llm_string)
if key in self._cache and self._cache[key] == return_val:
return
self._cache[(prompt, llm_string)] = return_val
data = zlib.compress(dill.dumps({"key": key, "value": return_val}))
conn = sqlite3.connect(self.database_path)
cursor = conn.cursor()
with self.lock:
cursor.execute("INSERT INTO saved_llm_calls (data) VALUES (?)", (data,))
conn.commit()
conn.close()
def clear(self, **kwargs: Any) -> None:
"""Clear cache."""
conn = sqlite3.connect(self.database_path)
cursor = conn.cursor()
with self.lock:
cursor.execute('''CREATE TABLE IF NOT EXISTS saved_llm_calls
(id INTEGER PRIMARY KEY AUTOINCREMENT,
data TEXT)''')
cursor.execute("SELECT data FROM saved_llm_calls")
rows = cursor.fetchall()
conn.commit()
conn.close()
datas = [
dill.loads(
zlib.decompress(row[0])
) for row in rows
]
self._cache = {
d["key"]: d["value"]
for d in datas
}
async def alookup(self, prompt: str, llm_string: str) -> Optional[RETURN_VAL_TYPE]:
"""Look up based on prompt and llm_string."""
return self.lookup(prompt, llm_string)
async def aupdate(
self, prompt: str, llm_string: str, return_val: RETURN_VAL_TYPE
) -> None:
"""Update cache based on prompt and llm_string."""
self.update(prompt, llm_string, return_val)
async def aclear(self, **kwargs: Any) -> None:
"""Clear cache."""
self.clear() |
|
is this issue fixed? |
I don't think so but I made my own workaround. Either from what I mentionned above or the updated version there: https://github.com/thiswillbeyourgithub/wdoc/blob/main/wdoc/utils/customs/fix_llm_caching.py |
|
I have recently started using caches and facing this same issue even in langchain 0.3 version. Can we reopen the issue and fix it? If not, is there any workaround other than writing my custom cache as suggested above? |
|
+1 same issue here |
Checked other resources
Example Code
Error Message and Stack Trace (if applicable)
Description
System Info
System Information
Package Information
Packages not installed (Not Necessarily a Problem)
The following packages were not found:
litellm==1.39.6
The text was updated successfully, but these errors were encountered: