Update PMAT and necessary changes for new langchain and small changes…

… to be compatible with PMA (#103)

prediction_prophet/autonolas/research.py

@@ -36,6 +36,7 @@
 from prediction_prophet.functions.cache import persistent_inmemory_cache
 from prediction_prophet.functions.parallelism import par_map
 from pydantic.types import SecretStr
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 load_dotenv()
 
@@ -1186,7 +1187,7 @@ def make_prediction(
 
     prediction_prompt = ChatPromptTemplate.from_template(template=PREDICTION_PROMPT)
 
-    llm = ChatOpenAI(model=engine, temperature=temperature, api_key=api_key.get_secret_value() if api_key else None)
+    llm = ChatOpenAI(model=engine, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key))
     formatted_messages = prediction_prompt.format_messages(user_prompt=prompt, additional_information=additional_information, timestamp=formatted_time_utc)
     generation = llm.generate([formatted_messages], logprobs=True, top_logprobs=5)
 

prediction_prophet/functions/create_embeddings_from_results.py

@@ -14,13 +14,14 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from pydantic.types import SecretStr
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 def create_embeddings_from_results(results: list[WebScrapeResult], text_splitter: RecursiveCharacterTextSplitter, api_key: SecretStr | None = None) -> Chroma:
     if api_key == None:
         api_key = secret_str_from_env("OPENAI_API_KEY")
 
-    collection = Chroma(embedding_function=OpenAIEmbeddings(api_key=api_key.get_secret_value() if api_key else None))
+    collection = Chroma(embedding_function=OpenAIEmbeddings(api_key=secretstr_to_v1_secretstr(api_key)))
     texts = []
     metadatas = []
 

prediction_prophet/functions/debate_prediction.py

@@ -10,6 +10,7 @@
 from langchain_openai import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 PREDICTION_PROMPT = """
@@ -92,7 +93,7 @@ def make_debated_prediction(prompt: str, additional_information: str, api_key: S
 
     prediction_chain = (
         prediction_prompt |
-        ChatOpenAI(model="gpt-4-0125-preview", api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model="gpt-4-0125-preview", api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 
@@ -127,7 +128,7 @@ def make_debated_prediction(prompt: str, additional_information: str, api_key: S
 
     extraction_chain = (
         extraction_prompt |
-        ChatOpenAI(model="gpt-3.5-turbo-0125", api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model="gpt-3.5-turbo-0125", api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 

prediction_prophet/functions/evaluate_question.py

@@ -5,6 +5,7 @@
 from prediction_prophet.functions.cache import persistent_inmemory_cache
 from pydantic.types import SecretStr
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 # I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
@@ -50,7 +51,7 @@ def is_predictable(
 
     if api_key == None:
         api_key = secret_str_from_env("OPENAI_API_KEY")
-    llm = ChatOpenAI(model=engine, temperature=0.0, api_key=api_key.get_secret_value() if api_key else None)
+    llm = ChatOpenAI(model=engine, temperature=0.0, api_key=secretstr_to_v1_secretstr(api_key))
 
     prompt = ChatPromptTemplate.from_template(template=prompt_template)
     messages = prompt.format_messages(question=question)

prediction_prophet/functions/generate_subqueries.py

@@ -4,6 +4,7 @@
 from langchain.prompts import ChatPromptTemplate
 from pydantic.types import SecretStr
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 subquery_generation_template = """
@@ -22,7 +23,7 @@ def generate_subqueries(query: str, limit: int, model: str, api_key: SecretStr |
 
     subquery_generation_chain = (
         subquery_generation_prompt |
-        ChatOpenAI(model=model, api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
         CommaSeparatedListOutputParser()
     )
 

prediction_prophet/functions/is_predictable_and_binary.py

@@ -5,7 +5,7 @@
 from prediction_prophet.functions.cache import persistent_inmemory_cache
 from pydantic.types import SecretStr
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
-
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 # I tried to make it return a JSON, but it didn't work well in combo with asking it to do chain of thought.
@@ -52,7 +52,7 @@ def is_predictable_and_binary(
 
     if api_key == None:
         api_key = secret_str_from_env("OPENAI_API_KEY")
-    llm = ChatOpenAI(model=engine, temperature=0.0, api_key=api_key.get_secret_value() if api_key else None)
+    llm = ChatOpenAI(model=engine, temperature=0.0, api_key=secretstr_to_v1_secretstr(api_key))
 
     prompt = ChatPromptTemplate.from_template(template=prompt_template)
     messages = prompt.format_messages(question=question)

prediction_prophet/functions/prepare_report.py

@@ -8,6 +8,7 @@
 from prediction_prophet.functions.utils import trim_to_n_tokens
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
 from pydantic.types import SecretStr
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 
 @persistent_inmemory_cache
@@ -28,7 +29,7 @@ def prepare_summary(goal: str, content: str, model: str, api_key: SecretStr | No
 
     research_evaluation_chain = (
         evaluation_prompt |
-        ChatOpenAI(model=model, api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 
@@ -69,7 +70,7 @@ def prepare_report(goal: str, scraped: list[str], model: str, api_key: SecretStr
 
     research_evaluation_chain = (
         evaluation_prompt |
-        ChatOpenAI(model=model, api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 

prediction_prophet/functions/rerank_subqueries.py

@@ -4,6 +4,7 @@
 from langchain.schema.output_parser import StrOutputParser
 from pydantic.types import SecretStr
 from prediction_market_agent_tooling.tools.utils import secret_str_from_env
+from prediction_market_agent_tooling.gtypes import secretstr_to_v1_secretstr
 
 rerank_queries_template = """
 I will present you with a list of queries to search the web for, for answers to the question: {goal}.
@@ -23,7 +24,7 @@ def rerank_subqueries(queries: list[str], goal: str, model: str, api_key: Secret
 
     rerank_results_chain = (
         rerank_results_prompt |
-        ChatOpenAI(model=model, api_key=api_key.get_secret_value() if api_key else None) |
+        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 

prediction_prophet/functions/research.py

@@ -1,4 +1,5 @@
 import logging
+import typing as t
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from prediction_prophet.functions.create_embeddings_from_results import create_embeddings_from_results
@@ -10,6 +11,9 @@
 from prediction_prophet.functions.search import search
 from pydantic.types import SecretStr
 
+if t.TYPE_CHECKING:
+    from loguru import Logger
+
 def research(
     goal: str,
     use_summaries: bool,
@@ -22,7 +26,7 @@ def research(
     use_tavily_raw_content: bool = False,
     openai_api_key: SecretStr | None = None,
     tavily_api_key: SecretStr | None = None,
-    logger: logging.Logger = logging.getLogger()
+    logger: t.Union[logging.Logger, "Logger"] = logging.getLogger()
 ) -> str:
     logger.info("Started subqueries generation")
     queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, api_key=openai_api_key)

prediction_prophet/functions/summarize.py

@@ -3,7 +3,9 @@
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.chains.llm import LLMChain
-from langchain.chains.summarize import ReduceDocumentsChain, StuffDocumentsChain, MapReduceDocumentsChain
+from langchain.chains.combine_documents.reduce import ReduceDocumentsChain
+from langchain.chains.combine_documents.stuff import  StuffDocumentsChain
+from langchain.chains.combine_documents.map_reduce import MapReduceDocumentsChain
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 def summarize(objective: str, content: str) -> str:

pyproject.toml

@@ -7,22 +7,21 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "~3.10"
-langchain = "^0.1.9"
+langchain = "^0.2.6"
 beautifulsoup4 = "^4.12.3"
 click = "^8.0.2"
 markdownify = "0.11.6"
 pandas = "2.1.1"
-pytest = "^8.0.0"
 openai = "^1.10.0"
-chromadb = "0.4.22"
+chromadb = "0.4.24"
 spacy = "3.7.5"
 en_core_web_md = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.7.1/en_core_web_md-3.7.1-py3-none-any.whl" }
 google-api-python-client = "2.95.0"
-tiktoken = "0.5.2"
+tiktoken = "^0.7.0"
 tavily-python = "^0.3.0"
 tabulate = "^0.9.0"
 pysqlite3-binary = {version="^0.5.2.post3", markers = "sys_platform == 'linux'"}
-langchain-openai = "^0.0.5"
+langchain-openai = "^0.1.0"
 tenacity = "^8.2.3"
 joblib = "^1.3.2"
 streamlit = "^1.30.0"
@@ -31,15 +30,17 @@ scipy = "^1.12.0"
 scikit-learn = "^1.4.0"
 typer = ">=0.9.0,<1.0.0"
 types-requests = "^2.31.0.20240125"
-types-python-dateutil = "^2.8.19.20240106"
-prediction-market-agent-tooling = { version = "^0.40.0", extras = ["langchain", "google"] }
-langchain-community = "^0.0.32"
+types-python-dateutil = "^2.9.0"
+prediction-market-agent-tooling = { version = ">=0.43.0,<1", extras = ["langchain", "google"] }
+langchain-community = "^0.2.6"
 memory-profiler = "^0.61.0"
 matplotlib = "^3.8.3"
 pyautogen = "^0.2.19"
+python-dateutil = "^2.9.0.post0"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.8.0"
+pytest = "^8.0.0"
 
 [tool.poetry.scripts]
 research= "prediction_prophet.main:research"