Allow to change temperature of research

prediction_prophet/app.py

@@ -20,20 +20,21 @@ def research(
     goal: str,
     tavily_api_key: SecretStr,
     model: str = "gpt-4-0125-preview",
+    temperature: float = 0.7, 
     initial_subqueries_limit: int = 20,
     subqueries_limit: int = 4,
     scrape_content_split_chunk_size: int = 800,
     scrape_content_split_chunk_overlap: int = 225,
     top_k_per_query: int = 8
 ) -> str:
     with st.status("Generating subqueries"):
-        queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model)
+        queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, temperature=temperature)
 
         stringified_queries = '\n- ' + '\n- '.join(queries)
         st.write(f"Generated subqueries: {stringified_queries}")
 
     with st.status("Reranking subqueries"):
-        queries = rerank_subqueries(queries=queries, goal=goal, model=model)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else queries
+        queries = rerank_subqueries(queries=queries, goal=goal, model=model, temperature=temperature)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else queries
 
         stringified_queries = '\n- ' + '\n- '.join(queries)
         st.write(f"Reranked subqueries. Will use top {subqueries_limit}: {stringified_queries}")
@@ -85,7 +86,7 @@ def research(
         st.write(f"Found {len(vector_result_texts)} relevant information chunks")
 
     with st.status(f"Preparing report"):
-        report = prepare_report(goal, vector_result_texts, model=model)
+        report = prepare_report(goal, vector_result_texts, model=model, temperature=temperature)
         st.markdown(report)
 
     return report

prediction_prophet/autonolas/research.py

@@ -1182,6 +1182,8 @@ def make_prediction(
     additional_information: str,
     temperature: float = 0.7,
     engine: str = "gpt-3.5-turbo-0125",
+    log_probs: bool = False,
+    top_logprobs: int = 5,
     api_key: SecretStr | None = None,
 ) -> Prediction:
     if api_key == None:
@@ -1194,21 +1196,22 @@ def make_prediction(
 
     llm = ChatOpenAI(model=engine, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key))
     formatted_messages = prediction_prompt.format_messages(user_prompt=prompt, additional_information=additional_information, timestamp=formatted_time_utc)
-    generation = llm.generate([formatted_messages], logprobs=True, top_logprobs=5, callbacks=[langfuse_context.get_current_langchain_handler()])
+    generation = llm.generate([formatted_messages], logprobs=log_probs, top_logprobs=top_logprobs, callbacks=[langfuse_context.get_current_langchain_handler()])
 
     completion = generation.generations[0][0].text
 
     # Get probability that is based on the token's top logprobs.
     decision, probability = None, None
-    for token in check_not_none(generation.generations[0][0].generation_info)["logprobs"]["content"]:  
-        # Check if the token is a decision token, we prompt the model for it to be there, so it is in 99% of cases.
-        if token["token"] in ("y", "n"):
-            decision = token["token"]
-            probability = math.exp(token["logprob"])
-            break
+    if log_probs:
+        for token in check_not_none(generation.generations[0][0].generation_info)["logprobs"]["content"]:  
+            # Check if the token is a decision token, we prompt the model for it to be there, so it is in 99% of cases.
+            if token["token"] in ("y", "n"):
+                decision = token["token"]
+                probability = math.exp(token["logprob"])
+                break
 
-    if decision is None or probability is None:
-        raise ValueError(f"No decision found in completion from {engine=}, {completion=}, {formatted_messages=}")
+        if decision is None or probability is None:
+            raise ValueError(f"No decision found in completion from {engine=}, {completion=}, {formatted_messages=}")
 
     response: Prediction = json.loads(clean_completion_json(completion))
     response["decision"] = decision

prediction_prophet/benchmark/agents.py

@@ -156,7 +156,8 @@ class PredictionProphetAgent(AbstractBenchmarkedAgent):
     def __init__(
         self,
         model: str,
-        temperature: float = 0.0,
+        research_temperature: float = 0.7,
+        prediction_temperature: float = 0.0,
         agent_name: str = "prediction_prophet",
         use_summaries: bool = False,
         use_tavily_raw_content: bool = False,
@@ -166,7 +167,8 @@ def __init__(
     ):
         super().__init__(agent_name=agent_name, max_workers=max_workers)
         self.model: str = model
-        self.temperature = temperature
+        self.research_temperature = research_temperature
+        self.prediction_temperature = prediction_temperature
         self.use_summaries = use_summaries
         self.use_tavily_raw_content = use_tavily_raw_content
         self.tavily_storage = tavily_storage
@@ -184,6 +186,7 @@ def research(self, market_question: str) -> Research:
         return prophet_research(
             goal=market_question,
             model=self.model,
+            temperature=self.research_temperature,
             use_summaries=self.use_summaries,
             use_tavily_raw_content=self.use_tavily_raw_content,
             tavily_storage=self.tavily_storage,
@@ -197,7 +200,7 @@ def predict(self, market_question: str) -> Prediction:
                 market_question=market_question,
                 additional_information=research.report,
                 engine=self.model,
-                temperature=self.temperature,
+                temperature=self.prediction_temperature,
         )
         except ValueError as e:
             print(f"Error in PredictionProphet's predict: {e}")

prediction_prophet/functions/generate_subqueries.py

@@ -17,7 +17,7 @@
 Limit your searches to {search_limit}.
 """
 @observe()
-def generate_subqueries(query: str, limit: int, model: str, api_key: SecretStr | None = None) -> list[str]:
+def generate_subqueries(query: str, limit: int, model: str, temperature: float, api_key: SecretStr | None = None) -> list[str]:
     if limit == 0:
         return [query]
 
@@ -28,7 +28,7 @@ def generate_subqueries(query: str, limit: int, model: str, api_key: SecretStr |
 
     subquery_generation_chain = (
         subquery_generation_prompt |
-        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
+        ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
         CommaSeparatedListOutputParser()
     )
 

prediction_prophet/functions/prepare_report.py

@@ -43,7 +43,7 @@ def prepare_summary(goal: str, content: str, model: str, api_key: SecretStr | No
 
 
 @observe()
-def prepare_report(goal: str, scraped: list[str], model: str, api_key: SecretStr | None = None) -> str:
+def prepare_report(goal: str, scraped: list[str], model: str, temperature: float, api_key: SecretStr | None = None) -> str:
     if api_key == None:
         api_key = secret_str_from_env("OPENAI_API_KEY")
 
@@ -72,7 +72,7 @@ def prepare_report(goal: str, scraped: list[str], model: str, api_key: SecretStr
 
     research_evaluation_chain = (
         evaluation_prompt |
-        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
+        ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 

prediction_prophet/functions/rerank_subqueries.py

@@ -18,15 +18,15 @@
 Queries: {queries}
 """
 @observe()
-def rerank_subqueries(queries: list[str], goal: str, model: str, api_key: SecretStr | None = None) -> list[str]:
+def rerank_subqueries(queries: list[str], goal: str, model: str, temperature: float, api_key: SecretStr | None = None) -> list[str]:
     if api_key == None:
         api_key = secret_str_from_env("OPENAI_API_KEY")
 
     rerank_results_prompt = ChatPromptTemplate.from_template(template=rerank_queries_template)
 
     rerank_results_chain = (
         rerank_results_prompt |
-        ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
+        ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
         StrOutputParser()
     )
 

prediction_prophet/functions/research.py

@@ -31,6 +31,7 @@ def research(
     goal: str,
     use_summaries: bool,
     model: str = "gpt-4-0125-preview",
+    temperature: float = 0.7,
     initial_subqueries_limit: int = 20,
     subqueries_limit: int = 4,
     max_results_per_search: int = 5,
@@ -53,13 +54,13 @@ def research(
         )
 
     logger.info("Started subqueries generation")
-    all_queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, api_key=openai_api_key)
+    all_queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, temperature=temperature, api_key=openai_api_key)
 
     stringified_queries = '\n- ' + '\n- '.join(all_queries)
     logger.info(f"Generated subqueries: {stringified_queries}")
 
     logger.info("Started subqueries reranking")
-    queries = rerank_subqueries(queries=all_queries, goal=goal, model=model, api_key=openai_api_key)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else all_queries
+    queries = rerank_subqueries(queries=all_queries, goal=goal, model=model, temperature=temperature, api_key=openai_api_key)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else all_queries
 
     stringified_queries = '\n- ' + '\n- '.join(queries)
     logger.info(f"Reranked subqueries. Will use top {subqueries_limit}: {stringified_queries}")
@@ -145,7 +146,7 @@ def research(
         logger.info(f"Information summarized")
 
     logger.info(f"Started preparing report")
-    report = prepare_report(goal, vector_result_texts, model=model, api_key=openai_api_key)
+    report = prepare_report(goal, vector_result_texts, model=model, temperature=temperature, api_key=openai_api_key)
     logger.info(f"Report prepared")
     logger.info(report)