Skip to content

Commit

Permalink
Allow to change temperature of research (#112)
Browse files Browse the repository at this point in the history
kongzii authored Sep 16, 2024
1 parent 69592d0 commit 32bdf95
Showing 7 changed files with 32 additions and 24 deletions.
7 changes: 4 additions & 3 deletions prediction_prophet/app.py
Original file line number Diff line number Diff line change
@@ -20,20 +20,21 @@ def research(
goal: str,
tavily_api_key: SecretStr,
model: str = "gpt-4-0125-preview",
temperature: float = 0.7,
initial_subqueries_limit: int = 20,
subqueries_limit: int = 4,
scrape_content_split_chunk_size: int = 800,
scrape_content_split_chunk_overlap: int = 225,
top_k_per_query: int = 8
) -> str:
with st.status("Generating subqueries"):
queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model)
queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, temperature=temperature)

stringified_queries = '\n- ' + '\n- '.join(queries)
st.write(f"Generated subqueries: {stringified_queries}")

with st.status("Reranking subqueries"):
queries = rerank_subqueries(queries=queries, goal=goal, model=model)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else queries
queries = rerank_subqueries(queries=queries, goal=goal, model=model, temperature=temperature)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else queries

stringified_queries = '\n- ' + '\n- '.join(queries)
st.write(f"Reranked subqueries. Will use top {subqueries_limit}: {stringified_queries}")
@@ -85,7 +86,7 @@ def research(
st.write(f"Found {len(vector_result_texts)} relevant information chunks")

with st.status(f"Preparing report"):
report = prepare_report(goal, vector_result_texts, model=model)
report = prepare_report(goal, vector_result_texts, model=model, temperature=temperature)
st.markdown(report)

return report
21 changes: 12 additions & 9 deletions prediction_prophet/autonolas/research.py
Original file line number Diff line number Diff line change
@@ -1182,6 +1182,8 @@ def make_prediction(
additional_information: str,
temperature: float = 0.7,
engine: str = "gpt-3.5-turbo-0125",
log_probs: bool = False,
top_logprobs: int = 5,
api_key: SecretStr | None = None,
) -> Prediction:
if api_key == None:
@@ -1194,21 +1196,22 @@ def make_prediction(

llm = ChatOpenAI(model=engine, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key))
formatted_messages = prediction_prompt.format_messages(user_prompt=prompt, additional_information=additional_information, timestamp=formatted_time_utc)
generation = llm.generate([formatted_messages], logprobs=True, top_logprobs=5, callbacks=[langfuse_context.get_current_langchain_handler()])
generation = llm.generate([formatted_messages], logprobs=log_probs, top_logprobs=top_logprobs if log_probs else None, callbacks=[langfuse_context.get_current_langchain_handler()])

completion = generation.generations[0][0].text

# Get probability that is based on the token's top logprobs.
decision, probability = None, None
for token in check_not_none(generation.generations[0][0].generation_info)["logprobs"]["content"]:
# Check if the token is a decision token, we prompt the model for it to be there, so it is in 99% of cases.
if token["token"] in ("y", "n"):
decision = token["token"]
probability = math.exp(token["logprob"])
break
if log_probs:
for token in check_not_none(generation.generations[0][0].generation_info)["logprobs"]["content"]:
# Check if the token is a decision token, we prompt the model for it to be there, so it is in 99% of cases.
if token["token"] in ("y", "n"):
decision = token["token"]
probability = math.exp(token["logprob"])
break

if decision is None or probability is None:
raise ValueError(f"No decision found in completion from {engine=}, {completion=}, {formatted_messages=}")
if decision is None or probability is None:
raise ValueError(f"No decision found in completion from {engine=}, {completion=}, {formatted_messages=}")

response: Prediction = json.loads(clean_completion_json(completion))
response["decision"] = decision
9 changes: 6 additions & 3 deletions prediction_prophet/benchmark/agents.py
Original file line number Diff line number Diff line change
@@ -156,7 +156,8 @@ class PredictionProphetAgent(AbstractBenchmarkedAgent):
def __init__(
self,
model: str,
temperature: float = 0.0,
research_temperature: float = 0.7,
prediction_temperature: float = 0.0,
agent_name: str = "prediction_prophet",
use_summaries: bool = False,
use_tavily_raw_content: bool = False,
@@ -166,7 +167,8 @@ def __init__(
):
super().__init__(agent_name=agent_name, max_workers=max_workers)
self.model: str = model
self.temperature = temperature
self.research_temperature = research_temperature
self.prediction_temperature = prediction_temperature
self.use_summaries = use_summaries
self.use_tavily_raw_content = use_tavily_raw_content
self.tavily_storage = tavily_storage
@@ -184,6 +186,7 @@ def research(self, market_question: str) -> Research:
return prophet_research(
goal=market_question,
model=self.model,
temperature=self.research_temperature,
use_summaries=self.use_summaries,
use_tavily_raw_content=self.use_tavily_raw_content,
tavily_storage=self.tavily_storage,
@@ -197,7 +200,7 @@ def predict(self, market_question: str) -> Prediction:
market_question=market_question,
additional_information=research.report,
engine=self.model,
temperature=self.temperature,
temperature=self.prediction_temperature,
)
except ValueError as e:
print(f"Error in PredictionProphet's predict: {e}")
4 changes: 2 additions & 2 deletions prediction_prophet/functions/generate_subqueries.py
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
Limit your searches to {search_limit}.
"""
@observe()
def generate_subqueries(query: str, limit: int, model: str, api_key: SecretStr | None = None) -> list[str]:
def generate_subqueries(query: str, limit: int, model: str, temperature: float, api_key: SecretStr | None = None) -> list[str]:
if limit == 0:
return [query]

@@ -28,7 +28,7 @@ def generate_subqueries(query: str, limit: int, model: str, api_key: SecretStr |

subquery_generation_chain = (
subquery_generation_prompt |
ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
CommaSeparatedListOutputParser()
)

4 changes: 2 additions & 2 deletions prediction_prophet/functions/prepare_report.py
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ def prepare_summary(goal: str, content: str, model: str, api_key: SecretStr | No


@observe()
def prepare_report(goal: str, scraped: list[str], model: str, api_key: SecretStr | None = None) -> str:
def prepare_report(goal: str, scraped: list[str], model: str, temperature: float, api_key: SecretStr | None = None) -> str:
if api_key == None:
api_key = secret_str_from_env("OPENAI_API_KEY")

@@ -72,7 +72,7 @@ def prepare_report(goal: str, scraped: list[str], model: str, api_key: SecretStr

research_evaluation_chain = (
evaluation_prompt |
ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
StrOutputParser()
)

4 changes: 2 additions & 2 deletions prediction_prophet/functions/rerank_subqueries.py
Original file line number Diff line number Diff line change
@@ -18,15 +18,15 @@
Queries: {queries}
"""
@observe()
def rerank_subqueries(queries: list[str], goal: str, model: str, api_key: SecretStr | None = None) -> list[str]:
def rerank_subqueries(queries: list[str], goal: str, model: str, temperature: float, api_key: SecretStr | None = None) -> list[str]:
if api_key == None:
api_key = secret_str_from_env("OPENAI_API_KEY")

rerank_results_prompt = ChatPromptTemplate.from_template(template=rerank_queries_template)

rerank_results_chain = (
rerank_results_prompt |
ChatOpenAI(model=model, api_key=secretstr_to_v1_secretstr(api_key)) |
ChatOpenAI(model=model, temperature=temperature, api_key=secretstr_to_v1_secretstr(api_key)) |
StrOutputParser()
)

7 changes: 4 additions & 3 deletions prediction_prophet/functions/research.py
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ def research(
goal: str,
use_summaries: bool,
model: str = "gpt-4-0125-preview",
temperature: float = 0.7,
initial_subqueries_limit: int = 20,
subqueries_limit: int = 4,
max_results_per_search: int = 5,
@@ -53,13 +54,13 @@ def research(
)

logger.info("Started subqueries generation")
all_queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, api_key=openai_api_key)
all_queries = generate_subqueries(query=goal, limit=initial_subqueries_limit, model=model, temperature=temperature, api_key=openai_api_key)

stringified_queries = '\n- ' + '\n- '.join(all_queries)
logger.info(f"Generated subqueries: {stringified_queries}")

logger.info("Started subqueries reranking")
queries = rerank_subqueries(queries=all_queries, goal=goal, model=model, api_key=openai_api_key)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else all_queries
queries = rerank_subqueries(queries=all_queries, goal=goal, model=model, temperature=temperature, api_key=openai_api_key)[:subqueries_limit] if initial_subqueries_limit > subqueries_limit else all_queries

stringified_queries = '\n- ' + '\n- '.join(queries)
logger.info(f"Reranked subqueries. Will use top {subqueries_limit}: {stringified_queries}")
@@ -145,7 +146,7 @@ def research(
logger.info(f"Information summarized")

logger.info(f"Started preparing report")
report = prepare_report(goal, vector_result_texts, model=model, api_key=openai_api_key)
report = prepare_report(goal, vector_result_texts, model=model, temperature=temperature, api_key=openai_api_key)
logger.info(f"Report prepared")
logger.info(report)

0 comments on commit 32bdf95

Please sign in to comment.