Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

oai assistant multiple actions #13068

Merged
merged 5 commits into from
Nov 8, 2023
Merged

oai assistant multiple actions #13068

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
32a16b5
wip
baskaryan Nov 8, 2023
e1c4035
nit
baskaryan Nov 8, 2023
29b49ee
comment
baskaryan Nov 8, 2023
e8ea72c
example
baskaryan Nov 8, 2023
f7495e8
nb
baskaryan Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 44 additions & 73 deletions cookbook/openai_v1_cookbook.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
"metadata": {},
"outputs": [],
"source": [
"!pip install -U openai \"langchain>=0.0.331rc1\" langchain-experimental"
"!pip install -U openai \"langchain>=0.0.331rc2\" langchain-experimental"
]
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 1,
"id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136",
"metadata": {},
"outputs": [],
Expand All @@ -43,17 +43,17 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 2,
"id": "1c8c3965-d3c9-4186-b5f3-5e67855ef916",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content='This image appears to be a diagram representing the architecture or components of a software system or platform named \"LangChain.\" The diagram is organized into various blocks that represent different layers or aspects of the system. Here\\'s a breakdown of the labeled parts:\\n\\n1. **LangSmith**: This seems to be a tool or component related to testing, evaluation, monitoring, feedback, and annotation, as well as debugging.\\n\\n2. **LangServe**: This component chains as REST API, suggesting it may serve as an interface for the system\\'s functionality to be accessed over the web using RESTful API calls.\\n\\n3. **Templates**: Reference applications are mentioned here, which might be pre-built applications or use cases provided as starting points for users of the system.\\n\\n4. **Chains, agents, agent executors**: This part refers to common application logic, indicating modular components that can be combined or chained together to create complex workflows.\\n\\n5. **Model I/O**: This includes elements such as prompt, example selector, model, and output parser, which may deal with the input and output processes of a machine learning model or similar.\\n\\n6. **Retrieval**: Components under this section, like document loader, text splitter, embedding model, vector store, and retriever, imply functionality related to')"
"AIMessage(content='The image appears to be a diagram representing the architecture or components of a software system or framework related to language processing, possibly named LangChain or associated with a project or product called LangChain, based on the prominent appearance of that term. The diagram is organized into several layers or aspects, each containing various elements or modules:\\n\\n1. **Protocol**: This may be the foundational layer, which includes \"LCEL\" and terms like parallelization, fallbacks, tracing, batching, streaming, async, and composition. These seem related to communication and execution protocols for the system.\\n\\n2. **Integrations Components**: This layer includes \"Model I/O\" with elements such as the model, output parser, prompt, and example selector. It also has a \"Retrieval\" section with a document loader, retriever, embedding model, vector store, and text splitter. Lastly, there\\'s an \"Agent Tooling\" section. These components likely deal with the interaction with external data, models, and tools.\\n\\n3. **Application**: The application layer features \"LangChain\" with chains, agents, agent executors, and common application logic. This suggests that the system uses a modular approach with chains and agents to process language tasks.\\n\\n4. **Deployment**: This contains \"Lang')"
]
},
"execution_count": 26,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -103,27 +103,27 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 1,
"id": "a9064bbe-d9f7-4a29-a7b3-73933b3197e7",
"metadata": {},
"outputs": [],
"source": [
"from langchain_experimental.openai_assistant import OpenAIAssistantRunnable\n"
"from langchain_experimental.openai_assistant import OpenAIAssistantRunnable"
]
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 2,
"id": "7a20a008-49ac-46d2-aa26-b270118af5ea",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[ThreadMessage(id='msg_RGOsJ2RBYp79rILrZ0NsAX68', assistant_id='asst_9Xb1ZgefoAbp2V5ddRlDGisy', content=[MessageContentText(text=Text(annotations=[], value='\\\\( 10 - 4^{2.7} \\\\) is approximately -32.2243.'), type='text')], created_at=1699385426, file_ids=[], metadata={}, object='thread.message', role='assistant', run_id='run_E2PRoP04ryly4p5ds5ek2qxs', thread_id='thread_pu9CpsYIWWZtxemZxpbYfhm4')]"
"[ThreadMessage(id='msg_g9OJv0rpPgnc3mHmocFv7OVd', assistant_id='asst_hTwZeNMMphxzSOqJ01uBMsJI', content=[MessageContentText(text=Text(annotations=[], value='The result of \\\\(10 - 4^{2.7}\\\\) is approximately \\\\(-32.224\\\\).'), type='text')], created_at=1699460600, file_ids=[], metadata={}, object='thread.message', role='assistant', run_id='run_nBIT7SiAwtUfSCTrQNSPLOfe', thread_id='thread_14n4GgXwxgNL0s30WJW5F6p0')]"
]
},
"execution_count": 5,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -151,26 +151,36 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"id": "ee4cc355-f2d6-4c51-bcf7-f502868357d3",
"metadata": {},
"outputs": [],
"source": [
"!pip install e2b duckduckgo-search"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "48681ac7-b267-48d4-972c-8a7df8393a21",
"metadata": {},
"outputs": [],
"source": [
"from langchain.tools import E2BDataAnalysisTool\n",
"from langchain.tools import E2BDataAnalysisTool, DuckDuckGoSearchRun\n",
"\n",
"tools = [E2BDataAnalysisTool(api_key=\"...\")]"
"tools = [E2BDataAnalysisTool(api_key=\"...\"), DuckDuckGoSearchRun()]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 4,
"id": "1c01dd79-dd3e-4509-a2e2-009a7f99f16a",
"metadata": {},
"outputs": [],
"source": [
"agent = OpenAIAssistantRunnable.create_assistant(\n",
" name=\"langchain assistant e2b tool\",\n",
" instructions=\"You are a personal math tutor. Write and run code to answer math questions.\",\n",
" instructions=\"You are a personal math tutor. Write and run code to answer math questions. You can also search the internet.\",\n",
" tools=tools,\n",
" model=\"gpt-4-1106-preview\",\n",
" as_agent=True\n",
Expand All @@ -187,18 +197,18 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 5,
"id": "1f137f94-801f-4766-9ff5-2de9df5e8079",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'content': \"What's 10 - 4 raised to the 2.7\",\n",
" 'output': 'The result of \\\\(10 - 4\\\\) raised to the power of \\\\(2.7\\\\) is approximately \\\\(126.19\\\\).'}"
"{'content': \"What's the weather in SF today divided by 2.7\",\n",
" 'output': \"The weather in San Francisco today is reported to have temperatures as high as 66 °F. To get the temperature divided by 2.7, we will calculate that:\\n\\n66 °F / 2.7 = 24.44 °F\\n\\nSo, when the high temperature of 66 °F is divided by 2.7, the result is approximately 24.44 °F. Please note that this doesn't have a meteorological meaning; it's purely a mathematical operation based on the given temperature.\"}"
]
},
"execution_count": 9,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -207,7 +217,7 @@
"from langchain.agents import AgentExecutor\n",
"\n",
"agent_executor = AgentExecutor(agent=agent, tools=tools)\n",
"agent_executor.invoke({\"content\": \"What's 10 - 4 raised to the 2.7\"})"
"agent_executor.invoke({\"content\": \"What's the weather in SF today divided by 2.7\"})"
]
},
{
Expand All @@ -220,7 +230,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 6,
"id": "c0475fa7-b6c1-4331-b8e2-55407466c724",
"metadata": {},
"outputs": [],
Expand All @@ -236,7 +246,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 7,
"id": "b76cb669-6aba-4827-868f-00aa960026f2",
"metadata": {},
"outputs": [],
Expand All @@ -259,17 +269,17 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 8,
"id": "7946116a-b82f-492e-835e-ca958a8949a5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"e2b_data_analysis {'python_code': 'print((10 - 4) ** 2.7)'} {\"stdout\": \"126.18518711065899\", \"stderr\": \"\", \"artifacts\": []}\n",
"e2b_data_analysis {'python_code': 'print(10 - 4 ** 2.7)'} {\"stdout\": \"-32.22425314473263\", \"stderr\": \"\", \"artifacts\": []}\n",
"\n",
"\\( 10 - 4 \\) raised to the power of 2.7 is approximately 126.185.\n"
"\\( 10 - 4^{2.7} \\) is approximately \\(-32.22425314473263\\).\n"
]
}
],
Expand All @@ -280,17 +290,17 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 9,
"id": "f2744a56-9f4f-4899-827a-fa55821c318c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"e2b_data_analysis {'python_code': 'result = (10 - 4) ** 2.7 + 17.241\\nprint(result)'} {\"stdout\": \"143.426187110659\", \"stderr\": \"\", \"artifacts\": []}\n",
"e2b_data_analysis {'python_code': 'result = 10 - 4 ** 2.7\\nprint(result + 17.241)'} {\"stdout\": \"-14.983253144732629\", \"stderr\": \"\", \"artifacts\": []}\n",
"\n",
"\\( (10 - 4)^{2.7} + 17.241 \\) is approximately 143.426.\n"
"When you add \\( 17.241 \\) to \\( 10 - 4^{2.7} \\), the result is approximately \\( -14.98325314473263 \\).\n"
]
}
],
Expand All @@ -313,29 +323,10 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"id": "db6072c4-f3f3-415d-872b-71ea9f3c02bb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"companies\": [\n",
" {\n",
" \"name\": \"Google\",\n",
" \"origin\": \"USA\"\n",
" },\n",
" {\n",
" \"name\": \"Deepmind\",\n",
" \"origin\": \"UK\"\n",
" }\n",
" ]\n",
"}\n"
]
}
],
"outputs": [],
"source": [
"chat = ChatOpenAI(model=\"gpt-3.5-turbo-1106\").bind(\n",
" response_format={\"type\": \"json_object\"}\n",
Expand All @@ -356,22 +347,10 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": null,
"id": "08e00ccf-b991-4249-846b-9500a0ccbfa0",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'companies': [{'name': 'Google', 'origin': 'USA'},\n",
" {'name': 'Deepmind', 'origin': 'UK'}]}"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"import json\n",
"\n",
Expand All @@ -390,18 +369,10 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"id": "1281883c-bf8f-4665-89cd-4f33ccde69ab",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'token_usage': {'completion_tokens': 43, 'prompt_tokens': 49, 'total_tokens': 92}, 'model_name': 'gpt-3.5-turbo-1106', 'system_fingerprint': 'fp_eeff13170a'}\n"
]
}
],
"outputs": [],
"source": [
"chat = ChatOpenAI(model=\"gpt-3.5-turbo-1106\")\n",
"output = chat.generate(\n",
Expand Down
47 changes: 32 additions & 15 deletions libs/experimental/langchain_experimental/openai_assistant/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import json
from time import sleep
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union

from langchain.pydantic_v1 import Field
from langchain.schema.agent import AgentAction, AgentFinish
Expand Down Expand Up @@ -212,8 +212,12 @@ def invoke(
will return OpenAI types
Union[List[ThreadMessage], List[RequiredActionFunctionToolCall]].
"""
input = self._parse_input(input)
if "thread_id" not in input:
# Being run within AgentExecutor and there are tool outputs to submit.
if self.as_agent and input.get("intermediate_steps"):
tool_outputs = self._parse_intermediate_steps(input["intermediate_steps"])
run = self.client.beta.threads.runs.submit_tool_outputs(**tool_outputs)
# Starting a new thread and a new run.
elif "thread_id" not in input:
thread = {
"messages": [
{
Expand All @@ -226,6 +230,7 @@ def invoke(
"metadata": input.get("thread_metadata"),
}
run = self._create_thread_and_run(input, thread)
# Starting a new run in an existing thread.
elif "run_id" not in input:
_ = self.client.beta.threads.messages.create(
input["thread_id"],
Expand All @@ -235,21 +240,31 @@ def invoke(
metadata=input.get("message_metadata"),
)
run = self._create_run(input)
# Submitting tool outputs to an existing run, outside the AgentExecutor
# framework.
else:
run = self.client.beta.threads.runs.submit_tool_outputs(**input)
return self._get_response(run.id, run.thread_id)

def _parse_input(self, input: dict) -> dict:
if self.as_agent and input.get("intermediate_steps"):
last_action, last_output = input["intermediate_steps"][-1]
input = {
"tool_outputs": [
{"output": last_output, "tool_call_id": last_action.tool_call_id}
],
"run_id": last_action.run_id,
"thread_id": last_action.thread_id,
}
return input
def _parse_intermediate_steps(
self, intermediate_steps: List[Tuple[OpenAIAssistantAction, str]]
) -> dict:
last_action, last_output = intermediate_steps[-1]
run = self._wait_for_run(last_action.run_id, last_action.thread_id)
required_tool_call_ids = {
tc.id for tc in run.required_action.submit_tool_outputs.tool_calls
}
tool_outputs = [
{"output": output, "tool_call_id": action.tool_call_id}
for action, output in intermediate_steps
if action.tool_call_id in required_tool_call_ids
]
submit_tool_outputs = {
"tool_outputs": tool_outputs,
"run_id": last_action.run_id,
"thread_id": last_action.thread_id,
}
return submit_tool_outputs

def _create_run(self, input: dict) -> Any:
params = {
Expand Down Expand Up @@ -307,6 +322,8 @@ def _get_response(self, run_id: str, thread_id: str) -> Any:
for tool_call in run.required_action.submit_tool_outputs.tool_calls:
function = tool_call.function
args = json.loads(function.arguments)
if len(args) == 1 and "__arg1" in args:
args = args["__arg1"]
actions.append(
OpenAIAssistantAction(
tool=function.name,
Expand All @@ -321,7 +338,7 @@ def _get_response(self, run_id: str, thread_id: str) -> Any:
else:
run_info = json.dumps(run.dict(), indent=2)
raise ValueError(
f"Unknown run status {run.status}. Full run info:\n\n{run_info})"
f"Unexpected run status: {run.status}. Full run info:\n\n{run_info})"
)

def _wait_for_run(self, run_id: str, thread_id: str) -> Any:
Expand Down