Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: enforce valid JSON response in interior design assistant #126

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

fix: enforce valid JSON response in interior design assistant #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b601006
fix: import enforce_response_format in api.py
Anirudh2112 Nov 22, 2024
f7fdf0e
fix: integrate enforce_response_format and add tests
Anirudh2112 Nov 22, 2024
ac98196
fix: enforce response_format for structured JSON outputs
Anirudh2112 Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions examples/interior_design_assistant/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@ async def list_items(self, file_path: str) -> List[str]:
assert (
self.agent_id is not None
), "Agent not initialized, call initialize() first"

response_format = {
"type": "object",
"properties": {
"description": {"type": "string"},
"items": {"type": "array", "items": {"type": "string"}},
},
"required": ["description", "items"]
}

text = textwrap.dedent(
"""
Analyze the image to provide a 4 sentence description of the architecture and furniture items present in it.
Expand Down Expand Up @@ -95,6 +105,7 @@ async def list_items(self, file_path: str) -> List[str]:
session_id=resposne.session_id,
messages=[message],
stream=True,
response_format=response_format,
)

result = ""
Expand All @@ -106,8 +117,8 @@ async def list_items(self, file_path: str) -> List[str]:

# print(turn.output_message.content)
result = turn.output_message.content
d = json.loads(result.strip())
return d
return json.loads(result)


async def suggest_alternatives(
self, file_path: str, item: str, n: int = 3
Expand All @@ -116,6 +127,16 @@ async def suggest_alternatives(
Analyze the image using multimodal llm
and return possible alternative descriptions for the provided item.
"""
response_format = {
"type": "array",
"items": {
"type": "object",
"properties": {
"description": {"type": "string"}
},
"required": ["description"]
}
}
prompt = textwrap.dedent(
"""
For the given image, your task is to carefully examine the image to provide alternative suggestions for {item}.
Expand Down Expand Up @@ -154,11 +175,13 @@ async def suggest_alternatives(
agent_id=self.agent_id,
session_name=uuid.uuid4().hex,
)

generator = self.client.agents.turn.create(
agent_id=self.agent_id,
session_id=resposne.session_id,
messages=[message],
stream=True,
response_format=response_format,
)
result = ""
for chunk in generator:
Expand All @@ -167,8 +190,8 @@ async def suggest_alternatives(
turn = payload.turn

result = turn.output_message.content
print(result)
return [r["description"].strip() for r in json.loads(result.strip())]
print(result)
return [r["description"] for r in json.loads(result)]

async def retrieve_images(self, description: str) -> List[ImageMedia]:
"""
Expand Down
1 change: 1 addition & 0 deletions examples/interior_design_assistant/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import mimetypes
import uuid

import json

# TODO: This should move into a common util as will be needed by all apps
def data_url_from_image(file_path):
Expand Down