Skip to content

Commit

Permalink
add async stream
Browse files Browse the repository at this point in the history
RexWzh committed Sep 25, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent f352c96 commit 7ebd718
Showing 3 changed files with 33 additions and 4 deletions.
2 changes: 1 addition & 1 deletion chattool/__init__.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@

__author__ = """Rex Wang"""
__email__ = '[email protected]'
__version__ = '3.3.3'
__version__ = '3.3.4'

import os, sys, requests, json
from .chattype import Chat, Resp
33 changes: 31 additions & 2 deletions chattool/chattype.py
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@
from .functioncall import generate_json_schema, delete_dialogue_assist
from pprint import pformat
from loguru import logger
import asyncio

class Chat():
def __init__( self
@@ -263,6 +264,35 @@ async def async_stream_responses( self
self.api_key, self.chat_url, self.chat_log, self.model, timeout=timeout, **options):
yield resp.delta_content if textonly else resp

def stream_responses(self, timeout:int=0, textonly:bool=False, **options):
"""Post request synchronously and stream the responses
Args:
timeout (int, optional): timeout for the API call. Defaults to 0(no timeout).
textonly (bool, optional): whether to only return the text. Defaults to False.
options (dict, optional): other options like `temperature`, `top_p`, etc.
Returns:
str: response text
"""

async_gen = self.async_stream_responses(timeout=timeout, textonly=textonly, **options)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)

try:
# Synchronously get responses from the async generator
while True:
try:
# Run the async generator to get each response
response = loop.run_until_complete(async_gen.__anext__())
yield response
except StopAsyncIteration:
# End the generator when the async generator is exhausted
break
finally:
loop.close()

# Part3: tool call
def iswaiting(self):
"""Whether the response is waiting"""
@@ -397,8 +427,7 @@ def get_valid_models(self, gpt_only:bool=True)->List[str]:
elif self.base_url:
model_url = os.path.join(self.base_url, 'v1/models')
model_list = valid_models(self.api_key, model_url, gpt_only=gpt_only)
model_list.sort()
return model_list
return sorted(set(model_list))

def get_curl(self, use_env_key:bool=False, **options):
"""Get the curl command
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
with open('README.md') as readme_file:
readme = readme_file.read()

VERSION = '3.3.3'
VERSION = '3.3.4'

requirements = [
'Click>=7.0', 'requests>=2.20', "responses>=0.23", 'aiohttp>=3.8',

0 comments on commit 7ebd718

Please sign in to comment.