From d9e437eb0041447e7ca4f79172beaac3c0103e50 Mon Sep 17 00:00:00 2001 From: ygarg25 Date: Tue, 1 Oct 2024 14:13:26 +0530 Subject: [PATCH] added/feature: Portfolio News Agent --- .../news_agent_benchmarks/README.md | 20 +++ .../news_agent_benchmarks/__init__.py | 0 .../news_agent_benchmarks/benchmarks.py | 38 +++++ .../news_agent_benchmarks/config.py | 14 ++ .../news_agent_benchmarks/helpers.py | 48 +++++++ .../moragents_dockers/agents/requirements.txt | 6 +- .../moragents_dockers/agents/src/config.py | 7 + .../agents/src/news_agent/src/__init__.py | 0 .../agents/src/news_agent/src/agent.py | 130 ++++++++++++++++++ .../agents/src/news_agent/src/config.py | 129 +++++++++++++++++ .../agents/src/news_agent/src/tools.py | 71 ++++++++++ 11 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 submodules/benchmarks/news_agent_benchmarks/README.md create mode 100644 submodules/benchmarks/news_agent_benchmarks/__init__.py create mode 100644 submodules/benchmarks/news_agent_benchmarks/benchmarks.py create mode 100644 submodules/benchmarks/news_agent_benchmarks/config.py create mode 100644 submodules/benchmarks/news_agent_benchmarks/helpers.py create mode 100644 submodules/moragents_dockers/agents/src/news_agent/src/__init__.py create mode 100644 submodules/moragents_dockers/agents/src/news_agent/src/agent.py create mode 100644 submodules/moragents_dockers/agents/src/news_agent/src/config.py create mode 100644 submodules/moragents_dockers/agents/src/news_agent/src/tools.py diff --git a/submodules/benchmarks/news_agent_benchmarks/README.md b/submodules/benchmarks/news_agent_benchmarks/README.md new file mode 100644 index 0000000..d1c4b90 --- /dev/null +++ b/submodules/benchmarks/news_agent_benchmarks/README.md @@ -0,0 +1,20 @@ +# Benchmarking & Testing News Agent Guide + + +## How to Run the Tests: +1) In the parent directory: +- ```cd submodules/moragents_dockers/agents``` + +2) ```docker build -t agent .``` + +2. NOTE: If you are using Apple Silicon then you may experience problems due to the base image not being build for arm64. We have included a separate Dockerfile in order to deal with this issue, run: + +- ```docker build . -t agent -f Dockerfile-apple``` + +3) To run the agent: + +- ```docker run --name agent -p 5000:5000 agent``` + +4) Check if the agent is up and running on docker or not +5) If it is running, navigate to `submodules/news_agent_benchmarks` +6) run `pytest benchmarks.py` \ No newline at end of file diff --git a/submodules/benchmarks/news_agent_benchmarks/__init__.py b/submodules/benchmarks/news_agent_benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submodules/benchmarks/news_agent_benchmarks/benchmarks.py b/submodules/benchmarks/news_agent_benchmarks/benchmarks.py new file mode 100644 index 0000000..843c33d --- /dev/null +++ b/submodules/benchmarks/news_agent_benchmarks/benchmarks.py @@ -0,0 +1,38 @@ +# submodules/benchmarks/news_agent_benchmarks/benchmarks.py + +import pytest +import logging +from submodules.benchmarks.news_agent_benchmarks.config import Config +from submodules.benchmarks.news_agent_benchmarks.helpers import ask_news_agent, extract_classification + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def test_news_classification(): + for i, test_case in enumerate(Config.TEST_CASES): + article_text = test_case["article_text"] + expected_classification = test_case["expected_classification"] + + logger.info(f"Testing article classification (Test {i + 1})") + + # Ask the news agent to classify the article + response = ask_news_agent(article_text, Config.LOCAL_AGENT_URL) + + # Extract the classification from the response + classification = extract_classification(response) + + if classification == "UNKNOWN": + logger.warning(f"Test case {i + 1} resulted in UNKNOWN classification. Response: {response}") + assert False, f"Test case {i + 1} failed: Could not determine classification" + else: + # Check if the classification matches the expected classification + assert classification == expected_classification, f"Test case {i + 1} failed: Expected {expected_classification}, but got {classification}" + + logger.info(f"Test case {i + 1} passed: Correctly classified as {classification}") + + logger.info("All test cases passed successfully") + + +if __name__ == "__main__": + pytest.main() diff --git a/submodules/benchmarks/news_agent_benchmarks/config.py b/submodules/benchmarks/news_agent_benchmarks/config.py new file mode 100644 index 0000000..2868b17 --- /dev/null +++ b/submodules/benchmarks/news_agent_benchmarks/config.py @@ -0,0 +1,14 @@ +class Config: + TEST_CASES = [ + { + "article_text": "ETH Prices zooms 10 percent today while Bitcoin's price surged to $70,000 today, breaking all previous records. Analysts attribute this to increased institutional adoption and positive regulatory news from several countries.", + "expected_classification": "RELEVANT", + }, + { + "article_text": "A new Tesla facility has opened in Texas, utilizing 100% renewable energy.", + "expected_classification": "NOT RELEVANT", + }, + # Add more test cases as needed + ] + + LOCAL_AGENT_URL = "http://127.0.0.1:5000/" diff --git a/submodules/benchmarks/news_agent_benchmarks/helpers.py b/submodules/benchmarks/news_agent_benchmarks/helpers.py new file mode 100644 index 0000000..7ce2623 --- /dev/null +++ b/submodules/benchmarks/news_agent_benchmarks/helpers.py @@ -0,0 +1,48 @@ +# submodules/benchmarks/news_agent_benchmarks/helpers.py + +import requests +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def ask_news_agent(article_text: str, url: str) -> dict: + headers = {'Content-Type': 'application/json'} + payload = { + "prompt": { + "role": "user", + "content": f"Classify if this article is relevant to cryptocurrency price movements: {article_text}" + } + } + response = requests.post(url, headers=headers, json=payload) + if response.status_code == 200: + return response.json() + else: + raise Exception(f"Request failed with status code {response.status_code}: {response.text}") + + +def extract_classification(response: dict) -> str: + if not isinstance(response, dict): + logger.warning(f"Unexpected response type: {type(response)}") + return "UNKNOWN" + + content = response.get('content') + + if content is None: + logger.warning("Response content is None") + return "UNKNOWN" + + if not isinstance(content, str): + logger.warning(f"Unexpected content type: {type(content)}") + return "UNKNOWN" + + content = content.upper() + + if "NOT RELEVANT" in content: + return "NOT RELEVANT" + elif "RELEVANT" in content: + return "RELEVANT" + else: + logger.warning(f"Could not determine relevance from content: {content}") + return "NOT RELEVANT" \ No newline at end of file diff --git a/submodules/moragents_dockers/agents/requirements.txt b/submodules/moragents_dockers/agents/requirements.txt index e87e422..0987f5a 100644 --- a/submodules/moragents_dockers/agents/requirements.txt +++ b/submodules/moragents_dockers/agents/requirements.txt @@ -14,4 +14,8 @@ langchain-text-splitters==0.2.2 langchain-core==0.2.24 langchain-community==0.2.10 torch -tweepy \ No newline at end of file +tweepy +feedparser +python-dateutil +pytz +pyshorteners \ No newline at end of file diff --git a/submodules/moragents_dockers/agents/src/config.py b/submodules/moragents_dockers/agents/src/config.py index f52f542..1e194da 100644 --- a/submodules/moragents_dockers/agents/src/config.py +++ b/submodules/moragents_dockers/agents/src/config.py @@ -58,5 +58,12 @@ class Config: "name": "reward agent", "upload_required": False, }, + { + "path": "news_agent.src.agent", + "class": "NewsAgent", + "description": "Fetches and analyzes cryptocurrency news for potential price impacts.", + "name": "crypto news agent", + "upload_required": False, + } ] } diff --git a/submodules/moragents_dockers/agents/src/news_agent/src/__init__.py b/submodules/moragents_dockers/agents/src/news_agent/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/submodules/moragents_dockers/agents/src/news_agent/src/agent.py b/submodules/moragents_dockers/agents/src/news_agent/src/agent.py new file mode 100644 index 0000000..4dbb93c --- /dev/null +++ b/submodules/moragents_dockers/agents/src/news_agent/src/agent.py @@ -0,0 +1,130 @@ +import json +import logging +import re +import urllib.parse +from news_agent.src.config import Config +from news_agent.src.tools import clean_html, is_within_time_window, fetch_rss_feed +import pyshorteners + +logger = logging.getLogger(__name__) + +class NewsAgent: + def __init__(self, agent_info, llm, llm_ollama, embeddings, flask_app): + self.agent_info = agent_info + self.llm = llm + self.flask_app = flask_app + self.tools_provided = self.get_tools() + self.url_shortener = pyshorteners.Shortener() + logger.info("NewsAgent initialized") + + def get_tools(self): + return [ + { + "type": "function", + "function": { + "name": "fetch_crypto_news", + "description": "Fetch and analyze cryptocurrency news for potential price impacts", + "parameters": { + "type": "object", + "properties": { + "coins": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of cryptocurrency symbols to fetch news for" + } + }, + "required": ["coins"] + } + } + } + ] + + def check_relevance_and_summarize(self, title, content, coin): + logger.info(f"Checking relevance for {coin}: {title}") + prompt = Config.RELEVANCE_PROMPT.format(coin=coin, title=title, content=content) + result = self.llm.create_chat_completion( + messages=[{"role": "user", "content": prompt}], + max_tokens=Config.LLM_MAX_TOKENS, + temperature=Config.LLM_TEMPERATURE + ) + return result['choices'][0]['message']['content'].strip() + + def process_rss_feed(self, feed_url, coin): + logger.info(f"Processing RSS feed for {coin}: {feed_url}") + feed = fetch_rss_feed(feed_url) + results = [] + for entry in feed.entries: + published_time = entry.get('published') or entry.get('updated') + if is_within_time_window(published_time): + title = clean_html(entry.title) + content = clean_html(entry.summary) + logger.info(f"Checking relevance for article: {title}") + result = self.check_relevance_and_summarize(title, content, coin) + if not result.upper().startswith("NOT RELEVANT"): + results.append({ + "Title": title, + "Summary": result, + "Link": entry.link + }) + if len(results) >= Config.ARTICLES_PER_TOKEN: + break + else: + logger.info(f"Skipping article: {entry.title} (published: {published_time})") + logger.info(f"Found {len(results)} relevant articles for {coin}") + return results + + def fetch_crypto_news(self, coins): + logger.info(f"Fetching news for coins: {coins}") + all_news = [] + for coin in coins: + logger.info(f"Processing news for {coin}") + coin_name = Config.CRYPTO_DICT.get(coin.upper(), coin) + google_news_url = Config.GOOGLE_NEWS_BASE_URL.format(coin_name) + results = self.process_rss_feed(google_news_url, coin_name) + all_news.extend([{"Coin": coin, **result} for result in results[:Config.ARTICLES_PER_TOKEN]]) + + logger.info(f"Total news items fetched: {len(all_news)}") + return all_news + + def chat(self, request): + try: + data = request.get_json() + if 'prompt' in data: + prompt = data['prompt'] + if isinstance(prompt, dict) and 'content' in prompt: + prompt = prompt['content'] + + # Updated coin detection logic + coins = re.findall(r'\b(' + '|'.join(re.escape(key) for key in Config.CRYPTO_DICT.keys()) + r')\b', + prompt.upper()) + + if not coins: + return {"role": "assistant", + "content": "I couldn't identify any cryptocurrency symbols in your message. Please specify the cryptocurrencies you want news for.", + "next_turn_agent": None} + + news = self.fetch_crypto_news(coins) + + if not news: + return {"role": "assistant", + "content": "No relevant news found for the specified cryptocurrencies in the last 24 hours.", + "next_turn_agent": None} + + response = "Here are the latest news items relevant to changes in price movement of the mentioned tokens in the last 24 hours:\n\n" + for index, item in enumerate(news, start=1): + coin_name = Config.CRYPTO_DICT.get(item['Coin'], item['Coin']) + short_url = self.url_shortener.tinyurl.short(item['Link']) + response += f"{index}. ***{coin_name} News***:\n" + response += f"{item['Title']}\n" + response += f"{item['Summary']}\n" + response += f"Read more: {short_url}\n\n" + + return {"role": "assistant", "content": response, "next_turn_agent": None} + else: + return {"role": "assistant", "content": "Missing required parameters", "next_turn_agent": None} + + except Exception as e: + logger.error(f"Error in chat method: {str(e)}", exc_info=True) + return {"role": "assistant", "content": f"An error occurred: {str(e)}", "next_turn_agent": None} \ No newline at end of file diff --git a/submodules/moragents_dockers/agents/src/news_agent/src/config.py b/submodules/moragents_dockers/agents/src/news_agent/src/config.py new file mode 100644 index 0000000..c8a399b --- /dev/null +++ b/submodules/moragents_dockers/agents/src/news_agent/src/config.py @@ -0,0 +1,129 @@ +import logging + +logging.basicConfig(level=logging.INFO) + +class Config: + # RSS Feed URL + GOOGLE_NEWS_BASE_URL = "https://news.google.com/rss/search?q={}&hl=en-US&gl=US&ceid=US:en" + + # Time window for news (in hours) + NEWS_TIME_WINDOW = 24 + + # Number of articles to show per token + ARTICLES_PER_TOKEN = 1 + + # LLM configuration + LLM_MAX_TOKENS = 150 + LLM_TEMPERATURE = 0.3 + + # Prompts + RELEVANCE_PROMPT = ( + "Consider the following news article about {coin}:\n\n" + "Title: {title}\n\nContent: {content}\n\n" + "Is this article relevant to potential price impacts on the cryptocurrency? " + "If yes, provide a concise summary focused on how it might impact trading or prices. " + "If it's not relevant or only about price movements, respond with 'NOT RELEVANT'." + ) + + # Dictionary of top 100 popular tickers and their crypto names + CRYPTO_DICT = { + 'BTC': 'Bitcoin', + 'ETH': 'Ethereum', + 'USDT': 'Tether', + 'BNB': 'BNB', + 'SOL': 'Solana', + 'USDC': 'USDC', + 'XRP': 'XRP', + 'STETH': 'Lido Staked Ether', + 'DOGE': 'Dogecoin', + 'TON': 'Toncoin', + 'ADA': 'Cardano', + 'TRX': 'TRON', + 'AVAX': 'Avalanche', + 'WSTETH': 'Wrapped stETH', + 'SHIB': 'Shiba Inu', + 'WBTC': 'Wrapped Bitcoin', + 'WETH': 'Binance-Peg WETH', + 'LINK': 'Chainlink', + 'BCH': 'Bitcoin Cash', + 'DOT': 'Polkadot', + 'NEAR': 'NEAR Protocol', + 'UNI': 'Uniswap', + 'LEO': 'LEO Token', + 'DAI': 'Dai', + 'SUI': 'Sui', + 'LTC': 'Litecoin', + 'PEPE': 'Pepe', + 'ICP': 'Internet Computer', + 'WEETH': 'Wrapped eETH', + 'TAO': 'Bittensor', + 'FET': 'Artificial Superintelligence Alliance', + 'APT': 'Aptos', + 'KAS': 'Kaspa', + 'POL': 'POL (ex-MATIC)', + 'XLM': 'Stellar', + 'ETC': 'Ethereum Classic', + 'STX': 'Stacks', + 'FDUSD': 'First Digital USD', + 'IMX': 'Immutable', + 'XMR': 'Monero', + 'RENDER': 'Render', + 'WIF': 'dogwifhat', + 'USDE': 'Ethena USDe', + 'OKB': 'OKB', + 'AAVE': 'Aave', + 'INJ': 'Injective', + 'OP': 'Optimism', + 'FIL': 'Filecoin', + 'CRO': 'Cronos', + 'ARB': 'Arbitrum', + 'HBAR': 'Hedera', + 'FTM': 'Fantom', + 'MNT': 'Mantle', + 'VET': 'VeChain', + 'ATOM': 'Cosmos Hub', + 'RUNE': 'THORChain', + 'BONK': 'Bonk', + 'GRT': 'The Graph', + 'SEI': 'Sei', + 'WBT': 'WhiteBIT Coin', + 'FLOKI': 'FLOKI', + 'AR': 'Arweave', + 'THETA': 'Theta Network', + 'RETH': 'Rocket Pool ETH', + 'BGB': 'Bitget Token', + 'MKR': 'Maker', + 'HNT': 'Helium', + 'METH': 'Mantle Staked Ether', + 'SOLVBTC': 'Solv Protocol SolvBTC', + 'PYTH': 'Pyth Network', + 'TIA': 'Celestia', + 'JUP': 'Jupiter', + 'LDO': 'Lido DAO', + 'MATIC': 'Polygon', + 'ONDO': 'Ondo', + 'ALGO': 'Algorand', + 'GT': 'Gate', + 'JASMY': 'JasmyCoin', + 'QNT': 'Quant', + 'OM': 'MANTRA', + 'BEAM': 'Beam', + 'POPCAT': 'Popcat', + 'BSV': 'Bitcoin SV', + 'KCS': 'KuCoin', + 'EZETH': 'Renzo Restaked ETH', + 'CORE': 'Core', + 'BRETT': 'Brett', + 'WLD': 'Worldcoin', + 'GALA': 'GALA', + 'BTT': 'BitTorrent', + 'FLOW': 'Flow', + 'NOT': 'Notcoin', + 'STRK': 'Starknet', + 'EETH': 'ether.fi Staked ETH', + 'MSOL': 'Marinade Staked SOL', + 'EIGEN': 'Eigenlayer', + 'ORDI': 'ORDI', + 'CFX': 'Conflux', + 'W': 'Wormhole' + } \ No newline at end of file diff --git a/submodules/moragents_dockers/agents/src/news_agent/src/tools.py b/submodules/moragents_dockers/agents/src/news_agent/src/tools.py new file mode 100644 index 0000000..607f2fc --- /dev/null +++ b/submodules/moragents_dockers/agents/src/news_agent/src/tools.py @@ -0,0 +1,71 @@ +import feedparser +from datetime import datetime, timedelta +import pytz +from dateutil import parser +import re +from html import unescape +from news_agent.src.config import Config +import logging +import urllib.parse + +logger = logging.getLogger(__name__) + + +def clean_html(raw_html): + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, '', raw_html) + cleantext = unescape(cleantext) + cleantext = ' '.join(cleantext.split()) + return cleantext + + +def is_within_time_window(published_time, hours=24): + if not published_time: + return False + try: + pub_date = parser.parse(published_time, fuzzy=True) + now = datetime.now(pytz.UTC) + if pub_date.tzinfo is None: + pub_date = pub_date.replace(tzinfo=pytz.UTC) + return (now - pub_date) <= timedelta(hours=hours) + except Exception as e: + logger.error(f"Error parsing date: {str(e)} for date {published_time}") + return False + + +def fetch_rss_feed(feed_url): + # URL encode the query parameter + parsed_url = urllib.parse.urlparse(feed_url) + query_params = urllib.parse.parse_qs(parsed_url.query) + if 'q' in query_params: + query_params['q'] = [urllib.parse.quote(q) for q in query_params['q']] + encoded_query = urllib.parse.urlencode(query_params, doseq=True) + encoded_url = urllib.parse.urlunparse(parsed_url._replace(query=encoded_query)) + + return feedparser.parse(encoded_url) + + +def get_tools(): + """Return a list of tools for the agent.""" + return [ + { + "type": "function", + "function": { + "name": "fetch_crypto_news", + "description": "Fetch and analyze cryptocurrency news for potential price impacts", + "parameters": { + "type": "object", + "properties": { + "coins": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of cryptocurrency symbols to fetch news for" + } + }, + "required": ["coins"] + } + } + } + ] \ No newline at end of file