From 9d7e637855d093aa14a1d0e29121f57832c39ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Arce?= Date: Tue, 2 Jan 2024 16:48:28 -0300 Subject: [PATCH 1/5] feat: handle orders when mm does not have liquidity --- mm-bot/src/main.py | 25 +++++++++++++++++++++++++ mm-bot/src/models/order_status.py | 1 + mm-bot/src/persistence/order_dao.py | 17 ++++++++++++++++- mm-bot/src/services/ethereum.py | 14 ++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/mm-bot/src/main.py b/mm-bot/src/main.py index 28ed5110..cfdafc85 100644 --- a/mm-bot/src/main.py +++ b/mm-bot/src/main.py @@ -1,6 +1,8 @@ import asyncio import logging +import schedule + from config import constants from config.database_config import get_db from config.logging_config import setup_logger @@ -17,6 +19,7 @@ setup_logger() logger = logging.getLogger(__name__) SLEEP_TIME = 5 +PROCESS_NO_BALANCE_ORDERS_MINUTES_TIMER = 5 def using_herodotus(): @@ -32,6 +35,8 @@ async def run(): block_dao = BlockDao(get_db()) eth_lock = asyncio.Lock() herodotus_semaphore = asyncio.Semaphore(100) + (schedule.every(PROCESS_NO_BALANCE_ORDERS_MINUTES_TIMER).minutes + .do(no_balance_orders_job, order_dao, eth_lock, herodotus_semaphore)) try: # 1 Get all orders that are not completed from the db @@ -56,6 +61,7 @@ async def run(): # 5. Update latest block block_dao.update_latest_block(await starknet.get_latest_block()) + schedule.run_pending() await asyncio.sleep(SLEEP_TIME) except Exception as e: logger.error(f"[-] Error: {e}") @@ -99,6 +105,10 @@ async def process_order(order: Order, order_dao: OrderDao, # (bridging is complete for the user) if order.status in [OrderStatus.PROCESSING, OrderStatus.TRANSFERRING]: async with eth_lock: + if not ethereum.has_funds(order.get_int_amount()): + order_dao.set_order_no_balance(order) + logger.info(f"[+] Order {order.order_id} has no balance") + return if order.status is OrderStatus.PROCESSING: try: await transfer(order, order_dao) @@ -129,6 +139,21 @@ async def process_order(order: Order, order_dao: OrderDao, logger.info(f"[+] Order {order.order_id} completed") +def no_balance_orders_job(order_dao: OrderDao, + eth_lock: asyncio.Lock, herodotus_semaphore: asyncio.Semaphore): + asyncio.create_task(process_no_balance_orders(order_dao, eth_lock, herodotus_semaphore), name="No-balance-orders") + + +async def process_no_balance_orders(order_dao: OrderDao, + eth_lock: asyncio.Lock, herodotus_semaphore: asyncio.Semaphore): + logger.debug(f"[+] Processing no balance orders") + orders = order_dao.get_no_balance_orders() + for order in orders: + order_dao.set_order_processing(order) + create_order_task(order, order_dao, eth_lock, herodotus_semaphore) + # logger.info(f"[+] Processing no balance order: {order}") + + async def transfer(order: Order, order_dao: OrderDao): logger.info(f"[+] Transferring eth on ethereum") # in case it's processed on ethereum, but not processed on starknet diff --git a/mm-bot/src/models/order_status.py b/mm-bot/src/models/order_status.py index 830777dd..2dc3b56a 100644 --- a/mm-bot/src/models/order_status.py +++ b/mm-bot/src/models/order_status.py @@ -10,3 +10,4 @@ class OrderStatus(Enum): PROVED = "PROVED" COMPLETED = "COMPLETED" FAILED = "FAILED" + NO_BALANCE = "NO_BALANCE" diff --git a/mm-bot/src/persistence/order_dao.py b/mm-bot/src/persistence/order_dao.py index 95dc4032..dcaff5fa 100644 --- a/mm-bot/src/persistence/order_dao.py +++ b/mm-bot/src/persistence/order_dao.py @@ -61,7 +61,22 @@ def set_order_proved(self, order: Order) -> Order: def set_order_completed(self, order: Order) -> Order: return self.update_order(order, OrderStatus.COMPLETED) + def set_order_no_balance(self, order: Order) -> Order: + return self.update_order(order, OrderStatus.NO_BALANCE) + + """ + An order is incomplete if it's not completed, failed or has no balance + An order in NO_BALANCE state is considered COMPLETED and will be re-processed in the next iteration + when the balance is enough + """ def get_incomplete_orders(self): return (self.db.query(Order) - .filter(and_(Order.status != OrderStatus.COMPLETED, Order.status != OrderStatus.FAILED)) + .filter(and_(Order.status != OrderStatus.COMPLETED, + Order.status != OrderStatus.FAILED, + Order.status != OrderStatus.NO_BALANCE)) + .all()) + + def get_no_balance_orders(self): + return (self.db.query(Order) + .filter(Order.status == OrderStatus.NO_BALANCE) .all()) diff --git a/mm-bot/src/services/ethereum.py b/mm-bot/src/services/ethereum.py index e24b57a4..43bea768 100644 --- a/mm-bot/src/services/ethereum.py +++ b/mm-bot/src/services/ethereum.py @@ -49,6 +49,20 @@ def get_is_used_order(order_id, recipient_address, amount) -> bool: raise Exception("Failed to get is used order from all nodes") +def get_balance() -> int: + for w3 in w3_clients: + try: + return w3.eth.get_balance(main_account.address) + except Exception as exception: + logger.warning(f"[-] Failed to get balance from node: {exception}") + logger.error(f"[-] Failed to get balance from all nodes") + raise Exception("Failed to get balance from all nodes") + + +def has_funds(amount: int) -> bool: + return get_balance() >= amount + + def transfer(deposit_id, dst_addr, amount): dst_addr_bytes = int(dst_addr, 0) deposit_id = Web3.to_int(deposit_id) From 68d0bb9089148e7ec99b3ef627ac121b8503dfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Arce?= Date: Tue, 2 Jan 2024 16:52:25 -0300 Subject: [PATCH 2/5] docs: get_incomplete_orders description --- mm-bot/src/persistence/order_dao.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm-bot/src/persistence/order_dao.py b/mm-bot/src/persistence/order_dao.py index dcaff5fa..6c54c3ca 100644 --- a/mm-bot/src/persistence/order_dao.py +++ b/mm-bot/src/persistence/order_dao.py @@ -65,10 +65,11 @@ def set_order_no_balance(self, order: Order) -> Order: return self.update_order(order, OrderStatus.NO_BALANCE) """ - An order is incomplete if it's not completed, failed or has no balance + An order is incomplete if it's not completed, not failed or has balance An order in NO_BALANCE state is considered COMPLETED and will be re-processed in the next iteration when the balance is enough """ + def get_incomplete_orders(self): return (self.db.query(Order) .filter(and_(Order.status != OrderStatus.COMPLETED, From e2d4334b4770c166de6f712905bac7d2c9d150c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Arce?= Date: Tue, 2 Jan 2024 16:54:27 -0300 Subject: [PATCH 3/5] fix: log better message when mm does not have liquidity --- mm-bot/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm-bot/src/main.py b/mm-bot/src/main.py index cfdafc85..d921d01c 100644 --- a/mm-bot/src/main.py +++ b/mm-bot/src/main.py @@ -107,7 +107,7 @@ async def process_order(order: Order, order_dao: OrderDao, async with eth_lock: if not ethereum.has_funds(order.get_int_amount()): order_dao.set_order_no_balance(order) - logger.info(f"[+] Order {order.order_id} has no balance") + logger.info(f"[+] MM does not have balance for order {order.order_id}") return if order.status is OrderStatus.PROCESSING: try: From 16897b0668b1cced7147cae3361a9ed71756eb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Arce?= Date: Tue, 2 Jan 2024 17:06:17 -0300 Subject: [PATCH 4/5] fix: rename accounts to account_rpc and add comments --- mm-bot/src/services/ethereum.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mm-bot/src/services/ethereum.py b/mm-bot/src/services/ethereum.py index 43bea768..b5789c0c 100644 --- a/mm-bot/src/services/ethereum.py +++ b/mm-bot/src/services/ethereum.py @@ -9,9 +9,13 @@ fallback_w3 = Web3(Web3.HTTPProvider(constants.ETH_FALLBACK_RPC_URL)) w3_clients = [main_w3, fallback_w3] -main_account = main_w3.eth.account.from_key(constants.ETH_PRIVATE_KEY) -fallback_account = fallback_w3.eth.account.from_key(constants.ETH_PRIVATE_KEY) -accounts = [main_account, fallback_account] +""" +accounts are instances from the same account but from different nodes +So if a node is down we can use the other one +""" +main_account_rpc = main_w3.eth.account.from_key(constants.ETH_PRIVATE_KEY) +fallback_account_rpc = fallback_w3.eth.account.from_key(constants.ETH_PRIVATE_KEY) +accounts_rpc = [main_account_rpc, fallback_account_rpc] # get only the abi not the entire file abi = json.load(open(os.getcwd() + '/abi/YABTransfer.json'))['abi'] @@ -50,9 +54,9 @@ def get_is_used_order(order_id, recipient_address, amount) -> bool: def get_balance() -> int: - for w3 in w3_clients: + for index, w3 in enumerate(w3_clients): try: - return w3.eth.get_balance(main_account.address) + return w3.eth.get_balance(accounts_rpc[index].address) except Exception as exception: logger.warning(f"[-] Failed to get balance from node: {exception}") logger.error(f"[-] Failed to get balance from all nodes") @@ -80,11 +84,11 @@ def create_transfer(deposit_id, dst_addr_bytes, amount): try: unsent_tx = contracts_rpc[index].functions.transfer(deposit_id, dst_addr_bytes, amount).build_transaction({ "chainId": 5, - "from": accounts[index].address, - "nonce": get_nonce(w3, accounts[index].address), + "from": accounts_rpc[index].address, + "nonce": get_nonce(w3, accounts_rpc[index].address), "value": amount, }) - signed_tx = w3.eth.account.sign_transaction(unsent_tx, private_key=accounts[index].key) + signed_tx = w3.eth.account.sign_transaction(unsent_tx, private_key=accounts_rpc[index].key) return signed_tx except Exception as exception: logger.warning(f"[-] Failed to create transfer eth on node: {exception}") @@ -108,11 +112,11 @@ def create_withdraw(deposit_id, dst_addr_bytes, amount): try: unsent_tx = contracts_rpc[index].functions.withdraw(deposit_id, dst_addr_bytes, amount).build_transaction({ "chainId": 5, - "from": accounts[index].address, - "nonce": get_nonce(w3, accounts[index].address), + "from": accounts_rpc[index].address, + "nonce": get_nonce(w3, accounts_rpc[index].address), "value": amount, }) - signed_tx = w3.eth.account.sign_transaction(unsent_tx, private_key=accounts[index].key) + signed_tx = w3.eth.account.sign_transaction(unsent_tx, private_key=accounts_rpc[index].key) return signed_tx except Exception as exception: logger.warning(f"[-] Failed to create withdraw eth on node: {exception}") From ea31e6d01dfd906fc70c93d1566c6032f6462734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juli=C3=A1n=20Arce?= Date: Tue, 2 Jan 2024 17:12:04 -0300 Subject: [PATCH 5/5] fix: add logs for reprocessing no balance orders --- mm-bot/src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm-bot/src/main.py b/mm-bot/src/main.py index d921d01c..9d60900e 100644 --- a/mm-bot/src/main.py +++ b/mm-bot/src/main.py @@ -150,8 +150,8 @@ async def process_no_balance_orders(order_dao: OrderDao, orders = order_dao.get_no_balance_orders() for order in orders: order_dao.set_order_processing(order) + logger.info(f"[+] Processing no balance order: {order}") create_order_task(order, order_dao, eth_lock, herodotus_semaphore) - # logger.info(f"[+] Processing no balance order: {order}") async def transfer(order: Order, order_dao: OrderDao):