Merge branch 'hummingbot:development' into development

docker-compose.yml

@@ -16,7 +16,7 @@ services:
       driver: "json-file"
       options:
         max-size: "10m"
-        max-file: 5
+        max-file: "5"
     tty: true
     stdin_open: true
     network_mode: host

hummingbot/VERSION

@@ -1 +1 @@
-dev-1.20.0
+dev-1.21.0

hummingbot/connector/connector_status.py

@@ -51,7 +51,6 @@
     'xswap': 'bronze',
     'dexalot': 'silver',
     'kucoin_perpetual': 'silver',
-    'kucoin_perpetual_testnet': 'silver',
     'injective_perpetual': 'bronze',
     'bit_com_perpetual': 'bronze',
     'bit_com_perpetual_testnet': 'bronze',

hummingbot/connector/derivative/kucoin_perpetual/kucoin_perpetual_constants.py

@@ -10,13 +10,9 @@
 
 DEFAULT_TIME_IN_FORCE = "GTC"
 
-REST_URLS = {"kucoin_perpetual_main": "https://api-futures.kucoin.com/",
-             "kucoin_perpetual_testnet": "https://api-sandbox-futures.kucoin.com/"}
-WSS_PUBLIC_URLS = {"kucoin_perpetual_main": "wss://stream.kucoin.com/realtime_public",
-                   "kucoin_perpetual_testnet": "wss://stream-testnet.kucoin.com/realtime_public"}
-WSS_PRIVATE_URLS = {"kucoin_perpetual_main": "wss://stream.kucoin.com/realtime_private",
-                    "kucoin_perpetual_testnet": "wss://stream-testnet.kucoin.com/realtime_private"}
-
+REST_URLS = {"kucoin_perpetual_main": "https://api-futures.kucoin.com/"}
+WSS_PUBLIC_URLS = {"kucoin_perpetual_main": "wss://stream.kucoin.com/realtime_public"}
+WSS_PRIVATE_URLS = {"kucoin_perpetual_main": "wss://stream.kucoin.com/realtime_private"}
 REST_API_VERSION = "api/v1"
 
 HB_PARTNER_ID = "Hummingbot"

hummingbot/connector/derivative/kucoin_perpetual/kucoin_perpetual_derivative.py

@@ -419,7 +419,7 @@ async def _update_balances(self):
         self._account_balances.clear()
 
         if wallet_balance["data"] is not None:
-            if type(wallet_balance["data"]) == list:
+            if isinstance(wallet_balance["data"], list):
                 for balance_data in wallet_balance["data"]:
                     currency = str(balance_data["currency"])
                     self._account_balances[currency] = Decimal(str(balance_data["marginBalance"]))
@@ -456,7 +456,7 @@ async def _update_positions(self):
             data = position
             ex_trading_pair = data.get("symbol")
             hb_trading_pair = await self.trading_pair_associated_to_exchange_symbol(ex_trading_pair)
-            amount = self.get_value_of_contracts(hb_trading_pair, int(str(data["currentQty"])))
+            amount = self.get_value_of_contracts(hb_trading_pair, int(data["currentQty"]))
             position_side = PositionSide.SHORT if amount < 0 else PositionSide.LONG
             unrealized_pnl = Decimal(str(data["unrealisedPnl"]))
             entry_price = Decimal(str(data["avgEntryPrice"]))
@@ -591,7 +591,7 @@ async def _user_stream_event_listener(self):
                         self._order_tracker.process_order_update(order_update=order_update)
 
                 elif endpoint == CONSTANTS.WS_SUBSCRIPTION_WALLET_ENDPOINT_NAME:
-                    if type(payload) == list:
+                    if isinstance(payload, list):
                         for wallet_msg in payload:
                             self._process_wallet_event_message(wallet_msg)
                     else:
@@ -615,7 +615,7 @@ async def _process_account_position_event(self, position_msg: Dict[str, Any]):
         if "changeReason" in position_msg and position_msg["changeReason"] != "markPriceChange":
             ex_trading_pair = position_msg["symbol"]
             trading_pair = await self.trading_pair_associated_to_exchange_symbol(symbol=ex_trading_pair)
-            amount = Decimal(str(position_msg["currentQty"]))
+            amount = self.get_value_of_contracts(trading_pair, int(position_msg["currentQty"]))
             position_side = PositionSide.SHORT if amount < 0 else PositionSide.LONG
             entry_price = Decimal(str(position_msg["avgEntryPrice"]))
             leverage = Decimal(str(position_msg["realLeverage"]))
@@ -830,8 +830,7 @@ async def _get_last_traded_price(self, trading_pair: str) -> float:
             path_url=CONSTANTS.LATEST_SYMBOL_INFORMATION_ENDPOINT.format(symbol=exchange_symbol),
             limit_id=CONSTANTS.LATEST_SYMBOL_INFORMATION_ENDPOINT,
         )
-
-        if type(resp_json["data"]) == list:
+        if isinstance(resp_json["data"], list):
             if "lastTradePrice" in resp_json["data"][0]:
                 price = float(resp_json["data"][0]["lastTradePrice"])
             else:

hummingbot/connector/derivative/kucoin_perpetual/kucoin_perpetual_utils.py

@@ -65,52 +65,3 @@ class Config:
 
 
 KEYS = KucoinPerpetualConfigMap.construct()
-
-OTHER_DOMAINS = ["kucoin_perpetual_testnet"]
-OTHER_DOMAINS_PARAMETER = {"kucoin_perpetual_testnet": "kucoin_perpetual_testnet"}
-OTHER_DOMAINS_EXAMPLE_PAIR = {"kucoin_perpetual_testnet": "BTC-USDT"}
-OTHER_DOMAINS_DEFAULT_FEES = {
-    "kucoin_perpetual_testnet": TradeFeeSchema(
-        maker_percent_fee_decimal=Decimal("-0.00025"),
-        taker_percent_fee_decimal=Decimal("0.00075"),
-    )
-}
-
-
-class KucoinPerpetualTestnetConfigMap(BaseConnectorConfigMap):
-    connector: str = Field(default="kucoin_perpetual_testnet", client_data=None)
-    kucoin_perpetual_testnet_api_key: SecretStr = Field(
-        default=...,
-        client_data=ClientFieldData(
-            prompt=lambda cm: "Enter your Kucoin Perpetual Testnet API key",
-            is_secure=True,
-            is_connect_key=True,
-            prompt_on_new=True,
-        )
-    )
-    kucoin_perpetual_testnet_secret_key: SecretStr = Field(
-        default=...,
-        client_data=ClientFieldData(
-            prompt=lambda cm: "Enter your Kucoin Perpetual Testnet secret key",
-            is_secure=True,
-            is_connect_key=True,
-            prompt_on_new=True,
-        )
-    )
-    kucoin_perpetual_testnet_passphrase: SecretStr = Field(
-        default=...,
-        client_data=ClientFieldData(
-            prompt=lambda cm: "Enter your KuCoin Perpetual Testnet passphrase",
-            is_secure=True,
-            is_connect_key=True,
-            prompt_on_new=True,
-        )
-    )
-
-    class Config:
-        title = "kucoin_perpetual_testnet"
-
-
-OTHER_DOMAINS_KEYS = {
-    "kucoin_perpetual_testnet": KucoinPerpetualTestnetConfigMap.construct()
-}

hummingbot/connector/exchange/bitmart/bitmart_constants.py

@@ -26,7 +26,7 @@
 CREATE_ORDER_PATH_URL = "spot/v1/submit_order"
 CANCEL_ORDER_PATH_URL = "spot/v2/cancel_order"
 GET_ACCOUNT_SUMMARY_PATH_URL = "spot/v1/wallet"
-GET_ORDER_DETAIL_PATH_URL = "spot/v1/order_detail"
+GET_ORDER_DETAIL_PATH_URL = "spot/v2/order_detail"
 GET_TRADE_DETAIL_PATH_URL = "spot/v1/trades"
 SERVER_TIME_PATH = "system/time"
 

hummingbot/connector/exchange/bitmart/bitmart_exchange.py

@@ -232,7 +232,6 @@ async def _format_trading_rules(self, symbols_details: Dict[str, Any]) -> List[T
                          "quote_currency":"BTC",
                          "quote_increment":"1.00000000",
                          "base_min_size":"1.00000000",
-                         "base_max_size":"10000000.00000000",
                          "price_min_precision":6,
                          "price_max_precision":8,
                          "expiration":"NA",
@@ -254,7 +253,6 @@ async def _format_trading_rules(self, symbols_details: Dict[str, Any]) -> List[T
                     price_step = Decimal("1") / Decimal(str(math.pow(10, price_decimals)))
                     result.append(TradingRule(trading_pair=trading_pair,
                                               min_order_size=Decimal(str(rule["base_min_size"])),
-                                              max_order_size=Decimal(str(rule["base_max_size"])),
                                               min_order_value=Decimal(str(rule["min_buy_amount"])),
                                               min_base_amount_increment=Decimal(str(rule["base_min_size"])),
                                               min_price_increment=price_step))
@@ -291,7 +289,7 @@ async def _update_balances(self):
     async def _request_order_update(self, order: InFlightOrder) -> Dict[str, Any]:
         return await self._api_get(
             path_url=CONSTANTS.GET_ORDER_DETAIL_PATH_URL,
-            params={"clientOrderId": order.client_order_id},
+            params={"order_id": order.exchange_order_id},
             is_auth_required=True)
 
     async def _request_order_fills(self, order: InFlightOrder) -> Dict[str, Any]:

scripts/adjusted_mid_price.py

@@ -117,7 +117,7 @@ def adjusted_mid_price(self):
         ask_result = self.connector.get_quote_volume_for_base_amount(self.strategy["pair"], True, self.strategy["test_volume"])
         bid_result = self.connector.get_quote_volume_for_base_amount(self.strategy["pair"], False, self.strategy["test_volume"])
         average_ask = ask_result.result_volume / ask_result.query_volume
-        average_bid = bid_result = bid_result.result_volume / bid_result.query_volume
+        average_bid = bid_result.result_volume / bid_result.query_volume
         return average_bid + ((average_ask - average_bid) / 2)
 
     def format_status(self) -> str:

test/hummingbot/connector/derivative/kucoin_perpetual/test_kucoin_perpetual_derivative.py

@@ -15,10 +15,11 @@
 from hummingbot.client.config.client_config_map import ClientConfigMap
 from hummingbot.client.config.config_helpers import ClientConfigAdapter
 from hummingbot.connector.derivative.kucoin_perpetual.kucoin_perpetual_derivative import KucoinPerpetualDerivative
+from hummingbot.connector.derivative.position import Position
 from hummingbot.connector.test_support.perpetual_derivative_test import AbstractPerpetualDerivativeTests
 from hummingbot.connector.trading_rule import TradingRule
 from hummingbot.connector.utils import combine_to_hb_trading_pair
-from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType
+from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType
 from hummingbot.core.data_type.funding_info import FundingInfo
 from hummingbot.core.data_type.in_flight_order import InFlightOrder
 from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount, TradeFeeBase
@@ -974,7 +975,7 @@ def trade_event_for_full_fill_websocket_update(self, order: InFlightOrder):
                 "size": float(order.amount) * 1000,
                 "fee": str(self.expected_fill_fee.percent),
                 "remainSize": "0",
-                "matchSize": float(order.amount) * 1000,
+                "matchSize": float(order.amount) * 1000000,
                 "canceledSize": "0",
                 "clientOid": order.client_order_id or "",
                 "orderTime": 1545914149935808589,
@@ -1018,7 +1019,7 @@ def position_event_for_full_fill_websocket_update(self, order: InFlightOrder, un
                 "changeReason": "positionChange",
                 "currentCost": str(position_value),
                 "openingTimestamp": 1558433191000,
-                "currentQty": -float(order.amount),
+                "currentQty": -int(order.amount),
                 "delevPercentage": 0.52,
                 "currentComm": 0.00000271,
                 "realisedGrossCost": 0E-8,
@@ -1555,3 +1556,158 @@ def test_start_network_update_trading_rules(self, mock_api):
         self.assertEqual(1, len(self.exchange.trading_rules))
         self.assertIn(self.trading_pair, self.exchange.trading_rules)
         self.assertEqual(repr(self.expected_trading_rule), repr(self.exchange.trading_rules[self.trading_pair]))
+
+    @aioresponses()
+    def test_user_stream_update_for_order_full_fill(self, mock_api):
+        self.exchange._set_current_timestamp(1640780000)
+        self._simulate_trading_rules_initialized()
+        leverage = 2
+        self.exchange._perpetual_trading.set_leverage(self.trading_pair, leverage)
+        self.exchange.start_tracking_order(
+            order_id=self.client_order_id_prefix + "1",
+            exchange_order_id=self.exchange_order_id_prefix + "1",
+            trading_pair=self.trading_pair,
+            order_type=OrderType.LIMIT,
+            trade_type=TradeType.SELL,
+            price=Decimal("10000"),
+            amount=Decimal("1"),
+            position_action=PositionAction.OPEN,
+        )
+        order = self.exchange.in_flight_orders[self.client_order_id_prefix + "1"]
+
+        order_event = self.order_event_for_full_fill_websocket_update(order=order)
+        trade_event = self.trade_event_for_full_fill_websocket_update(order=order)
+        expected_unrealized_pnl = 12
+        position_event = self.position_event_for_full_fill_websocket_update(
+            order=order, unrealized_pnl=expected_unrealized_pnl
+        )
+
+        mock_queue = AsyncMock()
+        event_messages = []
+        if trade_event:
+            event_messages.append(trade_event)
+        if order_event:
+            event_messages.append(order_event)
+        if position_event:
+            event_messages.append(position_event)
+        event_messages.append(asyncio.CancelledError)
+        mock_queue.get.side_effect = event_messages
+        self.exchange._user_stream_tracker._user_stream = mock_queue
+
+        if self.is_order_fill_http_update_executed_during_websocket_order_event_processing:
+            self.configure_full_fill_trade_response(
+                order=order,
+                mock_api=mock_api)
+
+        try:
+            self.async_run_with_timeout(self.exchange._user_stream_event_listener())
+        except asyncio.CancelledError:
+            pass
+        # Execute one more synchronization to ensure the async task that processes the update is finished
+        self.async_run_with_timeout(order.wait_until_completely_filled())
+
+        fill_event = self.order_filled_logger.event_log[0]
+        self.assertEqual(self.exchange.current_timestamp, fill_event.timestamp)
+        self.assertEqual(order.client_order_id, fill_event.order_id)
+        self.assertEqual(order.trading_pair, fill_event.trading_pair)
+        self.assertEqual(order.trade_type, fill_event.trade_type)
+        self.assertEqual(order.order_type, fill_event.order_type)
+        self.assertEqual(order.price, fill_event.price)
+        self.assertEqual(order.amount, fill_event.amount)
+        expected_fee = self.expected_fill_fee
+        self.assertEqual(expected_fee, fill_event.trade_fee)
+        self.assertEqual(leverage, fill_event.leverage)
+        self.assertEqual(PositionAction.OPEN.value, fill_event.position)
+
+        sell_event = self.sell_order_completed_logger.event_log[0]
+        self.assertEqual(self.exchange.current_timestamp, sell_event.timestamp)
+        self.assertEqual(order.client_order_id, sell_event.order_id)
+        self.assertEqual(order.base_asset, sell_event.base_asset)
+        self.assertEqual(order.quote_asset, sell_event.quote_asset)
+        self.assertEqual(order.amount, sell_event.base_asset_amount)
+        self.assertEqual(order.amount * fill_event.price, sell_event.quote_asset_amount)
+        self.assertEqual(order.order_type, sell_event.order_type)
+        self.assertEqual(order.exchange_order_id, sell_event.exchange_order_id)
+        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)
+        self.assertTrue(order.is_filled)
+        self.assertTrue(order.is_done)
+
+        self.assertTrue(
+            self.is_logged(
+                "INFO",
+                f"SELL order {order.client_order_id} completely filled."
+            )
+        )
+
+        self.assertEqual(1, len(self.exchange.account_positions))
+
+        position: Position = self.exchange.account_positions[self.trading_pair]
+        self.assertEqual(self.trading_pair, position.trading_pair)
+        self.assertEqual(PositionSide.SHORT, position.position_side)
+        self.assertEqual(expected_unrealized_pnl, position.unrealized_pnl)
+        self.assertEqual(fill_event.price, position.entry_price)
+        self.assertEqual(-fill_event.amount, (self.exchange.get_quantity_of_contracts(self.trading_pair, position.amount)))
+        self.assertEqual(leverage, position.leverage)
+
+    @aioresponses()
+    def test_lost_order_user_stream_full_fill_events_are_processed(self, mock_api):
+        self.exchange._set_current_timestamp(1640780000)
+        self._simulate_trading_rules_initialized()
+        self.exchange.start_tracking_order(
+            order_id=self.client_order_id_prefix + "1",
+            exchange_order_id=str(self.expected_exchange_order_id),
+            trading_pair=self.trading_pair,
+            order_type=OrderType.LIMIT,
+            trade_type=TradeType.BUY,
+            price=Decimal("10000"),
+            amount=Decimal("1"),
+        )
+        order = self.exchange.in_flight_orders[self.client_order_id_prefix + "1"]
+
+        for _ in range(self.exchange._order_tracker._lost_order_count_limit + 1):
+            self.async_run_with_timeout(
+                self.exchange._order_tracker.process_order_not_found(client_order_id=order.client_order_id))
+
+        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)
+
+        order_event = self.order_event_for_full_fill_websocket_update(order=order)
+        trade_event = self.trade_event_for_full_fill_websocket_update(order=order)
+
+        mock_queue = AsyncMock()
+        event_messages = []
+        if trade_event:
+            event_messages.append(trade_event)
+        if order_event:
+            event_messages.append(order_event)
+        event_messages.append(asyncio.CancelledError)
+        mock_queue.get.side_effect = event_messages
+        self.exchange._user_stream_tracker._user_stream = mock_queue
+
+        if self.is_order_fill_http_update_executed_during_websocket_order_event_processing:
+            self.configure_full_fill_trade_response(
+                order=order,
+                mock_api=mock_api)
+
+        try:
+            self.async_run_with_timeout(self.exchange._user_stream_event_listener())
+        except asyncio.CancelledError:
+            pass
+        # Execute one more synchronization to ensure the async task that processes the update is finished
+        self.async_run_with_timeout(order.wait_until_completely_filled())
+
+        fill_event = self.order_filled_logger.event_log[0]
+        self.assertEqual(self.exchange.current_timestamp, fill_event.timestamp)
+        self.assertEqual(order.client_order_id, fill_event.order_id)
+        self.assertEqual(order.trading_pair, fill_event.trading_pair)
+        self.assertEqual(order.trade_type, fill_event.trade_type)
+        self.assertEqual(order.order_type, fill_event.order_type)
+        self.assertEqual(order.price, fill_event.price)
+        self.assertEqual(order.amount, fill_event.amount)
+        expected_fee = self.expected_fill_fee
+        self.assertEqual(expected_fee, fill_event.trade_fee)
+
+        self.assertEqual(0, len(self.buy_order_completed_logger.event_log))
+        self.assertNotIn(order.client_order_id, self.exchange.in_flight_orders)
+        self.assertNotIn(order.client_order_id, self.exchange._order_tracker.lost_orders)
+        self.assertTrue(order.is_filled)
+        self.assertTrue(order.is_failure)

test/hummingbot/connector/derivative/kucoin_perpetual/test_kucoin_perpetual_web_utils.py

@@ -9,6 +9,3 @@ def test_get_rest_url_for_endpoint(self):
 
         url = web_utils.get_rest_url_for_endpoint(endpoint, domain="kucoin_perpetual_main")
         self.assertEqual("https://api-futures.kucoin.com/testEndpoint", url)
-
-        url = web_utils.get_rest_url_for_endpoint(endpoint, domain="kucoin_perpetual_testnet")
-        self.assertEqual("https://api-sandbox-futures.kucoin.com/testEndpoint", url)