Merge branch 'hummingbot:development' into development

hummingbot/client/command/config_command.py

@@ -74,6 +74,7 @@
                              "gateway_api_port",
                              "rate_oracle_source",
                              "extra_tokens",
+                             "fetch_pairs_from_all_exchanges",
                              "global_token",
                              "global_token_name",
                              "global_token_symbol",

hummingbot/client/command/connect_command.py

@@ -9,6 +9,7 @@
 from hummingbot.client.ui.interface_utils import format_df_for_printout
 from hummingbot.connector.connector_status import get_connector_status
 from hummingbot.core.utils.async_utils import safe_ensure_future
+from hummingbot.core.utils.trading_pair_fetcher import TradingPairFetcher
 from hummingbot.user.user_balances import UserBalances
 
 if TYPE_CHECKING:
@@ -143,6 +144,7 @@ async def _perform_connect(self, connector_config: ClientConfigAdapter, previous
         err_msg = await self.validate_n_connect_connector(connector_name)
         if err_msg is None:
             self.notify(f"\nYou are now connected to {connector_name}.")
+            safe_ensure_future(TradingPairFetcher.get_instance(client_config_map=ClientConfigAdapter).fetch_all(client_config_map=ClientConfigAdapter))
         else:
             self.notify(f"\nError: {err_msg}")
             if previous_keys is not None:

hummingbot/client/config/client_config_map.py

@@ -932,6 +932,13 @@ class ClientConfigMap(BaseClientModel):
             prompt=lambda cm: "Instance UID of the bot",
         ),
     )
+    fetch_pairs_from_all_exchanges: bool = Field(
+        default=False,
+        description="Fetch trading pairs from all exchanges if True, otherwise fetch only from connected exchanges.",
+        client_data=ClientFieldData(
+            prompt=lambda cm: "Would you like to fetch from all exchanges? (True/False)",
+        ),
+    )
     log_level: str = Field(default="INFO")
     debug_console: bool = Field(default=False)
     strategy_report_interval: float = Field(default=900)
@@ -1144,7 +1151,7 @@ def validate_telegram_mode(cls, v: Union[(str, Dict) + tuple(TELEGRAM_MODES.valu
             sub_model = TELEGRAM_MODES[v].construct()
         return sub_model
 
-    @validator("send_error_logs", pre=True)
+    @validator("send_error_logs", "fetch_pairs_from_all_exchanges", pre=True)
     def validate_bool(cls, v: str):
         """Used for client-friendly error output."""
         if isinstance(v, str):

hummingbot/client/config/config_helpers.py

@@ -88,6 +88,10 @@ def __eq__(self, other):
     def hb_config(self) -> BaseClientModel:
         return self._hb_config
 
+    @property
+    def fetch_pairs_from_all_exchanges(self) -> bool:
+        return ClientConfigMap.fetch_pairs_from_all_exchanges
+
     @property
     def title(self) -> str:
         return self._hb_config.Config.title

hummingbot/client/hummingbot_application.py

@@ -131,6 +131,10 @@ def __init__(self, client_config_map: Optional[ClientConfigAdapter] = None):
     def instance_id(self) -> str:
         return self.client_config_map.instance_id
 
+    @property
+    def fetch_pairs_from_all_exchanges(self) -> bool:
+        return self.client_config_map.fetch_pairs_from_all_exchanges
+
     @property
     def gateway_config_keys(self) -> List[str]:
         return self._gateway_monitor.gateway_config_keys

hummingbot/client/settings.py

@@ -192,6 +192,10 @@ def uses_gateway_generic_connector(self) -> bool:
         non_gateway_connectors_types = [ConnectorType.Exchange, ConnectorType.Derivative, ConnectorType.Connector]
         return self.type not in non_gateway_connectors_types
 
+    def connector_connected(self) -> str:
+        from hummingbot.client.config.security import Security
+        return True if Security.connector_config_file_exists(self.name) else False
+
     def uses_clob_connector(self) -> bool:
         return self.type in [ConnectorType.CLOB_SPOT, ConnectorType.CLOB_PERP]
 

hummingbot/connector/exchange/binance/binance_constants.py

@@ -69,25 +69,25 @@
 
 RATE_LIMITS = [
     # Pools
-    RateLimit(limit_id=REQUEST_WEIGHT, limit=1200, time_interval=ONE_MINUTE),
+    RateLimit(limit_id=REQUEST_WEIGHT, limit=6000, time_interval=ONE_MINUTE),
     RateLimit(limit_id=ORDERS, limit=50, time_interval=10 * ONE_SECOND),
     RateLimit(limit_id=ORDERS_24HR, limit=160000, time_interval=ONE_DAY),
-    RateLimit(limit_id=RAW_REQUESTS, limit=6100, time_interval= 5 * ONE_MINUTE),
+    RateLimit(limit_id=RAW_REQUESTS, limit=61000, time_interval= 5 * ONE_MINUTE),
     # Weighted Limits
     RateLimit(limit_id=TICKER_PRICE_CHANGE_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 1),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 2),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=TICKER_BOOK_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 2),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 4),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=EXCHANGE_INFO_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 10),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 20),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=SNAPSHOT_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 50),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 100),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=BINANCE_USER_STREAM_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 1),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 2),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=SERVER_TIME_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
               linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 1),
@@ -96,13 +96,13 @@
               linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 1),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=ACCOUNTS_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 10),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 20),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=MY_TRADES_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 10),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 20),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)]),
     RateLimit(limit_id=ORDER_PATH_URL, limit=MAX_REQUEST, time_interval=ONE_MINUTE,
-              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 2),
+              linked_limits=[LinkedLimitWeightPair(REQUEST_WEIGHT, 4),
                              LinkedLimitWeightPair(ORDERS, 1),
                              LinkedLimitWeightPair(ORDERS_24HR, 1),
                              LinkedLimitWeightPair(RAW_REQUESTS, 1)])

hummingbot/core/utils/trading_pair_fetcher.py

@@ -28,6 +28,7 @@ def get_instance(cls, client_config_map: Optional["ClientConfigAdapter"] = None)
     def __init__(self, client_config_map: ClientConfigAdapter):
         self.ready = False
         self.trading_pairs: Dict[str, Any] = {}
+        self.fetch_pairs_from_all_exchanges = client_config_map.fetch_pairs_from_all_exchanges
         self._fetch_task = safe_ensure_future(self.fetch_all(client_config_map))
 
     def _fetch_pairs_from_connector_setting(
@@ -49,14 +50,16 @@ async def fetch_all(self, client_config_map: ClientConfigAdapter):
                         connector_setting=connector_settings[conn_setting.parent_name],
                         connector_name=conn_setting.name
                     )
+                elif not self.fetch_pairs_from_all_exchanges:
+                    if conn_setting.connector_connected():
+                        self._fetch_pairs_from_connector_setting(connector_setting=conn_setting)
                 else:
                     self._fetch_pairs_from_connector_setting(connector_setting=conn_setting)
             except ModuleNotFoundError:
                 continue
             except Exception:
                 self.logger().exception(f"An error occurred when fetching trading pairs for {conn_setting.name}."
                                         "Please check the logs")
-
         self.ready = True
 
     async def call_fetch_pairs(self, fetch_fn: Callable[[], Awaitable[List[str]]], exchange_name: str):

hummingbot/data_feed/candles_feed/binance_spot_candles/constants.py

@@ -30,6 +30,6 @@
 REQUEST_WEIGHT = "REQUEST_WEIGHT"
 
 RATE_LIMITS = [
-    RateLimit(REQUEST_WEIGHT, limit=1200, time_interval=60),
+    RateLimit(REQUEST_WEIGHT, limit=6000, time_interval=60),
     RateLimit(CANDLES_ENDPOINT, limit=1200, time_interval=60, linked_limits=[LinkedLimitWeightPair("raw", 1)]),
     RateLimit(HEALTH_CHECK_ENDPOINT, limit=1200, time_interval=60, linked_limits=[LinkedLimitWeightPair("raw", 1)])]

hummingbot/strategy/directional_strategy_base.py

@@ -31,7 +31,7 @@ class DirectionalStrategyBase(ScriptStrategyBase):
         take_profit (float): The take profit percentage.
         time_limit (int): The time limit for the position.
         open_order_type (OrderType): The order type for opening the position.
-        open_order_slippage_buffer (int): The slippage buffer for the opening order.
+        open_order_slippage_buffer (float): The slippage buffer for the opening order.
         take_profit_order_type (OrderType): The order type for the take profit order.
         stop_loss_order_type (OrderType): The order type for the stop loss order.
         time_limit_order_type (OrderType): The order type for the time limit order.
@@ -59,7 +59,7 @@ class DirectionalStrategyBase(ScriptStrategyBase):
     take_profit: float = 0.01
     time_limit: int = 120
     open_order_type = OrderType.MARKET
-    open_order_slippage_buffer: int = 0.001
+    open_order_slippage_buffer: float = 0.001
     take_profit_order_type: OrderType = OrderType.MARKET
     stop_loss_order_type: OrderType = OrderType.MARKET
     time_limit_order_type: OrderType = OrderType.MARKET

setup/environment.yml

@@ -45,7 +45,7 @@ dependencies:
     - flake8==3.7.9
     - gql
     - importlib-metadata==0.23
-    - injective-py==0.8.*
+    - injective-py==0.9.*
     - jsonpickle==3.0.1
     - mypy-extensions==0.4.3
     - pandas_ta==0.3.14b

test/hummingbot/client/command/test_config_command.py

@@ -64,6 +64,7 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock):
                            "    | Key                               | Value                |\n"
                            "    |-----------------------------------+----------------------|\n"
                            "    | instance_id                       | TEST_ID              |\n"
+                           "    | fetch_pairs_from_all_exchanges    | False                |\n"
                            "    | kill_switch_mode                  | kill_switch_disabled |\n"
                            "    | autofill_import                   | disabled             |\n"
                            "    | telegram_mode                     | telegram_disabled    |\n"

test/hummingbot/core/utils/test_trading_pair_fetcher.py

@@ -10,6 +10,7 @@
 from hummingbot.client.config.client_config_map import ClientConfigMap
 from hummingbot.client.config.config_helpers import ClientConfigAdapter
 from hummingbot.client.config.config_var import ConfigVar
+from hummingbot.client.config.security import Security
 from hummingbot.client.settings import ConnectorSetting, ConnectorType
 from hummingbot.connector.exchange.binance import binance_constants as CONSTANTS, binance_web_utils
 from hummingbot.core.data_type.trade_fee import TradeFeeSchema
@@ -37,7 +38,7 @@ def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1):
 
     class MockConnectorSetting(MagicMock):
         def __init__(self, name, parent_name=None, connector=None, *args, **kwargs) -> None:
-            super().__init__(*args, **kwargs)
+            super().__init__(name, *args, **kwargs)
             self._name = name
             self._parent_name = parent_name
             self._connector = connector
@@ -53,6 +54,9 @@ def parent_name(self) -> str:
         def base_name(self) -> str:
             return self.name
 
+        def connector_connected(self) -> bool:
+            return True
+
         def add_domain_parameter(*_, **__) -> Dict[str, Any]:
             return {}
 
@@ -82,17 +86,41 @@ def test_fetched_connector_trading_pairs(self, _, mock_connector_settings):
         }
 
         client_config_map = ClientConfigAdapter(ClientConfigMap())
+        client_config_map.fetch_pairs_from_all_exchanges = True
         trading_pair_fetcher = TradingPairFetcher(client_config_map)
         self.async_run_with_timeout(self.wait_until_trading_pair_fetcher_ready(trading_pair_fetcher), 1.0)
         trading_pairs = trading_pair_fetcher.trading_pairs
         self.assertEqual(2, len(trading_pairs))
         self.assertEqual({"mockConnector": ["MOCK-HBOT"], "mock_paper_trade": ["MOCK-HBOT"]}, trading_pairs)
 
+    @patch("hummingbot.core.utils.trading_pair_fetcher.TradingPairFetcher._all_connector_settings")
+    @patch("hummingbot.core.utils.trading_pair_fetcher.TradingPairFetcher._sf_shared_instance")
+    @patch("hummingbot.client.config.security.Security.connector_config_file_exists")
+    @patch("hummingbot.client.config.security.Security.wait_til_decryption_done")
+    def test_fetched_connected_trading_pairs(self, _, __: MagicMock, ___: AsyncMock, mock_connector_settings):
+        connector = AsyncMock()
+        connector.all_trading_pairs.return_value = ["MOCK-HBOT"]
+        mock_connector_settings.return_value = {
+            "mock_exchange_1": self.MockConnectorSetting(name="binance", connector=connector),
+            "mock_paper_trade": self.MockConnectorSetting(name="mock_paper_trade", parent_name="mock_exchange_1")
+        }
+
+        client_config_map = ClientConfigAdapter(ClientConfigMap())
+        client_config_map.fetch_pairs_from_all_exchanges = False
+        self.assertTrue(Security.connector_config_file_exists("binance"))
+        trading_pair_fetcher = TradingPairFetcher(client_config_map)
+        self.async_run_with_timeout(self.wait_until_trading_pair_fetcher_ready(trading_pair_fetcher), 1.0)
+        trading_pairs = trading_pair_fetcher.trading_pairs
+        self.assertEqual(2, len(trading_pairs))
+        self.assertEqual({"binance": ["MOCK-HBOT"], "mock_paper_trade": ["MOCK-HBOT"]}, trading_pairs)
+
     @aioresponses()
     @patch("hummingbot.core.utils.trading_pair_fetcher.TradingPairFetcher._all_connector_settings")
     @patch("hummingbot.core.gateway.gateway_http_client.GatewayHttpClient.get_perp_markets")
     @patch("hummingbot.client.settings.GatewayConnectionSetting.get_connector_spec_from_market_name")
     def test_fetch_all(self, mock_api, con_spec_mock, perp_market_mock, all_connector_settings_mock, ):
+        client_config_map = ClientConfigAdapter(ClientConfigMap())
+        client_config_map.fetch_pairs_from_all_exchanges = True
         all_connector_settings_mock.return_value = {
             "binance": ConnectorSetting(
                 name='binance',
@@ -135,7 +163,6 @@ def test_fetch_all(self, mock_api, con_spec_mock, perp_market_mock, all_connecto
         }
 
         url = binance_web_utils.public_rest_url(path_url=CONSTANTS.EXCHANGE_INFO_PATH_URL)
-
         mock_response: Dict[str, Any] = {
             "timezone": "UTC",
             "serverTime": 1639598493658,
@@ -239,7 +266,6 @@ def test_fetch_all(self, mock_api, con_spec_mock, perp_market_mock, all_connecto
             "wallet_address": "0x..."
         }
 
-        client_config_map = ClientConfigAdapter(ClientConfigMap())
         fetcher = TradingPairFetcher(client_config_map)
         asyncio.get_event_loop().run_until_complete(fetcher._fetch_task)
         trading_pairs = fetcher.trading_pairs