Merge branch 'development' into feat/gateway-v2

README.md

@@ -31,6 +31,7 @@ We created hummingbot to promote **decentralized market-making**: enabling membe
 | <img src="assets/blocktane-logo.jpg" alt="Blocktane" width="90" /> | blocktane | [Blocktane](https://blocktane.io/) | 2 | [API](https://blocktane.io/api) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+)  |
 | <img src="assets/bybit-logo.jpg" alt="Bybit Perpetual" width="90" /> | bybit_perpetual | [Bybit Perpetual](https://www.bybit.com/en-US/) | 2 | [API](https://bybit-exchange.github.io/docs/linear/#t-introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+)  |
 | <img src="assets/coinbase_pro-logo.jpg" alt="Coinbase Pro" width="90" /> | coinbase_pro | [Coinbase Pro](https://pro.coinbase.com/) | * | [API](https://docs.pro.coinbase.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) |
+| <img src="assets/coinflex_logo.png" alt="CoinFLEX" width="90" /> | coinflex | [CoinFLEX](https://coinflex.com/) | 2 | [API](https://coinflex.com/api/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) |
 | <img src="assets/coinzoom-logo.jpg" alt="CoinZoom" width="90" /> | coinzoom | [CoinZoom](https://trade.coinzoom.com/landing) | * | [API](https://api-docs.coinzoom.com/) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) |
 | <img src="assets/cryptocom-logo.jpg" alt="Crypto.com" width="90" /> | crypto_com | [Crypto.com](https://crypto.com/exchange) | 2 | [API](https://exchange-docs.crypto.com/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) |
 | <img src="assets/digifinex-logo.jpg" alt="Digifinex" width="90" /> | digifinex | [Digifinex](https://www.digifinex.com/en-ww) | 3 | [API](https://docs.digifinex.com/en-ww/v3/#introduction) |![YELLOW](https://via.placeholder.com/15/ffff00/?text=+) |

bin/hummingbot.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-import path_util        # noqa: F401
 import asyncio
 from typing import (
     List,
@@ -11,23 +10,23 @@
     ReferenceType,
 )
 
-from hummingbot.client.hummingbot_application import HummingbotApplication
-from hummingbot.client.config.global_config_map import global_config_map
+import path_util  # noqa: F401
+from hummingbot import (
+    chdir_to_data_directory,
+    init_logging,
+)
 from hummingbot.client.config.config_helpers import (
     create_yml_files,
     read_system_configs_from_yml,
     write_config_to_yml,
 )
-from hummingbot import (
-    init_logging,
-    check_dev_mode,
-    chdir_to_data_directory
-)
-from hummingbot.client.ui import login_prompt
+from hummingbot.client.config.global_config_map import global_config_map
+from hummingbot.client.hummingbot_application import HummingbotApplication
 from hummingbot.client.settings import AllConnectorSettings
-from hummingbot.core.gateway import start_existing_gateway_container
-from hummingbot.core.event.events import HummingbotUIEvent
+from hummingbot.client.ui import login_prompt
 from hummingbot.core.event.event_listener import EventListener
+from hummingbot.core.event.events import HummingbotUIEvent
+from hummingbot.core.gateway import start_existing_gateway_container
 from hummingbot.core.utils.async_utils import safe_gather
 from hummingbot.core.utils import detect_available_port
 
@@ -47,12 +46,8 @@ def hummingbot_app(self) -> HummingbotApplication:
         return self._hb_ref()
 
     async def ui_start_handler(self):
-        dev_mode: bool = check_dev_mode()
         hb: HummingbotApplication = self.hummingbot_app
 
-        if dev_mode:
-            hb.app.log("Running from dev branches. Full remote logging will be enabled.")
-
         if hb.strategy_file_name is not None and hb.strategy_name is not None:
             await write_config_to_yml(hb.strategy_name, hb.strategy_file_name)
             hb.start(global_config_map.get("log_level").value)

bin/hummingbot_quickstart.py

@@ -1,18 +1,18 @@
 #!/usr/bin/env python
 
-import path_util        # noqa: F401
 import argparse
 import asyncio
 import logging
 import os
 from pathlib import Path
 import pwd
+import subprocess
 from typing import (
     Coroutine,
     List,
 )
-import subprocess
 
+import path_util        # noqa: F401
 from bin.hummingbot import (
     detect_available_port,
     UIStartListener,
@@ -31,7 +31,7 @@
 from hummingbot.core.gateway import start_existing_gateway_container
 from hummingbot.core.management.console import start_management_console
 from hummingbot.core.utils.async_utils import safe_gather
-from hummingbot.client.settings import CONF_FILE_PATH, AllConnectorSettings
+from hummingbot.client.settings import AllConnectorSettings, CONF_FILE_PATH
 from hummingbot.client.config.security import Security
 
 from bin.docker_connection import fork_and_start

conf/__init__.py

@@ -125,6 +125,10 @@
 altmarkets_api_key = os.getenv("ALTMARKETS_API_KEY")
 altmarkets_secret_key = os.getenv("ALTMARKETS_SECRET_KEY")
 
+# CoinFLEX Test
+coinflex_api_key = os.getenv("COINFLEX_API_KEY")
+coinflex_api_secret = os.getenv("COINFLEX_API_SECRET")
+
 # Wallet Tests
 test_erc20_token_address = os.getenv("TEST_ERC20_TOKEN_ADDRESS")
 web3_test_private_key_a = os.getenv("TEST_WALLET_PRIVATE_KEY_A")

hummingbot/__init__.py

@@ -1,21 +1,11 @@
-#!/usr/bin/env python
-from typing import (
-    List,
-    Optional
-)
 import logging
 import subprocess
-from concurrent.futures import ThreadPoolExecutor
-from os import (
-    listdir,
-    path,
-)
 import sys
+from concurrent.futures import ThreadPoolExecutor
+from os import listdir, path
+from typing import List, Optional
 
-from hummingbot.logger.struct_logger import (
-    StructLogRecord,
-    StructLogger
-)
+from hummingbot.logger.struct_logger import StructLogger, StructLogRecord
 
 STRUCT_LOGGER_SET = False
 DEV_STRATEGY_PREFIX = "dev"
@@ -118,26 +108,8 @@ def chdir_to_data_directory():
     set_prefix_path(app_data_dir)
 
 
-def add_remote_logger_handler(loggers):
-    from hummingbot.logger.reporting_proxy_handler import ReportingProxyHandler
-    root_logger = logging.getLogger()
-    try:
-        from hummingbot.client.config.global_config_map import global_config_map
-        log_server_url = global_config_map.get("log_server_url", "https://api.coinalpha.com/reporting-proxy")
-        remote_logger = ReportingProxyHandler(level="ERROR",
-                                              proxy_url=log_server_url,
-                                              capacity=5)
-        root_logger.addHandler(remote_logger)
-        for logger_name in loggers:
-            logger = logging.getLogger(logger_name)
-            logger.addHandler(remote_logger)
-    except Exception:
-        root_logger.error("Error adding remote log handler.", exc_info=True)
-
-
 def init_logging(conf_filename: str,
                  override_log_level: Optional[str] = None,
-                 dev_mode: bool = False,
                  strategy_file_path: str = "hummingbot"):
     import io
     import logging.config
@@ -175,9 +147,6 @@ def init_logging(conf_filename: str,
                         logger in global_config_map["logger_override_whitelist"].value:
                     config_dict["loggers"][logger]["level"] = override_log_level
         logging.config.dictConfig(config_dict)
-        # add remote logging to logger if in dev mode
-        if dev_mode:
-            add_remote_logger_handler(config_dict.get("loggers", []))
 
 
 def get_strategy_list() -> List[str]:

hummingbot/client/command/balance_command.py

@@ -80,6 +80,7 @@ def balance(self,
 
     async def show_balances(self):
         total_col_name = f'Total ({RateOracle.global_token_symbol})'
+        sum_not_for_show_name = "sum_not_for_show"
         self.notify("Updating balances, please wait...")
         network_timeout = float(global_config_map["other_commands_timeout"].value)
         try:
@@ -103,10 +104,14 @@ async def show_balances(self):
             if df.empty:
                 self.notify("You have no balance on this exchange.")
             else:
-                lines = ["    " + line for line in df.to_string(index=False).split("\n")]
+                lines = ["    " + line for line in df.drop(sum_not_for_show_name, axis=1).to_string(index=False).split("\n")]
                 self.notify("\n".join(lines))
-                self.notify(f"\n  Total: {RateOracle.global_token_symbol} {PerformanceMetrics.smart_round(df[total_col_name].sum())}    "
-                            f"Allocated: {allocated_total / df[total_col_name].sum():.2%}")
+                self.notify(f"\n  Total: {RateOracle.global_token_symbol} "
+                            f"{PerformanceMetrics.smart_round(df[total_col_name].sum())}")
+                allocated_percentage = 0
+                if df[sum_not_for_show_name].sum() != Decimal("0"):
+                    allocated_percentage = allocated_total / df[sum_not_for_show_name].sum()
+                self.notify(f"Allocated: {allocated_percentage:.2%}")
                 exchanges_total += df[total_col_name].sum()
 
         self.notify(f"\n\nExchanges Total: {RateOracle.global_token_symbol} {exchanges_total:.0f}    ")

hummingbot/client/config/global_config_map.py

@@ -1,19 +1,16 @@
+import os.path
 import random
 import re
-from typing import Callable, Optional, Dict
 from decimal import Decimal
-import os.path
+from typing import Callable, Optional, Dict
 
 from tabulate import tabulate_formats
 
-from hummingbot.client.config.config_var import ConfigVar
 from hummingbot.client.config.config_methods import using_exchange as using_exchange_pointer
-from hummingbot.client.config.config_validators import (
-    validate_bool,
-    validate_decimal
-)
+from hummingbot.client.config.config_validators import validate_bool, validate_decimal
+from hummingbot.client.config.config_var import ConfigVar
 from hummingbot.client.settings import AllConnectorSettings, DEFAULT_KEY_FILE_PATH, DEFAULT_LOG_FILE_PATH
-from hummingbot.core.rate_oracle.rate_oracle import RateOracleSource, RateOracle
+from hummingbot.core.rate_oracle.rate_oracle import RateOracle, RateOracleSource
 
 
 def generate_client_id() -> str:
@@ -238,17 +235,17 @@ def global_token_symbol_on_validated(value: str):
                   type_str="str",
                   required_if=lambda: False,
                   default="5000"),
-    "heartbeat_enabled":
-        ConfigVar(key="heartbeat_enabled",
-                  prompt="Do you want to enable aggregated order and trade data collection? >>> ",
+    "anonymized_metrics_enabled":
+        ConfigVar(key="anonymized_metrics_enabled",
+                  prompt="Do you want to report aggregated, anonymized trade volume by exchange to "
+                         "Hummingbot Foundation? >>> ",
                   required_if=lambda: False,
                   type_str="bool",
                   validator=validate_bool,
                   default=True),
-    "heartbeat_interval_min":
-        ConfigVar(key="heartbeat_interval_min",
-                  prompt="How often do you want Hummingbot to send aggregated order and trade data (in minutes, "
-                         "e.g. enter 5 for once every 5 minutes)? >>> ",
+    "anonymized_metrics_interval_min":
+        ConfigVar(key="anonymized_metrics_interval_min",
+                  prompt="How often do you want to send the anonymized metrics (Enter 5 for 5 minutes)? >>> ",
                   required_if=lambda: False,
                   type_str="decimal",
                   validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False),

hummingbot/client/ui/hummingbot_cli.py

@@ -1,42 +1,42 @@
-#!/usr/bin/env python
-
 import asyncio
-from contextlib import ExitStack
 import logging
 import threading
+from contextlib import ExitStack
+from typing import Any, Callable, Dict, Optional, TYPE_CHECKING
+
 from prompt_toolkit.application import Application
 from prompt_toolkit.clipboard.pyperclip import PyperclipClipboard
 from prompt_toolkit.completion import Completer
 from prompt_toolkit.document import Document
-from prompt_toolkit.layout.processors import BeforeInput, PasswordProcessor
 from prompt_toolkit.key_binding import KeyBindings
-from typing import Callable, Optional, Dict, Any, TYPE_CHECKING
+from prompt_toolkit.layout.processors import BeforeInput, PasswordProcessor
 
-from hummingbot import init_logging, check_dev_mode
-if TYPE_CHECKING:
-    from hummingbot.client.hummingbot_application import HummingbotApplication
+from hummingbot import init_logging
+from hummingbot.client.config.global_config_map import global_config_map
+from hummingbot.client.tab.data_types import CommandTab
+from hummingbot.client.ui.interface_utils import start_process_monitor, start_timer, start_trade_monitor
 from hummingbot.client.ui.layout import (
     create_input_field,
+    create_live_field,
     create_log_field,
     create_log_toggle,
     create_output_field,
+    create_process_monitor,
     create_search_field,
-    generate_layout,
+    create_tab_button,
     create_timer,
-    create_process_monitor,
     create_trade_monitor,
-    create_live_field,
-    create_tab_button,
+    generate_layout,
 )
-from hummingbot.client.config.global_config_map import global_config_map
-from hummingbot.client.tab.data_types import CommandTab
-from hummingbot.client.ui.interface_utils import start_timer, start_process_monitor, start_trade_monitor
 from hummingbot.client.ui.stdout_redirection import patch_stdout
 from hummingbot.client.ui.style import load_style
 from hummingbot.core.event.events import HummingbotUIEvent
 from hummingbot.core.pubsub import PubSub
 from hummingbot.core.utils.async_utils import safe_ensure_future
 
+if TYPE_CHECKING:
+    from hummingbot.client.hummingbot_application import HummingbotApplication
+
 
 # Monkey patching here as _handle_exception gets the UI hanged into Press ENTER screen mode
 def _handle_exception_patch(self, loop, context):
@@ -99,10 +99,8 @@ def did_start_ui(self):
         self._stdout_redirect_context.enter_context(patch_stdout(log_field=self.log_field))
 
         log_level = global_config_map.get("log_level").value
-        dev_mode = check_dev_mode()
         init_logging("hummingbot_logs.yml",
-                     override_log_level=log_level,
-                     dev_mode=dev_mode)
+                     override_log_level=log_level)
 
         self.trigger_event(HummingbotUIEvent.Start, self)
 

hummingbot/connector/connector_base.pxd

@@ -1,5 +1,5 @@
-from hummingbot.core.event.event_reporter cimport EventReporter
 from hummingbot.core.event.event_logger cimport EventLogger
+from hummingbot.core.event.event_reporter cimport EventReporter
 from hummingbot.core.network_iterator cimport NetworkIterator
 
 cdef class ConnectorBase(NetworkIterator):
@@ -15,6 +15,7 @@ cdef class ConnectorBase(NetworkIterator):
         public set _current_trade_fills
         public dict _exchange_order_ids
         public object _trade_fee_schema
+        public object _trade_volume_metric_collector
 
     cdef str c_buy(self, str trading_pair, object amount, object order_type=*, object price=*, dict kwargs=*)
     cdef str c_sell(self, str trading_pair, object amount, object order_type=*, object price=*, dict kwargs=*)

hummingbot/connector/connector_base.pyx

@@ -4,13 +4,16 @@ from typing import Dict, List, Set, Tuple
 
 from hummingbot.client.config.global_config_map import global_config_map
 from hummingbot.client.config.trade_fee_schema_loader import TradeFeeSchemaLoader
+from hummingbot.connector.connector_metrics_collector import TradeVolumeMetricCollector
 from hummingbot.connector.in_flight_order_base import InFlightOrderBase
 from hummingbot.connector.utils import split_hb_trading_pair, TradeFillOrderDetails
+from hummingbot.core.clock cimport Clock
 from hummingbot.core.data_type.cancellation_result import CancellationResult
 from hummingbot.core.data_type.common import OrderType, TradeType
 from hummingbot.core.event.event_logger import EventLogger
 from hummingbot.core.event.events import MarketEvent, OrderFilledEvent
 from hummingbot.core.network_iterator import NetworkIterator
+from hummingbot.core.rate_oracle.rate_oracle import RateOracle
 from hummingbot.core.utils.estimate_fee import estimate_fee
 
 NaN = float("nan")
@@ -60,6 +63,9 @@ cdef class ConnectorBase(NetworkIterator):
         self._current_trade_fills = set()
         self._exchange_order_ids = dict()
         self._trade_fee_schema = None
+        self._trade_volume_metric_collector = TradeVolumeMetricCollector.from_configuration(
+            connector=self,
+            rate_provider=RateOracle.get_instance())
 
     @property
     def real_time_balance_update(self) -> bool:
@@ -211,6 +217,15 @@ cdef class ConnectorBase(NetworkIterator):
     cdef c_tick(self, double timestamp):
         NetworkIterator.c_tick(self, timestamp)
         self.tick(timestamp)
+        self._trade_volume_metric_collector.process_tick(timestamp)
+
+    cdef c_start(self, Clock clock, double timestamp):
+        NetworkIterator.c_start(self, clock, timestamp)
+        self._trade_volume_metric_collector.start()
+
+    cdef c_stop(self, Clock clock):
+        NetworkIterator.c_stop(self, clock)
+        self._trade_volume_metric_collector.stop()
 
     async def cancel_all(self, timeout_seconds: float) -> List[CancellationResult]:
         """