Merge branch 'hummingbot:development' into development

bin/hummingbot.py

@@ -7,6 +7,7 @@
 import path_util  # noqa: F401
 
 from hummingbot import chdir_to_data_directory, init_logging
+from hummingbot.client.config.client_config_map import ClientConfigMap
 from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger
 from hummingbot.client.config.config_helpers import (
     ClientConfigAdapter,
@@ -85,8 +86,12 @@ def main():
         ev_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
         asyncio.set_event_loop(ev_loop)
 
-    client_config_map = load_client_config_map_from_file()
-    if login_prompt(secrets_manager_cls, style=load_style(client_config_map)):
+    # We need to load a default style for the login screen because the password is required to load the
+    # real configuration now that it can include secret parameters
+    style = load_style(ClientConfigAdapter(ClientConfigMap()))
+
+    if login_prompt(secrets_manager_cls, style=style):
+        client_config_map = load_client_config_map_from_file()
         ev_loop.run_until_complete(main_async(client_config_map))
 
 

hummingbot/client/config/client_config_map.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
 
-from pydantic import BaseModel, Field, root_validator, validator
+from pydantic import BaseModel, Field, SecretStr, root_validator, validator
 from tabulate import tabulate_formats
 
 from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData
@@ -805,6 +805,87 @@ def post_validations(cls, values: Dict):
         return values
 
 
+class CoinCapRateSourceMode(RateSourceModeBase):
+    name: str = Field(
+        default="coin_cap",
+        const=True,
+        client_data=None,
+    )
+    assets_map: Dict[str, str] = Field(
+        default=",".join(
+            [
+                ":".join(pair) for pair in {
+                    "BTC": "bitcoin",
+                    "ETH": "ethereum",
+                    "USDT": "tether",
+                    "CONV": "convergence",
+                    "FIRO": "zcoin",
+                    "BUSD": "binance-usd",
+                    "ONE": "harmony",
+                    "PDEX": "polkadex",
+                }.items()
+            ]
+        ),
+        description=(
+            "The symbol-to-asset ID map for CoinCap. Assets IDs can be found by selecting a symbol"
+            " on https://coincap.io/ and extracting the last segment of the URL path."
+        ),
+        client_data=ClientFieldData(
+            prompt=lambda cm: (
+                "CoinCap symbol-to-asset ID map (e.g. 'BTC:bitcoin,ETH:ethereum', find IDs on https://coincap.io/"
+                " by selecting a symbol and extracting the last segment of the URL path)"
+            ),
+            is_connect_key=True,
+            prompt_on_new=True,
+        ),
+    )
+    api_key: SecretStr = Field(
+        default=SecretStr(""),
+        description="API key to use to request information from CoinCap (if empty public requests will be used)",
+        client_data=ClientFieldData(
+            prompt=lambda cm: "CoinCap API key (optional, but improves rate limits)",
+            is_secure=True,
+            is_connect_key=True,
+            prompt_on_new=True,
+        ),
+    )
+
+    class Config:
+        title = "coin_cap"
+
+    def build_rate_source(self) -> RateSourceBase:
+        rate_source = RATE_ORACLE_SOURCES["coin_cap"](
+            assets_map=self.assets_map, api_key=self.api_key.get_secret_value()
+        )
+        return rate_source
+
+    @validator("assets_map", pre=True)
+    def validate_extra_tokens(cls, value: Union[str, Dict[str, str]]):
+        if isinstance(value, str):
+            value = {key: val for key, val in [v.split(":") for v in value.split(",")]}
+        return value
+
+    # === post-validations ===
+
+    @root_validator()
+    def post_validations(cls, values: Dict):
+        cls.rate_oracle_source_on_validated(values)
+        return values
+
+    @classmethod
+    def rate_oracle_source_on_validated(cls, values: Dict):
+        RateOracle.get_instance().source = cls._build_rate_source_cls(
+            assets_map=values["assets_map"], api_key=values["api_key"]
+        )
+
+    @classmethod
+    def _build_rate_source_cls(cls, assets_map: Dict[str, str], api_key: SecretStr) -> RateSourceBase:
+        rate_source = RATE_ORACLE_SOURCES["coin_cap"](
+            assets_map=assets_map, api_key=api_key.get_secret_value()
+        )
+        return rate_source
+
+
 class KuCoinRateSourceMode(ExchangeRateSourceModeBase):
     name: str = Field(
         default="kucoin",
@@ -831,6 +912,7 @@ class Config:
     AscendExRateSourceMode.Config.title: AscendExRateSourceMode,
     BinanceRateSourceMode.Config.title: BinanceRateSourceMode,
     CoinGeckoRateSourceMode.Config.title: CoinGeckoRateSourceMode,
+    CoinCapRateSourceMode.Config.title: CoinCapRateSourceMode,
     KuCoinRateSourceMode.Config.title: KuCoinRateSourceMode,
     GateIoRateSourceMode.Config.title: GateIoRateSourceMode,
 }
@@ -1153,7 +1235,5 @@ def post_validations(cls, values: Dict):
     @classmethod
     def rate_oracle_source_on_validated(cls, values: Dict):
         rate_source_mode: RateSourceModeBase = values["rate_oracle_source"]
-        rate_source_name = rate_source_mode.Config.title
-        if rate_source_name != RateOracle.get_instance().source.name:
-            RateOracle.get_instance().source = rate_source_mode.build_rate_source()
+        RateOracle.get_instance().source = rate_source_mode.build_rate_source()
         RateOracle.get_instance().quote_token = values["global_token"].global_token_name

hummingbot/client/config/config_helpers.py

@@ -38,63 +38,6 @@
     AllConnectorSettings,
 )
 
-# Use ruamel.yaml to preserve order and comments in .yml file
-yaml_parser = ruamel.yaml.YAML()  # legacy
-
-
-def decimal_representer(dumper: SafeDumper, data: Decimal):
-    return dumper.represent_float(float(data))
-
-
-def enum_representer(dumper: SafeDumper, data: ClientConfigEnum):
-    return dumper.represent_str(str(data))
-
-
-def date_representer(dumper: SafeDumper, data: date):
-    return dumper.represent_date(data)
-
-
-def time_representer(dumper: SafeDumper, data: time):
-    return dumper.represent_str(data.strftime("%H:%M:%S"))
-
-
-def datetime_representer(dumper: SafeDumper, data: datetime):
-    return dumper.represent_datetime(data)
-
-
-def path_representer(dumper: SafeDumper, data: Path):
-    return dumper.represent_str(str(data))
-
-
-def command_shortcut_representer(dumper: SafeDumper, data: CommandShortcutModel):
-    return dumper.represent_dict(data.__dict__)
-
-
-yaml.add_representer(
-    data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper
-)
-yaml.add_multi_representer(
-    data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=date, representer=date_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=time, representer=time_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=datetime, representer=datetime_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=Path, representer=path_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=PosixPath, representer=path_representer, Dumper=SafeDumper
-)
-yaml.add_representer(
-    data_type=CommandShortcutModel, representer=command_shortcut_representer, Dumper=SafeDumper
-)
-
 
 class ConfigValidationError(Exception):
     pass
@@ -244,12 +187,18 @@ def generate_yml_output_str_with_comments(self) -> str:
         return yml_str
 
     def validate_model(self) -> List[str]:
-        results = validate_model(type(self._hb_config), json.loads(self._hb_config.json()))
+        input_data = self._hb_config.dict()
+        results = validate_model(model=type(self._hb_config), input_data=input_data)  # coerce types
         conf_dict = results[0]
         for key, value in conf_dict.items():
             self.setattr_no_validation(key, value)
-        self._decrypt_all_internal_secrets()
+        self.decrypt_all_secure_data()
+        input_data = self._hb_config.dict()
+        results = validate_model(model=type(self._hb_config), input_data=input_data)  # validate decrypted values
+        conf_dict = results[0]
         errors = results[2]
+        for key, value in conf_dict.items():
+            self.setattr_no_validation(key, value)
         validation_errors = []
         if errors is not None:
             errors = errors.errors()
@@ -266,6 +215,29 @@ def setattr_no_validation(self, attr: str, value: Any):
     def full_copy(self):
         return self.__class__(hb_config=self._hb_config.copy(deep=True))
 
+    def decrypt_all_secure_data(self):
+        from hummingbot.client.config.security import Security  # avoids circular import
+
+        secure_config_items = (
+            traversal_item
+            for traversal_item in self.traverse()
+            if traversal_item.client_field_data is not None and traversal_item.client_field_data.is_secure
+        )
+        for traversal_item in secure_config_items:
+            value = traversal_item.value
+            if isinstance(value, SecretStr):
+                value = value.get_secret_value()
+            if value == "" or Security.secrets_manager is None:
+                decrypted_value = value
+            else:
+                decrypted_value = Security.secrets_manager.decrypt_secret_value(attr=traversal_item.attr, value=value)
+            *intermediate_items, final_config_element = traversal_item.config_path.split(".")
+            config_model = self
+            if len(intermediate_items) > 0:
+                for attr in intermediate_items:
+                    config_model = config_model.__getattr__(attr)
+            setattr(config_model, final_config_element, decrypted_value)
+
     @contextlib.contextmanager
     def _disable_validation(self):
         self._hb_config.Config.validate_assignment = False
@@ -387,6 +359,79 @@ def lock_config(cls, config_map: ClientConfigMap):
         return cls(config_map._hb_config)
 
 
+# Use ruamel.yaml to preserve order and comments in .yml file
+yaml_parser = ruamel.yaml.YAML()  # legacy
+
+
+def decimal_representer(dumper: SafeDumper, data: Decimal):
+    return dumper.represent_float(float(data))
+
+
+def enum_representer(dumper: SafeDumper, data: ClientConfigEnum):
+    return dumper.represent_str(str(data))
+
+
+def date_representer(dumper: SafeDumper, data: date):
+    return dumper.represent_date(data)
+
+
+def time_representer(dumper: SafeDumper, data: time):
+    return dumper.represent_str(data.strftime("%H:%M:%S"))
+
+
+def datetime_representer(dumper: SafeDumper, data: datetime):
+    return dumper.represent_datetime(data)
+
+
+def path_representer(dumper: SafeDumper, data: Path):
+    return dumper.represent_str(str(data))
+
+
+def command_shortcut_representer(dumper: SafeDumper, data: CommandShortcutModel):
+    return dumper.represent_dict(data.__dict__)
+
+
+def client_config_adapter_representer(dumper: SafeDumper, data: ClientConfigAdapter):
+    return dumper.represent_dict(data._dict_in_conf_order())
+
+
+def base_client_model_representer(dumper: SafeDumper, data: BaseClientModel):
+    dictionary_representation = ClientConfigAdapter(data)._dict_in_conf_order()
+    return dumper.represent_dict(dictionary_representation)
+
+
+yaml.add_representer(
+    data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper
+)
+yaml.add_multi_representer(
+    data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=date, representer=date_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=time, representer=time_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=datetime, representer=datetime_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=Path, representer=path_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=PosixPath, representer=path_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=CommandShortcutModel, representer=command_shortcut_representer, Dumper=SafeDumper
+)
+yaml.add_representer(
+    data_type=ClientConfigAdapter, representer=client_config_adapter_representer, Dumper=SafeDumper
+)
+yaml.add_multi_representer(
+    data_type=BaseClientModel, multi_representer=base_client_model_representer, Dumper=SafeDumper
+)
+
+
 def parse_cvar_value(cvar: ConfigVar, value: Any) -> Any:
     """
     Based on the target type specified in `ConfigVar.type_str`, parses a string value into the target type.