From 3ca449ac784927541ce20a9d1481aab4c5b284c4 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 22 Feb 2024 20:10:30 -0300 Subject: [PATCH] Refactor modules --- lean/commands/backtest.py | 39 +-- lean/commands/cloud/live/deploy.py | 127 ++++---- lean/commands/cloud/status.py | 8 +- lean/commands/live/deploy.py | 295 ++++-------------- lean/commands/optimize.py | 29 +- lean/commands/research.py | 21 +- lean/components/api/live_client.py | 6 +- lean/components/config/lean_config_manager.py | 6 +- lean/components/util/json_modules_handler.py | 158 +++++++--- lean/constants.py | 15 + lean/models/__init__.py | 2 +- lean/models/addon_modules/__init__.py | 22 -- lean/models/addon_modules/addon_module.py | 20 -- lean/models/brokerages/__init__.py | 12 - lean/models/brokerages/cloud/__init__.py | 40 --- .../brokerages/cloud/cloud_brokerage.py | 90 ------ lean/models/brokerages/local/__init__.py | 38 --- lean/models/brokerages/local/data_feed.py | 29 -- .../brokerages/local/local_brokerage.py | 29 -- lean/models/cli/__init__.py | 43 +++ lean/models/click_options.py | 42 +-- .../{data_providers => cloud}/__init__.py | 22 +- lean/models/configuration.py | 80 +---- lean/models/data_providers/data_provider.py | 26 -- lean/models/json_module.py | 171 ++++++---- lean/models/lean_config_configurer.py | 121 ------- 26 files changed, 479 insertions(+), 1012 deletions(-) delete mode 100644 lean/models/addon_modules/__init__.py delete mode 100644 lean/models/addon_modules/addon_module.py delete mode 100644 lean/models/brokerages/__init__.py delete mode 100644 lean/models/brokerages/cloud/__init__.py delete mode 100644 lean/models/brokerages/cloud/cloud_brokerage.py delete mode 100644 lean/models/brokerages/local/__init__.py delete mode 100644 lean/models/brokerages/local/data_feed.py delete mode 100644 lean/models/brokerages/local/local_brokerage.py create mode 100644 lean/models/cli/__init__.py rename lean/models/{data_providers => cloud}/__init__.py (50%) delete mode 100644 lean/models/data_providers/data_provider.py delete mode 100644 lean/models/lean_config_configurer.py diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index d3d781bc..442623d2 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -18,11 +18,9 @@ from lean.click import LeanCommand, PathParameter from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH from lean.container import container, Logger -from lean.models.api import QCMinimalOrganization from lean.models.utils import DebuggingMethod -from lean.models.logger import Option -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module +from lean.models.cli import cli_data_downloaders, cli_addon_modules +from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name from lean.models.click_options import options_from_json, get_configs_for_options # The _migrate_* methods automatically update launch configurations for a given debugging method. @@ -33,6 +31,7 @@ # # These methods checks if the project has outdated configurations, and if so, update them to keep it working. + def _migrate_python_pycharm(logger: Logger, project_dir: Path) -> None: from os import path from click import Abort @@ -225,20 +224,6 @@ def _migrate_csharp_csproj(project_dir: Path) -> None: csproj_path.write_text(xml_manager.to_string(current_content), encoding="utf-8") -def _select_organization() -> QCMinimalOrganization: - """Asks the user for the organization that should be charged when downloading data. - - :return: the selected organization - """ - api_client = container.api_client - - organizations = api_client.organizations.get_all() - options = [Option(id=organization, label=organization.name) for organization in organizations] - - logger = container.logger - return logger.prompt_list("Select the organization to purchase and download data with", options) - - @command(cls=LeanCommand, requires_lean_config=True, requires_docker=True) @argument("project", type=PathParameter(exists=True, file_okay=True, dir_okay=True)) @option("--output", @@ -252,7 +237,7 @@ def _select_organization() -> QCMinimalOrganization: type=Choice(["pycharm", "ptvsd", "vsdbg", "rider", "local-platform"], case_sensitive=False), help="Enable a certain debugging method (see --help for more information)") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("backtest")) @@ -338,6 +323,7 @@ def backtest(project: Path, if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + environment_name = "backtesting" debugging_method = None if debug == "pycharm": debugging_method = DebuggingMethod.PyCharm @@ -360,17 +346,17 @@ def backtest(project: Path, if algorithm_file.name.endswith(".cs"): _migrate_csharp_csproj(algorithm_file.parent) - lean_config = lean_config_manager.get_complete_lean_config("backtesting", algorithm_file, debugging_method) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, debugging_method) if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" organization_id = container.organization_manager.try_get_working_organization_id() if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(organization_id) - data_provider_configurer.configure(lean_config, "backtesting") + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -407,11 +393,12 @@ def backtest(project: Path, lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' # Configure addon modules - build_and_configure_modules(addon_module, organization_id, lean_config, logger, "backtesting") + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) lean_runner = container.lean_runner lean_runner.run_lean(lean_config, - "backtesting", + environment_name, algorithm_file, output, engine_image, diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index ad9bb347..174cd6d7 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -11,25 +11,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path -from typing import Any, Dict, List, Tuple, Optional +from typing import List, Tuple, Optional from click import prompt, option, argument, Choice, confirm from lean.click import LeanCommand, ensure_options from lean.components.api.api_client import APIClient +from lean.components.util.json_modules_handler import non_interactive_config_build_for_name, save_settings, \ + interactive_config_build from lean.components.util.logger import Logger from lean.container import container from lean.models.api import (QCEmailNotificationMethod, QCNode, QCNotificationMethod, QCSMSNotificationMethod, QCWebhookNotificationMethod, QCTelegramNotificationMethod, QCProject) from lean.models.json_module import LiveInitialStateInput from lean.models.logger import Option -from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage -from lean.models.configuration import InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerage_data_feeds +from lean.models.cloud import cloud_brokerages, cloud_data_queue_handlers from lean.commands.cloud.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively + def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: """Logs a list of notification methods.""" logger = container.logger @@ -98,37 +98,7 @@ def _prompt_notification_method() -> QCNotificationMethod: return QCSMSNotificationMethod(phoneNumber=phone_number) -def _configure_brokerage(lean_config: Dict[str, Any], logger: Logger, user_provided_options: Dict[str, Any], show_secrets: bool) -> CloudBrokerage: - """Interactively configures the brokerage to use. - - :param lean_config: the LEAN configuration that should be used - :param logger: the logger to use - :param user_provided_options: the dictionary containing user provided options - :param show_secrets: whether to show secrets on input - :return: the cloud brokerage the user configured - """ - brokerage_options = [Option(id=b, label=b.get_name()) for b in all_cloud_brokerages] - return logger.prompt_list("Select a brokerage", brokerage_options).build(lean_config, - logger, - user_provided_options, - hide_input=not show_secrets) - -def _configure_data_feed(brokerage: CloudBrokerage, logger: Logger) -> None: - """Configures the live data provider to use based on the brokerage given. - - :param brokerage: the cloud brokerage - :param logger: the logger to use - """ - if len(cloud_brokerage_data_feeds[brokerage]) != 0: - data_feed_selected = logger.prompt_list("Select a live data provider", [ - Option(id=data_feed, label=data_feed) for data_feed in cloud_brokerage_data_feeds[brokerage] - ], multiple=False) - data_feed_property_name = [name for name in brokerage.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] - data_feed_property_name = data_feed_property_name[0] if len(data_feed_property_name) != 0 else "" - brokerage.update_value_for_given_config(data_feed_property_name, data_feed_selected) - - -def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: +def _configure_live_node(node: str, logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use. :param logger: the logger to use @@ -137,6 +107,15 @@ def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: Q :return: the live node the user wants to start live trading on """ nodes = api_client.nodes.get_all(cloud_project.organizationId) + if node is not None: + live_node = next((n for n in nodes.live if n.id == node or n.name == node), None) + + if live_node is None: + raise RuntimeError(f"You have no live node with name or id '{node}'") + + if live_node.busy: + raise RuntimeError(f"The live node named '{live_node.name}' is already in use by '{live_node.usedBy}'") + return live_node live_nodes = [node for node in nodes.live if not node.busy] if len(live_nodes) == 0: @@ -187,8 +166,12 @@ def _configure_auto_restart(logger: Logger) -> bool: @live.command(cls=LeanCommand, default_command=True, name="deploy") @argument("project", type=str) @option("--brokerage", - type=Choice([b.get_name() for b in all_cloud_brokerages], case_sensitive=False), + type=Choice([b.get_name() for b in cloud_brokerages], case_sensitive=False), help="The brokerage to use") +@option("--data-provider-live", + type=Choice([d.get_name() for d in cloud_data_queue_handlers], case_sensitive=False), + multiple=True, + help="The live data provider to use") @options_from_json(get_configs_for_options("live-cloud")) @option("--node", type=str, help="The name or id of the live node to run on") @option("--auto-restart", type=bool, help="Whether automatic algorithm restarting must be enabled") @@ -213,7 +196,7 @@ def _configure_auto_restart(logger: Logger) -> bool: @option("--push", is_flag=True, default=False, - help="Push local modifications to the cloud before starting live trading") + help="Push cli modifications to the cloud before starting live trading") @option("--open", "open_browser", is_flag=True, default=False, @@ -221,6 +204,7 @@ def _configure_auto_restart(logger: Logger) -> bool: @option("--show-secrets", is_flag=True, show_default=True, default=False, help="Show secrets as they are input") def deploy(project: str, brokerage: str, + data_provider_live: Optional[str], node: str, auto_restart: bool, notify_order_events: Optional[bool], @@ -254,34 +238,15 @@ def deploy(project: str, cloud_runner = container.cloud_runner finished_compile = cloud_runner.compile_project(cloud_project) + live_data_provider_settings = {} + lean_config = container.lean_config_manager.get_lean_config() + if brokerage is not None: ensure_options(["brokerage", "node", "auto_restart", "notify_order_events", "notify_insights"]) - brokerage_instance = None - [brokerage_instance] = [cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage.get_name() == brokerage] - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_essential_properties()] - ensure_options(essential_properties) - essential_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in essential_properties} - brokerage_instance.update_configs(essential_properties_value) - # now required properties can be fetched as per historical data provider from essential properties - required_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_required_properties([InternalInputUserInput])] - ensure_options(required_properties) - required_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in required_properties} - brokerage_instance.update_configs(required_properties_value) - - all_nodes = api_client.nodes.get_all(cloud_project.organizationId) - live_node = next((n for n in all_nodes.live if n.id == node or n.name == node), None) - - if live_node is None: - raise RuntimeError(f"You have no live node with name or id '{node}'") - - if live_node.busy: - raise RuntimeError(f"The live node named '{live_node.name}' is already in use by '{live_node.usedBy}'") - + brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cloud_brokerages, + kwargs, logger) notify_methods = [] - if notify_emails is not None: for config in notify_emails.split(","): address, subject = config.split(":") @@ -320,10 +285,10 @@ def deploy(project: str, raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage_instance.get_name()}") else: - lean_config = container.lean_config_manager.get_lean_config() - brokerage_instance = _configure_brokerage(lean_config, logger, kwargs, show_secrets=show_secrets) - _configure_data_feed(brokerage_instance, logger) - live_node = _configure_live_node(logger, api_client, cloud_project) + # let the user choose the brokerage + brokerage_instance = interactive_config_build(lean_config, cloud_brokerages, logger, kwargs, show_secrets, + "Select a brokerage", multiple=False) + notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project) @@ -332,15 +297,31 @@ def deploy(project: str, if holdings_option != LiveInitialStateInput.NotSupported: live_holdings = _configure_initial_holdings_interactively(logger, holdings_option, last_holdings) + live_node = _configure_live_node(node, logger, api_client, cloud_project) + + if data_provider_live is not None and len(data_provider_live) > 0: + # the user sent the live data provider to use + for data_provider in data_provider_live: + data_provider_instance = non_interactive_config_build_for_name(lean_config, data_provider, + cloud_data_queue_handlers, kwargs, logger) + + live_data_provider_settings.update({data_provider_instance.get_id(): data_provider_instance.get_settings()}) + else: + # let's ask the user which live data providers to use + data_feed_instances = interactive_config_build(lean_config, cloud_data_queue_handlers, logger, kwargs, + show_secrets, "Select a live data feed", multiple=True) + for data_feed in data_feed_instances: + settings = data_feed.get_settings() + + live_data_provider_settings.update({data_feed.get_id(): settings}) + brokerage_settings = brokerage_instance.get_settings() - price_data_handler = brokerage_instance.get_price_data_handler() - logger.info(f"Brokerage: {brokerage_instance.get_name()}") + logger.info(f"Brokerage: {brokerage_settings}") logger.info(f"Project id: {cloud_project.projectId}") - logger.info(f"Environment: {brokerage_settings['environment'].title()}") logger.info(f"Server name: {live_node.name}") logger.info(f"Server type: {live_node.sku}") - logger.info(f"Live data provider: {price_data_handler.replace('Handler', '')}") + logger.info(f"Live data providers: {live_data_provider_settings}") logger.info(f"LEAN version: {cloud_project.leanVersionId}") logger.info(f"Order event notifications: {'Yes' if notify_order_events else 'No'}") logger.info(f"Insight notifications: {'Yes' if notify_insights else 'No'}") @@ -357,11 +338,15 @@ def deploy(project: str, default=False, abort=True) + # save them for next time + save_settings(live_data_provider_settings) + save_settings(brokerage_settings) + live_algorithm = api_client.live.start(cloud_project.projectId, finished_compile.compileId, live_node.id, brokerage_settings, - price_data_handler, + live_data_provider_settings, auto_restart, cloud_project.leanVersionId, notify_order_events, diff --git a/lean/commands/cloud/status.py b/lean/commands/cloud/status.py index 06737d8b..70b62c3b 100644 --- a/lean/commands/cloud/status.py +++ b/lean/commands/cloud/status.py @@ -16,8 +16,7 @@ from lean.click import LeanCommand from lean.container import container from lean.models.api import QCLiveAlgorithmStatus -from lean.models.brokerages.cloud import all_cloud_brokerages, PaperTradingBrokerage - +from lean.models.brokerage import cloud_brokerages @command(cls=LeanCommand) @argument("project", type=str) @@ -49,12 +48,9 @@ def status(project: str) -> None: QCLiveAlgorithmStatus.LoggingIn: "Logging in" }.get(live_algorithm.status, live_algorithm.status.value) - brokerage_name = next((b.get_name() for b in all_cloud_brokerages if b.get_id() == live_algorithm.brokerage), + brokerage_name = next((b.get_name() for b in cloud_brokerages if b.get_id() == live_algorithm.brokerage), live_algorithm.brokerage) - if brokerage_name == "PaperBrokerage": - brokerage_name = PaperTradingBrokerage.get_name() - logger.info(f"Live status: {live_status}") logger.info(f"Live id: {live_algorithm.deployId}") logger.info(f"Live url: {live_algorithm.get_url()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 4f406dba..ab43c505 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -14,22 +14,19 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from click import option, argument, Choice -from lean.click import LeanCommand, PathParameter, ensure_options +from lean.click import LeanCommand, PathParameter from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container -from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds +from lean.models.cli import cli_brokerages, cli_data_queue_handlers, cli_data_downloaders, cli_addon_modules from lean.models.errors import MoreInfoError -from lean.models.lean_config_configurer import LeanConfigConfigurer -from lean.models.logger import Option -from lean.models.configuration import ConfigurationsEnvConfiguration, InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.json_module import LiveInitialStateInput +from lean.models.json_module import LiveInitialStateInput, JsonModule from lean.commands.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively -from lean.models.data_providers import all_data_providers -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available +from lean.components.util.json_modules_handler import build_and_configure_modules, \ + non_interactive_config_build_for_name, save_settings, interactive_config_build _environment_skeleton = { "live-mode": True, @@ -40,82 +37,6 @@ } -def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]: - """Returns the configurable modules from the given environment. - - :param lean_config: the LEAN configuration that should be used - :param environment_name: the name of the environment - :return: the configurable modules from the given environment - """ - environment = lean_config["environments"][environment_name] - for key in ["live-mode-brokerage", "data-queue-handler"]: - if key not in environment: - raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", - "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") - - brokerage = environment["live-mode-brokerage"] - data_queue_handlers = environment["data-queue-handler"] - [brokerage_configurer] = [local_brokerage - for local_brokerage in all_local_brokerages - if _get_brokerage_base_name(local_brokerage.get_live_name()) == _get_brokerage_base_name(brokerage)] - data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] - data_feed_configurers = [local_data_feed - for local_data_feed in all_local_data_feeds - if _get_brokerage_base_name(local_data_feed.get_live_name()) in data_queue_handlers_base_names] - return brokerage_configurer, data_feed_configurers - - -def _get_brokerage_base_name(brokerage: str) -> str: - """Returns the base name of the brokerage. - - :param brokerage: the name of the brokerage - :return: the base name of the brokerage - """ - return brokerage.split('.')[-1] - -def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None: - """Raises an error if any of the given modules are not installed. - - :param modules: the modules to check - """ - for module in modules: - if module is None or not module._installs: - continue - organization_id = container.organization_manager.try_get_working_organization_id() - module.ensure_module_installed(organization_id) - - -def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None: - """Raises an error if any required properties are missing. - - :param lean_config: the LEAN configuration that should be used - :param environment_name: the name of the environment - :param lean_config_path: the path to the LEAN configuration file - """ - brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name) - brokerage_properties = brokerage_configurer.get_required_properties(include_optionals=False) - data_queue_handler_properties = [] - for data_feed_configurer in data_feed_configurers: - data_queue_handler_properties.extend(data_feed_configurer.get_required_properties(include_optionals=False)) - required_properties = list(set(brokerage_properties + data_queue_handler_properties)) - missing_properties = [p for p in required_properties if p not in lean_config or lean_config[p] == ""] - missing_properties = set(missing_properties) - if len(missing_properties) == 0: - return - - properties_str = "properties" if len(missing_properties) > 1 else "property" - these_str = "these" if len(missing_properties) > 1 else "this" - - missing_properties = "\n".join(f"- {p}" for p in missing_properties) - - raise RuntimeError(f""" -Please configure the following missing {properties_str} in {lean_config_path}: -{missing_properties} -Go to the following url for documentation on {these_str} {properties_str}: -https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading - """.strip()) - - def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], environment_name: str) -> None: """Starts IQConnect if the given environment uses IQFeed as data queue handler. @@ -147,80 +68,17 @@ def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], environment_name: sleep(10) -def _configure_lean_config_interactively(lean_config: Dict[str, Any], - environment_name: str, - properties: Dict[str, Any], - show_secrets: bool) -> None: - """Interactively configures the Lean config to use. - - Asks the user all questions required to set up the Lean config for local live trading. - - :param lean_config: the base lean config to use - :param environment_name: the name of the environment to configure - :param properties: the properties to use to configure lean - :param show_secrets: whether to show secrets on input - """ - logger = container.logger - - lean_config["environments"] = { - environment_name: _environment_skeleton - } - - brokerage = logger.prompt_list("Select a brokerage", [ - Option(id=brokerage, label=brokerage.get_name()) for brokerage in all_local_brokerages - ]) - - brokerage.build(lean_config, logger, properties, hide_input=not show_secrets).configure(lean_config, environment_name) - - data_feeds = logger.prompt_list("Select a live data provider", [ - Option(id=data_feed, label=data_feed.get_name()) for data_feed in local_brokerage_data_feeds[brokerage] - ], multiple= True) - for data_feed in data_feeds: - if brokerage._id == data_feed._id: - # update essential properties, so that other dependent values can be fetched. - essential_properties_value = {brokerage.convert_lean_key_to_variable(config._id): config._value - for config in brokerage.get_essential_configs()} - properties.update(essential_properties_value) - logger.debug(f"live.deploy._configure_lean_config_interactively(): essential_properties_value: {brokerage._id} {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from essential properties - required_properties_value = {brokerage.convert_lean_key_to_variable(config._id): config._value - for config in brokerage.get_required_configs([InternalInputUserInput])} - properties.update(required_properties_value) - logger.debug(f"live.deploy._configure_lean_config_interactively(): required_properties_value: {required_properties_value}") - data_feed.build(lean_config, logger, properties, hide_input=not show_secrets).configure(lean_config, environment_name) - - _cached_lean_config = None + def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: """ Get name for historical data provider based on data provider live (if exist) :param data_provider_historical_name: the current (default) data provider historical :param data_provider_live_name: the current data provider live name """ - return next((live_data_historical.get_name() for live_data_historical in all_data_providers - if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) - - -# being used by lean.models.click_options.get_the_correct_type_default_value() -def _get_default_value(key: str) -> Optional[Any]: - """Returns the default value for an option based on the Lean config. - - :param key: the name of the property in the Lean config that supplies the default value of an option - :return: the value of the property in the Lean config, or None if there is none - """ - global _cached_lean_config - if _cached_lean_config is None: - _cached_lean_config = container.lean_config_manager.get_lean_config() - - if key not in _cached_lean_config: - return None - - value = _cached_lean_config[key] - if value == "": - return None - - return value + return next((live_data_historical.get_name() for live_data_historical in cli_data_downloaders + if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) @live.command(cls=LeanCommand, requires_lean_config=True, requires_docker=True, default_command=True, name="deploy") @@ -236,16 +94,16 @@ def _get_default_value(key: str) -> Optional[Any]: default=False, help="Run the live deployment in a detached Docker container and return immediately") @option("--brokerage", - type=Choice([b.get_name() for b in all_local_brokerages], case_sensitive=False), + type=Choice([b.get_name() for b in cli_brokerages], case_sensitive=False), help="The brokerage to use") @option("--data-provider-live", - type=Choice([d.get_name() for d in all_local_data_feeds], case_sensitive=False), + type=Choice([d.get_name() for d in cli_data_queue_handlers], case_sensitive=False), multiple=True, help="The live data provider to use") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders if dp.get_id() != "TerminalLinkBrokerage"], case_sensitive=False), help="Update the Lean configuration file to retrieve data from the given historical provider") -@options_from_json(get_configs_for_options("live-local")) +@options_from_json(get_configs_for_options("live-cli")) @option("--release", is_flag=True, default=False, @@ -285,7 +143,7 @@ def _get_default_value(key: str) -> Optional[Any]: @option("--no-update", is_flag=True, default=False, - help="Use the local LEAN engine image instead of pulling the latest version") + help="Use the cli LEAN engine image instead of pulling the latest version") def deploy(project: Path, environment: Optional[str], output: Optional[Path], @@ -342,99 +200,71 @@ def deploy(project: Path, lean_config_manager = container.lean_config_manager + brokerage_instance: JsonModule + data_provider_live_instances: [JsonModule] = [] + data_provider_historical_instances: JsonModule if environment is not None and (brokerage is not None or len(data_provider_live) > 0): raise RuntimeError("--environment and --brokerage + --data-provider-live are mutually exclusive") + environment_name = "lean-cli" if environment is not None: environment_name = environment lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) - lean_environment = lean_config["environments"][environment_name] - for key in ["live-mode-brokerage", "data-queue-handler"]: - if key not in lean_environment: - raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", - "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") - - brokerage = lean_environment["live-mode-brokerage"] - data_queue_handlers = lean_environment["data-queue-handler"] - data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] - data_feed_configurers = [] - - for local_brokerage in all_local_brokerages: - configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_brokerage._lean_configs if config._is_type_configurations_env] - for configuration_environment in configuration_environments: - configuration_environment_values = list(configuration_environment._env_and_values.values())[0] - if any(True for x in configuration_environment_values if x["name"] == "live-mode-brokerage" and _get_brokerage_base_name(x["value"]) == _get_brokerage_base_name(brokerage)): - brokerage_configurer = local_brokerage - # fill essential properties - for condition in configuration_environment._filter._conditions: - if condition._type != "exact-match": - continue - property_name_to_fill = local_brokerage.convert_lean_key_to_variable(condition._dependent_config_id) - property_value_to_fill = condition._pattern - kwargs[property_name_to_fill] = property_value_to_fill - lean_config[condition._dependent_config_id] = property_value_to_fill - break - - for local_data_feed in all_local_data_feeds: - configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_data_feed._lean_configs if config._is_type_configurations_env] - for configuration_environment in configuration_environments: - configuration_environment_values = list(configuration_environment._env_and_values.values())[0] - if any(True for x in configuration_environment_values if x["name"] == "data-queue-handler" and _get_brokerage_base_name(x["value"]) in data_queue_handlers_base_names): - data_feed_configurers.append(local_data_feed) - # fill essential properties - for condition in configuration_environment._filter._conditions: - if condition._type != "exact-match": - continue - property_name_to_fill = local_data_feed.convert_lean_key_to_variable(condition._dependent_config_id) - property_value_to_fill = condition._pattern - kwargs[property_name_to_fill] = property_value_to_fill - lean_config[condition._dependent_config_id] = property_value_to_fill - - [update_essential_properties_available([brokerage_configurer], kwargs)] - [update_essential_properties_available(data_feed_configurers, kwargs)] - - elif brokerage is not None or len(data_provider_live) > 0: - ensure_options(["brokerage", "data_provider_live"]) - - environment_name = "lean-cli" + if environment_name in lean_config["environments"]: + lean_environment = lean_config["environments"][environment_name] + for key in ["live-mode-brokerage", "data-queue-handler"]: + if key not in lean_environment: + raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", + "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") + + # todo figure out data downloader, 'data_provider_historical' from env + brokerage = lean_environment["live-mode-brokerage"] + data_provider_live = lean_environment["data-queue-handler"] + logger.debug(f'Deploy(): loading env \'{environment_name}\'. Brokerage: \'{brokerage}\'. IDQHs: [{data_provider_live}]') + else: + logger.info(f'Environment \'{environment_name}\' not found, creating from scratch') + lean_config["environments"] = {environment_name: copy(_environment_skeleton)} + else: lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) + lean_config["environments"] = {environment_name: copy(_environment_skeleton)} - lean_config["environments"] = { - environment_name: copy(_environment_skeleton) - } - - [brokerage_configurer] = [get_and_build_module(brokerage, all_local_brokerages, kwargs, logger)] - brokerage_configurer.configure(lean_config, environment_name) - - for df in data_provider_live: - [data_feed_configurer] = [get_and_build_module(df, all_local_data_feeds, kwargs, logger)] - data_feed_configurer.configure(lean_config, environment_name) - + if brokerage: + # user provided brokerage, check all arguments were provided + brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cli_brokerages, kwargs, + logger, environment_name) else: - environment_name = "lean-cli" - lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) - _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) + # let the user choose the brokerage + brokerage_instance = interactive_config_build(lean_config, cli_brokerages, logger, kwargs, show_secrets, + "Select a brokerage", multiple=False, + environment_name=environment_name) + + if data_provider_live and len(data_provider_live) > 0: + for data_feed_name in data_provider_live: + data_feed = non_interactive_config_build_for_name(lean_config, data_feed_name, cli_data_queue_handlers, + kwargs, logger, environment_name) + data_provider_live_instances.append(data_feed) + else: + data_provider_live_instances = interactive_config_build(lean_config, cli_data_queue_handlers, logger, kwargs, + show_secrets, "Select a live data feed", multiple=True, + environment_name=environment_name) if data_provider_historical is None: data_provider_historical = _try_get_data_historical_name("Local", data_provider_live) - [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] - data_provider_configurer.configure(lean_config, environment_name) + data_provider_historical_instances = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, + environment_name) - if "environments" not in lean_config or environment_name not in lean_config["environments"]: - lean_config_path = lean_config_manager.get_lean_config_path() - raise MoreInfoError(f"{lean_config_path} does not contain an environment named '{environment_name}'", - "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") + organization_id = container.organization_manager.try_get_working_organization_id() + for module in data_provider_live_instances + [data_provider_historical_instances, brokerage_instance]: + module.ensure_module_installed(organization_id) + logger.debug(f'Deploy(): lean_config: {lean_config}') + save_settings(lean_config) if not lean_config["environments"][environment_name]["live-mode"]: raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) - _install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs) - - _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) - project_config_manager = container.project_config_manager cli_config_manager = container.cli_config_manager @@ -451,7 +281,7 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, env_brokerage, + cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, brokerage_instance, project_config.get("cloud-id", None), project) if environment is None and brokerage is None and len(data_provider_live) == 0: # condition for using interactive panel @@ -498,7 +328,8 @@ def deploy(project: Path, lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output, given_algorithm_id)}" # Configure addon modules - build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, environment_name) + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) if container.platform_manager.is_host_arm(): if "InteractiveBrokersBrokerage" in lean_config["environments"][environment_name]["live-mode-brokerage"] \ @@ -506,4 +337,4 @@ def deploy(project: Path, raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts") lean_runner = container.lean_runner - lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) \ No newline at end of file + lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 05e02f27..8e365078 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -23,10 +23,10 @@ from lean.container import container from lean.models.api import QCParameter, QCBacktest from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.data_providers import all_data_providers, QuantConnectDataProvider, DataProvider +from lean.models.cli import cli_data_downloaders, cli_addon_modules from lean.models.errors import MoreInfoError from lean.models.optimizer import OptimizationTarget -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module +from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name def _get_latest_backtest_runtime(algorithm_directory: Path) -> timedelta: @@ -98,7 +98,7 @@ def get_filename_timestamp(path: Path) -> datetime: multiple=True, help="The 'statistic operator value' pairs configuring the constraints of the optimization") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @option("--download-data", @@ -206,7 +206,7 @@ def optimize(project: Path, from math import floor should_detach = detach and not estimate - environment = "backtesting" + environment_name = "backtesting" project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(project) @@ -294,18 +294,18 @@ def optimize(project: Path, logger.warn(f'A custom engine image: "{engine_image}" is being used!') lean_config_manager = container.lean_config_manager - lean_config = lean_config_manager.get_complete_lean_config(environment, algorithm_file, None) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) organization_id = container.organization_manager.try_get_working_organization_id() if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(organization_id) - data_provider_configurer.configure(lean_config, environment) - logger.info(lean_config) + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) + lean_config["environments"][environment_name].update(data_provider.get_settings()) if not output.exists(): output.mkdir(parents=True) @@ -317,9 +317,9 @@ def optimize(project: Path, # Set extra config for key, value in extra_config: - if "environments" in lean_config and environment in lean_config["environments"] \ - and key in lean_config["environments"][environment]: - lean_config["environments"][environment][key] = value + if "environments" in lean_config and environment_name in lean_config["environments"] \ + and key in lean_config["environments"][environment_name]: + lean_config["environments"][environment_name][key] = value else: lean_config[key] = value @@ -327,7 +327,8 @@ def optimize(project: Path, lean_config["algorithm-id"] = str(output_config_manager.get_optimization_id(output)) # Configure addon modules - build_and_configure_modules(addon_module, organization_id, lean_config, logger, environment) + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach) diff --git a/lean/commands/research.py b/lean/commands/research.py index d8c17d35..1d4dd0ee 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -18,9 +18,9 @@ from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH from lean.container import container -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider +from lean.models.cli import cli_data_downloaders from lean.components.util.name_extraction import convert_to_class_name -from lean.components.util.json_modules_handler import get_and_build_module +from lean.components.util.json_modules_handler import non_interactive_config_build_for_name from lean.models.click_options import options_from_json, get_configs_for_options def _check_docker_output(chunk: str, port: int) -> None: @@ -38,14 +38,14 @@ def _check_docker_output(chunk: str, port: int) -> None: @argument("project", type=PathParameter(exists=True, file_okay=False, dir_okay=True)) @option("--port", type=int, default=8888, help="The port to run Jupyter Lab on (defaults to 8888)") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("research")) @option("--download-data", is_flag=True, default=False, - help=f"Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical {QuantConnectDataProvider.get_name()}") + help=f"Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect") @option("--data-purchase-limit", type=int, help="The maximum amount of QCC to spend on downloading data during the research session when using QuantConnect as historical data provider") @@ -104,19 +104,20 @@ def research(project: Path, algorithm_file = project_manager.find_algorithm_file(project) algorithm_name = convert_to_class_name(project) + environment_name = "backtesting" lean_config_manager = container.lean_config_manager - lean_config = lean_config_manager.get_complete_lean_config("backtesting", algorithm_file, None) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) lean_config["composer-dll-directory"] = LEAN_ROOT_PATH lean_config["research-object-store-name"] = algorithm_name if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(container.organization_manager.try_get_working_organization_id()) - data_provider_configurer.configure(lean_config, "backtesting") - + organization_id = container.organization_manager.try_get_working_organization_id() + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) lean_runner = container.lean_runner diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index 7548bc4f..c3590467 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -57,7 +57,7 @@ def start(self, compile_id: str, node_id: str, brokerage_settings: Dict[str, Any], - price_data_handler: str, + live_data_providers_settings: Dict[str, Any], automatic_redeploy: bool, version_id: int, notify_order_events: bool, @@ -71,7 +71,7 @@ def start(self, :param compile_id: the id of the compile to use for live trading :param node_id: the id of the node to start live trading on :param brokerage_settings: the brokerage settings to use - :param price_data_handler: the live data provider to use + :param live_data_providers_settings: the live data providers settings to use :param automatic_redeploy: whether automatic redeploys are enabled :param version_id: the id of the LEAN version to use :param notify_order_events: whether notifications should be sent on order events @@ -92,7 +92,7 @@ def start(self, "compileId": compile_id, "nodeId": node_id, "brokerage": brokerage_settings, - "dataHandler": price_data_handler, + "dataProviders": live_data_providers_settings, "automaticRedeploy": automatic_redeploy, "versionId": version_id } diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 26622d5d..992e5205 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -155,10 +155,14 @@ def set_properties(self, updates: Dict[str, Any]) -> None: for key, value in reversed(list(updates.items())): json_value = dumps(value) + json_value = json_value.replace("\\", "/") # We can only use regex to set the property because converting the config back to JSON drops all comments if key in config: - config_text = sub(fr'"{key}":\s*("?[^",]*"?)', f'"{key}": {json_value}', config_text) + if json_value.startswith("["): + config_text = sub(fr'"{key}":\s*(\[.*?])', f'"{key}": {json_value}', config_text) + else: + config_text = sub(fr'"{key}":\s*("?[^",]*"?)', f'"{key}": {json_value}', config_text) else: config_text = config_text.replace("{", f'{{\n "{key}": {json_value},', 1) diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 85713a55..e81e755d 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -12,56 +12,124 @@ # limitations under the License. from typing import Any, Dict, List -from lean.models.addon_modules.addon_module import AddonModule -from lean.models.addon_modules import all_addon_modules from lean.components.util.logger import Logger -from lean.models.configuration import InternalInputUserInput from lean.models.json_module import JsonModule -from lean.click import ensure_options +from lean.models.logger import Option -def build_and_configure_modules(modules: List[AddonModule], organization_id: str, lean_config: Dict[str, Any], logger: Logger, environment_name: str) -> Dict[str, Any]: - """Capitalizes the given word. - :param word: the word to capitalize - :return: the word with the first letter capitalized (any other uppercase characters are preserved) +def build_and_configure_modules(target_modules: List[str], module_list: List[JsonModule], organization_id: str, + lean_config: Dict[str, Any], properties: Dict[str, Any], logger: Logger, + environment_name: str): + """Builds and configures the given modules + + :param target_modules: the requested modules + :param module_list: the available modules + :param organization_id: the organization id + :param lean_config: the current lean configs + :param properties: the user provided arguments + :param logger: the logger instance + :param environment_name: the environment name to use """ - for given_module in modules: - try: - found_module = next((module for module in all_addon_modules if module.get_name().lower() == given_module.lower()), None) - if found_module: - found_module.build(lean_config, logger).configure(lean_config, environment_name) - found_module.ensure_module_installed(organization_id) - else: - logger.error(f"Addon module '{given_module}' not found") - except Exception as e: - logger.error(f"Addon module '{given_module}' failed to configure: {e}") - return lean_config - - -def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger) -> JsonModule: - [target_module] = [module for module in module_list if module.get_name() == target_module_name] - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()] - ensure_options(essential_properties) - essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties} - target_module.update_configs(essential_properties_value) - logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: essential_properties_value with module {target_module_name}: {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from essential properties - required_properties: List[str] = [] - for config in target_module.get_required_configs([InternalInputUserInput]): - required_properties.append(target_module.convert_lean_key_to_variable(config._id)) - ensure_options(required_properties) - required_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in required_properties} - target_module.update_configs(required_properties_value) - logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: required_properties_value with module {target_module_name}: {required_properties_value}") + for target_module_name in target_modules: + module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties, + logger, environment_name) + module.ensure_module_installed(organization_id) + lean_config["environments"][environment_name].update(module.get_settings()) + + +def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, + module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger, + environment_name: str = None) -> JsonModule: + target_module: JsonModule = None + for module in module_list: + if module.get_id() == target_module_name or module.get_name() == target_module_name: + target_module = module + break + else: + index = target_module_name.rfind('.') + if (index != -1 and module.get_id() == target_module_name[index + 1:] + or module.get_name() == target_module_name[index + 1:]): + target_module = module + break + + if not target_module: + raise RuntimeError(f"""Failed to resolve module for name: '{target_module_name}'""") + else: + logger.debug(f'Found module \'{target_module_name}\' from given name') + + target_module.config_build(lean_config, logger, interactive=False, properties=properties, + environment_name=environment_name) + _update_settings(logger, environment_name, target_module, lean_config) return target_module -def update_essential_properties_available(module_list: List[JsonModule], properties: Dict[str, Any]) -> JsonModule: - for target_module in module_list: - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()] - essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties if properties[prop] is not None} - target_module.update_configs(essential_properties_value) +def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], logger: Logger, + user_provided_options: Dict[str, Any], show_secrets: bool, select_message: str, + multiple: bool, environment_name: str = None) -> [JsonModule]: + """Interactively configures the brokerage to use. + + :param lean_config: the LEAN configuration that should be used + :param models: the modules to choose from + :param logger: the logger to use + :param user_provided_options: the dictionary containing user provided options + :param show_secrets: whether to show secrets on input + :param select_message: the user facing selection message + :param multiple: true if multiple selections are allowed + :param environment_name: the target environment name + :return: the brokerage the user configured + """ + options = [Option(id=b, label=b.get_name()) for b in models] + + modules: [JsonModule] = [] + if multiple: + modules = logger.prompt_list(select_message, options, multiple=True) + else: + module = logger.prompt_list(select_message, options, multiple=False) + modules.append(module) + + for module in modules: + module.config_build(lean_config, logger, interactive=True, properties=user_provided_options, + hide_input=not show_secrets, environment_name=environment_name) + _update_settings(logger, environment_name, module, lean_config) + if multiple: + return modules + return modules[-1] + + +def save_settings(settings: Dict[str, Any]) -> None: + """Persistently save properties in the Lean configuration. + + :param settings: the dict containing settings to save + """ + return + from lean.container import container + container.lean_config_manager.set_properties(settings) + + +def _update_settings(logger: Logger, environment_name: str, module: JsonModule, + lean_config: Dict[str, Any]) -> None: + settings = module.get_settings() + logger.debug(f'_update_settings({module}): Settings: {settings}') + + if environment_name: + if environment_name not in lean_config["environments"]: + lean_config["environments"][environment_name] = {} + target = lean_config["environments"][environment_name] + else: + target = lean_config["environments"] + + for key, value in settings.items(): + if key in target and target[key].startswith("["): + # it already exists, and it's an array we need to merge + from json import loads, dumps + logger.debug(f'_update_settings({module}): target[key]: {target[key]}') + existing_value = loads(target[key]) + if value.startswith("["): + # the new value is also an array, merge them + existing_value = existing_value + loads(value) + else: + existing_value.append(value) + target[key] = dumps(existing_value) + else: + target[key] = value + diff --git a/lean/constants.py b/lean/constants.py index 121f9a71..53e4ff33 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -96,3 +96,18 @@ # The name of the Docker network which all Lean CLI containers are ran on DOCKER_NETWORK = "lean_cli" + +# Module constants +MODULE_TYPE = "type" +MODULE_PLATFORM = "platform" + +# types +MODULE_ADDON = "addon-module" +MODULE_BROKERAGE = "brokerage" +MODULE_DATA_DOWNLOADER = "data-downloader" +MODULE_HISTORY_PROVIDER = "history-provider" +MODULE_DATA_QUEUE_HANDLER = "data-queue-handler" + +# platforms +MODULE_CLI_PLATFORM = "cli" +MODULE_CLOUD_PLATFORM = "cloud" diff --git a/lean/models/__init__.py b/lean/models/__init__.py index 2eba3c18..952d30e5 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -17,7 +17,7 @@ from time import time json_modules = {} -file_name = "modules-1.12.json" +file_name = "modules-1.13.json" directory = Path(__file__).parent file_path = directory.parent / file_name diff --git a/lean/models/addon_modules/__init__.py b/lean/models/addon_modules/__init__.py deleted file mode 100644 index 2541abf3..00000000 --- a/lean/models/addon_modules/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List -from lean.models.addon_modules.addon_module import AddonModule -from lean.models import json_modules - -all_addon_modules: List[AddonModule] = [] - -for json_module in json_modules: - if "addon-module" in json_module["type"]: - all_addon_modules.append(AddonModule(json_module)) diff --git a/lean/models/addon_modules/addon_module.py b/lean/models/addon_modules/addon_module.py deleted file mode 100644 index 4fc94fc3..00000000 --- a/lean/models/addon_modules/addon_module.py +++ /dev/null @@ -1,20 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from lean.models.lean_config_configurer import LeanConfigConfigurer - -class AddonModule(LeanConfigConfigurer): - """A JsonModule implementation for add on modules.""" - - - diff --git a/lean/models/brokerages/__init__.py b/lean/models/brokerages/__init__.py deleted file mode 100644 index 9af974c0..00000000 --- a/lean/models/brokerages/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py deleted file mode 100644 index 72867060..00000000 --- a/lean/models/brokerages/cloud/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage -from lean.models import json_modules -from typing import Dict, Type, List -from lean.models.brokerages.local.data_feed import DataFeed - -all_cloud_brokerages: List[CloudBrokerage] = [] -all_cloud_data_feeds: List[DataFeed] = [] -cloud_brokerage_data_feeds: Dict[Type[CloudBrokerage], - List[Type[DataFeed]]] = {} - -for json_module in json_modules: - if "cloud-brokerage" in json_module["type"]: - all_cloud_brokerages.append(CloudBrokerage(json_module)) - if "data-queue-handler" in json_module["type"]: - all_cloud_data_feeds.append((DataFeed(json_module))) - -for cloud_brokerage in all_cloud_brokerages: - data_feed_property_found = False - for x in cloud_brokerage.get_all_input_configs(): - if "data-feed" in x.__getattribute__("_id"): - data_feed_property_found = True - cloud_brokerage_data_feeds[cloud_brokerage] = x.__getattribute__("_choices") - if not data_feed_property_found: - cloud_brokerage_data_feeds[cloud_brokerage] = [] - -[PaperTradingBrokerage] = [ - cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py deleted file mode 100644 index 1c30aea7..00000000 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ /dev/null @@ -1,90 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Any -from lean.models.json_module import JsonModule -from lean.models.configuration import InternalInputUserInput, TradingEnvConfiguration - - -class CloudBrokerage(JsonModule): - """A JsonModule implementation for the cloud brokerages.""" - - def __init__(self, json_cloud_brokerage_data: Dict[str, Any]) -> None: - super().__init__(json_cloud_brokerage_data) - - def get_id(self) -> str: - """Returns the id of the brokerage. - :return: the id of this brokerage as it is expected by the live/create API endpoint - """ - return self._id - - def _get_settings(self) -> Dict[str, str]: - """Returns all settings for this brokerage, except for the id. - :return: the settings of this brokerage excluding the id - """ - settings = {} - for config in self.get_required_configs(): - value = None - if not config._cloud_id: - continue - # TODO: handle cases where tranding env config is not present, environment will still be required. - if type(config) == TradingEnvConfiguration: - value = "paper" if config._value.lower() in [ - "practice", "demo", "beta", "paper"] else "live" - elif type(config) is InternalInputUserInput: - if not config._is_conditional: - value = config._value - else: - for option in config._value_options: - if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): - value = option._value - break - if not value: - options_to_log = set([(opt._condition._dependent_config_id, - self.get_config_value_from_name(opt._condition._dependent_config_id)) - for opt in config._value_options]) - raise ValueError( - f'No condition matched among present options for "{config._cloud_id}". ' - f'Please review ' + - ', '.join([f'"{x[0]}"' for x in options_to_log]) + - f' given value{"s" if len(options_to_log) > 1 else ""} ' + - ', '.join([f'"{x[1]}"' for x in options_to_log])) - else: - value = config._value - settings[config._cloud_id] = value - return settings - - def get_settings(self) -> Dict[str, str]: - """Returns all settings for this brokerage. - :return: the settings to set in the "brokerage" property of the live/create API endpoint - """ - settings = self._get_settings() - if "environment" not in settings.keys(): - settings["environment"] = "live" - settings["id"] = self.get_id() - return settings - - def get_price_data_handler(self) -> str: - """Returns the price live data provider handler to use. - :return: the value to assign to the "dataHandler" property of the live/create API endpoint - """ - # TODO: Handle this case with json conditions - property_name = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] - property_name = property_name[0] if len(property_name) != 0 else "" - brokerage_name = self.get_name().replace(" ", "") - if property_name != "": - if "QuantConnect +" in self.get_config_value_from_name(property_name): - return "quantconnecthandler+" + brokerage_name.lower() + "handler" - else: - return self.get_config_value_from_name(property_name).replace(" ", "") + "Handler" - return "QuantConnectHandler" diff --git a/lean/models/brokerages/local/__init__.py b/lean/models/brokerages/local/__init__.py deleted file mode 100644 index b46bdf12..00000000 --- a/lean/models/brokerages/local/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import environ -from typing import Dict, Type, List -from lean.container import container -from lean.models.brokerages.local.local_brokerage import LocalBrokerage -from lean.models.brokerages.local.data_feed import DataFeed -from lean.models import json_modules - -all_local_brokerages: List[LocalBrokerage] = [] -all_local_data_feeds: List[DataFeed] = [] -local_brokerage_data_feeds: Dict[Type[LocalBrokerage], - List[Type[DataFeed]]] = {} - -for json_module in json_modules: - if "local-brokerage" in json_module["type"]: - all_local_brokerages.append(LocalBrokerage(json_module)) - if "data-queue-handler" in json_module["type"]: - all_local_data_feeds.append(DataFeed(json_module)) - -# Remove IQFeed DataFeed for other than windows machines -if not [container.platform_manager.is_host_windows() or environ.get("__README__", "false") == "true"]: - all_local_data_feeds = [ - data_feed for data_feed in all_local_data_feeds if data_feed._id != "IQFeed"] - -for local_brokerage in all_local_brokerages: - local_brokerage_data_feeds[local_brokerage] = all_local_data_feeds diff --git a/lean/models/brokerages/local/data_feed.py b/lean/models/brokerages/local/data_feed.py deleted file mode 100644 index 10a8dec8..00000000 --- a/lean/models/brokerages/local/data_feed.py +++ /dev/null @@ -1,29 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - - -class DataFeed(LeanConfigConfigurer): - """A JsonModule implementation for the Json live data provider module.""" - - def __init__(self, json_datafeed_data: Dict[str, Any]) -> None: - super().__init__(json_datafeed_data) - - def get_live_name(self) -> str: - live_name = self._id - environment_obj = self.get_configurations_env_values() - if environment_obj: - [live_name] = [x["value"] for x in environment_obj if x["name"] == "data-queue-handler"] - return live_name diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py deleted file mode 100644 index c22f7fe7..00000000 --- a/lean/models/brokerages/local/local_brokerage.py +++ /dev/null @@ -1,29 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - - -class LocalBrokerage(LeanConfigConfigurer): - """A JsonModule implementation for the Json brokerage module.""" - - def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: - super().__init__(json_brokerage_data) - - def get_live_name(self) -> str: - live_name = self._id - environment_obj = self.get_configurations_env_values() - if environment_obj: - [live_name] = [x["value"] for x in environment_obj if x["name"] == "live-mode-brokerage"] - return live_name diff --git a/lean/models/cli/__init__.py b/lean/models/cli/__init__.py new file mode 100644 index 00000000..576f6456 --- /dev/null +++ b/lean/models/cli/__init__.py @@ -0,0 +1,43 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from lean.constants import MODULE_BROKERAGE, MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM, \ + MODULE_DATA_DOWNLOADER, MODULE_HISTORY_PROVIDER, MODULE_DATA_QUEUE_HANDLER, MODULE_ADDON +from lean.models import json_modules +from lean.models.json_module import JsonModule + +# load the modules +cli_brokerages: List[JsonModule] = [] +cli_addon_modules: List[JsonModule] = [] +cli_data_downloaders: List[JsonModule] = [] +cli_history_provider: List[JsonModule] = [] +cli_data_queue_handlers: List[JsonModule] = [] + +for json_module in json_modules: + module_type = json_module[MODULE_TYPE] + platform = json_module[MODULE_PLATFORM] + + if MODULE_CLI_PLATFORM in platform: + if MODULE_BROKERAGE in module_type: + cli_brokerages.append(JsonModule(json_module, MODULE_BROKERAGE, MODULE_CLI_PLATFORM)) + if MODULE_DATA_DOWNLOADER in module_type: + cli_data_downloaders.append(JsonModule(json_module, MODULE_DATA_DOWNLOADER, MODULE_CLI_PLATFORM)) + if MODULE_HISTORY_PROVIDER in module_type: + cli_history_provider.append(JsonModule(json_module, MODULE_HISTORY_PROVIDER, MODULE_CLI_PLATFORM)) + if MODULE_DATA_QUEUE_HANDLER in module_type: + cli_data_queue_handlers.append(JsonModule(json_module, MODULE_DATA_QUEUE_HANDLER, MODULE_CLI_PLATFORM)) + if MODULE_ADDON in module_type: + cli_addon_modules.append(JsonModule(json_module, MODULE_ADDON, MODULE_CLI_PLATFORM)) + + diff --git a/lean/models/click_options.py b/lean/models/click_options.py index 6c8f8e40..5e56f444 100644 --- a/lean/models/click_options.py +++ b/lean/models/click_options.py @@ -12,26 +12,25 @@ # limitations under the License. -from typing import Any, List, Dict +from typing import List, Dict from click import option, Choice from lean.click import PathParameter -from lean.models.configuration import Configuration -from lean.models.brokerages.cloud import all_cloud_brokerages -from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds -from lean.models.data_providers import all_data_providers +from lean.models.cli import cli_brokerages, cli_data_downloaders, cli_data_queue_handlers +from lean.models.cloud import cloud_brokerages, cloud_data_queue_handlers from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput + def get_configs_for_options(env: str) -> List[Configuration]: if env == "live-cloud": - brokerage = all_cloud_brokerages - elif env == "live-local": - brokerage = all_local_brokerages + all_local_data_feeds + all_data_providers + brokerage = cloud_brokerages + cloud_data_queue_handlers + elif env == "live-cli": + brokerage = cli_brokerages + cli_data_queue_handlers + cli_data_downloaders elif env == "backtest": - brokerage = all_data_providers + brokerage = cli_data_downloaders elif env == "research": - brokerage = all_data_providers + brokerage = cli_data_downloaders else: - raise ValueError("Acceptable values for 'env' are: 'live-cloud', 'live-local', 'backtest', 'research'") + raise ValueError("Acceptable values for 'env' are: 'live-cloud', 'live-cli', 'backtest', 'research'") run_options: Dict[str, Configuration] = {} config_with_module_id: Dict[str, str] = {} @@ -81,32 +80,11 @@ def get_attribute_type(configuration: Configuration): return str -def get_the_correct_type_default_value(default_lean_config_key: str, default_input_value: str, expected_type: Any, - choices: List[str] = None): - from lean.commands.live.deploy import _get_default_value - lean_value = _get_default_value(default_lean_config_key) - if lean_value is None and default_input_value is not None: - lean_value = default_input_value - # This handles backwards compatibility for the old modules.json values. - if lean_value is not None and type(lean_value) != expected_type and type(lean_value) == bool: - if choices and "true" in choices and "false" in choices: - # Backwards compatibility for zeroha-history-subscription. - lean_value = "true" if lean_value else "false" - else: - # Backwards compatibility for tradier-use-sandbox - lean_value = "paper" if lean_value else "live" - return lean_value - - def get_options_attributes(configuration: Configuration, default_lean_config_key=None): options_attributes = { "type": get_click_option_type(configuration), "help": configuration._help } - default_input_value = configuration._input_default if configuration._is_required_from_user else None - options_attributes["default"] = lambda: get_the_correct_type_default_value( - default_lean_config_key, default_input_value, get_attribute_type(configuration), - configuration._choices if configuration._input_method == "choice" else None) return options_attributes diff --git a/lean/models/data_providers/__init__.py b/lean/models/cloud/__init__.py similarity index 50% rename from lean/models/data_providers/__init__.py rename to lean/models/cloud/__init__.py index fb0c47e2..e9dfe040 100644 --- a/lean/models/data_providers/__init__.py +++ b/lean/models/cloud/__init__.py @@ -10,17 +10,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from typing import List -from lean.models.data_providers.data_provider import DataProvider + +from lean.constants import MODULE_BROKERAGE, MODULE_TYPE, MODULE_CLOUD_PLATFORM, MODULE_PLATFORM, \ + MODULE_DATA_QUEUE_HANDLER from lean.models import json_modules +from lean.models.json_module import JsonModule -all_data_providers: List[DataProvider] = [] +# load the modules +cloud_brokerages: List[JsonModule] = [] +cloud_data_queue_handlers: List[JsonModule] = [] for json_module in json_modules: - if "data-provider" in json_module["type"]: - all_data_providers.append(DataProvider(json_module)) + module_type = json_module[MODULE_TYPE] + platform = json_module[MODULE_PLATFORM] -# QuantConnect DataProvider -[QuantConnectDataProvider] = [ - data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] \ No newline at end of file + if MODULE_CLOUD_PLATFORM in platform: + if MODULE_BROKERAGE in module_type: + cloud_brokerages.append(JsonModule(json_module, MODULE_BROKERAGE, MODULE_CLOUD_PLATFORM)) + if MODULE_DATA_QUEUE_HANDLER in module_type: + cloud_data_queue_handlers.append(JsonModule(json_module, MODULE_DATA_QUEUE_HANDLER, MODULE_CLOUD_PLATFORM)) diff --git a/lean/models/configuration.py b/lean/models/configuration.py index aed2d549..783d93d0 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -93,12 +93,8 @@ def __init__(self, config_json_object): self._id: str = config_json_object["id"] self._config_type: str = config_json_object["type"] self._value: str = config_json_object["value"] - self._is_cloud_property: bool = "cloud-id" in config_json_object self._is_required_from_user = False self._save_persistently_in_lean = False - self._is_type_configurations_env: bool = type( - self) is ConfigurationsEnvConfiguration - self._is_type_trading_env: bool = type(self) is TradingEnvConfiguration self._log_message: str = "" if "log-message" in config_json_object.keys(): self._log_message = config_json_object["log-message"] @@ -107,6 +103,10 @@ def __init__(self, config_json_object): else: self._filter = Filter([]) + self._input_default = None + if "input-default" in config_json_object: + self._input_default = config_json_object["input-default"] + def factory(config_json_object) -> 'Configuration': """Creates an instance of the child classes. @@ -115,20 +115,21 @@ def factory(config_json_object) -> 'Configuration': :return: An instance of Configuration. """ - if config_json_object["type"] in ["info", "configurations-env"]: + if config_json_object["type"] in ["info"]: return InfoConfiguration.factory(config_json_object) elif config_json_object["type"] in ["input", "internal-input"]: return UserInputConfiguration.factory(config_json_object) elif config_json_object["type"] == "filter-env": return BrokerageEnvConfiguration.factory(config_json_object) - elif config_json_object["type"] == "trading-env": - return TradingEnvConfiguration.factory(config_json_object) else: raise ValueError( f'Undefined input method type {config_json_object["type"]}') + def __repr__(self): + return f'{self._id}: {self._value}' -class Filter(): + +class Filter: """This class handles the conditional filters added to configurations. """ @@ -154,23 +155,7 @@ def factory(config_json_object) -> 'InfoConfiguration': :param config_json_object: the json object dict with configuration info :return: An instance of InfoConfiguration. """ - if config_json_object["type"] == "configurations-env": - return ConfigurationsEnvConfiguration(config_json_object) - else: - return InfoConfiguration(config_json_object) - - -class ConfigurationsEnvConfiguration(InfoConfiguration): - """Configuration class used for environment properties. - - Doesn't support user prompt inputs. - Values of this configuration isn't persistently saved in the Lean configuration. - """ - - def __init__(self, config_json_object): - super().__init__(config_json_object) - self._env_and_values = { - env_obj["name"]: env_obj["value"] for env_obj in self._value} + return InfoConfiguration(config_json_object) class UserInputConfiguration(Configuration, ABC): @@ -186,7 +171,6 @@ def __init__(self, config_json_object): self._is_required_from_user = True self._save_persistently_in_lean = True self._input_method = self._prompt_info = self._help = "" - self._input_default = self._cloud_id = None self._optional = False if "input-method" in config_json_object: self._input_method = config_json_object["input-method"] @@ -194,10 +178,6 @@ def __init__(self, config_json_object): self._prompt_info = config_json_object["prompt-info"] if "help" in config_json_object: self._help = config_json_object["help"] - if "input-default" in config_json_object: - self._input_default = config_json_object["input-default"] - if "cloud-id" in config_json_object: - self._cloud_id = config_json_object["cloud-id"] if "save-persistently-in-lean" in config_json_object: self._save_persistently_in_lean = config_json_object["save-persistently-in-lean"] if "optional" in config_json_object: @@ -411,46 +391,6 @@ def ask_user_for_input(self, default_value, logger: Logger, hide_input: bool = F raise ValueError(f"Undefined input method type {self._input_method}") -class TradingEnvConfiguration(PromptUserInput, ChoiceUserInput, ConfirmUserInput): - """This class adds trading-mode/envirionment based user filters. - - Normalizes the value of envrionment values(live/paper) for cloud live. - """ - - def __init__(self, config_json_object): - super().__init__(config_json_object) - - def factory(config_json_object) -> 'TradingEnvConfiguration': - """Creates an instance of the child classes. - - :param config_json_object: the json object dict with configuration info - :return: An instance of TradingEnvConfiguration. - """ - if config_json_object["type"] == "trading-env": - return TradingEnvConfiguration(config_json_object) - else: - raise ValueError( - f'Undefined input method type {config_json_object["type"]}') - - def ask_user_for_input(self, default_value, logger: Logger, hide_input: bool = False): - """Prompts user to provide input while validating the type of input - against the expected type - - :param default_value: The default to prompt to the user. - :param logger: The instance of logger class. - :param hide_input: Whether to hide the input (not used for this type of input, which is never hidden). - :return: The value provided by the user. - """ - # NOTE: trading envrionment config should not use old boolean value as default - if type(default_value) == bool: - default_value = "paper" if default_value else "live" - if self._input_method == "confirm": - raise ValueError( - f'input method -- {self._input_method} is not allowed with {self.__class__.__name__}') - else: - return BrokerageEnvConfiguration.ask_user_for_input(self, default_value, logger) - - class FilterEnvConfiguration(BrokerageEnvConfiguration): """This class adds extra filters to user filters.""" diff --git a/lean/models/data_providers/data_provider.py b/lean/models/data_providers/data_provider.py deleted file mode 100644 index 9bc37372..00000000 --- a/lean/models/data_providers/data_provider.py +++ /dev/null @@ -1,26 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - -class DataProvider(LeanConfigConfigurer): - """A JsonModule implementation for the Json historical data provider module.""" - - def __init__(self, json_data_provider_data: Dict[str, Any]) -> None: - super().__init__(json_data_provider_data) - - def configure_credentials(self, lean_config: Dict[str, Any]) -> None: - super().configure_credentials(lean_config) - self._save_properties( - lean_config, self.get_non_user_required_properties()) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 41ee9967..430de732 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -18,6 +18,8 @@ from click.core import ParameterSource from lean.components.util.logger import Logger +from lean.constants import MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM +from lean.container import container from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput from copy import copy from abc import ABC @@ -26,18 +28,20 @@ class JsonModule(ABC): """The JsonModule class is the base class extended for all json modules.""" - def __init__(self, json_module_data: Dict[str, Any]) -> None: - self._type: List[str] = json_module_data["type"] + def __init__(self, json_module_data: Dict[str, Any], module_type: str, platform: str) -> None: + self._module_type: str = module_type + self._platform: str = platform self._product_id: int = json_module_data["product-id"] self._id: str = json_module_data["id"] self._display_name: str = json_module_data["display-id"] - self._installs: bool = json_module_data["installs"] + self._installs: bool = json_module_data["installs"] and platform == MODULE_CLI_PLATFORM self._lean_configs: List[Configuration] = [] for config in json_module_data["configurations"]: self._lean_configs.append(Configuration.factory(config)) self._lean_configs = self.sort_configs() self._is_module_installed: bool = False - self._initial_cash_balance: LiveInitialStateInput = LiveInitialStateInput(json_module_data["live-cash-balance-state"]) \ + self._initial_cash_balance: LiveInitialStateInput = LiveInitialStateInput( + json_module_data["live-cash-balance-state"]) \ if "live-cash-balance-state" in json_module_data \ else None self._initial_holdings: LiveInitialStateInput = LiveInitialStateInput(json_module_data["live-holdings-state"]) \ @@ -45,6 +49,9 @@ def __init__(self, json_module_data: Dict[str, Any]) -> None: else False self._minimum_seat = json_module_data["minimum-seat"] if "minimum-seat" in json_module_data else None + def get_id(self): + return self._id + def sort_configs(self) -> List[Configuration]: sorted_configs = [] brokerage_configs = [] @@ -64,44 +71,23 @@ def get_name(self) -> str: def check_if_config_passes_filters(self, config: Configuration) -> bool: for condition in config._filter._conditions: - if condition._dependent_config_id == "module-type": - target_value = self.__class__.__name__ + if condition._dependent_config_id == MODULE_TYPE: + target_value = self._module_type + elif condition._dependent_config_id == MODULE_PLATFORM: + target_value = self._platform else: - target_value = self.get_config_value_from_name( - condition._dependent_config_id) + target_value = self.get_config_value_from_name(condition._dependent_config_id) if not condition.check(target_value): return False return True - def check_if_config_passes_module_filter(self, config: Configuration) -> bool: - for condition in config._filter._conditions: - if condition._dependent_config_id == "module-type": - target_value = self.__class__.__name__ - if not condition.check(target_value): - return False - return True - def update_configs(self, key_and_values: Dict[str, str]): for key, value in key_and_values.items(): self.update_value_for_given_config(key, value) - def get_configurations_env_values(self) -> List[Dict[str, str]]: - env_config_values = [] - [env_config] = [config for config in self._lean_configs if - config._is_type_configurations_env and self.check_if_config_passes_filters( - config) - ] or [None] - if env_config is not None: - # Always get the first one, since we only expect one env config in the json modules file - env_config_values = list(env_config._env_and_values.values())[0] - return env_config_values - - def get_config_from_type(self, config_type: Configuration) -> str: - return [copy(config) for config in self._lean_configs if type(config) is config_type] - def update_value_for_given_config(self, target_name: str, value: Any) -> None: [idx] = [i for i in range(len(self._lean_configs)) - if self._lean_configs[i]._id == target_name] + if self._lean_configs[i]._id == target_name and self.check_if_config_passes_filters(self._lean_configs[i])] self._lean_configs[idx]._value = value def get_config_value_from_name(self, target_name: str) -> str: @@ -109,24 +95,49 @@ def get_config_value_from_name(self, target_name: str) -> str: if self._lean_configs[i]._id == target_name] return self._lean_configs[idx]._value - def get_non_user_required_properties(self) -> List[str]: - return [config._id for config in self._lean_configs if not config._is_required_from_user and not - config._is_type_configurations_env and self.check_if_config_passes_filters(config)] - def get_required_properties(self, filters: List[Type[Configuration]] = [], include_optionals: bool = True) -> List[str]: return [config._id for config in self.get_required_configs(filters, include_optionals)] - def get_required_configs(self, - filters: List[Type[Configuration]] = [], - include_optionals: bool = True) -> List[Configuration]: + def get_required_configs(self, filters: List[Type[Configuration]] = [], include_optionals: bool = True) ->\ + List[Configuration]: + required_configs = [copy(config) for config in self._lean_configs if config._is_required_from_user and type(config) not in filters and self.check_if_config_passes_filters(config) and (include_optionals or not getattr(config, '_optional', False))] return required_configs + def get_settings(self) -> Dict[str, str]: + settings: Dict[str, str] = {"id": self._id} + + # we build these after the rest, because they might depend on their values + for config in self._lean_configs: + if type(config) is InternalInputUserInput: + if config._is_conditional: + for option in config._value_options: + if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): + config._value = option._value + break + if not config._value: + options_to_log = set([(opt._condition._dependent_config_id, + self.get_config_value_from_name(opt._condition._dependent_config_id)) + for opt in config._value_options]) + raise ValueError( + f'No condition matched among present options for "{config._id}". ' + f'Please review ' + + ', '.join([f'"{x[0]}"' for x in options_to_log]) + + f' given value{"s" if len(options_to_log) > 1 else ""} ' + + ', '.join([f'"{x[1]}"' for x in options_to_log])) + + for configuration in self._lean_configs: + if not self.check_if_config_passes_filters(configuration): + continue + settings[configuration._id] = str(configuration._value).replace("\\", "/") + + return settings + def get_persistent_save_properties(self, filters: List[Type[Configuration]] = []) -> List[str]: return [config._id for config in self.get_required_configs(filters) if config._save_persistently_in_lean] @@ -139,7 +150,7 @@ def get_essential_configs(self) -> List[Configuration]: def get_all_input_configs(self, filters: List[Type[Configuration]] = []) -> List[Configuration]: return [copy(config) for config in self._lean_configs if config._is_required_from_user if type(config) not in filters - and self.check_if_config_passes_module_filter(config)] + and self.check_if_config_passes_filters(config)] def convert_lean_key_to_variable(self, lean_key: str) -> str: """Replaces hyphens with underscore to follow python naming convention. @@ -155,28 +166,38 @@ def convert_variable_to_lean_key(self, variable_key: str) -> str: """ return variable_key.replace('_', '-') - def build(self, - lean_config: Dict[str, Any], - logger: Logger, - properties: Dict[str, Any] = {}, - hide_input: bool = False) -> 'JsonModule': + def config_build(self, + lean_config: Dict[str, Any], + logger: Logger, + interactive: bool, + properties: Dict[str, Any] = {}, + hide_input: bool = False, + environment_name: str = None) -> 'JsonModule': """Builds a new instance of this class, prompting the user for input when necessary. :param lean_config: the Lean configuration dict to read defaults from :param logger: the logger to use + :param interactive: true if running in interactive mode :param properties: the properties that passed as options :param hide_input: whether to hide secrets inputs - :return: a LeanConfigConfigurer instance containing all the details needed to configure the Lean config + :param environment_name: the target environment name + :return: self """ - logger.info(f'Configure credentials for {self._display_name}') + logger.debug(f'Configuring {self._display_name}') + + # filter properties that were not passed as command line arguments, + # so that we prompt the user for them only when they don't have a value in the Lean config + context = get_current_context() + user_provided_options = {k: v for k, v in properties.items() + if context.get_parameter_source(k) == ParameterSource.COMMANDLINE} + + missing_options = [] for configuration in self._lean_configs: if not self.check_if_config_passes_filters(configuration): continue if not configuration._is_required_from_user: continue - if self.__class__.__name__ == 'CloudBrokerage' and not configuration._is_cloud_property: - continue - # Lets log messages for internal input configurations as well + # Let's log messages for internal input configurations as well if configuration._log_message is not None: log_message = configuration._log_message.strip() if log_message: @@ -184,37 +205,55 @@ def build(self, if type(configuration) is InternalInputUserInput: continue - # filter properties that were not passed as command line arguments, - # so that we prompt the user for them only when they don't have a value in the Lean config - context = get_current_context() - user_provided_options = {k: v for k, v in properties.items() - if context.get_parameter_source(k) == ParameterSource.COMMANDLINE} - - user_choice = None property_name = self.convert_lean_key_to_variable(configuration._id) # Only ask for user input if the config wasn't given as an option if property_name in user_provided_options and user_provided_options[property_name]: user_choice = user_provided_options[property_name] + logger.debug( + f'JsonModule({self._display_name}): user provided \'{user_choice}\' for \'{property_name._id}\'') else: - # Let's try to get the value from the lean config and use it as the user choice, without prompting - # TODO: use type(class) equality instead of class name (str) - if self.__class__.__name__ != 'CloudBrokerage': - user_choice = self._get_default(lean_config, configuration._id) - # Try to get the values from lean config - elif lean_config is not None and configuration._id in lean_config: - user_choice = lean_config[configuration._id] + logger.debug(f'JsonModule({self._display_name}): Configuration not provided \'{configuration._id}\'') + user_choice = self.get_default(lean_config, configuration._id, environment_name, logger) # There's no value in the lean config, let's use the module default value instead and prompt the user # NOTE: using "not" instead of "is None" because the default value can be false, # in which case we still want to prompt the user. if not user_choice: - default_value = configuration._input_default - user_choice = configuration.ask_user_for_input(default_value, logger, hide_input=hide_input) + if interactive: + default_value = configuration._input_default + user_choice = configuration.ask_user_for_input(default_value, logger, hide_input=hide_input) + else: + missing_options.append(configuration._id) - self.update_value_for_given_config(configuration._id, user_choice) + configuration._value = user_choice + if len(missing_options) > 0: + raise RuntimeError(f"""You are missing the following option{"s" if len(missing_options) > 1 else ""}: + {''.join(missing_options)}""".strip()) return self + def ensure_module_installed(self, organization_id: str) -> None: + if not self._is_module_installed and self._installs: + container.logger.debug(f"JsonModule.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") + container.module_manager.install_module( + self._product_id, organization_id) + self._is_module_installed = True + + def get_default(self, lean_config: Dict[str, Any], key: str, environment_name: str, logger: Logger): + user_choice = None + if lean_config is not None: + if (environment_name and "environments" in lean_config and environment_name in lean_config["environments"] + and key in lean_config["environments"][environment_name]): + user_choice = lean_config["environments"][environment_name][key] + logger.debug(f'JsonModule({self._display_name}): found \'{user_choice}\' for \'{key}\', in environment') + elif key in lean_config: + user_choice = lean_config[key] + logger.debug(f'JsonModule({self._display_name}): found \'{user_choice}\' for \'{key}\'') + return user_choice + + def __repr__(self): + return self.get_name() + class LiveInitialStateInput(str, Enum): Required = "required" diff --git a/lean/models/lean_config_configurer.py b/lean/models/lean_config_configurer.py deleted file mode 100644 index 7ff8f8f6..00000000 --- a/lean/models/lean_config_configurer.py +++ /dev/null @@ -1,121 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC - -from typing import Any, Dict, List, Optional -from lean.container import container -from lean.models.json_module import JsonModule -from lean.models.configuration import InternalInputUserInput -from copy import copy - -class LeanConfigConfigurer(JsonModule, ABC): - """The LeanConfigConfigurer class is the base class extended by all classes that update the Lean config.""" - - def configure(self, lean_config: Dict[str, Any], environment_name: str) -> None: - """Configures the Lean configuration for this brokerage. - - If the Lean configuration has been configured for this brokerage before, nothing will be changed. - Non-environment changes are saved persistently to disk so they can be used as defaults later. - - :param lean_config: the configuration dict to write to - :param environment_name: the name of the environment to configure - """ - self._configure_environment(lean_config, environment_name) - self.configure_credentials(lean_config) - - def _configure_environment(self, lean_config: Dict[str, Any], environment_name: str) -> None: - """Configures the environment in the Lean config for this brokerage. - :param lean_config: the Lean configuration dict to write to - :param environment_name: the name of the environment to update - """ - for environment_config in self.get_configurations_env_values(): - environment_config_name = environment_config["name"] - if self.__class__.__name__ == 'DataFeed': - if environment_config_name == "data-queue-handler": - previous_value = [] - if "data-queue-handler" in lean_config["environments"][environment_name]: - previous_value = copy(lean_config["environments"][environment_name][environment_config_name]) - previous_value.append(environment_config["value"]) - lean_config["environments"][environment_name][environment_config_name] = copy(previous_value) - elif self.__class__.__name__ == 'LocalBrokerage': - if environment_config_name != "data-queue-handler": - lean_config["environments"][environment_name][environment_config_name] = environment_config["value"] - - def configure_credentials(self, lean_config: Dict[str, Any]) -> None: - """Configures the credentials in the Lean config for this brokerage and saves them persistently to disk. - :param lean_config: the Lean configuration dict to write to - """ - if self._installs: - lean_config["job-organization-id"] = container.organization_manager.try_get_working_organization_id() - for configuration in self._lean_configs: - value = None - if configuration._is_type_configurations_env: - continue - elif not self.check_if_config_passes_filters(configuration): - continue - elif type(configuration) is InternalInputUserInput: - if not configuration._is_conditional: - value = configuration._value - else: - for option in configuration._value_options: - if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): - value = option._value - break - if not value: - options_to_log = set([(opt._condition._dependent_config_id, - self.get_config_value_from_name(opt._condition._dependent_config_id)) - for opt in configuration._value_options]) - raise ValueError( - f'No condition matched among present options for "{configuration._cloud_id}". ' - f'Please review ' + - ', '.join([f'"{x[0]}"' for x in options_to_log]) + - f' given value{"s" if len(options_to_log) > 1 else ""} ' + - ', '.join([f'"{x[1]}"' for x in options_to_log])) - else: - value = configuration._value - from pathlib import WindowsPath, PosixPath - if type(value) == WindowsPath or type(value) == PosixPath: - value = str(value).replace("\\", "/") - lean_config[configuration._id] = value - container.logger.debug(f"LeanConfigConfigurer.ensure_module_installed(): _save_properties for module {self._id}: {self.get_persistent_save_properties()}") - self._save_properties(lean_config, self.get_persistent_save_properties()) - - def ensure_module_installed(self, organization_id: str) -> None: - if not self._is_module_installed and self._installs: - container.logger.debug(f"LeanConfigConfigurer.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") - container.module_manager.install_module( - self._product_id, organization_id) - self._is_module_installed = True - - def _get_default(cls, lean_config: Dict[str, Any], key: str) -> Optional[Any]: - """Returns the default value for a property based on the current Lean configuration. - - :param lean_config: the current Lean configuration - :param key: the name of the property - :return: the default value for the property, or None if there is none - """ - if key not in lean_config or lean_config[key] == "": - return None - - return lean_config[key] - - def _save_properties(self, lean_config: Dict[str, Any], properties: List[str]) -> None: - """Persistently save properties in the Lean configuration. - - :param lean_config: the dict containing all properties - :param properties: the names of the properties to save persistently - """ - from lean.container import container - container.lean_config_manager.set_properties( - {key: lean_config[key] for key in properties})