diff --git a/README.md b/README.md index 34efd08e..4c3a5d64 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,8 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT Options: --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit] The brokerage to use + --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Polygon|IEX|CoinApi|Bybit] + The live data provider to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id --ib-password TEXT Your Interactive Brokers password @@ -292,15 +294,10 @@ Options: Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at this time, and will require 2FA verification. This is required by Interactive Brokers. Use this option explicitly to override the default value. - --ib-data-feed [QuantConnect|Interactive Brokers|QuantConnect + InteractiveBrokers] - The available price data feeds are: Interactive Brokers price data feed, QuantConnect - price data feed or QuantConnect + InteractiveBrokers price data feed --tradier-account-id TEXT Your Tradier account id --tradier-access-token TEXT Your Tradier access token --tradier-environment [live|paper] Whether the developer sandbox should be used - --tradier-data-feed [QuantConnect|Tradier Brokerage] - Whether the Tradier data feed must be used instead of the QuantConnect price data feed --oanda-account-id TEXT Your OANDA account id --oanda-access-token TEXT Your OANDA API token --oanda-environment [Practice|Trade] @@ -371,6 +368,13 @@ Options: --bybit-api-secret TEXT Your Bybit API secret --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5] Your Bybit VIP Level + --polygon-api-key TEXT Your Polygon.io API Key + --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key + --iex-price-plan [Launch|Grow|Enterprise] + Your IEX Cloud Price plan + --coinapi-api-key TEXT Your coinapi.io Api Key + --coinapi-product [Free|Startup|Streamer|Professional|Enterprise] + CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing) --node TEXT The name or id of the live node to run on --auto-restart BOOLEAN Whether automatic algorithm restarting must be enabled --notify-order-events BOOLEAN Whether notifications must be sent for order events @@ -1065,7 +1069,7 @@ Options: -d, --detach Run the live deployment in a detached Docker container and return immediately --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit] The brokerage to use - --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit] + --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit] The live data provider to use --data-provider-historical [IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local] Update the Lean configuration file to retrieve data from the given historical provider diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index d3d781bc..442623d2 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -18,11 +18,9 @@ from lean.click import LeanCommand, PathParameter from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH from lean.container import container, Logger -from lean.models.api import QCMinimalOrganization from lean.models.utils import DebuggingMethod -from lean.models.logger import Option -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module +from lean.models.cli import cli_data_downloaders, cli_addon_modules +from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name from lean.models.click_options import options_from_json, get_configs_for_options # The _migrate_* methods automatically update launch configurations for a given debugging method. @@ -33,6 +31,7 @@ # # These methods checks if the project has outdated configurations, and if so, update them to keep it working. + def _migrate_python_pycharm(logger: Logger, project_dir: Path) -> None: from os import path from click import Abort @@ -225,20 +224,6 @@ def _migrate_csharp_csproj(project_dir: Path) -> None: csproj_path.write_text(xml_manager.to_string(current_content), encoding="utf-8") -def _select_organization() -> QCMinimalOrganization: - """Asks the user for the organization that should be charged when downloading data. - - :return: the selected organization - """ - api_client = container.api_client - - organizations = api_client.organizations.get_all() - options = [Option(id=organization, label=organization.name) for organization in organizations] - - logger = container.logger - return logger.prompt_list("Select the organization to purchase and download data with", options) - - @command(cls=LeanCommand, requires_lean_config=True, requires_docker=True) @argument("project", type=PathParameter(exists=True, file_okay=True, dir_okay=True)) @option("--output", @@ -252,7 +237,7 @@ def _select_organization() -> QCMinimalOrganization: type=Choice(["pycharm", "ptvsd", "vsdbg", "rider", "local-platform"], case_sensitive=False), help="Enable a certain debugging method (see --help for more information)") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("backtest")) @@ -338,6 +323,7 @@ def backtest(project: Path, if output is None: output = algorithm_file.parent / "backtests" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + environment_name = "backtesting" debugging_method = None if debug == "pycharm": debugging_method = DebuggingMethod.PyCharm @@ -360,17 +346,17 @@ def backtest(project: Path, if algorithm_file.name.endswith(".cs"): _migrate_csharp_csproj(algorithm_file.parent) - lean_config = lean_config_manager.get_complete_lean_config("backtesting", algorithm_file, debugging_method) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, debugging_method) if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" organization_id = container.organization_manager.try_get_working_organization_id() if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(organization_id) - data_provider_configurer.configure(lean_config, "backtesting") + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -407,11 +393,12 @@ def backtest(project: Path, lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' # Configure addon modules - build_and_configure_modules(addon_module, organization_id, lean_config, logger, "backtesting") + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) lean_runner = container.lean_runner lean_runner.run_lean(lean_config, - "backtesting", + environment_name, algorithm_file, output, engine_image, diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index ad9bb347..0308ee30 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -11,25 +11,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path -from typing import Any, Dict, List, Tuple, Optional +from typing import List, Tuple, Optional from click import prompt, option, argument, Choice, confirm from lean.click import LeanCommand, ensure_options from lean.components.api.api_client import APIClient +from lean.components.util.json_modules_handler import non_interactive_config_build_for_name, save_settings, \ + interactive_config_build from lean.components.util.logger import Logger from lean.container import container from lean.models.api import (QCEmailNotificationMethod, QCNode, QCNotificationMethod, QCSMSNotificationMethod, QCWebhookNotificationMethod, QCTelegramNotificationMethod, QCProject) from lean.models.json_module import LiveInitialStateInput from lean.models.logger import Option -from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage -from lean.models.configuration import InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerage_data_feeds +from lean.models.cloud import cloud_brokerages, cloud_data_queue_handlers from lean.commands.cloud.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively + def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: """Logs a list of notification methods.""" logger = container.logger @@ -98,37 +98,7 @@ def _prompt_notification_method() -> QCNotificationMethod: return QCSMSNotificationMethod(phoneNumber=phone_number) -def _configure_brokerage(lean_config: Dict[str, Any], logger: Logger, user_provided_options: Dict[str, Any], show_secrets: bool) -> CloudBrokerage: - """Interactively configures the brokerage to use. - - :param lean_config: the LEAN configuration that should be used - :param logger: the logger to use - :param user_provided_options: the dictionary containing user provided options - :param show_secrets: whether to show secrets on input - :return: the cloud brokerage the user configured - """ - brokerage_options = [Option(id=b, label=b.get_name()) for b in all_cloud_brokerages] - return logger.prompt_list("Select a brokerage", brokerage_options).build(lean_config, - logger, - user_provided_options, - hide_input=not show_secrets) - -def _configure_data_feed(brokerage: CloudBrokerage, logger: Logger) -> None: - """Configures the live data provider to use based on the brokerage given. - - :param brokerage: the cloud brokerage - :param logger: the logger to use - """ - if len(cloud_brokerage_data_feeds[brokerage]) != 0: - data_feed_selected = logger.prompt_list("Select a live data provider", [ - Option(id=data_feed, label=data_feed) for data_feed in cloud_brokerage_data_feeds[brokerage] - ], multiple=False) - data_feed_property_name = [name for name in brokerage.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] - data_feed_property_name = data_feed_property_name[0] if len(data_feed_property_name) != 0 else "" - brokerage.update_value_for_given_config(data_feed_property_name, data_feed_selected) - - -def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: +def _configure_live_node(node: str, logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use. :param logger: the logger to use @@ -137,6 +107,15 @@ def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: Q :return: the live node the user wants to start live trading on """ nodes = api_client.nodes.get_all(cloud_project.organizationId) + if node is not None: + live_node = next((n for n in nodes.live if n.id == node or n.name == node), None) + + if live_node is None: + raise RuntimeError(f"You have no live node with name or id '{node}'") + + if live_node.busy: + raise RuntimeError(f"The live node named '{live_node.name}' is already in use by '{live_node.usedBy}'") + return live_node live_nodes = [node for node in nodes.live if not node.busy] if len(live_nodes) == 0: @@ -187,8 +166,12 @@ def _configure_auto_restart(logger: Logger) -> bool: @live.command(cls=LeanCommand, default_command=True, name="deploy") @argument("project", type=str) @option("--brokerage", - type=Choice([b.get_name() for b in all_cloud_brokerages], case_sensitive=False), + type=Choice([b.get_name() for b in cloud_brokerages], case_sensitive=False), help="The brokerage to use") +@option("--data-provider-live", + type=Choice([d.get_name() for d in cloud_data_queue_handlers], case_sensitive=False), + multiple=True, + help="The live data provider to use") @options_from_json(get_configs_for_options("live-cloud")) @option("--node", type=str, help="The name or id of the live node to run on") @option("--auto-restart", type=bool, help="Whether automatic algorithm restarting must be enabled") @@ -221,6 +204,7 @@ def _configure_auto_restart(logger: Logger) -> bool: @option("--show-secrets", is_flag=True, show_default=True, default=False, help="Show secrets as they are input") def deploy(project: str, brokerage: str, + data_provider_live: Optional[str], node: str, auto_restart: bool, notify_order_events: Optional[bool], @@ -254,34 +238,15 @@ def deploy(project: str, cloud_runner = container.cloud_runner finished_compile = cloud_runner.compile_project(cloud_project) + live_data_provider_settings = {} + lean_config = container.lean_config_manager.get_lean_config() + if brokerage is not None: ensure_options(["brokerage", "node", "auto_restart", "notify_order_events", "notify_insights"]) - brokerage_instance = None - [brokerage_instance] = [cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage.get_name() == brokerage] - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_essential_properties()] - ensure_options(essential_properties) - essential_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in essential_properties} - brokerage_instance.update_configs(essential_properties_value) - # now required properties can be fetched as per historical data provider from essential properties - required_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_required_properties([InternalInputUserInput])] - ensure_options(required_properties) - required_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in required_properties} - brokerage_instance.update_configs(required_properties_value) - - all_nodes = api_client.nodes.get_all(cloud_project.organizationId) - live_node = next((n for n in all_nodes.live if n.id == node or n.name == node), None) - - if live_node is None: - raise RuntimeError(f"You have no live node with name or id '{node}'") - - if live_node.busy: - raise RuntimeError(f"The live node named '{live_node.name}' is already in use by '{live_node.usedBy}'") - + brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cloud_brokerages, + kwargs, logger) notify_methods = [] - if notify_emails is not None: for config in notify_emails.split(","): address, subject = config.split(":") @@ -320,10 +285,10 @@ def deploy(project: str, raise RuntimeError(f"Custom portfolio holdings setting is not available for {brokerage_instance.get_name()}") else: - lean_config = container.lean_config_manager.get_lean_config() - brokerage_instance = _configure_brokerage(lean_config, logger, kwargs, show_secrets=show_secrets) - _configure_data_feed(brokerage_instance, logger) - live_node = _configure_live_node(logger, api_client, cloud_project) + # let the user choose the brokerage + brokerage_instance = interactive_config_build(lean_config, cloud_brokerages, logger, kwargs, show_secrets, + "Select a brokerage", multiple=False) + notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(api_client, brokerage_instance, cloud_project.projectId, project) @@ -332,15 +297,31 @@ def deploy(project: str, if holdings_option != LiveInitialStateInput.NotSupported: live_holdings = _configure_initial_holdings_interactively(logger, holdings_option, last_holdings) + live_node = _configure_live_node(node, logger, api_client, cloud_project) + + if data_provider_live is not None and len(data_provider_live) > 0: + # the user sent the live data provider to use + for data_provider in data_provider_live: + data_provider_instance = non_interactive_config_build_for_name(lean_config, data_provider, + cloud_data_queue_handlers, kwargs, logger) + + live_data_provider_settings.update({data_provider_instance.get_id(): data_provider_instance.get_settings()}) + else: + # let's ask the user which live data providers to use + data_feed_instances = interactive_config_build(lean_config, cloud_data_queue_handlers, logger, kwargs, + show_secrets, "Select a live data feed", multiple=True) + for data_feed in data_feed_instances: + settings = data_feed.get_settings() + + live_data_provider_settings.update({data_feed.get_id(): settings}) + brokerage_settings = brokerage_instance.get_settings() - price_data_handler = brokerage_instance.get_price_data_handler() - logger.info(f"Brokerage: {brokerage_instance.get_name()}") + logger.info(f"Brokerage: {brokerage_settings}") logger.info(f"Project id: {cloud_project.projectId}") - logger.info(f"Environment: {brokerage_settings['environment'].title()}") logger.info(f"Server name: {live_node.name}") logger.info(f"Server type: {live_node.sku}") - logger.info(f"Live data provider: {price_data_handler.replace('Handler', '')}") + logger.info(f"Live data providers: {live_data_provider_settings}") logger.info(f"LEAN version: {cloud_project.leanVersionId}") logger.info(f"Order event notifications: {'Yes' if notify_order_events else 'No'}") logger.info(f"Insight notifications: {'Yes' if notify_insights else 'No'}") @@ -357,11 +338,15 @@ def deploy(project: str, default=False, abort=True) + # save them for next time + save_settings(live_data_provider_settings) + save_settings(brokerage_settings) + live_algorithm = api_client.live.start(cloud_project.projectId, finished_compile.compileId, live_node.id, brokerage_settings, - price_data_handler, + live_data_provider_settings, auto_restart, cloud_project.leanVersionId, notify_order_events, diff --git a/lean/commands/cloud/status.py b/lean/commands/cloud/status.py index 06737d8b..70b62c3b 100644 --- a/lean/commands/cloud/status.py +++ b/lean/commands/cloud/status.py @@ -16,8 +16,7 @@ from lean.click import LeanCommand from lean.container import container from lean.models.api import QCLiveAlgorithmStatus -from lean.models.brokerages.cloud import all_cloud_brokerages, PaperTradingBrokerage - +from lean.models.brokerage import cloud_brokerages @command(cls=LeanCommand) @argument("project", type=str) @@ -49,12 +48,9 @@ def status(project: str) -> None: QCLiveAlgorithmStatus.LoggingIn: "Logging in" }.get(live_algorithm.status, live_algorithm.status.value) - brokerage_name = next((b.get_name() for b in all_cloud_brokerages if b.get_id() == live_algorithm.brokerage), + brokerage_name = next((b.get_name() for b in cloud_brokerages if b.get_id() == live_algorithm.brokerage), live_algorithm.brokerage) - if brokerage_name == "PaperBrokerage": - brokerage_name = PaperTradingBrokerage.get_name() - logger.info(f"Live status: {live_status}") logger.info(f"Live id: {live_algorithm.deployId}") logger.info(f"Live url: {live_algorithm.get_url()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 4f406dba..22673142 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -14,22 +14,19 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from click import option, argument, Choice -from lean.click import LeanCommand, PathParameter, ensure_options +from lean.click import LeanCommand, PathParameter from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container -from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds +from lean.models.cli import cli_brokerages, cli_data_queue_handlers, cli_data_downloaders, cli_addon_modules from lean.models.errors import MoreInfoError -from lean.models.lean_config_configurer import LeanConfigConfigurer -from lean.models.logger import Option -from lean.models.configuration import ConfigurationsEnvConfiguration, InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.json_module import LiveInitialStateInput +from lean.models.json_module import LiveInitialStateInput, JsonModule from lean.commands.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively -from lean.models.data_providers import all_data_providers -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available +from lean.components.util.json_modules_handler import build_and_configure_modules, \ + non_interactive_config_build_for_name, save_settings, interactive_config_build _environment_skeleton = { "live-mode": True, @@ -40,82 +37,6 @@ } -def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]: - """Returns the configurable modules from the given environment. - - :param lean_config: the LEAN configuration that should be used - :param environment_name: the name of the environment - :return: the configurable modules from the given environment - """ - environment = lean_config["environments"][environment_name] - for key in ["live-mode-brokerage", "data-queue-handler"]: - if key not in environment: - raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", - "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") - - brokerage = environment["live-mode-brokerage"] - data_queue_handlers = environment["data-queue-handler"] - [brokerage_configurer] = [local_brokerage - for local_brokerage in all_local_brokerages - if _get_brokerage_base_name(local_brokerage.get_live_name()) == _get_brokerage_base_name(brokerage)] - data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] - data_feed_configurers = [local_data_feed - for local_data_feed in all_local_data_feeds - if _get_brokerage_base_name(local_data_feed.get_live_name()) in data_queue_handlers_base_names] - return brokerage_configurer, data_feed_configurers - - -def _get_brokerage_base_name(brokerage: str) -> str: - """Returns the base name of the brokerage. - - :param brokerage: the name of the brokerage - :return: the base name of the brokerage - """ - return brokerage.split('.')[-1] - -def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None: - """Raises an error if any of the given modules are not installed. - - :param modules: the modules to check - """ - for module in modules: - if module is None or not module._installs: - continue - organization_id = container.organization_manager.try_get_working_organization_id() - module.ensure_module_installed(organization_id) - - -def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None: - """Raises an error if any required properties are missing. - - :param lean_config: the LEAN configuration that should be used - :param environment_name: the name of the environment - :param lean_config_path: the path to the LEAN configuration file - """ - brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name) - brokerage_properties = brokerage_configurer.get_required_properties(include_optionals=False) - data_queue_handler_properties = [] - for data_feed_configurer in data_feed_configurers: - data_queue_handler_properties.extend(data_feed_configurer.get_required_properties(include_optionals=False)) - required_properties = list(set(brokerage_properties + data_queue_handler_properties)) - missing_properties = [p for p in required_properties if p not in lean_config or lean_config[p] == ""] - missing_properties = set(missing_properties) - if len(missing_properties) == 0: - return - - properties_str = "properties" if len(missing_properties) > 1 else "property" - these_str = "these" if len(missing_properties) > 1 else "this" - - missing_properties = "\n".join(f"- {p}" for p in missing_properties) - - raise RuntimeError(f""" -Please configure the following missing {properties_str} in {lean_config_path}: -{missing_properties} -Go to the following url for documentation on {these_str} {properties_str}: -https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading - """.strip()) - - def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], environment_name: str) -> None: """Starts IQConnect if the given environment uses IQFeed as data queue handler. @@ -147,80 +68,17 @@ def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], environment_name: sleep(10) -def _configure_lean_config_interactively(lean_config: Dict[str, Any], - environment_name: str, - properties: Dict[str, Any], - show_secrets: bool) -> None: - """Interactively configures the Lean config to use. - - Asks the user all questions required to set up the Lean config for local live trading. - - :param lean_config: the base lean config to use - :param environment_name: the name of the environment to configure - :param properties: the properties to use to configure lean - :param show_secrets: whether to show secrets on input - """ - logger = container.logger - - lean_config["environments"] = { - environment_name: _environment_skeleton - } - - brokerage = logger.prompt_list("Select a brokerage", [ - Option(id=brokerage, label=brokerage.get_name()) for brokerage in all_local_brokerages - ]) - - brokerage.build(lean_config, logger, properties, hide_input=not show_secrets).configure(lean_config, environment_name) - - data_feeds = logger.prompt_list("Select a live data provider", [ - Option(id=data_feed, label=data_feed.get_name()) for data_feed in local_brokerage_data_feeds[brokerage] - ], multiple= True) - for data_feed in data_feeds: - if brokerage._id == data_feed._id: - # update essential properties, so that other dependent values can be fetched. - essential_properties_value = {brokerage.convert_lean_key_to_variable(config._id): config._value - for config in brokerage.get_essential_configs()} - properties.update(essential_properties_value) - logger.debug(f"live.deploy._configure_lean_config_interactively(): essential_properties_value: {brokerage._id} {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from essential properties - required_properties_value = {brokerage.convert_lean_key_to_variable(config._id): config._value - for config in brokerage.get_required_configs([InternalInputUserInput])} - properties.update(required_properties_value) - logger.debug(f"live.deploy._configure_lean_config_interactively(): required_properties_value: {required_properties_value}") - data_feed.build(lean_config, logger, properties, hide_input=not show_secrets).configure(lean_config, environment_name) - - _cached_lean_config = None + def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: """ Get name for historical data provider based on data provider live (if exist) :param data_provider_historical_name: the current (default) data provider historical :param data_provider_live_name: the current data provider live name """ - return next((live_data_historical.get_name() for live_data_historical in all_data_providers - if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) - - -# being used by lean.models.click_options.get_the_correct_type_default_value() -def _get_default_value(key: str) -> Optional[Any]: - """Returns the default value for an option based on the Lean config. - - :param key: the name of the property in the Lean config that supplies the default value of an option - :return: the value of the property in the Lean config, or None if there is none - """ - global _cached_lean_config - if _cached_lean_config is None: - _cached_lean_config = container.lean_config_manager.get_lean_config() - - if key not in _cached_lean_config: - return None - - value = _cached_lean_config[key] - if value == "": - return None - - return value + return next((live_data_historical.get_name() for live_data_historical in cli_data_downloaders + if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) @live.command(cls=LeanCommand, requires_lean_config=True, requires_docker=True, default_command=True, name="deploy") @@ -236,16 +94,16 @@ def _get_default_value(key: str) -> Optional[Any]: default=False, help="Run the live deployment in a detached Docker container and return immediately") @option("--brokerage", - type=Choice([b.get_name() for b in all_local_brokerages], case_sensitive=False), + type=Choice([b.get_name() for b in cli_brokerages], case_sensitive=False), help="The brokerage to use") @option("--data-provider-live", - type=Choice([d.get_name() for d in all_local_data_feeds], case_sensitive=False), + type=Choice([d.get_name() for d in cli_data_queue_handlers], case_sensitive=False), multiple=True, help="The live data provider to use") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders if dp.get_id() != "TerminalLinkBrokerage"], case_sensitive=False), help="Update the Lean configuration file to retrieve data from the given historical provider") -@options_from_json(get_configs_for_options("live-local")) +@options_from_json(get_configs_for_options("live-cli")) @option("--release", is_flag=True, default=False, @@ -342,99 +200,71 @@ def deploy(project: Path, lean_config_manager = container.lean_config_manager + brokerage_instance: JsonModule + data_provider_live_instances: [JsonModule] = [] + data_provider_historical_instances: JsonModule if environment is not None and (brokerage is not None or len(data_provider_live) > 0): raise RuntimeError("--environment and --brokerage + --data-provider-live are mutually exclusive") + environment_name = "lean-cli" if environment is not None: environment_name = environment lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) - lean_environment = lean_config["environments"][environment_name] - for key in ["live-mode-brokerage", "data-queue-handler"]: - if key not in lean_environment: - raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", - "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") - - brokerage = lean_environment["live-mode-brokerage"] - data_queue_handlers = lean_environment["data-queue-handler"] - data_queue_handlers_base_names = [_get_brokerage_base_name(data_queue_handler) for data_queue_handler in data_queue_handlers] - data_feed_configurers = [] - - for local_brokerage in all_local_brokerages: - configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_brokerage._lean_configs if config._is_type_configurations_env] - for configuration_environment in configuration_environments: - configuration_environment_values = list(configuration_environment._env_and_values.values())[0] - if any(True for x in configuration_environment_values if x["name"] == "live-mode-brokerage" and _get_brokerage_base_name(x["value"]) == _get_brokerage_base_name(brokerage)): - brokerage_configurer = local_brokerage - # fill essential properties - for condition in configuration_environment._filter._conditions: - if condition._type != "exact-match": - continue - property_name_to_fill = local_brokerage.convert_lean_key_to_variable(condition._dependent_config_id) - property_value_to_fill = condition._pattern - kwargs[property_name_to_fill] = property_value_to_fill - lean_config[condition._dependent_config_id] = property_value_to_fill - break - - for local_data_feed in all_local_data_feeds: - configuration_environments: List[ConfigurationsEnvConfiguration] = [config for config in local_data_feed._lean_configs if config._is_type_configurations_env] - for configuration_environment in configuration_environments: - configuration_environment_values = list(configuration_environment._env_and_values.values())[0] - if any(True for x in configuration_environment_values if x["name"] == "data-queue-handler" and _get_brokerage_base_name(x["value"]) in data_queue_handlers_base_names): - data_feed_configurers.append(local_data_feed) - # fill essential properties - for condition in configuration_environment._filter._conditions: - if condition._type != "exact-match": - continue - property_name_to_fill = local_data_feed.convert_lean_key_to_variable(condition._dependent_config_id) - property_value_to_fill = condition._pattern - kwargs[property_name_to_fill] = property_value_to_fill - lean_config[condition._dependent_config_id] = property_value_to_fill - - [update_essential_properties_available([brokerage_configurer], kwargs)] - [update_essential_properties_available(data_feed_configurers, kwargs)] - - elif brokerage is not None or len(data_provider_live) > 0: - ensure_options(["brokerage", "data_provider_live"]) - - environment_name = "lean-cli" + if environment_name in lean_config["environments"]: + lean_environment = lean_config["environments"][environment_name] + for key in ["live-mode-brokerage", "data-queue-handler"]: + if key not in lean_environment: + raise MoreInfoError(f"The '{environment_name}' environment does not specify a {rename_internal_config_to_user_friendly_format(key)}", + "https://www.lean.io/docs/v2/lean-cli/live-trading/algorithm-control") + + # todo figure out data downloader, 'data_provider_historical' from env + brokerage = lean_environment["live-mode-brokerage"] + data_provider_live = lean_environment["data-queue-handler"] + logger.debug(f'Deploy(): loading env \'{environment_name}\'. Brokerage: \'{brokerage}\'. IDQHs: [{data_provider_live}]') + else: + logger.info(f'Environment \'{environment_name}\' not found, creating from scratch') + lean_config["environments"] = {environment_name: copy(_environment_skeleton)} + else: lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) + lean_config["environments"] = {environment_name: copy(_environment_skeleton)} - lean_config["environments"] = { - environment_name: copy(_environment_skeleton) - } - - [brokerage_configurer] = [get_and_build_module(brokerage, all_local_brokerages, kwargs, logger)] - brokerage_configurer.configure(lean_config, environment_name) - - for df in data_provider_live: - [data_feed_configurer] = [get_and_build_module(df, all_local_data_feeds, kwargs, logger)] - data_feed_configurer.configure(lean_config, environment_name) - + if brokerage: + # user provided brokerage, check all arguments were provided + brokerage_instance = non_interactive_config_build_for_name(lean_config, brokerage, cli_brokerages, kwargs, + logger, environment_name) else: - environment_name = "lean-cli" - lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) - _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) + # let the user choose the brokerage + brokerage_instance = interactive_config_build(lean_config, cli_brokerages, logger, kwargs, show_secrets, + "Select a brokerage", multiple=False, + environment_name=environment_name) + + if data_provider_live and len(data_provider_live) > 0: + for data_feed_name in data_provider_live: + data_feed = non_interactive_config_build_for_name(lean_config, data_feed_name, cli_data_queue_handlers, + kwargs, logger, environment_name) + data_provider_live_instances.append(data_feed) + else: + data_provider_live_instances = interactive_config_build(lean_config, cli_data_queue_handlers, logger, kwargs, + show_secrets, "Select a live data feed", multiple=True, + environment_name=environment_name) if data_provider_historical is None: data_provider_historical = _try_get_data_historical_name("Local", data_provider_live) - [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] - data_provider_configurer.configure(lean_config, environment_name) + data_provider_historical_instances = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, + environment_name) - if "environments" not in lean_config or environment_name not in lean_config["environments"]: - lean_config_path = lean_config_manager.get_lean_config_path() - raise MoreInfoError(f"{lean_config_path} does not contain an environment named '{environment_name}'", - "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") + organization_id = container.organization_manager.try_get_working_organization_id() + for module in data_provider_live_instances + [data_provider_historical_instances, brokerage_instance]: + module.ensure_module_installed(organization_id) + logger.debug(f'Deploy(): lean_config: {lean_config}') + save_settings(lean_config) if not lean_config["environments"][environment_name]["live-mode"]: raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) - _install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs) - - _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) - project_config_manager = container.project_config_manager cli_config_manager = container.cli_config_manager @@ -451,7 +281,7 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, env_brokerage, + cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, brokerage_instance, project_config.get("cloud-id", None), project) if environment is None and brokerage is None and len(data_provider_live) == 0: # condition for using interactive panel @@ -498,7 +328,8 @@ def deploy(project: Path, lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output, given_algorithm_id)}" # Configure addon modules - build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, environment_name) + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) if container.platform_manager.is_host_arm(): if "InteractiveBrokersBrokerage" in lean_config["environments"][environment_name]["live-mode-brokerage"] \ @@ -506,4 +337,4 @@ def deploy(project: Path, raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts") lean_runner = container.lean_runner - lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) \ No newline at end of file + lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 05e02f27..8e365078 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -23,10 +23,10 @@ from lean.container import container from lean.models.api import QCParameter, QCBacktest from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.data_providers import all_data_providers, QuantConnectDataProvider, DataProvider +from lean.models.cli import cli_data_downloaders, cli_addon_modules from lean.models.errors import MoreInfoError from lean.models.optimizer import OptimizationTarget -from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module +from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name def _get_latest_backtest_runtime(algorithm_directory: Path) -> timedelta: @@ -98,7 +98,7 @@ def get_filename_timestamp(path: Path) -> datetime: multiple=True, help="The 'statistic operator value' pairs configuring the constraints of the optimization") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @option("--download-data", @@ -206,7 +206,7 @@ def optimize(project: Path, from math import floor should_detach = detach and not estimate - environment = "backtesting" + environment_name = "backtesting" project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(project) @@ -294,18 +294,18 @@ def optimize(project: Path, logger.warn(f'A custom engine image: "{engine_image}" is being used!') lean_config_manager = container.lean_config_manager - lean_config = lean_config_manager.get_complete_lean_config(environment, algorithm_file, None) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) organization_id = container.organization_manager.try_get_working_organization_id() if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(organization_id) - data_provider_configurer.configure(lean_config, environment) - logger.info(lean_config) + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) + lean_config["environments"][environment_name].update(data_provider.get_settings()) if not output.exists(): output.mkdir(parents=True) @@ -317,9 +317,9 @@ def optimize(project: Path, # Set extra config for key, value in extra_config: - if "environments" in lean_config and environment in lean_config["environments"] \ - and key in lean_config["environments"][environment]: - lean_config["environments"][environment][key] = value + if "environments" in lean_config and environment_name in lean_config["environments"] \ + and key in lean_config["environments"][environment_name]: + lean_config["environments"][environment_name][key] = value else: lean_config[key] = value @@ -327,7 +327,8 @@ def optimize(project: Path, lean_config["algorithm-id"] = str(output_config_manager.get_optimization_id(output)) # Configure addon modules - build_and_configure_modules(addon_module, organization_id, lean_config, logger, environment) + build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config, + kwargs, logger, environment_name) run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach) diff --git a/lean/commands/research.py b/lean/commands/research.py index d8c17d35..1d4dd0ee 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -18,9 +18,9 @@ from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH from lean.container import container -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider +from lean.models.cli import cli_data_downloaders from lean.components.util.name_extraction import convert_to_class_name -from lean.components.util.json_modules_handler import get_and_build_module +from lean.components.util.json_modules_handler import non_interactive_config_build_for_name from lean.models.click_options import options_from_json, get_configs_for_options def _check_docker_output(chunk: str, port: int) -> None: @@ -38,14 +38,14 @@ def _check_docker_output(chunk: str, port: int) -> None: @argument("project", type=PathParameter(exists=True, file_okay=False, dir_okay=True)) @option("--port", type=int, default=8888, help="The port to run Jupyter Lab on (defaults to 8888)") @option("--data-provider-historical", - type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), + type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False), default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("research")) @option("--download-data", is_flag=True, default=False, - help=f"Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical {QuantConnectDataProvider.get_name()}") + help=f"Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect") @option("--data-purchase-limit", type=int, help="The maximum amount of QCC to spend on downloading data during the research session when using QuantConnect as historical data provider") @@ -104,19 +104,20 @@ def research(project: Path, algorithm_file = project_manager.find_algorithm_file(project) algorithm_name = convert_to_class_name(project) + environment_name = "backtesting" lean_config_manager = container.lean_config_manager - lean_config = lean_config_manager.get_complete_lean_config("backtesting", algorithm_file, None) + lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) lean_config["composer-dll-directory"] = LEAN_ROOT_PATH lean_config["research-object-store-name"] = algorithm_name if download_data: - data_provider_historical = QuantConnectDataProvider.get_name() + data_provider_historical = "QuantConnect" if data_provider_historical is not None: - data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger) - data_provider_configurer.ensure_module_installed(container.organization_manager.try_get_working_organization_id()) - data_provider_configurer.configure(lean_config, "backtesting") - + organization_id = container.organization_manager.try_get_working_organization_id() + data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, + cli_data_downloaders, kwargs, logger, environment_name) + data_provider.ensure_module_installed(organization_id) lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) lean_runner = container.lean_runner diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index 7548bc4f..c3590467 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -57,7 +57,7 @@ def start(self, compile_id: str, node_id: str, brokerage_settings: Dict[str, Any], - price_data_handler: str, + live_data_providers_settings: Dict[str, Any], automatic_redeploy: bool, version_id: int, notify_order_events: bool, @@ -71,7 +71,7 @@ def start(self, :param compile_id: the id of the compile to use for live trading :param node_id: the id of the node to start live trading on :param brokerage_settings: the brokerage settings to use - :param price_data_handler: the live data provider to use + :param live_data_providers_settings: the live data providers settings to use :param automatic_redeploy: whether automatic redeploys are enabled :param version_id: the id of the LEAN version to use :param notify_order_events: whether notifications should be sent on order events @@ -92,7 +92,7 @@ def start(self, "compileId": compile_id, "nodeId": node_id, "brokerage": brokerage_settings, - "dataHandler": price_data_handler, + "dataProviders": live_data_providers_settings, "automaticRedeploy": automatic_redeploy, "versionId": version_id } diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index 26622d5d..992e5205 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -155,10 +155,14 @@ def set_properties(self, updates: Dict[str, Any]) -> None: for key, value in reversed(list(updates.items())): json_value = dumps(value) + json_value = json_value.replace("\\", "/") # We can only use regex to set the property because converting the config back to JSON drops all comments if key in config: - config_text = sub(fr'"{key}":\s*("?[^",]*"?)', f'"{key}": {json_value}', config_text) + if json_value.startswith("["): + config_text = sub(fr'"{key}":\s*(\[.*?])', f'"{key}": {json_value}', config_text) + else: + config_text = sub(fr'"{key}":\s*("?[^",]*"?)', f'"{key}": {json_value}', config_text) else: config_text = config_text.replace("{", f'{{\n "{key}": {json_value},', 1) diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 85713a55..e81e755d 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -12,56 +12,124 @@ # limitations under the License. from typing import Any, Dict, List -from lean.models.addon_modules.addon_module import AddonModule -from lean.models.addon_modules import all_addon_modules from lean.components.util.logger import Logger -from lean.models.configuration import InternalInputUserInput from lean.models.json_module import JsonModule -from lean.click import ensure_options +from lean.models.logger import Option -def build_and_configure_modules(modules: List[AddonModule], organization_id: str, lean_config: Dict[str, Any], logger: Logger, environment_name: str) -> Dict[str, Any]: - """Capitalizes the given word. - :param word: the word to capitalize - :return: the word with the first letter capitalized (any other uppercase characters are preserved) +def build_and_configure_modules(target_modules: List[str], module_list: List[JsonModule], organization_id: str, + lean_config: Dict[str, Any], properties: Dict[str, Any], logger: Logger, + environment_name: str): + """Builds and configures the given modules + + :param target_modules: the requested modules + :param module_list: the available modules + :param organization_id: the organization id + :param lean_config: the current lean configs + :param properties: the user provided arguments + :param logger: the logger instance + :param environment_name: the environment name to use """ - for given_module in modules: - try: - found_module = next((module for module in all_addon_modules if module.get_name().lower() == given_module.lower()), None) - if found_module: - found_module.build(lean_config, logger).configure(lean_config, environment_name) - found_module.ensure_module_installed(organization_id) - else: - logger.error(f"Addon module '{given_module}' not found") - except Exception as e: - logger.error(f"Addon module '{given_module}' failed to configure: {e}") - return lean_config - - -def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger) -> JsonModule: - [target_module] = [module for module in module_list if module.get_name() == target_module_name] - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()] - ensure_options(essential_properties) - essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties} - target_module.update_configs(essential_properties_value) - logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: essential_properties_value with module {target_module_name}: {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from essential properties - required_properties: List[str] = [] - for config in target_module.get_required_configs([InternalInputUserInput]): - required_properties.append(target_module.convert_lean_key_to_variable(config._id)) - ensure_options(required_properties) - required_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in required_properties} - target_module.update_configs(required_properties_value) - logger.debug(f"json_module_handler.get_and_build_module(): non-interactive: required_properties_value with module {target_module_name}: {required_properties_value}") + for target_module_name in target_modules: + module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties, + logger, environment_name) + module.ensure_module_installed(organization_id) + lean_config["environments"][environment_name].update(module.get_settings()) + + +def non_interactive_config_build_for_name(lean_config: Dict[str, Any], target_module_name: str, + module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger, + environment_name: str = None) -> JsonModule: + target_module: JsonModule = None + for module in module_list: + if module.get_id() == target_module_name or module.get_name() == target_module_name: + target_module = module + break + else: + index = target_module_name.rfind('.') + if (index != -1 and module.get_id() == target_module_name[index + 1:] + or module.get_name() == target_module_name[index + 1:]): + target_module = module + break + + if not target_module: + raise RuntimeError(f"""Failed to resolve module for name: '{target_module_name}'""") + else: + logger.debug(f'Found module \'{target_module_name}\' from given name') + + target_module.config_build(lean_config, logger, interactive=False, properties=properties, + environment_name=environment_name) + _update_settings(logger, environment_name, target_module, lean_config) return target_module -def update_essential_properties_available(module_list: List[JsonModule], properties: Dict[str, Any]) -> JsonModule: - for target_module in module_list: - # update essential properties from brokerage to datafeed - # needs to be updated before fetching required properties - essential_properties = [target_module.convert_lean_key_to_variable(prop) for prop in target_module.get_essential_properties()] - essential_properties_value = {target_module.convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties if properties[prop] is not None} - target_module.update_configs(essential_properties_value) +def interactive_config_build(lean_config: Dict[str, Any], models: [JsonModule], logger: Logger, + user_provided_options: Dict[str, Any], show_secrets: bool, select_message: str, + multiple: bool, environment_name: str = None) -> [JsonModule]: + """Interactively configures the brokerage to use. + + :param lean_config: the LEAN configuration that should be used + :param models: the modules to choose from + :param logger: the logger to use + :param user_provided_options: the dictionary containing user provided options + :param show_secrets: whether to show secrets on input + :param select_message: the user facing selection message + :param multiple: true if multiple selections are allowed + :param environment_name: the target environment name + :return: the brokerage the user configured + """ + options = [Option(id=b, label=b.get_name()) for b in models] + + modules: [JsonModule] = [] + if multiple: + modules = logger.prompt_list(select_message, options, multiple=True) + else: + module = logger.prompt_list(select_message, options, multiple=False) + modules.append(module) + + for module in modules: + module.config_build(lean_config, logger, interactive=True, properties=user_provided_options, + hide_input=not show_secrets, environment_name=environment_name) + _update_settings(logger, environment_name, module, lean_config) + if multiple: + return modules + return modules[-1] + + +def save_settings(settings: Dict[str, Any]) -> None: + """Persistently save properties in the Lean configuration. + + :param settings: the dict containing settings to save + """ + return + from lean.container import container + container.lean_config_manager.set_properties(settings) + + +def _update_settings(logger: Logger, environment_name: str, module: JsonModule, + lean_config: Dict[str, Any]) -> None: + settings = module.get_settings() + logger.debug(f'_update_settings({module}): Settings: {settings}') + + if environment_name: + if environment_name not in lean_config["environments"]: + lean_config["environments"][environment_name] = {} + target = lean_config["environments"][environment_name] + else: + target = lean_config["environments"] + + for key, value in settings.items(): + if key in target and target[key].startswith("["): + # it already exists, and it's an array we need to merge + from json import loads, dumps + logger.debug(f'_update_settings({module}): target[key]: {target[key]}') + existing_value = loads(target[key]) + if value.startswith("["): + # the new value is also an array, merge them + existing_value = existing_value + loads(value) + else: + existing_value.append(value) + target[key] = dumps(existing_value) + else: + target[key] = value + diff --git a/lean/constants.py b/lean/constants.py index 121f9a71..53e4ff33 100644 --- a/lean/constants.py +++ b/lean/constants.py @@ -96,3 +96,18 @@ # The name of the Docker network which all Lean CLI containers are ran on DOCKER_NETWORK = "lean_cli" + +# Module constants +MODULE_TYPE = "type" +MODULE_PLATFORM = "platform" + +# types +MODULE_ADDON = "addon-module" +MODULE_BROKERAGE = "brokerage" +MODULE_DATA_DOWNLOADER = "data-downloader" +MODULE_HISTORY_PROVIDER = "history-provider" +MODULE_DATA_QUEUE_HANDLER = "data-queue-handler" + +# platforms +MODULE_CLI_PLATFORM = "cli" +MODULE_CLOUD_PLATFORM = "cloud" diff --git a/lean/models/__init__.py b/lean/models/__init__.py index 2eba3c18..952d30e5 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -17,7 +17,7 @@ from time import time json_modules = {} -file_name = "modules-1.12.json" +file_name = "modules-1.13.json" directory = Path(__file__).parent file_path = directory.parent / file_name diff --git a/lean/models/addon_modules/__init__.py b/lean/models/addon_modules/__init__.py deleted file mode 100644 index 2541abf3..00000000 --- a/lean/models/addon_modules/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import List -from lean.models.addon_modules.addon_module import AddonModule -from lean.models import json_modules - -all_addon_modules: List[AddonModule] = [] - -for json_module in json_modules: - if "addon-module" in json_module["type"]: - all_addon_modules.append(AddonModule(json_module)) diff --git a/lean/models/addon_modules/addon_module.py b/lean/models/addon_modules/addon_module.py deleted file mode 100644 index 4fc94fc3..00000000 --- a/lean/models/addon_modules/addon_module.py +++ /dev/null @@ -1,20 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from lean.models.lean_config_configurer import LeanConfigConfigurer - -class AddonModule(LeanConfigConfigurer): - """A JsonModule implementation for add on modules.""" - - - diff --git a/lean/models/brokerages/__init__.py b/lean/models/brokerages/__init__.py deleted file mode 100644 index 9af974c0..00000000 --- a/lean/models/brokerages/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py deleted file mode 100644 index 72867060..00000000 --- a/lean/models/brokerages/cloud/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage -from lean.models import json_modules -from typing import Dict, Type, List -from lean.models.brokerages.local.data_feed import DataFeed - -all_cloud_brokerages: List[CloudBrokerage] = [] -all_cloud_data_feeds: List[DataFeed] = [] -cloud_brokerage_data_feeds: Dict[Type[CloudBrokerage], - List[Type[DataFeed]]] = {} - -for json_module in json_modules: - if "cloud-brokerage" in json_module["type"]: - all_cloud_brokerages.append(CloudBrokerage(json_module)) - if "data-queue-handler" in json_module["type"]: - all_cloud_data_feeds.append((DataFeed(json_module))) - -for cloud_brokerage in all_cloud_brokerages: - data_feed_property_found = False - for x in cloud_brokerage.get_all_input_configs(): - if "data-feed" in x.__getattribute__("_id"): - data_feed_property_found = True - cloud_brokerage_data_feeds[cloud_brokerage] = x.__getattribute__("_choices") - if not data_feed_property_found: - cloud_brokerage_data_feeds[cloud_brokerage] = [] - -[PaperTradingBrokerage] = [ - cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py deleted file mode 100644 index 1c30aea7..00000000 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ /dev/null @@ -1,90 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, Any -from lean.models.json_module import JsonModule -from lean.models.configuration import InternalInputUserInput, TradingEnvConfiguration - - -class CloudBrokerage(JsonModule): - """A JsonModule implementation for the cloud brokerages.""" - - def __init__(self, json_cloud_brokerage_data: Dict[str, Any]) -> None: - super().__init__(json_cloud_brokerage_data) - - def get_id(self) -> str: - """Returns the id of the brokerage. - :return: the id of this brokerage as it is expected by the live/create API endpoint - """ - return self._id - - def _get_settings(self) -> Dict[str, str]: - """Returns all settings for this brokerage, except for the id. - :return: the settings of this brokerage excluding the id - """ - settings = {} - for config in self.get_required_configs(): - value = None - if not config._cloud_id: - continue - # TODO: handle cases where tranding env config is not present, environment will still be required. - if type(config) == TradingEnvConfiguration: - value = "paper" if config._value.lower() in [ - "practice", "demo", "beta", "paper"] else "live" - elif type(config) is InternalInputUserInput: - if not config._is_conditional: - value = config._value - else: - for option in config._value_options: - if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): - value = option._value - break - if not value: - options_to_log = set([(opt._condition._dependent_config_id, - self.get_config_value_from_name(opt._condition._dependent_config_id)) - for opt in config._value_options]) - raise ValueError( - f'No condition matched among present options for "{config._cloud_id}". ' - f'Please review ' + - ', '.join([f'"{x[0]}"' for x in options_to_log]) + - f' given value{"s" if len(options_to_log) > 1 else ""} ' + - ', '.join([f'"{x[1]}"' for x in options_to_log])) - else: - value = config._value - settings[config._cloud_id] = value - return settings - - def get_settings(self) -> Dict[str, str]: - """Returns all settings for this brokerage. - :return: the settings to set in the "brokerage" property of the live/create API endpoint - """ - settings = self._get_settings() - if "environment" not in settings.keys(): - settings["environment"] = "live" - settings["id"] = self.get_id() - return settings - - def get_price_data_handler(self) -> str: - """Returns the price live data provider handler to use. - :return: the value to assign to the "dataHandler" property of the live/create API endpoint - """ - # TODO: Handle this case with json conditions - property_name = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] - property_name = property_name[0] if len(property_name) != 0 else "" - brokerage_name = self.get_name().replace(" ", "") - if property_name != "": - if "QuantConnect +" in self.get_config_value_from_name(property_name): - return "quantconnecthandler+" + brokerage_name.lower() + "handler" - else: - return self.get_config_value_from_name(property_name).replace(" ", "") + "Handler" - return "QuantConnectHandler" diff --git a/lean/models/brokerages/local/__init__.py b/lean/models/brokerages/local/__init__.py deleted file mode 100644 index b46bdf12..00000000 --- a/lean/models/brokerages/local/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from os import environ -from typing import Dict, Type, List -from lean.container import container -from lean.models.brokerages.local.local_brokerage import LocalBrokerage -from lean.models.brokerages.local.data_feed import DataFeed -from lean.models import json_modules - -all_local_brokerages: List[LocalBrokerage] = [] -all_local_data_feeds: List[DataFeed] = [] -local_brokerage_data_feeds: Dict[Type[LocalBrokerage], - List[Type[DataFeed]]] = {} - -for json_module in json_modules: - if "local-brokerage" in json_module["type"]: - all_local_brokerages.append(LocalBrokerage(json_module)) - if "data-queue-handler" in json_module["type"]: - all_local_data_feeds.append(DataFeed(json_module)) - -# Remove IQFeed DataFeed for other than windows machines -if not [container.platform_manager.is_host_windows() or environ.get("__README__", "false") == "true"]: - all_local_data_feeds = [ - data_feed for data_feed in all_local_data_feeds if data_feed._id != "IQFeed"] - -for local_brokerage in all_local_brokerages: - local_brokerage_data_feeds[local_brokerage] = all_local_data_feeds diff --git a/lean/models/brokerages/local/data_feed.py b/lean/models/brokerages/local/data_feed.py deleted file mode 100644 index 10a8dec8..00000000 --- a/lean/models/brokerages/local/data_feed.py +++ /dev/null @@ -1,29 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - - -class DataFeed(LeanConfigConfigurer): - """A JsonModule implementation for the Json live data provider module.""" - - def __init__(self, json_datafeed_data: Dict[str, Any]) -> None: - super().__init__(json_datafeed_data) - - def get_live_name(self) -> str: - live_name = self._id - environment_obj = self.get_configurations_env_values() - if environment_obj: - [live_name] = [x["value"] for x in environment_obj if x["name"] == "data-queue-handler"] - return live_name diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py deleted file mode 100644 index c22f7fe7..00000000 --- a/lean/models/brokerages/local/local_brokerage.py +++ /dev/null @@ -1,29 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - - -class LocalBrokerage(LeanConfigConfigurer): - """A JsonModule implementation for the Json brokerage module.""" - - def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: - super().__init__(json_brokerage_data) - - def get_live_name(self) -> str: - live_name = self._id - environment_obj = self.get_configurations_env_values() - if environment_obj: - [live_name] = [x["value"] for x in environment_obj if x["name"] == "live-mode-brokerage"] - return live_name diff --git a/lean/models/cli/__init__.py b/lean/models/cli/__init__.py new file mode 100644 index 00000000..576f6456 --- /dev/null +++ b/lean/models/cli/__init__.py @@ -0,0 +1,43 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List + +from lean.constants import MODULE_BROKERAGE, MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM, \ + MODULE_DATA_DOWNLOADER, MODULE_HISTORY_PROVIDER, MODULE_DATA_QUEUE_HANDLER, MODULE_ADDON +from lean.models import json_modules +from lean.models.json_module import JsonModule + +# load the modules +cli_brokerages: List[JsonModule] = [] +cli_addon_modules: List[JsonModule] = [] +cli_data_downloaders: List[JsonModule] = [] +cli_history_provider: List[JsonModule] = [] +cli_data_queue_handlers: List[JsonModule] = [] + +for json_module in json_modules: + module_type = json_module[MODULE_TYPE] + platform = json_module[MODULE_PLATFORM] + + if MODULE_CLI_PLATFORM in platform: + if MODULE_BROKERAGE in module_type: + cli_brokerages.append(JsonModule(json_module, MODULE_BROKERAGE, MODULE_CLI_PLATFORM)) + if MODULE_DATA_DOWNLOADER in module_type: + cli_data_downloaders.append(JsonModule(json_module, MODULE_DATA_DOWNLOADER, MODULE_CLI_PLATFORM)) + if MODULE_HISTORY_PROVIDER in module_type: + cli_history_provider.append(JsonModule(json_module, MODULE_HISTORY_PROVIDER, MODULE_CLI_PLATFORM)) + if MODULE_DATA_QUEUE_HANDLER in module_type: + cli_data_queue_handlers.append(JsonModule(json_module, MODULE_DATA_QUEUE_HANDLER, MODULE_CLI_PLATFORM)) + if MODULE_ADDON in module_type: + cli_addon_modules.append(JsonModule(json_module, MODULE_ADDON, MODULE_CLI_PLATFORM)) + + diff --git a/lean/models/click_options.py b/lean/models/click_options.py index 6c8f8e40..5e56f444 100644 --- a/lean/models/click_options.py +++ b/lean/models/click_options.py @@ -12,26 +12,25 @@ # limitations under the License. -from typing import Any, List, Dict +from typing import List, Dict from click import option, Choice from lean.click import PathParameter -from lean.models.configuration import Configuration -from lean.models.brokerages.cloud import all_cloud_brokerages -from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds -from lean.models.data_providers import all_data_providers +from lean.models.cli import cli_brokerages, cli_data_downloaders, cli_data_queue_handlers +from lean.models.cloud import cloud_brokerages, cloud_data_queue_handlers from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput + def get_configs_for_options(env: str) -> List[Configuration]: if env == "live-cloud": - brokerage = all_cloud_brokerages - elif env == "live-local": - brokerage = all_local_brokerages + all_local_data_feeds + all_data_providers + brokerage = cloud_brokerages + cloud_data_queue_handlers + elif env == "live-cli": + brokerage = cli_brokerages + cli_data_queue_handlers + cli_data_downloaders elif env == "backtest": - brokerage = all_data_providers + brokerage = cli_data_downloaders elif env == "research": - brokerage = all_data_providers + brokerage = cli_data_downloaders else: - raise ValueError("Acceptable values for 'env' are: 'live-cloud', 'live-local', 'backtest', 'research'") + raise ValueError("Acceptable values for 'env' are: 'live-cloud', 'live-cli', 'backtest', 'research'") run_options: Dict[str, Configuration] = {} config_with_module_id: Dict[str, str] = {} @@ -81,32 +80,11 @@ def get_attribute_type(configuration: Configuration): return str -def get_the_correct_type_default_value(default_lean_config_key: str, default_input_value: str, expected_type: Any, - choices: List[str] = None): - from lean.commands.live.deploy import _get_default_value - lean_value = _get_default_value(default_lean_config_key) - if lean_value is None and default_input_value is not None: - lean_value = default_input_value - # This handles backwards compatibility for the old modules.json values. - if lean_value is not None and type(lean_value) != expected_type and type(lean_value) == bool: - if choices and "true" in choices and "false" in choices: - # Backwards compatibility for zeroha-history-subscription. - lean_value = "true" if lean_value else "false" - else: - # Backwards compatibility for tradier-use-sandbox - lean_value = "paper" if lean_value else "live" - return lean_value - - def get_options_attributes(configuration: Configuration, default_lean_config_key=None): options_attributes = { "type": get_click_option_type(configuration), "help": configuration._help } - default_input_value = configuration._input_default if configuration._is_required_from_user else None - options_attributes["default"] = lambda: get_the_correct_type_default_value( - default_lean_config_key, default_input_value, get_attribute_type(configuration), - configuration._choices if configuration._input_method == "choice" else None) return options_attributes diff --git a/lean/models/data_providers/__init__.py b/lean/models/cloud/__init__.py similarity index 50% rename from lean/models/data_providers/__init__.py rename to lean/models/cloud/__init__.py index fb0c47e2..e9dfe040 100644 --- a/lean/models/data_providers/__init__.py +++ b/lean/models/cloud/__init__.py @@ -10,17 +10,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from typing import List -from lean.models.data_providers.data_provider import DataProvider + +from lean.constants import MODULE_BROKERAGE, MODULE_TYPE, MODULE_CLOUD_PLATFORM, MODULE_PLATFORM, \ + MODULE_DATA_QUEUE_HANDLER from lean.models import json_modules +from lean.models.json_module import JsonModule -all_data_providers: List[DataProvider] = [] +# load the modules +cloud_brokerages: List[JsonModule] = [] +cloud_data_queue_handlers: List[JsonModule] = [] for json_module in json_modules: - if "data-provider" in json_module["type"]: - all_data_providers.append(DataProvider(json_module)) + module_type = json_module[MODULE_TYPE] + platform = json_module[MODULE_PLATFORM] -# QuantConnect DataProvider -[QuantConnectDataProvider] = [ - data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] \ No newline at end of file + if MODULE_CLOUD_PLATFORM in platform: + if MODULE_BROKERAGE in module_type: + cloud_brokerages.append(JsonModule(json_module, MODULE_BROKERAGE, MODULE_CLOUD_PLATFORM)) + if MODULE_DATA_QUEUE_HANDLER in module_type: + cloud_data_queue_handlers.append(JsonModule(json_module, MODULE_DATA_QUEUE_HANDLER, MODULE_CLOUD_PLATFORM)) diff --git a/lean/models/configuration.py b/lean/models/configuration.py index aed2d549..783d93d0 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -93,12 +93,8 @@ def __init__(self, config_json_object): self._id: str = config_json_object["id"] self._config_type: str = config_json_object["type"] self._value: str = config_json_object["value"] - self._is_cloud_property: bool = "cloud-id" in config_json_object self._is_required_from_user = False self._save_persistently_in_lean = False - self._is_type_configurations_env: bool = type( - self) is ConfigurationsEnvConfiguration - self._is_type_trading_env: bool = type(self) is TradingEnvConfiguration self._log_message: str = "" if "log-message" in config_json_object.keys(): self._log_message = config_json_object["log-message"] @@ -107,6 +103,10 @@ def __init__(self, config_json_object): else: self._filter = Filter([]) + self._input_default = None + if "input-default" in config_json_object: + self._input_default = config_json_object["input-default"] + def factory(config_json_object) -> 'Configuration': """Creates an instance of the child classes. @@ -115,20 +115,21 @@ def factory(config_json_object) -> 'Configuration': :return: An instance of Configuration. """ - if config_json_object["type"] in ["info", "configurations-env"]: + if config_json_object["type"] in ["info"]: return InfoConfiguration.factory(config_json_object) elif config_json_object["type"] in ["input", "internal-input"]: return UserInputConfiguration.factory(config_json_object) elif config_json_object["type"] == "filter-env": return BrokerageEnvConfiguration.factory(config_json_object) - elif config_json_object["type"] == "trading-env": - return TradingEnvConfiguration.factory(config_json_object) else: raise ValueError( f'Undefined input method type {config_json_object["type"]}') + def __repr__(self): + return f'{self._id}: {self._value}' -class Filter(): + +class Filter: """This class handles the conditional filters added to configurations. """ @@ -154,23 +155,7 @@ def factory(config_json_object) -> 'InfoConfiguration': :param config_json_object: the json object dict with configuration info :return: An instance of InfoConfiguration. """ - if config_json_object["type"] == "configurations-env": - return ConfigurationsEnvConfiguration(config_json_object) - else: - return InfoConfiguration(config_json_object) - - -class ConfigurationsEnvConfiguration(InfoConfiguration): - """Configuration class used for environment properties. - - Doesn't support user prompt inputs. - Values of this configuration isn't persistently saved in the Lean configuration. - """ - - def __init__(self, config_json_object): - super().__init__(config_json_object) - self._env_and_values = { - env_obj["name"]: env_obj["value"] for env_obj in self._value} + return InfoConfiguration(config_json_object) class UserInputConfiguration(Configuration, ABC): @@ -186,7 +171,6 @@ def __init__(self, config_json_object): self._is_required_from_user = True self._save_persistently_in_lean = True self._input_method = self._prompt_info = self._help = "" - self._input_default = self._cloud_id = None self._optional = False if "input-method" in config_json_object: self._input_method = config_json_object["input-method"] @@ -194,10 +178,6 @@ def __init__(self, config_json_object): self._prompt_info = config_json_object["prompt-info"] if "help" in config_json_object: self._help = config_json_object["help"] - if "input-default" in config_json_object: - self._input_default = config_json_object["input-default"] - if "cloud-id" in config_json_object: - self._cloud_id = config_json_object["cloud-id"] if "save-persistently-in-lean" in config_json_object: self._save_persistently_in_lean = config_json_object["save-persistently-in-lean"] if "optional" in config_json_object: @@ -411,46 +391,6 @@ def ask_user_for_input(self, default_value, logger: Logger, hide_input: bool = F raise ValueError(f"Undefined input method type {self._input_method}") -class TradingEnvConfiguration(PromptUserInput, ChoiceUserInput, ConfirmUserInput): - """This class adds trading-mode/envirionment based user filters. - - Normalizes the value of envrionment values(live/paper) for cloud live. - """ - - def __init__(self, config_json_object): - super().__init__(config_json_object) - - def factory(config_json_object) -> 'TradingEnvConfiguration': - """Creates an instance of the child classes. - - :param config_json_object: the json object dict with configuration info - :return: An instance of TradingEnvConfiguration. - """ - if config_json_object["type"] == "trading-env": - return TradingEnvConfiguration(config_json_object) - else: - raise ValueError( - f'Undefined input method type {config_json_object["type"]}') - - def ask_user_for_input(self, default_value, logger: Logger, hide_input: bool = False): - """Prompts user to provide input while validating the type of input - against the expected type - - :param default_value: The default to prompt to the user. - :param logger: The instance of logger class. - :param hide_input: Whether to hide the input (not used for this type of input, which is never hidden). - :return: The value provided by the user. - """ - # NOTE: trading envrionment config should not use old boolean value as default - if type(default_value) == bool: - default_value = "paper" if default_value else "live" - if self._input_method == "confirm": - raise ValueError( - f'input method -- {self._input_method} is not allowed with {self.__class__.__name__}') - else: - return BrokerageEnvConfiguration.ask_user_for_input(self, default_value, logger) - - class FilterEnvConfiguration(BrokerageEnvConfiguration): """This class adds extra filters to user filters.""" diff --git a/lean/models/data_providers/data_provider.py b/lean/models/data_providers/data_provider.py deleted file mode 100644 index 9bc37372..00000000 --- a/lean/models/data_providers/data_provider.py +++ /dev/null @@ -1,26 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict -from lean.models.lean_config_configurer import LeanConfigConfigurer - -class DataProvider(LeanConfigConfigurer): - """A JsonModule implementation for the Json historical data provider module.""" - - def __init__(self, json_data_provider_data: Dict[str, Any]) -> None: - super().__init__(json_data_provider_data) - - def configure_credentials(self, lean_config: Dict[str, Any]) -> None: - super().configure_credentials(lean_config) - self._save_properties( - lean_config, self.get_non_user_required_properties()) diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 41ee9967..4c9a1e2f 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -18,6 +18,8 @@ from click.core import ParameterSource from lean.components.util.logger import Logger +from lean.constants import MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM +from lean.container import container from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput from copy import copy from abc import ABC @@ -26,18 +28,21 @@ class JsonModule(ABC): """The JsonModule class is the base class extended for all json modules.""" - def __init__(self, json_module_data: Dict[str, Any]) -> None: - self._type: List[str] = json_module_data["type"] - self._product_id: int = json_module_data["product-id"] + def __init__(self, json_module_data: Dict[str, Any], module_type: str, platform: str) -> None: + self._module_type: str = module_type + self._platform: str = platform + self._product_id: int = json_module_data["product-id"] if "product-id" in json_module_data else 0 self._id: str = json_module_data["id"] self._display_name: str = json_module_data["display-id"] - self._installs: bool = json_module_data["installs"] + self._installs: bool = json_module_data["installs"] if ("installs" in json_module_data + and platform == MODULE_CLI_PLATFORM) else False self._lean_configs: List[Configuration] = [] for config in json_module_data["configurations"]: self._lean_configs.append(Configuration.factory(config)) self._lean_configs = self.sort_configs() self._is_module_installed: bool = False - self._initial_cash_balance: LiveInitialStateInput = LiveInitialStateInput(json_module_data["live-cash-balance-state"]) \ + self._initial_cash_balance: LiveInitialStateInput = LiveInitialStateInput( + json_module_data["live-cash-balance-state"]) \ if "live-cash-balance-state" in json_module_data \ else None self._initial_holdings: LiveInitialStateInput = LiveInitialStateInput(json_module_data["live-holdings-state"]) \ @@ -45,6 +50,9 @@ def __init__(self, json_module_data: Dict[str, Any]) -> None: else False self._minimum_seat = json_module_data["minimum-seat"] if "minimum-seat" in json_module_data else None + def get_id(self): + return self._id + def sort_configs(self) -> List[Configuration]: sorted_configs = [] brokerage_configs = [] @@ -62,84 +70,59 @@ def get_name(self) -> str: """ return self._display_name - def check_if_config_passes_filters(self, config: Configuration) -> bool: + def _check_if_config_passes_filters(self, config: Configuration, all_for_platform_type: bool) -> bool: for condition in config._filter._conditions: - if condition._dependent_config_id == "module-type": - target_value = self.__class__.__name__ + if condition._dependent_config_id == MODULE_TYPE: + target_value = self._module_type + elif condition._dependent_config_id == MODULE_PLATFORM: + target_value = self._platform else: - target_value = self.get_config_value_from_name( - condition._dependent_config_id) + if all_for_platform_type: + # skip, we want all configurations that match type and platform, for help + continue + target_value = self.get_config_value_from_name(condition._dependent_config_id) if not condition.check(target_value): return False return True - def check_if_config_passes_module_filter(self, config: Configuration) -> bool: - for condition in config._filter._conditions: - if condition._dependent_config_id == "module-type": - target_value = self.__class__.__name__ - if not condition.check(target_value): - return False - return True - - def update_configs(self, key_and_values: Dict[str, str]): - for key, value in key_and_values.items(): - self.update_value_for_given_config(key, value) - - def get_configurations_env_values(self) -> List[Dict[str, str]]: - env_config_values = [] - [env_config] = [config for config in self._lean_configs if - config._is_type_configurations_env and self.check_if_config_passes_filters( - config) - ] or [None] - if env_config is not None: - # Always get the first one, since we only expect one env config in the json modules file - env_config_values = list(env_config._env_and_values.values())[0] - return env_config_values - - def get_config_from_type(self, config_type: Configuration) -> str: - return [copy(config) for config in self._lean_configs if type(config) is config_type] - - def update_value_for_given_config(self, target_name: str, value: Any) -> None: - [idx] = [i for i in range(len(self._lean_configs)) - if self._lean_configs[i]._id == target_name] - self._lean_configs[idx]._value = value - def get_config_value_from_name(self, target_name: str) -> str: [idx] = [i for i in range(len(self._lean_configs)) if self._lean_configs[i]._id == target_name] return self._lean_configs[idx]._value - def get_non_user_required_properties(self) -> List[str]: - return [config._id for config in self._lean_configs if not config._is_required_from_user and not - config._is_type_configurations_env and self.check_if_config_passes_filters(config)] - - def get_required_properties(self, - filters: List[Type[Configuration]] = [], - include_optionals: bool = True) -> List[str]: - return [config._id for config in self.get_required_configs(filters, include_optionals)] - - def get_required_configs(self, - filters: List[Type[Configuration]] = [], - include_optionals: bool = True) -> List[Configuration]: - required_configs = [copy(config) for config in self._lean_configs if config._is_required_from_user - and type(config) not in filters - and self.check_if_config_passes_filters(config) - and (include_optionals or not getattr(config, '_optional', False))] - return required_configs + def get_settings(self) -> Dict[str, str]: + settings: Dict[str, str] = {"id": self._id} - def get_persistent_save_properties(self, filters: List[Type[Configuration]] = []) -> List[str]: - return [config._id for config in self.get_required_configs(filters) if config._save_persistently_in_lean] + # we build these after the rest, because they might depend on their values + for config in self._lean_configs: + if type(config) is InternalInputUserInput: + if config._is_conditional: + for option in config._value_options: + if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): + config._value = option._value + break + if not config._value: + options_to_log = set([(opt._condition._dependent_config_id, + self.get_config_value_from_name(opt._condition._dependent_config_id)) + for opt in config._value_options]) + raise ValueError( + f'No condition matched among present options for "{config._id}". ' + f'Please review ' + + ', '.join([f'"{x[0]}"' for x in options_to_log]) + + f' given value{"s" if len(options_to_log) > 1 else ""} ' + + ', '.join([f'"{x[1]}"' for x in options_to_log])) - def get_essential_properties(self) -> List[str]: - return [config._id for config in self.get_essential_configs()] + for configuration in self._lean_configs: + if not self._check_if_config_passes_filters(configuration, all_for_platform_type=False): + continue + settings[configuration._id] = str(configuration._value).replace("\\", "/") - def get_essential_configs(self) -> List[Configuration]: - return [copy(config) for config in self._lean_configs if isinstance(config, BrokerageEnvConfiguration)] + return settings def get_all_input_configs(self, filters: List[Type[Configuration]] = []) -> List[Configuration]: return [copy(config) for config in self._lean_configs if config._is_required_from_user if type(config) not in filters - and self.check_if_config_passes_module_filter(config)] + and self._check_if_config_passes_filters(config, all_for_platform_type=True)] def convert_lean_key_to_variable(self, lean_key: str) -> str: """Replaces hyphens with underscore to follow python naming convention. @@ -155,28 +138,38 @@ def convert_variable_to_lean_key(self, variable_key: str) -> str: """ return variable_key.replace('_', '-') - def build(self, - lean_config: Dict[str, Any], - logger: Logger, - properties: Dict[str, Any] = {}, - hide_input: bool = False) -> 'JsonModule': + def config_build(self, + lean_config: Dict[str, Any], + logger: Logger, + interactive: bool, + properties: Dict[str, Any] = {}, + hide_input: bool = False, + environment_name: str = None) -> 'JsonModule': """Builds a new instance of this class, prompting the user for input when necessary. :param lean_config: the Lean configuration dict to read defaults from :param logger: the logger to use + :param interactive: true if running in interactive mode :param properties: the properties that passed as options :param hide_input: whether to hide secrets inputs - :return: a LeanConfigConfigurer instance containing all the details needed to configure the Lean config + :param environment_name: the target environment name + :return: self """ - logger.info(f'Configure credentials for {self._display_name}') + logger.debug(f'Configuring {self._display_name}') + + # filter properties that were not passed as command line arguments, + # so that we prompt the user for them only when they don't have a value in the Lean config + context = get_current_context() + user_provided_options = {k: v for k, v in properties.items() + if context.get_parameter_source(k) == ParameterSource.COMMANDLINE} + + missing_options = [] for configuration in self._lean_configs: if not self.check_if_config_passes_filters(configuration): continue if not configuration._is_required_from_user: continue - if self.__class__.__name__ == 'CloudBrokerage' and not configuration._is_cloud_property: - continue - # Lets log messages for internal input configurations as well + # Let's log messages for internal input configurations as well if configuration._log_message is not None: log_message = configuration._log_message.strip() if log_message: @@ -184,37 +177,55 @@ def build(self, if type(configuration) is InternalInputUserInput: continue - # filter properties that were not passed as command line arguments, - # so that we prompt the user for them only when they don't have a value in the Lean config - context = get_current_context() - user_provided_options = {k: v for k, v in properties.items() - if context.get_parameter_source(k) == ParameterSource.COMMANDLINE} - - user_choice = None property_name = self.convert_lean_key_to_variable(configuration._id) # Only ask for user input if the config wasn't given as an option if property_name in user_provided_options and user_provided_options[property_name]: user_choice = user_provided_options[property_name] + logger.debug( + f'JsonModule({self._display_name}): user provided \'{user_choice}\' for \'{property_name}\'') else: - # Let's try to get the value from the lean config and use it as the user choice, without prompting - # TODO: use type(class) equality instead of class name (str) - if self.__class__.__name__ != 'CloudBrokerage': - user_choice = self._get_default(lean_config, configuration._id) - # Try to get the values from lean config - elif lean_config is not None and configuration._id in lean_config: - user_choice = lean_config[configuration._id] + logger.debug(f'JsonModule({self._display_name}): Configuration not provided \'{configuration._id}\'') + user_choice = self.get_default(lean_config, configuration._id, environment_name, logger) # There's no value in the lean config, let's use the module default value instead and prompt the user # NOTE: using "not" instead of "is None" because the default value can be false, # in which case we still want to prompt the user. if not user_choice: - default_value = configuration._input_default - user_choice = configuration.ask_user_for_input(default_value, logger, hide_input=hide_input) + if interactive: + default_value = configuration._input_default + user_choice = configuration.ask_user_for_input(default_value, logger, hide_input=hide_input) + else: + missing_options.append(configuration._id) - self.update_value_for_given_config(configuration._id, user_choice) + configuration._value = user_choice + if len(missing_options) > 0: + raise RuntimeError(f"""You are missing the following option{"s" if len(missing_options) > 1 else ""}: + {''.join(missing_options)}""".strip()) return self + def ensure_module_installed(self, organization_id: str) -> None: + if not self._is_module_installed and self._installs: + container.logger.debug(f"JsonModule.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") + container.module_manager.install_module( + self._product_id, organization_id) + self._is_module_installed = True + + def get_default(self, lean_config: Dict[str, Any], key: str, environment_name: str, logger: Logger): + user_choice = None + if lean_config is not None: + if (environment_name and "environments" in lean_config and environment_name in lean_config["environments"] + and key in lean_config["environments"][environment_name]): + user_choice = lean_config["environments"][environment_name][key] + logger.debug(f'JsonModule({self._display_name}): found \'{user_choice}\' for \'{key}\', in environment') + elif key in lean_config: + user_choice = lean_config[key] + logger.debug(f'JsonModule({self._display_name}): found \'{user_choice}\' for \'{key}\'') + return user_choice + + def __repr__(self): + return self.get_name() + class LiveInitialStateInput(str, Enum): Required = "required" diff --git a/lean/models/lean_config_configurer.py b/lean/models/lean_config_configurer.py deleted file mode 100644 index 7ff8f8f6..00000000 --- a/lean/models/lean_config_configurer.py +++ /dev/null @@ -1,121 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC - -from typing import Any, Dict, List, Optional -from lean.container import container -from lean.models.json_module import JsonModule -from lean.models.configuration import InternalInputUserInput -from copy import copy - -class LeanConfigConfigurer(JsonModule, ABC): - """The LeanConfigConfigurer class is the base class extended by all classes that update the Lean config.""" - - def configure(self, lean_config: Dict[str, Any], environment_name: str) -> None: - """Configures the Lean configuration for this brokerage. - - If the Lean configuration has been configured for this brokerage before, nothing will be changed. - Non-environment changes are saved persistently to disk so they can be used as defaults later. - - :param lean_config: the configuration dict to write to - :param environment_name: the name of the environment to configure - """ - self._configure_environment(lean_config, environment_name) - self.configure_credentials(lean_config) - - def _configure_environment(self, lean_config: Dict[str, Any], environment_name: str) -> None: - """Configures the environment in the Lean config for this brokerage. - :param lean_config: the Lean configuration dict to write to - :param environment_name: the name of the environment to update - """ - for environment_config in self.get_configurations_env_values(): - environment_config_name = environment_config["name"] - if self.__class__.__name__ == 'DataFeed': - if environment_config_name == "data-queue-handler": - previous_value = [] - if "data-queue-handler" in lean_config["environments"][environment_name]: - previous_value = copy(lean_config["environments"][environment_name][environment_config_name]) - previous_value.append(environment_config["value"]) - lean_config["environments"][environment_name][environment_config_name] = copy(previous_value) - elif self.__class__.__name__ == 'LocalBrokerage': - if environment_config_name != "data-queue-handler": - lean_config["environments"][environment_name][environment_config_name] = environment_config["value"] - - def configure_credentials(self, lean_config: Dict[str, Any]) -> None: - """Configures the credentials in the Lean config for this brokerage and saves them persistently to disk. - :param lean_config: the Lean configuration dict to write to - """ - if self._installs: - lean_config["job-organization-id"] = container.organization_manager.try_get_working_organization_id() - for configuration in self._lean_configs: - value = None - if configuration._is_type_configurations_env: - continue - elif not self.check_if_config_passes_filters(configuration): - continue - elif type(configuration) is InternalInputUserInput: - if not configuration._is_conditional: - value = configuration._value - else: - for option in configuration._value_options: - if option._condition.check(self.get_config_value_from_name(option._condition._dependent_config_id)): - value = option._value - break - if not value: - options_to_log = set([(opt._condition._dependent_config_id, - self.get_config_value_from_name(opt._condition._dependent_config_id)) - for opt in configuration._value_options]) - raise ValueError( - f'No condition matched among present options for "{configuration._cloud_id}". ' - f'Please review ' + - ', '.join([f'"{x[0]}"' for x in options_to_log]) + - f' given value{"s" if len(options_to_log) > 1 else ""} ' + - ', '.join([f'"{x[1]}"' for x in options_to_log])) - else: - value = configuration._value - from pathlib import WindowsPath, PosixPath - if type(value) == WindowsPath or type(value) == PosixPath: - value = str(value).replace("\\", "/") - lean_config[configuration._id] = value - container.logger.debug(f"LeanConfigConfigurer.ensure_module_installed(): _save_properties for module {self._id}: {self.get_persistent_save_properties()}") - self._save_properties(lean_config, self.get_persistent_save_properties()) - - def ensure_module_installed(self, organization_id: str) -> None: - if not self._is_module_installed and self._installs: - container.logger.debug(f"LeanConfigConfigurer.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") - container.module_manager.install_module( - self._product_id, organization_id) - self._is_module_installed = True - - def _get_default(cls, lean_config: Dict[str, Any], key: str) -> Optional[Any]: - """Returns the default value for a property based on the current Lean configuration. - - :param lean_config: the current Lean configuration - :param key: the name of the property - :return: the default value for the property, or None if there is none - """ - if key not in lean_config or lean_config[key] == "": - return None - - return lean_config[key] - - def _save_properties(self, lean_config: Dict[str, Any], properties: List[str]) -> None: - """Persistently save properties in the Lean configuration. - - :param lean_config: the dict containing all properties - :param properties: the names of the properties to save persistently - """ - from lean.container import container - container.lean_config_manager.set_properties( - {key: lean_config[key] for key in properties})