Refactor modules (#428)

QuantConnect · Feb 28, 2024 · c32b763 · c32b763
1 parent a031b02
commit c32b763
Show file tree

Hide file tree

Showing 37 changed files with 580 additions and 1,263 deletions.
diff --git a/README.md b/README.md
@@ -285,22 +285,19 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT
 Options:
   --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit]
                                   The brokerage to use
+  --data-provider-live [QuantConnect|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Polygon|IEX|CoinApi|Bybit]
+                                  The live data provider to use
   --ib-user-name TEXT             Your Interactive Brokers username
   --ib-account TEXT               Your Interactive Brokers account id
   --ib-password TEXT              Your Interactive Brokers password
   --ib-weekly-restart-utc-time TEXT
                                   Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at
                                   this time, and will require 2FA verification. This is required by Interactive Brokers.
                                   Use this option explicitly to override the default value.
-  --ib-data-feed [QuantConnect|Interactive Brokers|QuantConnect + InteractiveBrokers]
-                                  The available price data feeds are: Interactive Brokers price data feed, QuantConnect
-                                  price data feed or QuantConnect + InteractiveBrokers price data feed
   --tradier-account-id TEXT       Your Tradier account id
   --tradier-access-token TEXT     Your Tradier access token
   --tradier-environment [live|paper]
                                   Whether the developer sandbox should be used
-  --tradier-data-feed [QuantConnect|Tradier Brokerage]
-                                  Whether the Tradier data feed must be used instead of the QuantConnect price data feed
   --oanda-account-id TEXT         Your OANDA account id
   --oanda-access-token TEXT       Your OANDA API token
   --oanda-environment [Practice|Trade]
@@ -371,6 +368,13 @@ Options:
   --bybit-api-secret TEXT         Your Bybit API secret
   --bybit-vip-level [VIP0|VIP1|VIP2|VIP3|VIP4|VIP5|SupremeVIP|Pro1|Pro2|Pro3|Pro4|Pro5]
                                   Your Bybit VIP Level
+  --polygon-api-key TEXT          Your Polygon.io API Key
+  --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
+  --iex-price-plan [Launch|Grow|Enterprise]
+                                  Your IEX Cloud Price plan
+  --coinapi-api-key TEXT          Your coinapi.io Api Key
+  --coinapi-product [Free|Startup|Streamer|Professional|Enterprise]
+                                  CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing)
   --node TEXT                     The name or id of the live node to run on
   --auto-restart BOOLEAN          Whether automatic algorithm restarting must be enabled
   --notify-order-events BOOLEAN   Whether notifications must be sent for order events
@@ -1065,7 +1069,7 @@ Options:
   -d, --detach                    Run the live deployment in a detached Docker container and return immediately
   --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|Bybit]
                                   The brokerage to use
-  --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit]
+  --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit]
                                   The live data provider to use
   --data-provider-historical [IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local]
                                   Update the Lean configuration file to retrieve data from the given historical provider

diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py
@@ -18,11 +18,9 @@
 from lean.click import LeanCommand, PathParameter
 from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH
 from lean.container import container, Logger
-from lean.models.api import QCMinimalOrganization
 from lean.models.utils import DebuggingMethod
-from lean.models.logger import Option
-from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider
-from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module
+from lean.models.cli import cli_data_downloaders, cli_addon_modules
+from lean.components.util.json_modules_handler import build_and_configure_modules, non_interactive_config_build_for_name
 from lean.models.click_options import options_from_json, get_configs_for_options
 
 # The _migrate_* methods automatically update launch configurations for a given debugging method.
@@ -33,6 +31,7 @@
 #
 # These methods checks if the project has outdated configurations, and if so, update them to keep it working.
 
+
 def _migrate_python_pycharm(logger: Logger, project_dir: Path) -> None:
     from os import path
     from click import Abort
@@ -225,20 +224,6 @@ def _migrate_csharp_csproj(project_dir: Path) -> None:
     csproj_path.write_text(xml_manager.to_string(current_content), encoding="utf-8")
 
 
-def _select_organization() -> QCMinimalOrganization:
-    """Asks the user for the organization that should be charged when downloading data.
-
-    :return: the selected organization
-    """
-    api_client = container.api_client
-
-    organizations = api_client.organizations.get_all()
-    options = [Option(id=organization, label=organization.name) for organization in organizations]
-
-    logger = container.logger
-    return logger.prompt_list("Select the organization to purchase and download data with", options)
-
-
 @command(cls=LeanCommand, requires_lean_config=True, requires_docker=True)
 @argument("project", type=PathParameter(exists=True, file_okay=True, dir_okay=True))
 @option("--output",
@@ -252,7 +237,7 @@ def _select_organization() -> QCMinimalOrganization:
               type=Choice(["pycharm", "ptvsd", "vsdbg", "rider", "local-platform"], case_sensitive=False),
               help="Enable a certain debugging method (see --help for more information)")
 @option("--data-provider-historical",
-              type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False),
+              type=Choice([dp.get_name() for dp in cli_data_downloaders], case_sensitive=False),
               default="Local",
               help="Update the Lean configuration file to retrieve data from the given historical provider")
 @options_from_json(get_configs_for_options("backtest"))
@@ -338,6 +323,7 @@ def backtest(project: Path,
     if output is None:
         output = algorithm_file.parent / "backtests" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
 
+    environment_name = "backtesting"
     debugging_method = None
     if debug == "pycharm":
         debugging_method = DebuggingMethod.PyCharm
@@ -360,17 +346,18 @@ def backtest(project: Path,
     if algorithm_file.name.endswith(".cs"):
         _migrate_csharp_csproj(algorithm_file.parent)
 
-    lean_config = lean_config_manager.get_complete_lean_config("backtesting", algorithm_file, debugging_method)
+    lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, debugging_method)
 
     if download_data:
-        data_provider_historical = QuantConnectDataProvider.get_name()
+        data_provider_historical = "QuantConnect"
 
     organization_id = container.organization_manager.try_get_working_organization_id()
 
     if data_provider_historical is not None:
-        data_provider_configurer: DataProvider = get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)
-        data_provider_configurer.ensure_module_installed(organization_id)
-        data_provider_configurer.configure(lean_config, "backtesting")
+        data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
+                                                              cli_data_downloaders, kwargs, logger, environment_name)
+        data_provider.ensure_module_installed(organization_id)
+        container.lean_config_manager.set_properties(data_provider.get_settings())
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
@@ -407,11 +394,12 @@ def backtest(project: Path,
         lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}'
 
     # Configure addon modules
-    build_and_configure_modules(addon_module, organization_id, lean_config, logger, "backtesting")
+    build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
+                                kwargs, logger, environment_name)
 
     lean_runner = container.lean_runner
     lean_runner.run_lean(lean_config,
-                         "backtesting",
+                         environment_name,
                          algorithm_file,
                          output,
                          engine_image,