From 025a82a025ca6badef7602351676c8b5e0d50118 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 21 Dec 2023 17:17:13 -0400 Subject: [PATCH] Install modules on backtest and research (#389) * Install modules on backtest and research * Minor change --- lean/commands/backtest.py | 9 ++++++--- lean/commands/research.py | 5 +++-- lean/components/docker/lean_runner.py | 5 ----- lean/components/util/json_modules_handler.py | 3 ++- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index b81b1ffd..e8deda1c 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -21,7 +21,7 @@ from lean.models.api import QCMinimalOrganization from lean.models.utils import DebuggingMethod from lean.models.logger import Option -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers +from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module from lean.models.click_options import options_from_json, get_configs_for_options @@ -365,8 +365,11 @@ def backtest(project: Path, if download_data: data_provider = QuantConnectDataProvider.get_name() + organization_id = container.organization_manager.try_get_working_organization_id() + if data_provider is not None: - [data_provider_configurer] = [get_and_build_module(data_provider, all_data_providers, kwargs, logger)] + data_provider_configurer: DataProvider = get_and_build_module(data_provider, all_data_providers, kwargs, logger) + data_provider_configurer.ensure_module_installed(organization_id) data_provider_configurer.configure(lean_config, "backtesting") lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -404,7 +407,7 @@ def backtest(project: Path, lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' # Configure addon modules - build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, "backtesting") + build_and_configure_modules(addon_module, organization_id, lean_config, logger, "backtesting") lean_runner = container.lean_runner lean_runner.run_lean(lean_config, diff --git a/lean/commands/research.py b/lean/commands/research.py index 14b6d62d..19b60b6c 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -18,7 +18,7 @@ from lean.components.docker.lean_runner import LeanRunner from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH from lean.container import container -from lean.models.data_providers import QuantConnectDataProvider, all_data_providers +from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider from lean.components.util.name_extraction import convert_to_class_name from lean.components.util.json_modules_handler import get_and_build_module from lean.models.click_options import options_from_json, get_configs_for_options @@ -113,7 +113,8 @@ def research(project: Path, data_provider = QuantConnectDataProvider.get_name() if data_provider is not None: - [data_provider_configurer] = [get_and_build_module(data_provider, all_data_providers, kwargs, logger)] + data_provider_configurer: DataProvider = get_and_build_module(data_provider, all_data_providers, kwargs, logger) + data_provider_configurer.ensure_module_installed(container.organization_manager.try_get_working_organization_id()) data_provider_configurer.configure(lean_config, "backtesting") lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 894d4091..d52c94ae 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -195,11 +195,6 @@ def get_basic_docker_config(self, project_config = self._project_config_manager.get_project_config(project_dir) docker_project_config = project_config.get("docker", {}) - # Install the required modules when they're needed - if lean_config.get("data-provider", None) == "QuantConnect.Lean.Engine.DataFeeds.DownloaderDataProvider" \ - and lean_config.get("data-downloader", None) == "TerminalLinkDataDownloader": - self._module_manager.install_module(TERMINAL_LINK_PRODUCT_ID, lean_config["job-organization-id"]) - # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider data_dir = self._lean_config_manager.get_data_directory() if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider": diff --git a/lean/components/util/json_modules_handler.py b/lean/components/util/json_modules_handler.py index 6f54dae8..a8d53e06 100644 --- a/lean/components/util/json_modules_handler.py +++ b/lean/components/util/json_modules_handler.py @@ -37,7 +37,8 @@ def build_and_configure_modules(modules: List[AddonModule], organization_id: str logger.error(f"Addon module '{given_module}' failed to configure: {e}") return lean_config -def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger): + +def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger) -> JsonModule: [target_module] = [module for module in module_list if module.get_name() == target_module_name] # update essential properties from brokerage to datafeed # needs to be updated before fetching required properties