Install modules on backtest and research (#389)

* Install modules on backtest and research

* Minor change

lean/commands/backtest.py

@@ -21,7 +21,7 @@
 from lean.models.api import QCMinimalOrganization
 from lean.models.utils import DebuggingMethod
 from lean.models.logger import Option
-from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
+from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider
 from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module
 from lean.models.click_options import options_from_json, get_configs_for_options
 
@@ -365,8 +365,11 @@ def backtest(project: Path,
     if download_data:
         data_provider = QuantConnectDataProvider.get_name()
 
+    organization_id = container.organization_manager.try_get_working_organization_id()
+
     if data_provider is not None:
-        [data_provider_configurer] = [get_and_build_module(data_provider, all_data_providers, kwargs, logger)]
+        data_provider_configurer: DataProvider = get_and_build_module(data_provider, all_data_providers, kwargs, logger)
+        data_provider_configurer.ensure_module_installed(organization_id)
         data_provider_configurer.configure(lean_config, "backtesting")
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
@@ -404,7 +407,7 @@ def backtest(project: Path,
         lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}'
 
     # Configure addon modules
-    build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, "backtesting")
+    build_and_configure_modules(addon_module, organization_id, lean_config, logger, "backtesting")
 
     lean_runner = container.lean_runner
     lean_runner.run_lean(lean_config,

lean/commands/research.py

@@ -18,7 +18,7 @@
 from lean.components.docker.lean_runner import LeanRunner
 from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
 from lean.container import container
-from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
+from lean.models.data_providers import QuantConnectDataProvider, all_data_providers, DataProvider
 from lean.components.util.name_extraction import convert_to_class_name
 from lean.components.util.json_modules_handler import get_and_build_module
 from lean.models.click_options import options_from_json, get_configs_for_options
@@ -113,7 +113,8 @@ def research(project: Path,
         data_provider = QuantConnectDataProvider.get_name()
 
     if data_provider is not None:
-        [data_provider_configurer] = [get_and_build_module(data_provider, all_data_providers, kwargs, logger)]
+        data_provider_configurer: DataProvider = get_and_build_module(data_provider, all_data_providers, kwargs, logger)
+        data_provider_configurer.ensure_module_installed(container.organization_manager.try_get_working_organization_id())
         data_provider_configurer.configure(lean_config, "backtesting")
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)

lean/components/docker/lean_runner.py

@@ -195,11 +195,6 @@ def get_basic_docker_config(self,
         project_config = self._project_config_manager.get_project_config(project_dir)
         docker_project_config = project_config.get("docker", {})
 
-        # Install the required modules when they're needed
-        if lean_config.get("data-provider", None) == "QuantConnect.Lean.Engine.DataFeeds.DownloaderDataProvider" \
-            and lean_config.get("data-downloader", None) == "TerminalLinkDataDownloader":
-            self._module_manager.install_module(TERMINAL_LINK_PRODUCT_ID, lean_config["job-organization-id"])
-
         # Force the use of the LocalDisk map/factor providers if no recent zip present and not using ApiDataProvider
         data_dir = self._lean_config_manager.get_data_directory()
         if lean_config.get("data-provider", None) != "QuantConnect.Lean.Engine.DataFeeds.ApiDataProvider":

lean/components/util/json_modules_handler.py

@@ -37,7 +37,8 @@ def build_and_configure_modules(modules: List[AddonModule], organization_id: str
             logger.error(f"Addon module '{given_module}' failed to configure: {e}")
     return lean_config
 
-def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger):
+
+def get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any], logger: Logger) -> JsonModule:
     [target_module] = [module for module in module_list if module.get_name() == target_module_name]
     # update essential properties from brokerage to datafeed
     # needs to be updated before fetching required properties