Fix 479 no update command (#485)

* feat: use container label to download specific module version

* feat: module version in deploy/download/optimize/research

* feat: change Unknown to None in get lean version name

* remove: extra log in module manager

* refactor: independent collection on specific module version and replace with actual download one

* remove: extra logging row

* feat: specific log message if package has not found

* remove: duplication of downloading modules in build_and_config_modules
refactor: use module_version like required parameter

* refactor: sort packages returns from API by version to will be sure that we return last one version

* remove: optional type parameter in install_module

lean/commands/backtest.py

@@ -16,7 +16,7 @@
 from click import command, option, argument, Choice
 
 from lean.click import LeanCommand, PathParameter
-from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH
+from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH, CONTAINER_LABEL_LEAN_VERSION_NAME
 from lean.container import container, Logger
 from lean.models.utils import DebuggingMethod
 from lean.models.cli import cli_data_downloaders, cli_addon_modules
@@ -359,21 +359,24 @@ def backtest(project: Path,
     organization_id = container.organization_manager.try_get_working_organization_id()
     paths_to_mount = None
 
+    cli_config_manager = container.cli_config_manager
+    project_config_manager = container.project_config_manager
+
+    project_config = project_config_manager.get_project_config(algorithm_file.parent)
+    engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None))
+
+    container_module_version = container.docker_manager.get_image_label(engine_image,
+                                                                        CONTAINER_LABEL_LEAN_VERSION_NAME, None)
+
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
-        data_provider.ensure_module_installed(organization_id)
+        data_provider.ensure_module_installed(organization_id, container_module_version)
         container.lean_config_manager.set_properties(data_provider.get_settings())
         paths_to_mount = data_provider.get_paths_to_mount()
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
-    cli_config_manager = container.cli_config_manager
-    project_config_manager = container.project_config_manager
-
-    project_config = project_config_manager.get_project_config(algorithm_file.parent)
-    engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None))
-
     if str(engine_image) != DEFAULT_ENGINE_IMAGE:
         logger.warn(f'A custom engine image: "{engine_image}" is being used!')
 

lean/commands/data/download.py

@@ -18,7 +18,7 @@
 from click import command, option, confirm, pass_context, Context, Choice, prompt
 from lean.click import LeanCommand, ensure_options
 from lean.components.util.json_modules_handler import config_build_for_name
-from lean.constants import DEFAULT_ENGINE_IMAGE
+from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME
 from lean.container import container
 from lean.models.api import QCDataInformation, QCDataVendor, QCFullOrganization, QCDatasetDelivery, QCResolution, QCSecurityType, QCDataType
 from lean.models.click_options import get_configs_for_options, options_from_json
@@ -675,13 +675,6 @@ def download(ctx: Context,
         logger = container.logger
         lean_config = container.lean_config_manager.get_complete_lean_config(None, None, None)
 
-        data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(),
-                                                         cli_data_downloaders, kwargs, logger, interactive=True)
-        data_downloader_provider.ensure_module_installed(organization.id)
-        container.lean_config_manager.set_properties(data_downloader_provider.get_settings())
-        # mounting additional data_downloader config files
-        paths_to_mount = data_downloader_provider.get_paths_to_mount()
-
         engine_image = container.cli_config_manager.get_engine_image(image)
 
         if str(engine_image) != DEFAULT_ENGINE_IMAGE:
@@ -690,6 +683,16 @@ def download(ctx: Context,
 
         container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
+        container_module_version = container.docker_manager.get_image_label(engine_image,
+                                                                            CONTAINER_LABEL_LEAN_VERSION_NAME, None)
+
+        data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(),
+                                                         cli_data_downloaders, kwargs, logger, interactive=True)
+        data_downloader_provider.ensure_module_installed(organization.id, container_module_version)
+        container.lean_config_manager.set_properties(data_downloader_provider.get_settings())
+        # mounting additional data_downloader config files
+        paths_to_mount = data_downloader_provider.get_paths_to_mount()
+
         downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug"
 
         run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config,

lean/commands/live/deploy.py

@@ -16,7 +16,7 @@
 from click import option, argument, Choice
 from lean.click import LeanCommand, PathParameter
 from lean.components.util.name_rename import rename_internal_config_to_user_friendly_format
-from lean.constants import DEFAULT_ENGINE_IMAGE
+from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME
 from lean.container import container
 from lean.models.cli import (cli_brokerages, cli_data_queue_handlers, cli_data_downloaders,
                              cli_addon_modules, cli_history_provider)
@@ -271,23 +271,26 @@ def deploy(project: Path,
                                                                  kwargs, logger, interactive=True,
                                                                  environment_name=environment_name))
 
+    project_config_manager = container.project_config_manager
+    cli_config_manager = container.cli_config_manager
+
+    project_config = project_config_manager.get_project_config(algorithm_file.parent)
+    engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None))
+
+    container_module_version = container.docker_manager.get_image_label(engine_image,
+                                                                        CONTAINER_LABEL_LEAN_VERSION_NAME, None)
+
     organization_id = container.organization_manager.try_get_working_organization_id()
     paths_to_mount = {}
     for module in (data_provider_live_instances + [data_downloader_instances, brokerage_instance]
                    + history_providers_instances):
-        module.ensure_module_installed(organization_id)
+        module.ensure_module_installed(organization_id, container_module_version)
         paths_to_mount.update(module.get_paths_to_mount())
 
     if not lean_config["environments"][environment_name]["live-mode"]:
         raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)",
                             "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading")
 
-    project_config_manager = container.project_config_manager
-    cli_config_manager = container.cli_config_manager
-
-    project_config = project_config_manager.get_project_config(algorithm_file.parent)
-    engine_image = cli_config_manager.get_engine_image(image or project_config.get("engine-image", None))
-
     container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
     _start_iqconnect_if_necessary(lean_config, environment_name)

lean/commands/optimize.py

@@ -19,7 +19,7 @@
 
 from lean.click import LeanCommand, PathParameter, ensure_options
 from lean.components.docker.lean_runner import LeanRunner
-from lean.constants import DEFAULT_ENGINE_IMAGE
+from lean.constants import DEFAULT_ENGINE_IMAGE, CONTAINER_LABEL_LEAN_VERSION_NAME
 from lean.container import container
 from lean.models.api import QCParameter, QCBacktest
 from lean.models.click_options import options_from_json, get_configs_for_options
@@ -307,10 +307,13 @@ def optimize(project: Path,
 
     paths_to_mount = None
 
+    container_module_version = container.docker_manager.get_image_label(engine_image,
+                                                                        CONTAINER_LABEL_LEAN_VERSION_NAME, None)
+
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
-        data_provider.ensure_module_installed(organization_id)
+        data_provider.ensure_module_installed(organization_id, container_module_version)
         container.lean_config_manager.set_properties(data_provider.get_settings())
         paths_to_mount = data_provider.get_paths_to_mount()
 

lean/commands/research.py

@@ -16,7 +16,7 @@
 from click import command, argument, option, Choice
 from lean.click import LeanCommand, PathParameter
 from lean.components.docker.lean_runner import LeanRunner
-from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
+from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH, CONTAINER_LABEL_LEAN_VERSION_NAME
 from lean.container import container
 from lean.models.cli import cli_data_downloaders
 from lean.components.util.name_extraction import convert_to_class_name
@@ -113,13 +113,27 @@ def research(project: Path,
     if download_data:
         data_provider_historical = "QuantConnect"
 
+    project_config_manager = container.project_config_manager
+    cli_config_manager = container.cli_config_manager
+
+    project_config = project_config_manager.get_project_config(algorithm_file.parent)
+    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
+
+    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
+
+    container_module_version = container.docker_manager.get_image_label(research_image,
+                                                                        CONTAINER_LABEL_LEAN_VERSION_NAME, None)
+
+    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
+        logger.warn(f'A custom research image: "{research_image}" is being used!')
+
     paths_to_mount = None
 
     if data_provider_historical is not None:
         organization_id = container.organization_manager.try_get_working_organization_id()
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
-        data_provider.ensure_module_installed(organization_id)
+        data_provider.ensure_module_installed(organization_id, container_module_version)
         container.lean_config_manager.set_properties(data_provider.get_settings())
         paths_to_mount = data_provider.get_paths_to_mount()
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
@@ -131,17 +145,6 @@ def research(project: Path,
     for key, value in extra_config:
         lean_config[key] = value
 
-    project_config_manager = container.project_config_manager
-    cli_config_manager = container.cli_config_manager
-
-    project_config = project_config_manager.get_project_config(algorithm_file.parent)
-    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
-
-    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
-
-    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
-        logger.warn(f'A custom research image: "{research_image}" is being used!')
-
     run_options = lean_runner.get_basic_docker_config(lean_config,
                                                       algorithm_file,
                                                       temp_manager.create_temporary_directory(),

lean/components/cloud/module_manager.py

@@ -37,28 +37,48 @@ def __init__(self, logger: Logger, api_client: APIClient, http_client: HTTPClien
         self._installed_product_ids: Set[int] = set()
         self._installed_packages: Dict[int, List[NuGetPackage]] = {}
 
-    def install_module(self, product_id: int, organization_id: str) -> None:
-        """Installs a module into the global modules directory.
+    def install_module(self, product_id: int, organization_id: str, module_version: str) -> None:
+        """
+        Installs a module into the global modules' directory.
 
-        If an outdated version is already installed, it is automatically updated.
-        If the organization does not have a subscription for the given module, an error is raised.
+        If an outdated version is already installed, it is automatically updated. If a specific version
+        is provided and is different from the installed version, it will be updated. If the organization
+        does not have a subscription for the given module, an error is raised.
 
-        :param product_id: the product id of the module to download
-        :param organization_id: the id of the organization that has a license for the module
+        Args:
+        product_id (int): The product id of the module to download.
+        organization_id (str): The id of the organization that has a license for the module.
+        module_version (str): The specific version of the module to install. If None, installs the latest version.
         """
         if product_id in self._installed_product_ids:
             return
 
+        # Retrieve the list of module files for the specified product and organization
         module_files = self._api_client.modules.list_files(product_id, organization_id)
+        # Dictionaries to store the latest packages to download and specific version packages
         packages_to_download: Dict[str, NuGetPackage] = {}
+        packages_to_download_specific_version: Dict[str, NuGetPackage] = {}
 
-        for file_name in module_files:
-            package = NuGetPackage.parse(file_name)
+        # Parse the module files into NuGetPackage objects and sort them by version
+        packages = [NuGetPackage.parse(file_name) for file_name in module_files]
+        sorted_packages = sorted(packages, key=lambda p: p.version)
 
+        for package in sorted_packages:
+            # Store the latest version of each package
             if package.name not in packages_to_download or package.version > packages_to_download[package.name].version:
                 packages_to_download[package.name] = package
+                # If a specific version is requested, keep track of the highest version <= module_version
+                if module_version and package.version.split('.')[-1] <= module_version:
+                    packages_to_download_specific_version[package.name] = package
+
+        # Replace version packages based on module_version if available
+        for package_name, package_specific_version in packages_to_download_specific_version.items():
+            packages_to_download[package_name] = package_specific_version
 
         for package in packages_to_download.values():
+            if module_version and package.version.split('.')[-1] != module_version:
+                self._logger.debug(f'Package "{package.name}" does not have the specified version {module_version}. '
+                                   f'Using available version {package.version} instead.')
             self._download_file(product_id, organization_id, package)
 
         self._installed_product_ids.add(product_id)

lean/components/util/json_modules_handler.py

@@ -33,7 +33,6 @@ def build_and_configure_modules(target_modules: List[str], module_list: List[Jso
     for target_module_name in target_modules:
         module = non_interactive_config_build_for_name(lean_config, target_module_name, module_list, properties,
                                                        logger, environment_name)
-        module.ensure_module_installed(organization_id)
         lean_config["environments"][environment_name].update(module.get_settings())
 
 

lean/constants.py

@@ -26,6 +26,9 @@
 DEFAULT_LEAN_STRICT_PYTHON_VERSION = f"{DEFAULT_LEAN_PYTHON_VERSION}.7"
 DEFAULT_LEAN_DOTNET_FRAMEWORK = "net6.0"
 
+# Label name used in Docker containers to specify the version of Lean being used
+CONTAINER_LABEL_LEAN_VERSION_NAME = "lean_version"
+
 # The path to the root python directory in docker image
 DOCKER_PYTHON_SITE_PACKAGES_PATH = "/root/.local/lib/python{LEAN_PYTHON_VERSION}/site-packages"
 

lean/models/json_module.py

@@ -246,11 +246,21 @@ def get_paths_to_mount(self) -> Dict[str, str]:
                 if (isinstance(config, PathParameterUserInput)
                     and self._check_if_config_passes_filters(config, all_for_platform_type=False))}
 
-    def ensure_module_installed(self, organization_id: str) -> None:
+    def ensure_module_installed(self, organization_id: str, module_version: str) -> None:
+        """
+        Ensures that the specified module is installed. If the module is not installed, it will be installed.
+
+        Args:
+            organization_id (str): The ID of the organization where the module should be installed.
+            module_version (str): The version of the module to install. If not provided,
+            the latest version will be installed.
+
+        Returns:
+            None
+        """
         if not self._is_module_installed and self._installs:
             container.logger.debug(f"JsonModule.ensure_module_installed(): installing module {self}: {self._product_id}")
-            container.module_manager.install_module(
-                self._product_id, organization_id)
+            container.module_manager.install_module(self._product_id, organization_id, module_version)
             self._is_module_installed = True
 
     def get_default(self, lean_config: Dict[str, Any], key: str, environment_name: str, logger: Logger):