Remove hardcoded python and dotnet version

- Remove hardcoded python and dotnet version, these will be retrieved
  from the docker labels of the image being used

lean/commands/library/add.py

@@ -13,7 +13,6 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional, Tuple
-from lean.constants import LEAN_STRICT_PYTHON_VERSION
 from click import command, argument, option
 
 from lean.click import LeanCommand, PathParameter
@@ -124,7 +123,7 @@ def _is_pypi_file_compatible(file: Dict[str, Any], required_python_version) -> b
     return True
 
 
-def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
+def _get_pypi_package(name: str, version: Optional[str], python_version: str) -> Tuple[str, str]:
     """Retrieves the properly-capitalized name and the latest compatible version of a package from PyPI.
 
     If the version is already given, this method checks whether that version is compatible with the Docker images.
@@ -148,7 +147,7 @@ def _get_pypi_package(name: str, version: Optional[str]) -> Tuple[str, str]:
     pypi_data = loads(response.text)
     name = pypi_data["info"]["name"]
 
-    required_python_version = StrictVersion(LEAN_STRICT_PYTHON_VERSION)
+    required_python_version = StrictVersion(python_version)
 
     last_compatible_version = None
     last_compatible_version_upload_time = None
@@ -237,7 +236,11 @@ def _add_pypi_package_to_python_project(project_dir: Path, name: str, version: O
     else:
         logger.info("Retrieving latest compatible version from PyPI")
 
-    name, version = _get_pypi_package(name, version)
+    project_config = container.project_config_manager.get_project_config(project_dir)
+    engine_image = container.cli_config_manager.get_engine_image(project_config.get("engine-image", None))
+    python_version = container.docker_manager.get_image_label(engine_image, 'strict_python_version', '3.11.7')
+
+    name, version = _get_pypi_package(name, version, python_version)
 
     requirements_file = project_dir / "requirements.txt"
     logger.info(f"Adding {name} {version} to '{path_manager.get_relative_path(requirements_file)}'")

lean/commands/optimize.py

@@ -336,7 +336,10 @@ def optimize(project: Path,
     build_and_configure_modules(addon_module, cli_addon_modules, organization_id, lean_config,
                                 kwargs, logger, environment_name)
 
-    run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach)
+    container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
+
+    run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
+                                                      engine_image)
 
     run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
     run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}")
@@ -346,7 +349,6 @@ def optimize(project: Path,
               type="bind",
               read_only=True)
     )
-    container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
     # Add known additional run options from the extra docker config
     LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))

lean/commands/research.py

@@ -128,12 +128,24 @@ def research(project: Path,
     for key, value in extra_config:
         lean_config[key] = value
 
+    project_config_manager = container.project_config_manager
+    cli_config_manager = container.cli_config_manager
+
+    project_config = project_config_manager.get_project_config(algorithm_file.parent)
+    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
+
+    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
+
+    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
+        logger.warn(f'A custom research image: "{research_image}" is being used!')
+
     run_options = lean_runner.get_basic_docker_config(lean_config,
                                                       algorithm_file,
                                                       temp_manager.create_temporary_directory(),
                                                       None,
                                                       False,
-                                                      detach)
+                                                      detach,
+                                                      research_image)
 
     # Mount the config in the notebooks directory as well
     local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json"))
@@ -174,17 +186,6 @@ def research(project: Path,
     # Add known additional run options from the extra docker config
     LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))
 
-    project_config_manager = container.project_config_manager
-    cli_config_manager = container.cli_config_manager
-
-    project_config = project_config_manager.get_project_config(algorithm_file.parent)
-    research_image = cli_config_manager.get_research_image(image or project_config.get("research-image", None))
-
-    if str(research_image) != DEFAULT_RESEARCH_IMAGE:
-        logger.warn(f'A custom research image: "{research_image}" is being used!')
-
-    container.update_manager.pull_docker_image_if_necessary(research_image, update, no_update)
-
     try:
         container.docker_manager.run_image(research_image, **run_options)
     except APIError as error:

lean/components/docker/docker_manager.py

@@ -38,6 +38,16 @@ def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager:
         self._temp_manager = temp_manager
         self._platform_manager = platform_manager
 
+    def get_image_label(self, image: DockerImage, label: str, default: str) -> str:
+        docker_image = self._get_docker_client().images.get(image.name)
+
+        for name, value in docker_image.labels.items():
+            if name == label:
+                self._logger.debug(f"Label '{label}' found in image '{image.name}', value {value}")
+                return value
+        self._logger.info(f"Label '{label}' not found in image '{image.name}', using default {default}")
+        return default
+
     def pull_image(self, image: DockerImage) -> None:
         """Pulls a Docker image.
 
@@ -563,13 +573,13 @@ def _format_source_path(self, path: str) -> str:
                 break
 
         return path
-    
+
     def get_container_port(self, container_name: str, internal_port: str) -> Optional[int]:
         """
         Returns a containers external port for a mapped internal port
         :param container_name: Name of the container
         :param internal_port: The internal port of container. If protocol not included
-        we assume /tcp. ex. 5678/tcp 
+        we assume /tcp. ex. 5678/tcp
         :return: The external port that is linked to it, or None if it does not exist
         """
 

lean/components/docker/lean_runner.py

@@ -23,7 +23,7 @@
 from lean.components.util.project_manager import ProjectManager
 from lean.components.util.temp_manager import TempManager
 from lean.components.util.xml_manager import XMLManager
-from lean.constants import MODULES_DIRECTORY, TERMINAL_LINK_PRODUCT_ID, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME
+from lean.constants import MODULES_DIRECTORY, LEAN_ROOT_PATH, DEFAULT_DATA_DIRECTORY_NAME
 from lean.constants import DOCKER_PYTHON_SITE_PACKAGES_PATH
 from lean.models.docker import DockerImage
 from lean.models.utils import DebuggingMethod
@@ -97,7 +97,8 @@ def run_lean(self,
                                                    output_dir,
                                                    debugging_method,
                                                    release,
-                                                   detach)
+                                                   detach,
+                                                   image)
 
         # Add known additional run options from the extra docker config
         self.parse_extra_docker_config(run_options, extra_docker_config)
@@ -175,7 +176,8 @@ def get_basic_docker_config(self,
                                 output_dir: Path,
                                 debugging_method: Optional[DebuggingMethod],
                                 release: bool,
-                                detach: bool) -> Dict[str, Any]:
+                                detach: bool,
+                                image: DockerImage) -> Dict[str, Any]:
         """Creates a basic Docker config to run the engine with.
 
         This method constructs the parts of the Docker config that is the same for both the engine and the optimizer.
@@ -186,6 +188,7 @@ def get_basic_docker_config(self,
         :param debugging_method: the debugging method if debugging needs to be enabled, None if not
         :param release: whether C# projects should be compiled in release configuration instead of debug
         :param detach: whether LEAN should run in a detached container
+        :param image: The docker image that will be used
         :return: the Docker configuration containing basic configuration to run Lean
         """
         from docker.types import Mount
@@ -309,7 +312,9 @@ def get_basic_docker_config(self,
             # Create a C# project used to resolve the dependencies of the modules
             run_options["commands"].append("mkdir /ModulesProject")
             run_options["commands"].append("dotnet new sln -o /ModulesProject")
-            run_options["commands"].append("dotnet new classlib -o /ModulesProject -f net6.0 --no-restore")
+
+            framework_ver = self._docker_manager.get_image_label(image, 'target_framework', 'net6.0')
+            run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore")
             run_options["commands"].append("rm /ModulesProject/Class1.cs")
 
             # Add all modules to the project, automatically resolving all dependencies
@@ -325,7 +330,8 @@ def get_basic_docker_config(self,
 
         # Set up language-specific run options
         self.setup_language_specific_run_options(run_options, project_dir, algorithm_file,
-                                            set_up_common_csharp_options_called, release)
+                                                 set_up_common_csharp_options_called, release,
+                                                 image)
 
         # Save the final Lean config to a temporary file so we can mount it into the container
         config_path = self._temp_manager.create_temporary_directory() / "config.json"
@@ -359,11 +365,12 @@ def get_basic_docker_config(self,
 
         return run_options
 
-    def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) -> None:
+    def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None:
         """Sets up Docker run options specific to Python projects.
 
         :param project_dir: the path to the project directory
         :param run_options: the dictionary to append run options to
+        :param image: the docker image that will be used
         """
 
         from docker.types import Mount
@@ -410,10 +417,13 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
             "mode": "rw"
         }
 
+        python_version = self._docker_manager.get_image_label(image, 'python_version', '3.11')
+        site_packages_path = DOCKER_PYTHON_SITE_PACKAGES_PATH.replace('{LEAN_PYTHON_VERSION}', python_version)
+
         # Mount a volume to the user packages directory so we don't install packages every time
         site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt)
         run_options["volumes"][site_packages_volume] = {
-            "bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}",
+            "bind": f"{site_packages_path}",
             "mode": "rw"
         }
 
@@ -424,7 +434,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any])
         # We only need to do this if it hasn't already been done before for this site packages volume
         # To keep track of this we create a special file in the site packages directory after installation
         # If this file already exists we can skip pip install completely
-        marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done"
+        marker_file = f"{site_packages_path}/pip-install-done"
         run_options["commands"].extend([
             f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt",
             f"touch {marker_file}"
@@ -452,12 +462,14 @@ def _concat_python_requirements(self, requirements_files: List[Path]) -> str:
         requirements = sorted(set(requirements))
         return "\n".join(requirements)
 
-    def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool) -> None:
+    def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool,
+                              image: DockerImage) -> None:
         """Sets up Docker run options specific to C# projects.
 
         :param project_dir: the path to the project directory
         :param run_options: the dictionary to append run options to
         :param release: whether C# projects should be compiled in release configuration instead of debug
+        :param image: the docker image that will be used
         """
         compile_root = self._get_csharp_compile_root(project_dir)
 
@@ -474,11 +486,12 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any],
         for path in compile_root.rglob("*.csproj"):
             self._ensure_csproj_uses_correct_lean(compile_root, path, csproj_temp_dir, run_options)
 
+        framework_ver = self._docker_manager.get_image_label(image, 'target_framework', 'net6.0')
         # Set up the MSBuild properties
         msbuild_properties = {
             "Configuration": "Release" if release else "Debug",
             "Platform": "AnyCPU",
-            "TargetFramework": "net6.0",
+            "TargetFramework": framework_ver,
             "OutputPath": "/Compile/bin",
             "GenerateAssemblyInfo": "false",
             "GenerateTargetFrameworkAttribute": "false",
@@ -722,14 +735,14 @@ def _force_disk_provider_if_necessary(self,
             lean_config[config_key] = disk_provider
 
     def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file,
-                                            set_up_common_csharp_options_called, release) -> None:
+                                            set_up_common_csharp_options_called, release, image: DockerImage) -> None:
         # Set up language-specific run options
         if algorithm_file.name.endswith(".py"):
-            self.set_up_python_options(project_dir, run_options)
+            self.set_up_python_options(project_dir, run_options, image)
         else:
             if not set_up_common_csharp_options_called:
                 self.set_up_common_csharp_options(run_options)
-            self.set_up_csharp_options(project_dir, run_options, release)
+            self.set_up_csharp_options(project_dir, run_options, release, image)
 
     def format_error_before_logging(self, chunk: str):
         from lean.components.util import compiler

lean/components/util/project_manager.py

@@ -15,8 +15,10 @@
 from pathlib import Path
 from typing import List, Optional, Union, Tuple
 from lean.components import reserved_names
+from lean.components.config.cli_config_manager import CLIConfigManager
 from lean.components.config.lean_config_manager import LeanConfigManager
 from lean.components.config.project_config_manager import ProjectConfigManager
+from lean.components.docker.docker_manager import DockerManager
 from lean.components.util.logger import Logger
 from lean.components.util.path_manager import PathManager
 from lean.components.util.platform_manager import PlatformManager
@@ -25,6 +27,7 @@
 from lean.models.api import QCLanguage, QCProject, QCProjectLibrary
 from lean.models.utils import LeanLibraryReference
 
+
 class ProjectManager:
     """The ProjectManager class provides utilities for handling a single project."""
 
@@ -34,7 +37,9 @@ def __init__(self,
                  lean_config_manager: LeanConfigManager,
                  path_manager: PathManager,
                  xml_manager: XMLManager,
-                 platform_manager: PlatformManager) -> None:
+                 platform_manager: PlatformManager,
+                 cli_config_manager: CLIConfigManager,
+                 docker_manager: DockerManager) -> None:
         """Creates a new ProjectManager instance.
 
         :param logger: the logger to use to log messages with
@@ -50,6 +55,8 @@ def __init__(self,
         self._path_manager = path_manager
         self._xml_manager = xml_manager
         self._platform_manager = platform_manager
+        self._cli_config_manager = cli_config_manager
+        self._docker_manager = docker_manager
 
     def find_algorithm_file(self, input: Path) -> Path:
         """Returns the path to the file containing the algorithm.
@@ -186,7 +193,10 @@ def create_new_project(self, project_dir: Path, language: QCLanguage) -> None:
             self._generate_pycharm_config(project_dir)
         else:
             self._generate_vscode_csharp_config(project_dir)
-            self._generate_csproj(project_dir)
+
+            image = self._cli_config_manager.get_engine_image()
+            framework_ver = self._docker_manager.get_image_label(image, 'target_framework', 'net6.0')
+            self._generate_csproj(project_dir, framework_ver)
             self.generate_rider_config(project_dir)
 
     def delete_project(self, project_dir: Path) -> None:
@@ -677,12 +687,13 @@ def _generate_vscode_csharp_config(self, project_dir: Path) -> None:
 }}
         """)
 
-    def _generate_csproj(self, project_dir: Path) -> None:
+    def _generate_csproj(self, project_dir: Path, framework_version: str) -> None:
         """Generates a .csproj file for the given project and returns the path to it.
 
         :param project_dir: the path of the new project
         """
-        self._generate_file(project_dir / f"{project_dir.name}.csproj", self.get_csproj_file_default_content())
+        self._generate_file(project_dir / f"{project_dir.name}.csproj",
+                            self.get_csproj_file_default_content(framework_version))
 
     def generate_rider_config(self, project_dir: Path) -> bool:
         """Generates C# debugging configuration for Rider.
@@ -952,13 +963,13 @@ def get_cloud_projects_libraries(self,
         return list(set(libraries)), list(set(libraries_not_found))
 
     @staticmethod
-    def get_csproj_file_default_content() -> str:
+    def get_csproj_file_default_content(framework_version: str) -> str:
         return """
 <Project Sdk="Microsoft.NET.Sdk">
     <PropertyGroup>
         <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
         <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
-        <TargetFramework>net6.0</TargetFramework>
+        <TargetFramework>{TARGET_FRAMEWORK}</TargetFramework>
         <OutputPath>bin/$(Configuration)</OutputPath>
         <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
         <DefaultItemExcludes>$(DefaultItemExcludes);backtests/*/code/**;live/*/code/**;optimizations/*/code/**</DefaultItemExcludes>
@@ -969,7 +980,7 @@ def get_csproj_file_default_content() -> str:
         <PackageReference Include="QuantConnect.DataSource.Libraries" Version="2.5.*"/>
     </ItemGroup>
 </Project>
-        """
+        """.replace('{TARGET_FRAMEWORK}', framework_version)
 
     @staticmethod
     def get_csproj_file_path(project_dir: Path) -> Path: