From a902ac61764770d6908ce332f0c8e3b445b7ac0c Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 29 Apr 2024 21:35:16 +0300 Subject: [PATCH] fix: get all config for data downloader refactor: get_complete_lean_config without environment and algo_file remove: test, cuz we have used interactive input for user --- lean/commands/data/download.py | 16 ++---- lean/components/config/lean_config_manager.py | 54 ++++++++++--------- tests/commands/data/test_download.py | 15 ------ 3 files changed, 33 insertions(+), 52 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 1868546a..9f5427cb 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -633,10 +633,10 @@ def download(ctx: Context, raise ValueError("Historical start date cannot be greater than or equal to historical end date.") logger = container.logger - lean_config = container.lean_config_manager.get_lean_config() + lean_config = container.lean_config_manager.get_complete_lean_config(None, None, None) data_downloader_provider = config_build_for_name(lean_config, data_downloader_provider.get_name(), - cli_data_downloaders, kwargs, logger, interactive=False) + cli_data_downloaders, kwargs, logger, interactive=True) data_downloader_provider.ensure_module_installed(organization.id) container.lean_config_manager.set_properties(data_downloader_provider.get_settings()) # mounting additional data_downloader config files @@ -652,15 +652,7 @@ def download(ctx: Context, downloader_data_provider_path_dll = "/Lean/DownloaderDataProvider/bin/Debug" - # Create config dictionary with credentials - config: Dict[str, str] = { - "job-user-id": lean_config.get("job-user-id"), - "api-access-token": lean_config.get("api-access-token"), - "job-organization-id": organization.id - } - config.update(data_downloader_provider.get_settings()) - - run_options = container.lean_runner.get_basic_docker_config_without_algo(config, + run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, debugging_method=None, detach=False, image=engine_image, @@ -669,7 +661,7 @@ def download(ctx: Context, config_path = container.temp_manager.create_temporary_directory() / "config.json" with config_path.open("w+", encoding="utf-8") as file: - dump(config, file) + dump(lean_config, file) run_options["working_dir"] = downloader_data_provider_path_dll diff --git a/lean/components/config/lean_config_manager.py b/lean/components/config/lean_config_manager.py index a33e7bc7..5b533386 100644 --- a/lean/components/config/lean_config_manager.py +++ b/lean/components/config/lean_config_manager.py @@ -223,7 +223,9 @@ def get_complete_lean_config(self, """ config = self.get_lean_config() - config["environment"] = environment + if environment and len(environment) > 0: + config["environment"] = environment + config["close-automatically"] = True config["composer-dll-directory"] = "." @@ -241,7 +243,6 @@ def get_complete_lean_config(self, config_defaults = { "job-user-id": self._cli_config_manager.user_id.get_value(default="0"), "api-access-token": self._cli_config_manager.api_token.get_value(default=""), - "job-project-id": self._project_config_manager.get_local_id(algorithm_file.parent), "job-organization-id": get_organization(config), "ib-host": "127.0.0.1", @@ -256,29 +257,32 @@ def get_complete_lean_config(self, if config.get(key, "") == "": config[key] = value - if algorithm_file.name.endswith(".py"): - config["algorithm-type-name"] = algorithm_file.name.split(".")[0] - config["algorithm-language"] = "Python" - config["algorithm-location"] = f"/LeanCLI/{algorithm_file.name}" - else: - from re import findall - algorithm_text = algorithm_file.read_text(encoding="utf-8") - config["algorithm-type-name"] = findall(r"class\s*([^\s:]+)\s*:\s*QCAlgorithm", algorithm_text)[0] - config["algorithm-language"] = "CSharp" - config["algorithm-location"] = f"{algorithm_file.parent.name}.dll" - - project_config = self._project_config_manager.get_project_config(algorithm_file.parent) - config["parameters"] = project_config.get("parameters", {}) - - # Add libraries paths to python project - project_language = project_config.get("algorithm-language", None) - if project_language == "Python": - library_references = project_config.get("libraries", []) - python_paths = config.get("python-additional-paths", []) - python_paths.extend([(Path("/") / library["path"]).as_posix() for library in library_references]) - if len(python_paths) > 0: - python_paths.append("/Library") - config["python-additional-paths"] = python_paths + if algorithm_file and len(algorithm_file.name) > 0: + config.get("job-project-id", self._project_config_manager.get_local_id(algorithm_file.parent)) + + if algorithm_file.name.endswith(".py"): + config["algorithm-type-name"] = algorithm_file.name.split(".")[0] + config["algorithm-language"] = "Python" + config["algorithm-location"] = f"/LeanCLI/{algorithm_file.name}" + else: + from re import findall + algorithm_text = algorithm_file.read_text(encoding="utf-8") + config["algorithm-type-name"] = findall(r"class\s*([^\s:]+)\s*:\s*QCAlgorithm", algorithm_text)[0] + config["algorithm-language"] = "CSharp" + config["algorithm-location"] = f"{algorithm_file.parent.name}.dll" + + project_config = self._project_config_manager.get_project_config(algorithm_file.parent) + config["parameters"] = project_config.get("parameters", {}) + + # Add libraries paths to python project + project_language = project_config.get("algorithm-language", None) + if project_language == "Python": + library_references = project_config.get("libraries", []) + python_paths = config.get("python-additional-paths", []) + python_paths.extend([(Path("/") / library["path"]).as_posix() for library in library_references]) + if len(python_paths) > 0: + python_paths.append("/Library") + config["python-additional-paths"] = python_paths # No real limit for the object store by default if "storage-limit-mb" not in config: diff --git a/tests/commands/data/test_download.py b/tests/commands/data/test_download.py index abf1c2e6..48c8ce2c 100644 --- a/tests/commands/data/test_download.py +++ b/tests/commands/data/test_download.py @@ -146,21 +146,6 @@ def test_download_data_non_interactive(data_provider: str, market: str, is_crypt assert run_data_download.exit_code == 0 -@pytest.mark.parametrize("data_provider,market,is_crypto,security_type,missed_parameters", - [("Polygon", "NYSE", False, "Equity", "--polygon-api-key"), - ("Binance", "Binance", True, "Crypto", "--binance-exchange-name"), - ("Interactive Brokers", "USA", False, "Equity", - "--ib-user-name, --ib-account, --ib-password")]) -def test_download_data_non_interactive_data_provider_missed_param(data_provider: str, market: str, is_crypto: bool, - security_type: str, missed_parameters: str): - run_data_download = _create_lean_data_download(data_provider, "Trade", "Minute", security_type, ["AAPL"], - "20240101", "20240202", _get_data_provider_config(is_crypto), market) - assert run_data_download.exit_code == 1 - - error_msg = str(run_data_download.exc_info[1]) - assert missed_parameters in error_msg - - @pytest.mark.parametrize("data_type,resolution", [("Trade", "Hour"), ("trade", "hour"), ("TRADE", "HOUR"), ("TrAdE", "HoUr")]) def test_download_data_non_interactive_insensitive_input_param(data_type: str, resolution: str):