From 9b8a338068af1ecb31f17f08727c8753c8472652 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 16 Feb 2024 23:59:40 +0200 Subject: [PATCH 01/14] feat: optional download data provider --- lean/commands/live/deploy.py | 40 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 5ae1c916..3b851bb4 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -39,7 +39,7 @@ } -def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]: +def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer], LeanConfigConfigurer]: """Returns the configurable modules from the given environment. :param lean_config: the LEAN configuration that should be used @@ -61,7 +61,14 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if _get_brokerage_base_name(local_data_feed.get_live_name()) in data_queue_handlers_base_names] - return brokerage_configurer, data_feed_configurers + + data_downloader_preference = None + if "data-downloader" in lean_config: + data_downloader_base_name = _get_brokerage_base_name(lean_config["data-downloader"]) + data_downloader_preference = next((local_data_history for local_data_history in all_data_providers + if _get_brokerage_base_name(local_data_history.get_config_value_from_name('data-downloader')) in data_downloader_base_name), None) + + return brokerage_configurer, data_feed_configurers, data_downloader_preference def _get_brokerage_base_name(brokerage: str) -> str: @@ -78,7 +85,7 @@ def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, :param modules: the modules to check """ for module in modules: - if not module._installs: + if module is None or not module._installs: continue organization_id = container.organization_manager.try_get_working_organization_id() module.ensure_module_installed(organization_id) @@ -91,12 +98,19 @@ def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: :param environment_name: the name of the environment :param lean_config_path: the path to the LEAN configuration file """ - brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name) + brokerage_configurer, data_feed_configurers, data_downloader_preference = _get_configurable_modules_from_environment(lean_config, environment_name) brokerage_properties = brokerage_configurer.get_required_properties(include_optionals=False) data_queue_handler_properties = [] - for data_feed_configurer in data_feed_configurers: + for data_feed_configurer in data_feed_configurers: data_queue_handler_properties.extend(data_feed_configurer.get_required_properties(include_optionals=False)) + required_properties = list(set(brokerage_properties + data_queue_handler_properties)) + + if data_downloader_preference is not None: + for data_downloader_property in data_downloader_preference.get_required_properties(include_optionals=False): + if data_downloader_property not in required_properties: + required_properties.append(data_downloader_property) + missing_properties = [p for p in required_properties if p not in lean_config or lean_config[p] == ""] missing_properties = set(missing_properties) if len(missing_properties) == 0: @@ -191,6 +205,15 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], _cached_lean_config = None +def _try_get_data_downloader_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: + """ Get name for data downloader provider based on data provider live (if exist) + + :param data_provider_historical_name: the current (default) data provider historical + :param data_provider_live_name: the current data provider live name + """ + return next((live_data_downloader.get_name() for live_data_downloader in all_data_providers + if live_data_downloader.get_name() in data_provider_live_name), data_provider_historical_name) + # being used by lean.models.click_options.get_the_correct_type_default_value() def _get_default_value(key: str) -> Optional[Any]: @@ -408,6 +431,9 @@ def deploy(project: Path, _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) if data_provider_historical is not None: + # if default historical provider try to find + if data_provider_historical == "Local": + data_provider_historical = _try_get_data_downloader_name(data_provider_historical, data_provider_live) [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] data_provider_configurer.configure(lean_config, environment_name) @@ -420,8 +446,8 @@ def deploy(project: Path, raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) - _install_modules([env_brokerage] + env_data_queue_handlers, kwargs) + env_brokerage, env_data_queue_handlers, env_data_downloader = _get_configurable_modules_from_environment(lean_config, environment_name) + _install_modules([env_brokerage] + env_data_queue_handlers + [env_data_downloader], kwargs) _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) From 6daa4a5455adc88c66322d8d1fce37466e3f35f5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Sat, 17 Feb 2024 00:00:35 +0200 Subject: [PATCH 02/14] feat: test to download data provider with different params --- tests/commands/test_live.py | 157 ++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index c418abdb..d279b368 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1011,3 +1011,160 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage return assert args[0]["live-holdings"] == holding_list + +def test_live_non_interactive_deploy_with_live_and_historical_provider() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + api_client = mock.MagicMock() + container.initialize(docker_manager=mock.Mock(), api_client=api_client, lean_runner=mock.Mock()) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + provider_history_option = ["--data-provider-historical", "Polygon", + "--polygon-api-key", "123"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + *provider_history_option, + "Python Project", + ]) + + # validate amount of request to download packages from api + assert len(api_client.method_calls) > 2 + + product_ids = [] + for method_call in list(api_client.method_calls)[:2]: + product_ids.append(method_call.args[0]) + + assert "333" in product_ids + assert "306" in product_ids + assert result.exit_code == 0 + +def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_historical_not_optional_config() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + provider_history_option = ["--data-provider-historical", "Polygon"] + # "--polygon-api-key", "123"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + *provider_history_option, + "Python Project", + ]) + error_msg = str(result.exc_info[1]).split() + + assert "--polygon-api-key" in error_msg + assert "--iex-cloud-api-key" not in error_msg + + assert result.exit_code == 1 + +def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_live_not_optional_config() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123"] + #"--iex-price-plan", "Launch"] + + provider_history_option = ["--data-provider-historical", "Polygon", "--polygon-api-key", "123"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + *provider_history_option, + "Python Project", + ]) + + error_msg = str(result.exc_info[1]).split() + + assert "--iex-price-plan" in error_msg + assert "--polygon-api-key" not in error_msg + + assert result.exit_code == 1 + +def test_live_non_interactive_deploy_with_live_and_without_historical_provider() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + api_client = mock.MagicMock() + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client=api_client) + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", + *provider_live_option, + "Python Project", + ]) + + # validate amount of request to download packages from api + assert len(api_client.method_calls) == 2 + + assert "333" in api_client.method_calls[0].args[0] + assert result.exit_code == 0 + +def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client = mock.MagicMock()) + + # create fake environment has IB configs already + brokerage = ["--brokerage", "OANDA"] + + provider_live_option = ["--data-provider-live", "IEX", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + result = CliRunner().invoke(lean, ["live", "deploy" , + *brokerage, + *provider_live_option, + "Python Project", + ]) + + error_msg = str(result.exc_info[1]).split() + + assert "--oanda-account-id" in error_msg + assert "--oanda-access-token" in error_msg + assert "--oanda-environment" in error_msg + assert "--iex-price-plan" not in error_msg + + assert result.exit_code == 1 + +def test_live_non_interactive_deploy_with_interactive_brokerage() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + api_client = mock.MagicMock() + container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client=api_client) + + # create fake environment has IB configs already + brokerage = ["--brokerage", "Interactive Brokers"] + + provider_live_option = ["--data-provider-live", "Interactive Brokers", + "--iex-cloud-api-key", "123", + "--iex-price-plan", "Launch"] + + result = CliRunner().invoke(lean, ["live", "deploy" , + *brokerage, + *provider_live_option, + "Python Project", + ]) + assert result.exit_code == 0 + + # validate amount of request to download packages from api + assert len(api_client.method_calls) == 1 + + assert "181" in api_client.method_calls[0].args[0] From 80c89d95169f1924019b4aace6e4b2b7dbb65f8c Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 15:43:01 +0200 Subject: [PATCH 03/14] feat: installed data provider list --- lean/commands/live/deploy.py | 4 ++-- lean/models/data_providers/__init__.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 3b851bb4..efbdbe74 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -27,7 +27,7 @@ from lean.commands.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively -from lean.models.data_providers import all_data_providers +from lean.models.data_providers import all_data_providers, installed_data_providers from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available _environment_skeleton = { @@ -65,7 +65,7 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi data_downloader_preference = None if "data-downloader" in lean_config: data_downloader_base_name = _get_brokerage_base_name(lean_config["data-downloader"]) - data_downloader_preference = next((local_data_history for local_data_history in all_data_providers + data_downloader_preference = next((local_data_history for local_data_history in installed_data_providers if _get_brokerage_base_name(local_data_history.get_config_value_from_name('data-downloader')) in data_downloader_base_name), None) return brokerage_configurer, data_feed_configurers, data_downloader_preference diff --git a/lean/models/data_providers/__init__.py b/lean/models/data_providers/__init__.py index 268e9e34..5eb8752c 100644 --- a/lean/models/data_providers/__init__.py +++ b/lean/models/data_providers/__init__.py @@ -16,11 +16,16 @@ from lean.models import json_modules all_data_providers: List[DataProvider] = [] +installed_data_providers: List[DataProvider] = [] for json_module in json_modules: if "data-provider" in json_module["type"]: all_data_providers.append(DataProvider(json_module)) +for data_provider in all_data_providers: + if data_provider._installs: + installed_data_providers.append(data_provider) + # QuantConnect DataProvider [QuantConnectDataProvider] = [ data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] From 6229745fb2c76c9e2662f3e87a782d948189440c Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 15:45:22 +0200 Subject: [PATCH 04/14] feat: general tests --- tests/commands/test_live.py | 127 ++++++++++++++++++++++++------------ tests/test_helpers.py | 10 +++ 2 files changed, 96 insertions(+), 41 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index d279b368..25a2f461 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -25,8 +25,9 @@ from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.docker import DockerImage -from tests.test_helpers import create_fake_lean_cli_directory +from tests.test_helpers import create_fake_lean_cli_directory, reset_state_installed_modules from tests.conftest import initialize_container +from click.testing import Result ENGINE_IMAGE = DockerImage.parse(DEFAULT_ENGINE_IMAGE) @@ -413,7 +414,18 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme "Terminal Link": terminal_link_required_options, "Kraken": brokerage_required_options["Kraken"], "TDAmeritrade": brokerage_required_options["TDAmeritrade"], - "Bybit": brokerage_required_options["Bybit"] + "Bybit": brokerage_required_options["Bybit"], + "IEX": { + "iex-cloud-api-key": "123", + "iex-price-plan": "Launch", + }, + "Polygon": { + "polygon-api-key": "123", + }, + "AlphaVantage": { + "alpha-vantage-api-key": "111", + "alpha-vantage-price-plan": "Free" + } } @@ -1093,28 +1105,6 @@ def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_li assert result.exit_code == 1 -def test_live_non_interactive_deploy_with_live_and_without_historical_provider() -> None: - create_fake_lean_cli_directory() - create_fake_environment("live-paper", True) - - api_client = mock.MagicMock() - container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client=api_client) - - provider_live_option = ["--data-provider-live", "IEX", - "--iex-cloud-api-key", "123", - "--iex-price-plan", "Launch"] - - result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", - *provider_live_option, - "Python Project", - ]) - - # validate amount of request to download packages from api - assert len(api_client.method_calls) == 2 - - assert "333" in api_client.method_calls[0].args[0] - assert result.exit_code == 0 - def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) @@ -1133,38 +1123,93 @@ def test_live_non_interactive_deploy_with_real_brokerage_without_credentials() - *provider_live_option, "Python Project", ]) + assert result.exit_code == 1 error_msg = str(result.exc_info[1]).split() assert "--oanda-account-id" in error_msg assert "--oanda-access-token" in error_msg assert "--oanda-environment" in error_msg - assert "--iex-price-plan" not in error_msg + assert "--iex-price-plan" not in error_msg - assert result.exit_code == 1 - -def test_live_non_interactive_deploy_with_interactive_brokerage() -> None: +def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, api_client: any) -> Result: + reset_state_installed_modules() create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - api_client = mock.MagicMock() - container.initialize(docker_manager=mock.Mock(), lean_runner=mock.Mock(), api_client=api_client) + initialize_container(api_client_to_use=api_client) - # create fake environment has IB configs already - brokerage = ["--brokerage", "Interactive Brokers"] + option = ["--brokerage", brokerage_name] + for key, value in brokerage_required_options[brokerage_name].items(): + option.extend([f"--{key}", value]) - provider_live_option = ["--data-provider-live", "Interactive Brokers", - "--iex-cloud-api-key", "123", - "--iex-price-plan", "Launch"] + option.extend(["--data-provider-live", data_provider_live_name]) + for key, value in data_feed_required_options[data_provider_live_name].items(): + if f"--{key}" not in option: + option.extend([f"--{key}", value]) - result = CliRunner().invoke(lean, ["live", "deploy" , - *brokerage, - *provider_live_option, + if data_provider_historical_name is not None: + option.extend(["--data-provider-historical", data_provider_historical_name]) + for key, value in data_feed_required_options[data_provider_historical_name].items(): + if f"--{key}" not in option: + option.extend([f"--{key}", value]) + + result = CliRunner().invoke(lean, ["live", "deploy", + *option, "Python Project", ]) assert result.exit_code == 0 + return result + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_historical_name,brokerage_product_id,data_provider_live_product_id,data_provider_historical_id", + [("Interactive Brokers", "IEX", "Polygon", "181", "333", "306"), + ("Paper Trading", "IEX", "Polygon", None, "333", "306"), + ("Tradier", "IEX", "AlphaVantage", "185", "333", "334")]) +def test_live_deploy_with_different_brokerage_and_different_live_data_provider_and_historical_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, brokerage_product_id: str, data_provider_live_product_id: str, data_provider_historical_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client) - # validate amount of request to download packages from api - assert len(api_client.method_calls) == 1 + if brokerage_product_id is None: + assert len(api_client.method_calls) == 3 + assert data_provider_live_product_id in api_client.method_calls[0].args[0] + assert data_provider_historical_id in api_client.method_calls[1].args[0] + else: + assert len(api_client.method_calls) == 3 + assert brokerage_product_id in api_client.method_calls[0].args[0] + assert data_provider_live_product_id in api_client.method_calls[1].args[0] + assert data_provider_historical_id in f"{api_client.method_calls[2].args[0]}" + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id,data_provider_live_product_id", + [("Interactive Brokers", "IEX", "181", "333"), + ("Tradier", "IEX", "185", "333")]) +def test_live_non_interactive_deploy_with_different_brokerage_and_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str, data_provider_live_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + assert len(api_client.method_calls) == 2 + assert brokerage_product_id in api_client.method_calls[0].args[0] + assert data_provider_live_product_id in api_client.method_calls[1].args[0] + +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id", + [("Bybit", "Bybit", "305"), + ("Interactive Brokers", "Interactive Brokers", "181"), + ("Tradier", "Tradier", "185")]) +def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + if brokerage_name == "Interactive Brokers": + assert len(api_client.method_calls) == 1 + else: + assert len(api_client.method_calls) == 2 + assert brokerage_product_id in api_client.method_calls[0].args[0] - assert "181" in api_client.method_calls[0].args[0] +@pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id", + [("Paper Trading", "IEX", "333"), + ("Paper Trading", "Polygon", "306")]) +def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_live_product_id: str) -> None: + api_client = mock.MagicMock() + create_lean_option(brokerage_name, data_provider_live_name, None, api_client) + + assert len(api_client.method_calls) == 2 + assert data_provider_live_product_id in f"{api_client.method_calls[0].args[0]}" \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 7c979d98..746f1229 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -15,6 +15,8 @@ from datetime import datetime from pathlib import Path from typing import List +from lean.models.data_providers import all_data_providers +from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds from lean.commands.create_project import (DEFAULT_CSHARP_MAIN, DEFAULT_CSHARP_NOTEBOOK, DEFAULT_PYTHON_MAIN, DEFAULT_PYTHON_NOTEBOOK, LIBRARY_PYTHON_MAIN, LIBRARY_CSHARP_MAIN) @@ -224,3 +226,11 @@ def create_lean_environments() -> List[QCLeanEnvironment]: description="", public=True) ] + +def reset_state_installed_modules() -> None: + for data_provider in all_data_providers: + data_provider.__setattr__("_is_module_installed", False) + for local_brokerage in all_local_brokerages: + local_brokerage.__setattr__("_is_module_installed", False) + for local_data_feed in all_local_data_feeds: + local_data_feed.__setattr__("_is_module_installed", False) From 2200149a8ed4e09e8207e0d3b5b8c27cd619bd02 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 16:06:04 +0200 Subject: [PATCH 05/14] refactor: separate data-provider from data-feed --- tests/commands/test_live.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 25a2f461..1c22b721 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -415,6 +415,9 @@ def test_live_sets_dependent_configurations_from_modules_json_based_on_environme "Kraken": brokerage_required_options["Kraken"], "TDAmeritrade": brokerage_required_options["TDAmeritrade"], "Bybit": brokerage_required_options["Bybit"], +} + +data_provider_required_options = { "IEX": { "iex-cloud-api-key": "123", "iex-price-plan": "Launch", @@ -1143,6 +1146,8 @@ def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_p for key, value in brokerage_required_options[brokerage_name].items(): option.extend([f"--{key}", value]) + data_feed_required_options.update(data_provider_required_options) + option.extend(["--data-provider-live", data_provider_live_name]) for key, value in data_feed_required_options[data_provider_live_name].items(): if f"--{key}" not in option: From 169c41b2c2aae24eb0de98a6d283c52b917a550e Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 16:19:13 +0200 Subject: [PATCH 06/14] fix: validation of amount of request on api --- tests/commands/test_live.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 1c22b721..ab1ecd41 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1203,10 +1203,6 @@ def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live api_client = mock.MagicMock() create_lean_option(brokerage_name, data_provider_live_name, None, api_client) - if brokerage_name == "Interactive Brokers": - assert len(api_client.method_calls) == 1 - else: - assert len(api_client.method_calls) == 2 assert brokerage_product_id in api_client.method_calls[0].args[0] @pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id", From 7ee0ed2a3dfa7caad1cdf1e2881c178091fe9e59 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 17:41:54 +0200 Subject: [PATCH 07/14] feat: remove default value for data-provider-historical --- lean/commands/live/deploy.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index efbdbe74..e713f924 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -257,7 +257,6 @@ def _get_default_value(key: str) -> Optional[Any]: help="The live data provider to use") @option("--data-provider-historical", type=Choice([dp.get_name() for dp in all_data_providers if dp._id != "TerminalLinkBrokerage"], case_sensitive=False), - default="Local", help="Update the Lean configuration file to retrieve data from the given historical provider") @options_from_json(get_configs_for_options("live-local")) @option("--release", @@ -430,12 +429,10 @@ def deploy(project: Path, lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) - if data_provider_historical is not None: - # if default historical provider try to find - if data_provider_historical == "Local": - data_provider_historical = _try_get_data_downloader_name(data_provider_historical, data_provider_live) - [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] - data_provider_configurer.configure(lean_config, environment_name) + if data_provider_historical is None: + data_provider_historical = _try_get_data_downloader_name("Local", data_provider_live) + [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] + data_provider_configurer.configure(lean_config, environment_name) if "environments" not in lean_config or environment_name not in lean_config["environments"]: lean_config_path = lean_config_manager.get_lean_config_path() From 39dc8fad8ca4c65805e031288be80c568c0188c5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 19 Feb 2024 17:42:23 +0200 Subject: [PATCH 08/14] feat: test case with local historical provider --- tests/commands/test_live.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index ab1ecd41..e860480a 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1155,9 +1155,10 @@ def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_p if data_provider_historical_name is not None: option.extend(["--data-provider-historical", data_provider_historical_name]) - for key, value in data_feed_required_options[data_provider_historical_name].items(): - if f"--{key}" not in option: - option.extend([f"--{key}", value]) + if data_provider_historical_name is not "Local": + for key, value in data_feed_required_options[data_provider_historical_name].items(): + if f"--{key}" not in option: + option.extend([f"--{key}", value]) result = CliRunner().invoke(lean, ["live", "deploy", *option, @@ -1169,15 +1170,19 @@ def create_lean_option(brokerage_name: str, data_provider_live_name: str, data_p @pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_historical_name,brokerage_product_id,data_provider_live_product_id,data_provider_historical_id", [("Interactive Brokers", "IEX", "Polygon", "181", "333", "306"), ("Paper Trading", "IEX", "Polygon", None, "333", "306"), - ("Tradier", "IEX", "AlphaVantage", "185", "333", "334")]) + ("Tradier", "IEX", "AlphaVantage", "185", "333", "334"), + ("Paper Trading", "IEX", "Local", None, "333", "222")]) def test_live_deploy_with_different_brokerage_and_different_live_data_provider_and_historical_data_provider(brokerage_name: str, data_provider_live_name: str, data_provider_historical_name: str, brokerage_product_id: str, data_provider_live_product_id: str, data_provider_historical_id: str) -> None: api_client = mock.MagicMock() create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client) - if brokerage_product_id is None: + if brokerage_product_id is None and data_provider_historical_name != "Local": assert len(api_client.method_calls) == 3 assert data_provider_live_product_id in api_client.method_calls[0].args[0] assert data_provider_historical_id in api_client.method_calls[1].args[0] + elif brokerage_product_id is None and data_provider_historical_name == "Local": + assert len(api_client.method_calls) == 2 + assert data_provider_live_product_id in api_client.method_calls[0].args[0] else: assert len(api_client.method_calls) == 3 assert brokerage_product_id in api_client.method_calls[0].args[0] From e56cc39826f2125f056e0912c088b2064b24198b Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 00:47:05 +0200 Subject: [PATCH 09/14] remove: repeat test case refactor: assertion in test with different version of py --- tests/commands/test_live.py | 78 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index e860480a..cfb1f43b 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1027,37 +1027,6 @@ def test_live_passes_live_holdings_to_lean_runner_when_given_as_option(brokerage assert args[0]["live-holdings"] == holding_list -def test_live_non_interactive_deploy_with_live_and_historical_provider() -> None: - create_fake_lean_cli_directory() - create_fake_environment("live-paper", True) - - api_client = mock.MagicMock() - container.initialize(docker_manager=mock.Mock(), api_client=api_client, lean_runner=mock.Mock()) - - provider_live_option = ["--data-provider-live", "IEX", - "--iex-cloud-api-key", "123", - "--iex-price-plan", "Launch"] - - provider_history_option = ["--data-provider-historical", "Polygon", - "--polygon-api-key", "123"] - - result = CliRunner().invoke(lean, ["live", "deploy" , "--brokerage", "Paper Trading", - *provider_live_option, - *provider_history_option, - "Python Project", - ]) - - # validate amount of request to download packages from api - assert len(api_client.method_calls) > 2 - - product_ids = [] - for method_call in list(api_client.method_calls)[:2]: - product_ids.append(method_call.args[0]) - - assert "333" in product_ids - assert "306" in product_ids - assert result.exit_code == 0 - def test_live_non_interactive_deploy_with_live_and_historical_provider_missed_historical_not_optional_config() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) @@ -1176,18 +1145,27 @@ def test_live_deploy_with_different_brokerage_and_different_live_data_provider_a api_client = mock.MagicMock() create_lean_option(brokerage_name, data_provider_live_name, data_provider_historical_name, api_client) + is_exists = [] if brokerage_product_id is None and data_provider_historical_name != "Local": assert len(api_client.method_calls) == 3 - assert data_provider_live_product_id in api_client.method_calls[0].args[0] - assert data_provider_historical_id in api_client.method_calls[1].args[0] + for m_c, id in zip(api_client.method_calls, [data_provider_live_product_id, data_provider_historical_id]): + if id in m_c[1]: + is_exists.append(True) + assert is_exists + assert len(is_exists) == 2 elif brokerage_product_id is None and data_provider_historical_name == "Local": assert len(api_client.method_calls) == 2 - assert data_provider_live_product_id in api_client.method_calls[0].args[0] + if data_provider_live_product_id in api_client.method_calls[0][1]: + is_exists.append(True) + assert is_exists + assert len(is_exists) == 1 else: assert len(api_client.method_calls) == 3 - assert brokerage_product_id in api_client.method_calls[0].args[0] - assert data_provider_live_product_id in api_client.method_calls[1].args[0] - assert data_provider_historical_id in f"{api_client.method_calls[2].args[0]}" + for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id, data_provider_historical_id]): + if id in f"{m_c[1]}": + is_exists.append(True) + assert is_exists + assert len(is_exists) == 3 @pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id,data_provider_live_product_id", [("Interactive Brokers", "IEX", "181", "333"), @@ -1197,8 +1175,13 @@ def test_live_non_interactive_deploy_with_different_brokerage_and_different_live create_lean_option(brokerage_name, data_provider_live_name, None, api_client) assert len(api_client.method_calls) == 2 - assert brokerage_product_id in api_client.method_calls[0].args[0] - assert data_provider_live_product_id in api_client.method_calls[1].args[0] + is_exists = [] + for m_c, id in zip(api_client.method_calls, [brokerage_product_id, data_provider_live_product_id]): + if id in m_c[1]: + is_exists.append(True) + + assert is_exists + assert len(is_exists) == 2 @pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id", [("Bybit", "Bybit", "305"), @@ -1207,8 +1190,15 @@ def test_live_non_interactive_deploy_with_different_brokerage_and_different_live def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None: api_client = mock.MagicMock() create_lean_option(brokerage_name, data_provider_live_name, None, api_client) - - assert brokerage_product_id in api_client.method_calls[0].args[0] + + print(api_client.call_args_list) + print(api_client.call_args) + + for m_c in api_client.method_calls: + if brokerage_product_id in m_c[1]: + is_exist = True + + assert is_exist @pytest.mark.parametrize("brokerage_name,data_provider_live_name,data_provider_live_product_id", [("Paper Trading", "IEX", "333"), @@ -1218,4 +1208,8 @@ def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provide create_lean_option(brokerage_name, data_provider_live_name, None, api_client) assert len(api_client.method_calls) == 2 - assert data_provider_live_product_id in f"{api_client.method_calls[0].args[0]}" \ No newline at end of file + for m_c in api_client.method_calls: + if data_provider_live_product_id in m_c[1]: + is_exist = True + + assert is_exist \ No newline at end of file From e52b3a451c8db83297bade95ccd2b85d971ab8dd Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 16:10:21 +0200 Subject: [PATCH 10/14] fix: missed rename data_feed to data_provider --- lean/commands/live/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index e713f924..d0d805f8 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -408,7 +408,7 @@ def deploy(project: Path, [update_essential_properties_available(data_feed_configurers, kwargs)] elif brokerage is not None or len(data_provider_live) > 0: - ensure_options(["brokerage", "data_feed"]) + ensure_options(["brokerage", "data_provider_live"]) environment_name = "lean-cli" lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) From 4bf85588879e2fecacc607ea2d9679935e12e21f Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 16:11:53 +0200 Subject: [PATCH 11/14] feat: additional test case --- tests/commands/test_live.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index cfb1f43b..476b2403 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1185,6 +1185,7 @@ def test_live_non_interactive_deploy_with_different_brokerage_and_different_live @pytest.mark.parametrize("brokerage_name,data_provider_live_name,brokerage_product_id", [("Bybit", "Bybit", "305"), + ("Coinbase Advanced Trade", "Coinbase Advanced Trade", "183"), ("Interactive Brokers", "Interactive Brokers", "181"), ("Tradier", "Tradier", "185")]) def test_live_non_interactive_deploy_with_different_brokerage_with_the_same_live_data_provider(brokerage_name: str, data_provider_live_name: str, brokerage_product_id: str) -> None: From d4514fb22dee0ea54d9302295c1a9bc4b2121ad2 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 16:58:27 +0200 Subject: [PATCH 12/14] revert: data_provider -> data_feed commit: e52b3a4 --- lean/commands/live/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index d0d805f8..e713f924 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -408,7 +408,7 @@ def deploy(project: Path, [update_essential_properties_available(data_feed_configurers, kwargs)] elif brokerage is not None or len(data_provider_live) > 0: - ensure_options(["brokerage", "data_provider_live"]) + ensure_options(["brokerage", "data_feed"]) environment_name = "lean-cli" lean_config = lean_config_manager.get_complete_lean_config(environment_name, algorithm_file, None) From abee306fd0aade8b68a4612539a61ecbbb22fb16 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 17:19:39 +0200 Subject: [PATCH 13/14] revert: repeat code of finding data-provider --- lean/commands/live/deploy.py | 30 +++++++------------------- lean/models/data_providers/__init__.py | 7 +----- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index e713f924..269a8473 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -27,7 +27,7 @@ from lean.commands.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively -from lean.models.data_providers import all_data_providers, installed_data_providers +from lean.models.data_providers import all_data_providers from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module, update_essential_properties_available _environment_skeleton = { @@ -39,7 +39,7 @@ } -def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer], LeanConfigConfigurer]: +def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]: """Returns the configurable modules from the given environment. :param lean_config: the LEAN configuration that should be used @@ -61,14 +61,7 @@ def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], envi data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if _get_brokerage_base_name(local_data_feed.get_live_name()) in data_queue_handlers_base_names] - - data_downloader_preference = None - if "data-downloader" in lean_config: - data_downloader_base_name = _get_brokerage_base_name(lean_config["data-downloader"]) - data_downloader_preference = next((local_data_history for local_data_history in installed_data_providers - if _get_brokerage_base_name(local_data_history.get_config_value_from_name('data-downloader')) in data_downloader_base_name), None) - - return brokerage_configurer, data_feed_configurers, data_downloader_preference + return brokerage_configurer, data_feed_configurers def _get_brokerage_base_name(brokerage: str) -> str: @@ -98,19 +91,12 @@ def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: :param environment_name: the name of the environment :param lean_config_path: the path to the LEAN configuration file """ - brokerage_configurer, data_feed_configurers, data_downloader_preference = _get_configurable_modules_from_environment(lean_config, environment_name) + brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name) brokerage_properties = brokerage_configurer.get_required_properties(include_optionals=False) data_queue_handler_properties = [] - for data_feed_configurer in data_feed_configurers: + for data_feed_configurer in data_feed_configurers: data_queue_handler_properties.extend(data_feed_configurer.get_required_properties(include_optionals=False)) - required_properties = list(set(brokerage_properties + data_queue_handler_properties)) - - if data_downloader_preference is not None: - for data_downloader_property in data_downloader_preference.get_required_properties(include_optionals=False): - if data_downloader_property not in required_properties: - required_properties.append(data_downloader_property) - missing_properties = [p for p in required_properties if p not in lean_config or lean_config[p] == ""] missing_properties = set(missing_properties) if len(missing_properties) == 0: @@ -443,8 +429,8 @@ def deploy(project: Path, raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - env_brokerage, env_data_queue_handlers, env_data_downloader = _get_configurable_modules_from_environment(lean_config, environment_name) - _install_modules([env_brokerage] + env_data_queue_handlers + [env_data_downloader], kwargs) + env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) + _install_modules([env_brokerage] + env_data_queue_handlers + [data_provider_configurer], kwargs) _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) @@ -519,4 +505,4 @@ def deploy(project: Path, raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts") lean_runner = container.lean_runner - lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) + lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) \ No newline at end of file diff --git a/lean/models/data_providers/__init__.py b/lean/models/data_providers/__init__.py index 5eb8752c..fb0c47e2 100644 --- a/lean/models/data_providers/__init__.py +++ b/lean/models/data_providers/__init__.py @@ -16,16 +16,11 @@ from lean.models import json_modules all_data_providers: List[DataProvider] = [] -installed_data_providers: List[DataProvider] = [] for json_module in json_modules: if "data-provider" in json_module["type"]: all_data_providers.append(DataProvider(json_module)) -for data_provider in all_data_providers: - if data_provider._installs: - installed_data_providers.append(data_provider) - # QuantConnect DataProvider [QuantConnectDataProvider] = [ - data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] + data_provider for data_provider in all_data_providers if data_provider._id == "QuantConnect"] \ No newline at end of file From acf5bbe7818fffd1866fe80e33b705087b6e1954 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Feb 2024 17:34:47 +0200 Subject: [PATCH 14/14] rename: helper method more fit name --- lean/commands/live/deploy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 269a8473..e61700ea 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -191,14 +191,14 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], _cached_lean_config = None -def _try_get_data_downloader_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: - """ Get name for data downloader provider based on data provider live (if exist) +def _try_get_data_historical_name(data_provider_historical_name: str, data_provider_live_name: str) -> str: + """ Get name for historical data provider based on data provider live (if exist) :param data_provider_historical_name: the current (default) data provider historical :param data_provider_live_name: the current data provider live name """ - return next((live_data_downloader.get_name() for live_data_downloader in all_data_providers - if live_data_downloader.get_name() in data_provider_live_name), data_provider_historical_name) + return next((live_data_historical.get_name() for live_data_historical in all_data_providers + if live_data_historical.get_name() in data_provider_live_name), data_provider_historical_name) # being used by lean.models.click_options.get_the_correct_type_default_value() @@ -416,7 +416,7 @@ def deploy(project: Path, _configure_lean_config_interactively(lean_config, environment_name, kwargs, show_secrets=show_secrets) if data_provider_historical is None: - data_provider_historical = _try_get_data_downloader_name("Local", data_provider_live) + data_provider_historical = _try_get_data_historical_name("Local", data_provider_live) [data_provider_configurer] = [get_and_build_module(data_provider_historical, all_data_providers, kwargs, logger)] data_provider_configurer.configure(lean_config, environment_name)