From ef27f199df84f3dcea51538e96a2602ae65634d7 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 20 Jun 2024 15:09:01 +0300 Subject: [PATCH 01/14] fix: getting last_portfolio if project_Id is None --- lean/components/util/live_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index ab24d810..0fa51c35 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -25,7 +25,11 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from json import loads from datetime import datetime - cloud_deployment_list = api_client.get("live/read") + if not project_id: + # Project is not initialized in the cloud, hence project_id is None + return None + + cloud_deployment_list = api_client.get("live/read", { "projectId": project_id }) cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) for instance in cloud_deployment_list["live"] if instance["projectId"] == project_id] cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else utc.localize(datetime.min) From abdad22cfd356b82f90924b6e16ca35844daaa17 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 24 Jun 2024 16:34:00 +0300 Subject: [PATCH 02/14] feat: several logging and patches --- lean/components/api/api_client.py | 10 ++++++++-- lean/components/util/live_utils.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lean/components/api/api_client.py b/lean/components/api/api_client.py index c6787c75..48e578b3 100644 --- a/lean/components/api/api_client.py +++ b/lean/components/api/api_client.py @@ -142,6 +142,12 @@ def _request(self, method: str, endpoint: str, options: Dict[str, Any] = {}, ret version = 99999999 headers["User-Agent"] = f"Lean CLI {version}" + self._logger.info(f'full_url: {full_url}') + self._logger.info(f'headers: {headers}') + self._logger.info(f'self._user_id: {self._user_id}') + self._logger.info(f'password: {password}') + self._logger.info(f'options: {options}') + response = self._http_client.request(method, full_url, headers=headers, @@ -149,8 +155,8 @@ def _request(self, method: str, endpoint: str, options: Dict[str, Any] = {}, ret raise_for_status=False, **options) - if self._logger.debug_logging_enabled: - self._logger.debug(f"Request response: {response.text}") + self._logger.debug(f"Request response: {response.text}") + # if self._logger.debug_logging_enabled: if 500 <= response.status_code < 600 and retry_http_5xx: return self._request(method, endpoint, options, False) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 0fa51c35..311b8935 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -24,12 +24,14 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from os import listdir, path from json import loads from datetime import datetime + from lean.container import container + + cloud_deployment_list = api_client.get("live/read", { "projectId": project_id }) + container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment_list}') - if not project_id: - # Project is not initialized in the cloud, hence project_id is None + if "live" not in cloud_deployment_list: return None - cloud_deployment_list = api_client.get("live/read", { "projectId": project_id }) cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) for instance in cloud_deployment_list["live"] if instance["projectId"] == project_id] cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else utc.localize(datetime.min) @@ -68,10 +70,14 @@ def get_last_portfolio_cash_holdings(api_client: APIClient, brokerage_instance: :param project: the name of the project :return: the options of initial cash/holdings setting, and the latest portfolio cash/holdings from the last deployment """ + from lean.container import container last_cash = [] last_holdings = [] + container.logger.info(f'brokerage_instance: {brokerage_instance}') cash_balance_option = brokerage_instance._initial_cash_balance holdings_option = brokerage_instance._initial_holdings + container.logger.info(f'cash_balance_option: {cash_balance_option}') + container.logger.info(f'holdings_option: {holdings_option}') if cash_balance_option != LiveInitialStateInput.NotSupported or holdings_option != LiveInitialStateInput.NotSupported: last_portfolio = _get_last_portfolio(api_client, project_id, project) last_cash = last_portfolio["cash"] if last_portfolio else None From 2435368894adc835b94bf3bc6b8da0341718f550 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 10 Jul 2024 00:11:30 +0300 Subject: [PATCH 03/14] fix: (part) cloud status of project command --- lean/commands/cloud/status.py | 2 +- lean/components/api/live_client.py | 29 ++++++++++++++--------------- lean/components/util/live_utils.py | 9 +++++---- lean/models/api.py | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lean/commands/cloud/status.py b/lean/commands/cloud/status.py index 7dd8c340..d33d7927 100644 --- a/lean/commands/cloud/status.py +++ b/lean/commands/cloud/status.py @@ -31,7 +31,7 @@ def status(project: str) -> None: cloud_project_manager = container.cloud_project_manager cloud_project = cloud_project_manager.get_cloud_project(project, False) - live_algorithm = next((d for d in api_client.live.get_all() if d.projectId == cloud_project.projectId), None) + live_algorithm = api_client.live.get_project_by_id(cloud_project.projectId) logger.info(f"Project id: {cloud_project.projectId}") logger.info(f"Project name: {cloud_project.name}") diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index c3590467..f5872e6f 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -28,29 +28,28 @@ def __init__(self, api_client: 'APIClient') -> None: """ self._api = api_client - def get_all(self, - status: Optional[QCLiveAlgorithmStatus] = None, - # Values less than 86400 cause errors on Windows: https://bugs.python.org/issue37527 - start: datetime = datetime.fromtimestamp(86400), - end: datetime = datetime.now()) -> List[QCFullLiveAlgorithm]: + def get_project_by_id(self, + project_id: str, + status: Optional[QCLiveAlgorithmStatus] = None) -> QCFullLiveAlgorithm: """Retrieves all live algorithms. :param status: the status to filter by or None if no status filter should be applied - :param start: the earliest launch time the returned algorithms should have - :param end: the latest launch time the returned algorithms should have - :return: a list of live algorithms which match the given filters + :param project_id: the project id + :return: a live algorithm which match the given filters """ - from math import floor - parameters = { - "start": floor(start.timestamp()), - "end": floor(end.timestamp()) - } + from lean.container import container + + parameters = {"projectId": project_id} if status is not None: parameters["status"] = status.value - data = self._api.get("live/read", parameters) - return [QCFullLiveAlgorithm(**algorithm) for algorithm in data["live"]] + response = self._api.get("live/read", parameters) + + if response: + return QCFullLiveAlgorithm(data=response) + + return None def start(self, project_id: int, diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 311b8935..3b7ad8e3 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -26,12 +26,13 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from datetime import datetime from lean.container import container - cloud_deployment_list = api_client.get("live/read", { "projectId": project_id }) - container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment_list}') - - if "live" not in cloud_deployment_list: + if not project_id: + # Project is not initialized in the cloud, hence project_id is None return None + cloud_deployment_list = api_client.get("live/read", {"projectId": project_id}) + container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment_list}') + cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) for instance in cloud_deployment_list["live"] if instance["projectId"] == project_id] cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else utc.localize(datetime.min) diff --git a/lean/models/api.py b/lean/models/api.py index 9edb4ddd..9e3e02ee 100644 --- a/lean/models/api.py +++ b/lean/models/api.py @@ -279,7 +279,7 @@ class QCMinimalLiveAlgorithm(WrappedBaseModel): def get_url(self) -> str: """Returns the url of the live deployment in the cloud. - :return: a url which when visited opens an Algorithm Lab tab containing the live deployment + :return: an url which when visited opens an Algorithm Lab tab containing the live deployment """ return f"https://www.quantconnect.com/project/{self.projectId}/live" From ff1ba2ce7cb2b743ba983f46d2cb95f421cd715d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:42:37 -0500 Subject: [PATCH 04/14] First attempt to solve the bug --- lean/components/util/live_utils.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 3b7ad8e3..f93991cb 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -30,12 +30,9 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa # Project is not initialized in the cloud, hence project_id is None return None - cloud_deployment_list = api_client.get("live/read", {"projectId": project_id}) - container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment_list}') - - cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) for instance in cloud_deployment_list["live"] - if instance["projectId"] == project_id] - cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else utc.localize(datetime.min) + cloud_deployment = api_client.get("live/read", {"projectId": project_id}) + container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') + cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" @@ -45,7 +42,7 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa local_last_time = sorted(local_deployment_time, reverse = True)[0] if cloud_last_time > local_last_time: - last_state = api_client.get("live/read/portfolio", {"projectId": project_id}) + last_state = api_client.get("live/portfolio/read", {"projectId": project_id}) previous_portfolio_state = last_state["portfolio"] elif cloud_last_time < local_last_time: from lean.container import container @@ -91,8 +88,8 @@ def _configure_initial_cash_interactively(logger: Logger, cash_input_option: Liv previous_cash_balance = [] if previous_cash_state: for cash_state in previous_cash_state.values(): - currency = cash_state["Symbol"] - amount = cash_state["Amount"] + currency = cash_state["symbol"] + amount = cash_state["amount"] previous_cash_balance.append({"currency": currency, "amount": amount}) if cash_input_option == LiveInitialStateInput.Required or confirm("Do you want to set the initial cash balance?", default=False): @@ -140,10 +137,10 @@ def _configure_initial_holdings_interactively(logger: Logger, holdings_option: L last_holdings = [] if previous_holdings: for holding in previous_holdings.values(): - symbol = holding["Symbol"] - quantity = int(holding["Quantity"]) - avg_price = float(holding["AveragePrice"]) - last_holdings.append({"symbol": symbol["Value"], "symbolId": symbol["ID"], "quantity": quantity, "averagePrice": avg_price}) + symbol = holding["symbol"] + quantity = int(holding["quantity"]) + avg_price = float(holding["averagePrice"]) + last_holdings.append({"symbol": symbol["value"], "symbolId": symbol["id"], "quantity": quantity, "averagePrice": avg_price}) if holdings_option == LiveInitialStateInput.Required or confirm("Do you want to set the initial portfolio holdings?", default=False): if confirm(f"Do you want to use the last portfolio holdings? {last_holdings}", default=False): From 53bd7863ad06b6f9e90bae331d617a238fa5ead3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:05:02 -0500 Subject: [PATCH 05/14] Add fix --- lean/components/util/live_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index f93991cb..28ce2f32 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -32,7 +32,9 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa cloud_deployment = api_client.get("live/read", {"projectId": project_id}) container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') - cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + cloud_last_time = utc.localize(datetime.min) + if cloud_deployment["success"]: + cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" From 5f35241bf92168cfa4f7b53a5be270e375a8e479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Wed, 10 Jul 2024 17:41:54 -0500 Subject: [PATCH 06/14] address requested changes --- lean/components/api/api_client.py | 10 ++-------- lean/components/api/live_client.py | 2 -- lean/components/util/live_utils.py | 10 +++------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/lean/components/api/api_client.py b/lean/components/api/api_client.py index 48e578b3..c6787c75 100644 --- a/lean/components/api/api_client.py +++ b/lean/components/api/api_client.py @@ -142,12 +142,6 @@ def _request(self, method: str, endpoint: str, options: Dict[str, Any] = {}, ret version = 99999999 headers["User-Agent"] = f"Lean CLI {version}" - self._logger.info(f'full_url: {full_url}') - self._logger.info(f'headers: {headers}') - self._logger.info(f'self._user_id: {self._user_id}') - self._logger.info(f'password: {password}') - self._logger.info(f'options: {options}') - response = self._http_client.request(method, full_url, headers=headers, @@ -155,8 +149,8 @@ def _request(self, method: str, endpoint: str, options: Dict[str, Any] = {}, ret raise_for_status=False, **options) - self._logger.debug(f"Request response: {response.text}") - # if self._logger.debug_logging_enabled: + if self._logger.debug_logging_enabled: + self._logger.debug(f"Request response: {response.text}") if 500 <= response.status_code < 600 and retry_http_5xx: return self._request(method, endpoint, options, False) diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index f5872e6f..b04c6506 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -37,8 +37,6 @@ def get_project_by_id(self, :param project_id: the project id :return: a live algorithm which match the given filters """ - from lean.container import container - parameters = {"projectId": project_id} if status is not None: diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 28ce2f32..bf4c0eec 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -26,10 +26,6 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from datetime import datetime from lean.container import container - if not project_id: - # Project is not initialized in the cloud, hence project_id is None - return None - cloud_deployment = api_client.get("live/read", {"projectId": project_id}) container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') cloud_last_time = utc.localize(datetime.min) @@ -73,11 +69,11 @@ def get_last_portfolio_cash_holdings(api_client: APIClient, brokerage_instance: from lean.container import container last_cash = [] last_holdings = [] - container.logger.info(f'brokerage_instance: {brokerage_instance}') + container.logger.debug(f'brokerage_instance: {brokerage_instance}') cash_balance_option = brokerage_instance._initial_cash_balance holdings_option = brokerage_instance._initial_holdings - container.logger.info(f'cash_balance_option: {cash_balance_option}') - container.logger.info(f'holdings_option: {holdings_option}') + container.logger.debug(f'cash_balance_option: {cash_balance_option}') + container.logger.debug(f'holdings_option: {holdings_option}') if cash_balance_option != LiveInitialStateInput.NotSupported or holdings_option != LiveInitialStateInput.NotSupported: last_portfolio = _get_last_portfolio(api_client, project_id, project) last_cash = last_portfolio["cash"] if last_portfolio else None From 636699de103c451cd6d8afd32b7652951f927db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Wed, 10 Jul 2024 18:26:46 -0500 Subject: [PATCH 07/14] Fix bug --- lean/components/util/live_utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index bf4c0eec..5f79006b 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -26,11 +26,12 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from datetime import datetime from lean.container import container - cloud_deployment = api_client.get("live/read", {"projectId": project_id}) - container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') cloud_last_time = utc.localize(datetime.min) - if cloud_deployment["success"]: - cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + if project_id: + cloud_deployment = api_client.get("live/read", {"projectId": project_id}) + container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') + if cloud_deployment["success"]: + cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" From f768854c92b5f442baa3d9a79b3e898655c56c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:22:27 -0500 Subject: [PATCH 08/14] Fix bugs --- lean/components/util/live_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 5f79006b..7173301d 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -29,9 +29,8 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa cloud_last_time = utc.localize(datetime.min) if project_id: cloud_deployment = api_client.get("live/read", {"projectId": project_id}) - container.logger.info(f'----- After cloud_deployment_list: {cloud_deployment}') - if cloud_deployment["success"]: - cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + if cloud_deployment["success"] and cloud_deployment["status"] != "Undefined": + cloud_last_time = datetime.strptime(cloud_deployment["stopped"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" From 2fca260c9b57ed5f0bd21b310de693a2ceae8921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Fri, 12 Jul 2024 09:42:55 -0500 Subject: [PATCH 09/14] Fix bug #468 --- lean/commands/cloud/status.py | 4 --- lean/components/api/live_client.py | 3 +- lean/components/util/live_utils.py | 1 - lean/models/api.py | 2 -- .../cloud/live/test_cloud_live_commands.py | 30 +++++++++++++++---- tests/commands/test_live.py | 6 ++-- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lean/commands/cloud/status.py b/lean/commands/cloud/status.py index d33d7927..fb188fcd 100644 --- a/lean/commands/cloud/status.py +++ b/lean/commands/cloud/status.py @@ -59,7 +59,3 @@ def status(project: str) -> None: if live_algorithm.stopped is not None: logger.info(f"Stopped: {live_algorithm.stopped.strftime('%Y-%m-%d %H:%M:%S')} UTC") - - if live_algorithm.error != "": - logger.info("Error:") - logger.info(live_algorithm.error) diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index b04c6506..7e611710 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -45,7 +45,8 @@ def get_project_by_id(self, response = self._api.get("live/read", parameters) if response: - return QCFullLiveAlgorithm(data=response) + response["projectId"] = project_id + return QCFullLiveAlgorithm(**response) return None diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 7173301d..24b48efa 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -24,7 +24,6 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa from os import listdir, path from json import loads from datetime import datetime - from lean.container import container cloud_last_time = utc.localize(datetime.min) if project_id: diff --git a/lean/models/api.py b/lean/models/api.py index 9e3e02ee..4df959c4 100644 --- a/lean/models/api.py +++ b/lean/models/api.py @@ -291,8 +291,6 @@ class QCFullLiveAlgorithm(QCMinimalLiveAlgorithm): launched: datetime stopped: Optional[datetime] brokerage: str - subscription: str - error: str class QCEmailNotificationMethod(WrappedBaseModel): diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 9a503c43..d066ab89 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -65,7 +65,11 @@ def test_cloud_live_deploy() -> None: api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + api_client.get.return_value = { + "status": "stopped", + "stopped": "2024-07-10 19:12:20", + "success": True, + "portfolio": {"holdings": {}, "cash": {}, "success": True}} container.api_client = api_client cloud_project_manager = mock.Mock() @@ -98,7 +102,11 @@ def test_cloud_live_deploy_with_ib_using_hybrid_datafeed() -> None: api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + api_client.get.return_value = { + "status": "stopped", + "stopped": "2024-07-10 19:12:20", + "success": True, + "portfolio": {"holdings": {}, "cash": {}, "success": True}} container.api_client = api_client cloud_project_manager = mock.Mock() @@ -155,7 +163,11 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + api_client.get.return_value = { + "status": "stopped", + "stopped": "2024-07-10 19:12:20", + "success": True, + "portfolio": {"holdings": {}, "cash": {}, "success": True}} container.api_client = api_client cloud_project_manager = mock.Mock() @@ -238,7 +250,11 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'live': [], 'portfolio': {}} + api_client.get.return_value = { + "status": "stopped", + "stopped": "2024-07-10 19:12:20", + "success": True, + "portfolio": {}} container.api_client = api_client cloud_runner = mock.Mock() @@ -315,7 +331,11 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'live': [], 'portfolio': {}} + api_client.get.return_value = { + "status": "stopped", + "stopped": "2024-07-10 19:12:20", + "success": True, + "portfolio": {}} container.api_client = api_client cloud_runner = mock.Mock() diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index dfa12e6f..54ce1417 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -1182,14 +1182,14 @@ def test_live_deploy_with_different_brokerage_and_different_live_data_provider_a is_exists = [] if brokerage_product_id is None and data_provider_historical_name != "Local": - assert len(api_client.method_calls) == 3 + assert len(api_client.method_calls) == 2 for m_c, id in zip(api_client.method_calls, [data_provider_live_product_id, data_provider_historical_id]): if id in m_c[1]: is_exists.append(True) assert is_exists assert len(is_exists) == 2 elif brokerage_product_id is None and data_provider_historical_name == "Local": - assert len(api_client.method_calls) == 2 + assert len(api_client.method_calls) == 1 if data_provider_live_product_id in api_client.method_calls[0][1]: is_exists.append(True) assert is_exists @@ -1243,7 +1243,7 @@ def test_live_non_interactive_deploy_paper_brokerage_different_live_data_provide api_client = mock.MagicMock() create_lean_option(brokerage_name, data_provider_live_name, None, api_client) - assert len(api_client.method_calls) == 2 + assert len(api_client.method_calls) == 1 for m_c in api_client.method_calls: if data_provider_live_product_id in m_c[1]: is_exist = True From 1b97821f1813e9c0f8f15e1d73a616e3e012fb77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:48:53 -0500 Subject: [PATCH 10/14] Address requested changes --- lean/components/api/live_client.py | 7 +-- lean/components/util/live_utils.py | 48 ++++++++++++++----- .../cloud/live/test_cloud_live_commands.py | 4 +- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index 7e611710..3a53d9ff 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -29,8 +29,7 @@ def __init__(self, api_client: 'APIClient') -> None: self._api = api_client def get_project_by_id(self, - project_id: str, - status: Optional[QCLiveAlgorithmStatus] = None) -> QCFullLiveAlgorithm: + project_id: str) -> QCFullLiveAlgorithm: """Retrieves all live algorithms. :param status: the status to filter by or None if no status filter should be applied @@ -38,10 +37,6 @@ def get_project_by_id(self, :return: a live algorithm which match the given filters """ parameters = {"projectId": project_id} - - if status is not None: - parameters["status"] = status.value - response = self._api.get("live/read", parameters) if response: diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 24b48efa..a26370cd 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -18,6 +18,22 @@ from lean.components.api.api_client import APIClient from lean.components.util.logger import Logger from lean.models.json_module import LiveInitialStateInput, JsonModule +from collections import UserDict +from typing import Any + + +class InsensitiveCaseDict(UserDict): + def __getitem__(self, key: Any) -> Any: + if type(key) is str: + return super().__getitem__(key.lower()) + return super().__getitem__(key) + + def __setitem__(self, key: Any, item: Any) -> Any: + if type(key) is str: + self.data[key.lower()] = item + return + self.data[key] = item + def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Path) -> List[Dict[str, Any]]: from pytz import utc, UTC @@ -29,7 +45,10 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa if project_id: cloud_deployment = api_client.get("live/read", {"projectId": project_id}) if cloud_deployment["success"] and cloud_deployment["status"] != "Undefined": - cloud_last_time = datetime.strptime(cloud_deployment["stopped"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + if cloud_deployment["stopped"] is not None: + cloud_last_time = datetime.strptime(cloud_deployment["stopped"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + else: + cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" @@ -66,8 +85,8 @@ def get_last_portfolio_cash_holdings(api_client: APIClient, brokerage_instance: :return: the options of initial cash/holdings setting, and the latest portfolio cash/holdings from the last deployment """ from lean.container import container - last_cash = [] - last_holdings = [] + last_cash = {} + last_holdings = {} container.logger.debug(f'brokerage_instance: {brokerage_instance}') cash_balance_option = brokerage_instance._initial_cash_balance holdings_option = brokerage_instance._initial_holdings @@ -75,8 +94,15 @@ def get_last_portfolio_cash_holdings(api_client: APIClient, brokerage_instance: container.logger.debug(f'holdings_option: {holdings_option}') if cash_balance_option != LiveInitialStateInput.NotSupported or holdings_option != LiveInitialStateInput.NotSupported: last_portfolio = _get_last_portfolio(api_client, project_id, project) - last_cash = last_portfolio["cash"] if last_portfolio else None - last_holdings = last_portfolio["holdings"] if last_portfolio else None + if last_portfolio is not None: + for key, value in last_portfolio["cash"].items(): + last_cash[key] = InsensitiveCaseDict(value) + for key, value in last_portfolio["holdings"].items(): + last_holdings[key] = InsensitiveCaseDict(value) + last_holdings[key]["symbol"] = InsensitiveCaseDict(last_holdings[key]["symbol"]) + else: + last_cash = None + last_holdings = None return cash_balance_option, holdings_option, last_cash, last_holdings @@ -85,8 +111,8 @@ def _configure_initial_cash_interactively(logger: Logger, cash_input_option: Liv previous_cash_balance = [] if previous_cash_state: for cash_state in previous_cash_state.values(): - currency = cash_state["symbol"] - amount = cash_state["amount"] + currency = cash_state["Symbol"] + amount = cash_state["Amount"] previous_cash_balance.append({"currency": currency, "amount": amount}) if cash_input_option == LiveInitialStateInput.Required or confirm("Do you want to set the initial cash balance?", default=False): @@ -134,10 +160,10 @@ def _configure_initial_holdings_interactively(logger: Logger, holdings_option: L last_holdings = [] if previous_holdings: for holding in previous_holdings.values(): - symbol = holding["symbol"] - quantity = int(holding["quantity"]) - avg_price = float(holding["averagePrice"]) - last_holdings.append({"symbol": symbol["value"], "symbolId": symbol["id"], "quantity": quantity, "averagePrice": avg_price}) + symbol = holding["Symbol"] + quantity = int(holding["Quantity"]) + avg_price = float(holding["AveragePrice"]) + last_holdings.append({"symbol": symbol["Value"], "symbolId": symbol["ID"], "quantity": quantity, "averagePrice": avg_price}) if holdings_option == LiveInitialStateInput.Required or confirm("Do you want to set the initial portfolio holdings?", default=False): if confirm(f"Do you want to use the last portfolio holdings? {last_holdings}", default=False): diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index d066ab89..94c4864f 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -254,7 +254,7 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> "status": "stopped", "stopped": "2024-07-10 19:12:20", "success": True, - "portfolio": {}} + "portfolio": {"cash": {}, "holdings": {}}} container.api_client = api_client cloud_runner = mock.Mock() @@ -335,7 +335,7 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> "status": "stopped", "stopped": "2024-07-10 19:12:20", "success": True, - "portfolio": {}} + "portfolio": {"cash": {}, "holdings": {}}} container.api_client = api_client cloud_runner = mock.Mock() From 9ffe3ff2a780f70b16b05ade5222ca5dcdec3849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:33:09 -0500 Subject: [PATCH 11/14] fix bug in cloud_last_time --- lean/components/api/live_client.py | 1 - lean/components/util/live_utils.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index 3a53d9ff..39e2460a 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -32,7 +32,6 @@ def get_project_by_id(self, project_id: str) -> QCFullLiveAlgorithm: """Retrieves all live algorithms. - :param status: the status to filter by or None if no status filter should be applied :param project_id: the project id :return: a live algorithm which match the given filters """ diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index a26370cd..8907c884 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -46,9 +46,14 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa cloud_deployment = api_client.get("live/read", {"projectId": project_id}) if cloud_deployment["success"] and cloud_deployment["status"] != "Undefined": if cloud_deployment["stopped"] is not None: - cloud_last_time = datetime.strptime(cloud_deployment["stopped"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + cloud_last_time = datetime.strptime(cloud_deployment["stopped"], "%Y-%m-%d %H:%M:%S") else: - cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S").astimezone(UTC) + cloud_last_time = datetime.strptime(cloud_deployment["launched"], "%Y-%m-%d %H:%M:%S") + cloud_last_time = datetime(cloud_last_time.year, cloud_last_time.month, + cloud_last_time.day, cloud_last_time.hour, + cloud_last_time.minute, + cloud_last_time.second, + tzinfo=UTC) local_last_time = utc.localize(datetime.min) live_deployment_path = f"{project_name}/live" From 7dcc5b583a936d83fcf43c83d821588a32f93f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:13:40 -0500 Subject: [PATCH 12/14] Fix bugs - Since you must always choose a data feed when deploying a local live algo, data_provider_live will never be zero. - Since the date used to determine the last local deployment (in live_utils.py get_last_portfolio) comes from the output directory, we cannot create the output directory before calling get_last_portfolio because then we would get always the local holdings in the get_last_portfolio_method - The output directory for the previous state file (see _get_last_portfolio) comes with an "L-" prefix --- lean/commands/live/deploy.py | 8 ++++---- lean/components/util/live_utils.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 0fb2d178..91851ea1 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -292,16 +292,13 @@ def deploy(project: Path, _start_iqconnect_if_necessary(lean_config, environment_name) - if not output.exists(): - output.mkdir(parents=True) - if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, brokerage_instance, project_config.get("cloud-id", None), project) - if environment is None and brokerage is None and len(data_provider_live) == 0: # condition for using interactive panel + if environment is None and brokerage is None: # condition for using interactive panel if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = _configure_initial_cash_interactively(logger, cash_balance_option, last_cash) @@ -341,6 +338,9 @@ def deploy(project: Path, else: lean_config[key] = value + if not output.exists(): + output.mkdir(parents=True) + output_config_manager = container.output_config_manager lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output, given_algorithm_id)}" diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 8907c884..adf59a57 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -70,7 +70,7 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa output_directory = container.output_config_manager.get_latest_output_directory("live") if not output_directory: return None - previous_state_file = get_latest_result_json_file(output_directory) + previous_state_file = get_latest_result_json_file(output_directory, True) if not previous_state_file: return None previous_portfolio_state = {x.lower(): y for x, y in loads(open(previous_state_file, "r", encoding="utf-8").read()).items()} @@ -111,7 +111,7 @@ def get_last_portfolio_cash_holdings(api_client: APIClient, brokerage_instance: return cash_balance_option, holdings_option, last_cash, last_holdings -def _configure_initial_cash_interactively(logger: Logger, cash_input_option: LiveInitialStateInput, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: +def _configure_initial_cash_interactively(logger: Logger, cash_input_option: LiveInitialStateInput, previous_cash_state: Dict[str, Any]) -> List[Dict[str, float]]: cash_list = [] previous_cash_balance = [] if previous_cash_state: @@ -140,7 +140,7 @@ def _configure_initial_cash_interactively(logger: Logger, cash_input_option: Liv return [] -def configure_initial_cash_balance(logger: Logger, cash_input_option: LiveInitialStateInput, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]])\ +def configure_initial_cash_balance(logger: Logger, cash_input_option: LiveInitialStateInput, live_cash_balance: str, previous_cash_state: Dict[str, Any])\ -> List[Dict[str, float]]: """Interactively configures the intial cash balance. @@ -160,7 +160,7 @@ def configure_initial_cash_balance(logger: Logger, cash_input_option: LiveInitia return _configure_initial_cash_interactively(logger, cash_input_option, previous_cash_state) -def _configure_initial_holdings_interactively(logger: Logger, holdings_option: LiveInitialStateInput, previous_holdings: List[Dict[str, Any]]) -> List[Dict[str, float]]: +def _configure_initial_holdings_interactively(logger: Logger, holdings_option: LiveInitialStateInput, previous_holdings: Dict[str, Any]) -> List[Dict[str, float]]: holdings = [] last_holdings = [] if previous_holdings: @@ -192,7 +192,7 @@ def _configure_initial_holdings_interactively(logger: Logger, holdings_option: L return [] -def configure_initial_holdings(logger: Logger, holdings_option: LiveInitialStateInput, live_holdings: str, previous_holdings: List[Dict[str, Any]])\ +def configure_initial_holdings(logger: Logger, holdings_option: LiveInitialStateInput, live_holdings: str, previous_holdings: Dict[str, Any])\ -> List[Dict[str, float]]: """Interactively configures the intial portfolio holdings. @@ -212,7 +212,7 @@ def configure_initial_holdings(logger: Logger, holdings_option: LiveInitialState return _configure_initial_holdings_interactively(logger, holdings_option, previous_holdings) -def get_latest_result_json_file(output_directory: Path) -> Optional[Path]: +def get_latest_result_json_file(output_directory: Path, is_previous_state_file: bool = False) -> Optional[Path]: from lean.container import container output_config_manager = container.output_config_manager @@ -221,7 +221,11 @@ def get_latest_result_json_file(output_directory: Path) -> Optional[Path]: if output_id is None: return None - result_file = output_directory / f"{output_id}.json" + prefix = "" + if is_previous_state_file: + prefix = "L-" + + result_file = output_directory / f"{prefix}{output_id}.json" if not result_file.exists(): return None From 28a6368878ca12e016f0f155f0950f03154d1e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:17:37 -0500 Subject: [PATCH 13/14] Address remarks --- lean/commands/live/deploy.py | 9 ++++++--- lean/components/util/live_utils.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 91851ea1..3e6bc852 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -298,6 +298,12 @@ def deploy(project: Path, cash_balance_option, holdings_option, last_cash, last_holdings = get_last_portfolio_cash_holdings(container.api_client, brokerage_instance, project_config.get("cloud-id", None), project) + # We cannot create the output directory before calling get_last_portfolio_holdings, since then the most recently + # deployment would be always the local one (it has the current time in its name), and we would never be able to + # use the cash and holdings from a cloud deployment (see live_utils._get_last_portfolio() method) + if not output.exists(): + output.mkdir(parents=True) + if environment is None and brokerage is None: # condition for using interactive panel if cash_balance_option != LiveInitialStateInput.NotSupported: live_cash_balance = _configure_initial_cash_interactively(logger, cash_balance_option, last_cash) @@ -338,9 +344,6 @@ def deploy(project: Path, else: lean_config[key] = value - if not output.exists(): - output.mkdir(parents=True) - output_config_manager = container.output_config_manager lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output, given_algorithm_id)}" diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index adf59a57..106afdec 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -212,7 +212,7 @@ def configure_initial_holdings(logger: Logger, holdings_option: LiveInitialState return _configure_initial_holdings_interactively(logger, holdings_option, previous_holdings) -def get_latest_result_json_file(output_directory: Path, is_previous_state_file: bool = False) -> Optional[Path]: +def get_latest_result_json_file(output_directory: Path, is_live_trading: bool = False) -> Optional[Path]: from lean.container import container output_config_manager = container.output_config_manager @@ -222,7 +222,7 @@ def get_latest_result_json_file(output_directory: Path, is_previous_state_file: return None prefix = "" - if is_previous_state_file: + if is_live_trading: prefix = "L-" result_file = output_directory / f"{prefix}{output_id}.json" From 6ec7410beae908e22c636419470ed0227db96d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Mon, 15 Jul 2024 18:36:00 -0500 Subject: [PATCH 14/14] Nit change --- lean/components/util/live_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 106afdec..41e0d0dd 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -19,7 +19,6 @@ from lean.components.util.logger import Logger from lean.models.json_module import LiveInitialStateInput, JsonModule from collections import UserDict -from typing import Any class InsensitiveCaseDict(UserDict):