Merge pull request #442 from jhonabreul/feature-factset-support

Json modules path configs support

README.md

@@ -143,7 +143,7 @@ Options:
   -d, --detach                    Run the backtest in a detached Docker container and return immediately
   --debug [pycharm|ptvsd|vsdbg|rider|local-platform]
                                   Enable a certain debugging method (see --help for more information)
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures]
                                   Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures]
@@ -161,6 +161,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan
@@ -1081,7 +1083,7 @@ Options:
                                   The brokerage to use
   --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit]
                                   The live data provider to use
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --ib-user-name TEXT             Your Interactive Brokers username
   --ib-account TEXT               Your Interactive Brokers account id
@@ -1194,6 +1196,8 @@ Options:
   --coinapi-api-key TEXT          Your coinapi.io Api Key
   --coinapi-product [Free|Startup|Streamer|Professional|Enterprise]
                                   CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing)
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --alpha-vantage-api-key TEXT    Your Alpha Vantage Api Key
   --alpha-vantage-price-plan [Free|Plan30|Plan75|Plan150|Plan300|Plan600|Plan1200]
                                   Your Alpha Vantage Premium API Key plan
@@ -1486,7 +1490,7 @@ Options:
   --parameter <TEXT FLOAT FLOAT FLOAT>...
                                   The 'parameter min max step' pairs configuring the parameters to optimize
   --constraint TEXT               The 'statistic operator value' pairs configuring the constraints of the optimization
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --download-data                 Update the Lean configuration file to download data from the QuantConnect API, alias
                                   for --data-provider-historical QuantConnect
@@ -1517,6 +1521,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan
@@ -1641,7 +1647,7 @@ Usage: lean research [OPTIONS] PROJECT
 
 Options:
   --port INTEGER                  The port to run Jupyter Lab on (defaults to 8888)
-  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
+  --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link]
                                   Update the Lean configuration file to retrieve data from the given historical provider
   --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures]
                                   Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures]
@@ -1659,6 +1665,8 @@ Options:
   --iqfeed-version TEXT           The product version of your IQFeed developer account
   --iqfeed-host TEXT              The IQFeed host address
   --polygon-api-key TEXT          Your Polygon.io API Key
+  --factset-auth-config-file FILE
+                                  The path to the FactSet authentication configuration file
   --iex-cloud-api-key TEXT        Your iexcloud.io API token publishable key
   --iex-price-plan [Launch|Grow|Enterprise]
                                   Your IEX Cloud Price plan

lean/commands/backtest.py

@@ -352,12 +352,14 @@ def backtest(project: Path,
         data_provider_historical = "QuantConnect"
 
     organization_id = container.organization_manager.try_get_working_organization_id()
+    paths_to_mount = None
 
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
@@ -406,4 +408,5 @@ def backtest(project: Path,
                          debugging_method,
                          release,
                          detach,
-                         loads(extra_docker_config))
+                         loads(extra_docker_config),
+                         paths_to_mount)

lean/commands/cloud/object_store/__init__.py

@@ -16,9 +16,11 @@
 from lean.commands.cloud.object_store.set import set
 from lean.commands.cloud.object_store.list import list
 from lean.commands.cloud.object_store.delete import delete
+from lean.commands.cloud.object_store.properties import properties
 
 object_store.add_command(get)
 object_store.add_command(set)
 object_store.add_command(list)
 object_store.add_command(delete)
+object_store.add_command(properties)
 

lean/commands/cloud/object_store/get.py

@@ -17,39 +17,37 @@
 
 
 @command(cls=LeanCommand)
-@argument("key", type=str)
-def get(key: str) -> str:
+@argument("key", type=str, multiple=True, help=f"The desired key to fetch")
+@argument("destination_folder", type=str, default="",
+          help=f"The destination folder to download the object store values,"
+               f" if not provided will use to current directory")
+def get(key: [str], destination_folder: str):
     """
-    Get a value from the organization's cloud object store.
+    Download an object store value to disk from the organization's cloud object store.
 
     """
     organization_id = container.organization_manager.try_get_working_organization_id()
     api_client = container.api_client
     logger = container.logger
-    data = api_client.object_store.get(key, organization_id)
-
-    try:
-        headers = ["size", "modified", "key", "preview"]
-        display_headers = ["Bytes", "Modified", "Filename", "Preview"]
-        data_row = []
-        for header in headers:
-            if header == "preview":
-                value = str(data["metadata"].get(header, "N/A"))
-                data_row.append(_clean_up_preview(value))
-            else:
-                value = str(data["metadata"].get(header, ""))
-                data_row.append(value)
-        all_rows = [display_headers] + [data_row]
-        column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))]
-        for row in all_rows:
-            logger.info("  ".join(value.ljust(width) for value, width in zip(row, column_widths)))
-    except KeyError as e:
-        logger.error(f"Key {key} not found.")
-    except Exception as e:
-        logger.error(f"Error: {e}")
-
-
-def _clean_up_preview(preview: str) -> str:
-    return preview.rstrip()[:10]
+
+    logger.debug(f"Fetch object store download url")
+    url = api_client.object_store.get(key, organization_id)
+
+    logger.debug(f"Start downloading: {url}")
+    progress = logger.progress(suffix="{task.percentage:0.0f}% ({task.completed:,.0f}/{task.total:,.0f})")
+    progress_task = progress.add_task("", total=1)
+
+    from uuid import uuid4
+    temp_file = f"{str(uuid4())}.zip"
+    api_client.data.download_url(url, temp_file, lambda advance: progress.update(progress_task, advance=advance))
+
+    if not destination_folder:
+        from os import getcwd
+        destination_folder = getcwd()
+
+    logger.debug(f"Unzipping object store keys values into: '{destination_folder}'")
+    from zipfile import ZipFile
+    with ZipFile(temp_file, 'r') as zip_ref:
+        zip_ref.extractall(destination_folder)
 
 

lean/commands/cloud/object_store/properties.py

@@ -0,0 +1,55 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from click import command, argument
+from lean.click import LeanCommand
+from lean.container import container
+
+
+@command(cls=LeanCommand)
+@argument("key", type=str)
+def properties(key: str) -> str:
+    """
+    Get a value properties from the organization's cloud object store.
+
+    """
+    organization_id = container.organization_manager.try_get_working_organization_id()
+    api_client = container.api_client
+    logger = container.logger
+    data = api_client.object_store.properties(key, organization_id)
+
+    try:
+        headers = ["size", "modified", "key", "preview"]
+        display_headers = ["Bytes", "Modified", "Filename", "Preview"]
+        data_row = []
+        for header in headers:
+            if header == "preview":
+                value = str(data["metadata"].get(header, "N/A"))
+                data_row.append(_clean_up_preview(value))
+            else:
+                value = str(data["metadata"].get(header, ""))
+                data_row.append(value)
+        all_rows = [display_headers] + [data_row]
+        column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))]
+        for row in all_rows:
+            logger.info("  ".join(value.ljust(width) for value, width in zip(row, column_widths)))
+    except KeyError as e:
+        logger.error(f"Key {key} not found.")
+    except Exception as e:
+        logger.error(f"Error: {e}")
+
+
+def _clean_up_preview(preview: str) -> str:
+    return preview.rstrip()[:10]
+
+

lean/commands/live/deploy.py

@@ -282,9 +282,11 @@ def deploy(project: Path,
                                                                  environment_name=environment_name))
 
     organization_id = container.organization_manager.try_get_working_organization_id()
+    paths_to_mount = {}
     for module in (data_provider_live_instances + [data_downloader_instances, brokerage_instance]
                    + history_providers_instances):
         module.ensure_module_installed(organization_id)
+        paths_to_mount.update(module.get_paths_to_mount())
 
     if not lean_config["environments"][environment_name]["live-mode"]:
         raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)",
@@ -362,4 +364,13 @@ def deploy(project: Path,
             raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")
 
     lean_runner = container.lean_runner
-    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
+    lean_runner.run_lean(lean_config,
+                         environment_name,
+                         algorithm_file,
+                         output,
+                         engine_image,
+                         None,
+                         release,
+                         detach,
+                         loads(extra_docker_config),
+                         paths_to_mount)

lean/commands/object_store/__init__.py

@@ -16,9 +16,11 @@
 from lean.commands.object_store.set import set
 from lean.commands.object_store.list import list
 from lean.commands.object_store.delete import delete
+from lean.commands.object_store.properties import properties
 
 object_store.add_command(get)
 object_store.add_command(set)
 object_store.add_command(list)
 object_store.add_command(delete)
+object_store.add_command(properties)
 

lean/commands/object_store/properties.py

@@ -0,0 +1,30 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from click import command
+from lean.click import LeanCommand
+from lean.container import container
+from lean.components.util.object_store_helper import open_storage_directory_in_explorer
+
+
+@command(cls=LeanCommand)
+def properties() -> str:
+    """
+    Opens the local storage directory in the file explorer.
+
+    """
+    open_storage_directory_in_explorer(container.lean_config_manager)
+
+
+

lean/commands/optimize.py

@@ -305,11 +305,14 @@ def optimize(project: Path,
     if download_data:
         data_provider_historical = "QuantConnect"
 
+    paths_to_mount = None
+
     if data_provider_historical is not None:
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
 
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
@@ -339,7 +342,7 @@ def optimize(project: Path,
     container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
     run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach,
-                                                      engine_image)
+                                                      engine_image, paths_to_mount)
 
     run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug"
     run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}")

lean/commands/research.py

@@ -113,12 +113,15 @@ def research(project: Path,
     if download_data:
         data_provider_historical = "QuantConnect"
 
+    paths_to_mount = None
+
     if data_provider_historical is not None:
         organization_id = container.organization_manager.try_get_working_organization_id()
         data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical,
                                                               cli_data_downloaders, kwargs, logger, environment_name)
         data_provider.ensure_module_installed(organization_id)
         container.lean_config_manager.set_properties(data_provider.get_settings())
+        paths_to_mount = data_provider.get_paths_to_mount()
     lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit)
 
     lean_runner = container.lean_runner
@@ -145,7 +148,8 @@ def research(project: Path,
                                                       None,
                                                       False,
                                                       detach,
-                                                      research_image)
+                                                      research_image,
+                                                      paths_to_mount)
 
     # Mount the config in the notebooks directory as well
     local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json"))