diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..c50f3e3 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,27 @@ + + +#### Expected Behavior + + +#### Actual Behavior + + +#### Potential Solution + + +#### Reproducing the Problem + + +#### System Information + + +#### Checklist + + +- [ ] I have completely filled out this template +- [ ] I have confirmed that this issue exists on the current `master` branch +- [ ] I have confirmed that this is not a duplicate issue by searching [issues](https://github.com/QuantConnect/Lean/issues) + +- [ ] I have provided detailed steps to reproduce the issue + + \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..1418a96 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,42 @@ + + + +#### Description + + +#### Related Issue + + + + + +#### Motivation and Context + + +#### Requires Documentation Change + + +#### How Has This Been Tested? + + + + +#### Types of changes + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] Refactor (non-breaking change which improves implementation) +- [ ] Performance (non-breaking change which improves performance. Please add associated performance test and results) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Non-functional change (xml comments/documentation/etc) + +#### Checklist: + + +- [ ] My code follows the code style of this project. +- [ ] I have read the **CONTRIBUTING** [document](https://github.com/QuantConnect/Lean/blob/master/CONTRIBUTING.md). +- [ ] I have added tests to cover my changes. +- [ ] All new and existing tests passed. +- [ ] My branch follows the naming convention `bug--` or `feature--` + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09449fc..d20f2a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,18 +9,18 @@ on: jobs: build: runs-on: ubuntu-20.04 - + env: + QC_JOB_USER_ID: ${{ secrets.QC_JOB_USER_ID }} + QC_API_ACCESS_TOKEN: ${{ secrets.QC_API_ACCESS_TOKEN }} + QC_JOB_ORGANIZATION_ID: ${{ secrets.QC_JOB_ORGANIZATION_ID }} + QC_ALPHA_VANTAGE_API_KEY: ${{ secrets.QC_ALPHA_VANTAGE_API_KEY }} steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Free space run: df -h && rm -rf /opt/hostedtoolcache* && df -h - - name: Pull Foundation Image - uses: addnab/docker-run-action@v3 - with: - image: quantconnect/lean:foundation - - name: Checkout Lean Same Branch id: lean-same-branch uses: actions/checkout@v2 @@ -40,11 +40,17 @@ jobs: - name: Move Lean run: mv Lean ../Lean - - name: BuildDataSource - run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Pull Foundation Image + uses: addnab/docker-run-action@v3 + with: + image: quantconnect/lean:foundation + options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.AlphaVantage/Lean.DataSource.AlphaVantage -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_ALPHA_VANTAGE_API_KEY=${{ secrets.QC_ALPHA_VANTAGE_API_KEY }} + + - name: Build QuantConnect.AlphaVantage + run: dotnet build ./QuantConnect.AlphaVantage/QuantConnect.AlphaVantage.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: BuildTests - run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build QuantConnect.AlphaVantage.Tests + run: dotnet build ./QuantConnect.AlphaVantage.Tests/QuantConnect.AlphaVantage.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Run Tests - run: dotnet test ./tests/bin/Release/net6.0/Tests.dll + - name: Run QuantConnect.AlphaVantage.Tests + run: dotnet test ./QuantConnect.AlphaVantage.Tests/bin/Release/QuantConnect.AlphaVantage.Tests.dll \ No newline at end of file diff --git a/DataProcessing/CLRImports.py b/DataProcessing/CLRImports.py deleted file mode 100644 index fca9342..0000000 --- a/DataProcessing/CLRImports.py +++ /dev/null @@ -1,12 +0,0 @@ -# This file is used to import the environment and classes/methods of LEAN. -# so that any python file could be using LEAN's classes/methods. -from clr_loader import get_coreclr -from pythonnet import set_runtime - -# process.runtimeconfig.json is created when we build the DataProcessing Project: -# dotnet build .\DataProcessing\DataProcessing.csproj -set_runtime(get_coreclr('process.runtimeconfig.json')) - -from AlgorithmImports import * -from QuantConnect.Lean.Engine.DataFeeds import * -AddReference("Fasterflect") \ No newline at end of file diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj deleted file mode 100644 index 26b5466..0000000 --- a/DataProcessing/DataProcessing.csproj +++ /dev/null @@ -1,30 +0,0 @@ - - - Exe - net6.0 - process - true - - - - - - - - - - - - - - - PreserveNewest - - - - - - PreserveNewest - - - \ No newline at end of file diff --git a/DataProcessing/MyCustomDataDownloader.cs b/DataProcessing/MyCustomDataDownloader.cs deleted file mode 100644 index 1bfef32..0000000 --- a/DataProcessing/MyCustomDataDownloader.cs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json; -using QuantConnect.Configuration; -using QuantConnect.Data.Auxiliary; -using QuantConnect.DataSource; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.DataProcessing -{ - /// - /// MyCustomDataDownloader implementation. - /// - public class MyCustomDataDownloader : IDisposable - { - public const string VendorName = "VendorName"; - public const string VendorDataName = "VendorDataName"; - - private readonly string _destinationFolder; - private readonly string _universeFolder; - private readonly string _clientKey; - private readonly string _dataFolder = Globals.DataFolder; - private readonly bool _canCreateUniverseFiles; - private readonly int _maxRetries = 5; - private static readonly List _defunctDelimiters = new() - { - '-', - '_' - }; - private ConcurrentDictionary> _tempData = new(); - - private readonly JsonSerializerSettings _jsonSerializerSettings = new() - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc - }; - - /// - /// Control the rate of download per unit of time. - /// - private readonly RateGate _indexGate; - - /// - /// Creates a new instance of - /// - /// The folder where the data will be saved - /// The Vendor API key - public MyCustomDataDownloader(string destinationFolder, string apiKey = null) - { - _destinationFolder = Path.Combine(destinationFolder, VendorDataName); - _universeFolder = Path.Combine(_destinationFolder, "universe"); - _clientKey = apiKey ?? Config.Get("vendor-auth-token"); - _canCreateUniverseFiles = Directory.Exists(Path.Combine(_dataFolder, "equity", "usa", "map_files")); - - // Represents rate limits of 10 requests per 1.1 second - _indexGate = new RateGate(10, TimeSpan.FromSeconds(1.1)); - - Directory.CreateDirectory(_destinationFolder); - Directory.CreateDirectory(_universeFolder); - } - - /// - /// Runs the instance of the object. - /// - /// True if process all downloads successfully - public bool Run() - { - var stopwatch = Stopwatch.StartNew(); - var today = DateTime.UtcNow.Date; - - throw new NotImplementedException(); - - Log.Trace($"MyCustomDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); - return true; - } - - /// - /// Sends a GET request for the provided URL - /// - /// URL to send GET request for - /// Content as string - /// Failed to get data after exceeding retries - private async Task HttpRequester(string url) - { - for (var retries = 1; retries <= _maxRetries; retries++) - { - try - { - using (var client = new HttpClient()) - { - client.BaseAddress = new Uri(""); - client.DefaultRequestHeaders.Clear(); - - // You must supply your API key in the HTTP header, - // otherwise you will receive a 403 Forbidden response - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", _clientKey); - - // Responses are in JSON: you need to specify the HTTP header Accept: application/json - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - // Makes sure we don't overrun Quiver rate limits accidentally - _indexGate.WaitToProceed(); - - var response = await client.GetAsync(Uri.EscapeUriString(url)); - if (response.StatusCode == HttpStatusCode.NotFound) - { - Log.Error($"MyCustomDataDownloader.HttpRequester(): Files not found at url: {Uri.EscapeUriString(url)}"); - response.DisposeSafely(); - return string.Empty; - } - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - var finalRequestUri = response.RequestMessage.RequestUri; // contains the final location after following the redirect. - response = client.GetAsync(finalRequestUri).Result; // Reissue the request. The DefaultRequestHeaders configured on the client will be used, so we don't have to set them again. - } - - response.EnsureSuccessStatusCode(); - - var result = await response.Content.ReadAsStringAsync(); - response.DisposeSafely(); - - return result; - } - } - catch (Exception e) - { - Log.Error(e, $"MyCustomDataDownloader.HttpRequester(): Error at HttpRequester. (retry {retries}/{_maxRetries})"); - Thread.Sleep(1000); - } - } - - throw new Exception($"Request failed with no more retries remaining (retry {_maxRetries}/{_maxRetries})"); - } - - /// - /// Saves contents to disk, deleting existing zip files - /// - /// Final destination of the data - /// file name - /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) - { - name = name.ToLowerInvariant(); - var finalPath = Path.Combine(destinationFolder, $"{name}.csv"); - var finalFileExists = File.Exists(finalPath); - - var lines = new HashSet(contents); - if (finalFileExists) - { - foreach (var line in File.ReadAllLines(finalPath)) - { - lines.Add(line); - } - } - - var finalLines = destinationFolder.Contains("universe") ? - lines.OrderBy(x => x.Split(',').First()).ToList() : - lines - .OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) - .ToList(); - - var tempPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.tmp"); - File.WriteAllLines(tempPath, finalLines); - var tempFilePath = new FileInfo(tempPath); - tempFilePath.MoveTo(finalPath, true); - } - - /// - /// Tries to normalize a potentially defunct ticker into a normal ticker. - /// - /// Ticker as received from Estimize - /// Set as the non-defunct ticker - /// true for success, false for failure - private static bool TryNormalizeDefunctTicker(string ticker, out string nonDefunctTicker) - { - // The "defunct" indicator can be in any capitalization/case - if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0) - { - foreach (var delimChar in _defunctDelimiters) - { - var length = ticker.IndexOf(delimChar); - - // Continue until we exhaust all delimiters - if (length == -1) - { - continue; - } - - nonDefunctTicker = ticker[..length].Trim(); - return true; - } - - nonDefunctTicker = string.Empty; - return false; - } - - nonDefunctTicker = ticker; - return true; - } - - /// - /// Disposes of unmanaged resources - /// - public void Dispose() - { - _indexGate?.Dispose(); - } - } -} \ No newline at end of file diff --git a/DataProcessing/Program.cs b/DataProcessing/Program.cs deleted file mode 100644 index b203eb3..0000000 --- a/DataProcessing/Program.cs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.IO; -using QuantConnect.Configuration; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.DataProcessing -{ - /// - /// Entrypoint for the data downloader/converter - /// - public class Program - { - /// - /// Entrypoint of the program - /// - /// Exit code. 0 equals successful, and any other value indicates the downloader/converter failed. - public static void Main() - { - // Get the config values first before running. These values are set for us - // automatically to the value set on the website when defining this data type - var destinationDirectory = Path.Combine( - Config.Get("temp-output-directory", "/temp-output-directory"), - "alternative", - "vendorname"); - - MyCustomDataDownloader instance = null; - try - { - // Pass in the values we got from the configuration into the downloader/converter. - instance = new MyCustomDataDownloader(destinationDirectory); - } - catch (Exception err) - { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data failed to be constructed"); - Environment.Exit(1); - } - - // No need to edit anything below here for most use cases. - // The downloader/converter is ran and cleaned up for you safely here. - try - { - // Run the data downloader/converter. - var success = instance.Run(); - if (!success) - { - Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {MyCustomDataDownloader.VendorName} {MyCustomDataDownloader.VendorDataName} data"); - Environment.Exit(1); - } - } - catch (Exception err) - { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data exited unexpectedly"); - Environment.Exit(1); - } - finally - { - // Run cleanup of the downloader/converter once it has finished or crashed. - instance.DisposeSafely(); - } - - // The downloader/converter was successful - Environment.Exit(0); - } - } -} \ No newline at end of file diff --git a/DataProcessing/config.json b/DataProcessing/config.json deleted file mode 100644 index 1d1e2f2..0000000 --- a/DataProcessing/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "data-folder": "../../../Data/", - - "vendor-auth-token": "" -} \ No newline at end of file diff --git a/DataProcessing/process.sample.ipynb b/DataProcessing/process.sample.ipynb deleted file mode 100644 index 2493379..0000000 --- a/DataProcessing/process.sample.ipynb +++ /dev/null @@ -1,78 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "9b8eae46", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection)\n", - "# Requirements:\n", - "# python -m pip install clr-loader==0.1.7\n", - "# python -m pip install pythonnet==3.0.0a2\n", - "# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed\n", - "# dotnet build .\\DataProcessing\\\n", - "import os\n", - "from CLRImports import *\n", - "\n", - "# To use QuantBook, we need to set its internal handlers\n", - "# We download LEAN confif with the default settings \n", - "with open(\"quantbook.json\", 'w') as fp:\n", - " from requests import get\n", - " response = get(\"https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json\")\n", - " fp.write(response.text)\n", - "\n", - "Config.SetConfigurationFile(\"quantbook.json\")\n", - "Config.Set(\"composer-dll-directory\", os.path.abspath(''))\n", - "\n", - "# Set the data folder\n", - "Config.Set(\"data-folder\", '')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ddc2ed2-5690-422c-8c91-6e6f64dd45cb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# To generate the Security Identifier, we need to create and initialize the Map File Provider\n", - "# and call the SecurityIdentifier.GenerateEquity method\n", - "mapFileProvider = LocalZipMapFileProvider()\n", - "mapFileProvider.Initialize(DefaultDataProvider())\n", - "sid = SecurityIdentifier.GenerateEquity(\"SPY\", Market.USA, True, mapFileProvider, datetime(2022, 3, 1))\n", - "\n", - "qb = QuantBook()\n", - "symbol = Symbol(sid, \"SPY\")\n", - "history = qb.History(symbol, 3600, Resolution.Daily)\n", - "print(history)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DataProcessing/process.sample.py b/DataProcessing/process.sample.py deleted file mode 100644 index 5b8285d..0000000 --- a/DataProcessing/process.sample.py +++ /dev/null @@ -1,32 +0,0 @@ -# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection) -# Requirements: -# python -m pip install clr-loader==0.1.7 -# python -m pip install pythonnet==3.0.0a2 -# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed -# dotnet build .\DataProcessing\ -import os -from CLRImports import * - -# To use QuantBook, we need to set its internal handlers -# We download LEAN confif with the default settings -with open("quantbook.json", 'w') as fp: - from requests import get - response = get("https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json") - fp.write(response.text) - -Config.SetConfigurationFile("quantbook.json") -Config.Set("composer-dll-directory", os.path.dirname(os.path.realpath(__file__))) - -# Set the data folder -Config.Set("data-folder", '') - -# To generate the Security Identifier, we need to create and initialize the Map File Provider -# and call the SecurityIdentifier.GenerateEquity method -mapFileProvider = LocalZipMapFileProvider() -mapFileProvider.Initialize(DefaultDataProvider()) -sid = SecurityIdentifier.GenerateEquity("SPY", Market.USA, True, mapFileProvider, datetime(2022, 3, 1)) - -qb = QuantBook() -symbol = Symbol(sid, "SPY") -history = qb.History(symbol, 3600, Resolution.Daily) -print(history) \ No newline at end of file diff --git a/DataProcessing/process.sample.sh b/DataProcessing/process.sample.sh deleted file mode 100644 index e69de29..0000000 diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index fc0eb1f..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,77 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Data; -using QuantConnect.Util; -using QuantConnect.Orders; -using QuantConnect.Algorithm; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataAlgorithm : QCAlgorithm - { - private Symbol _customDataSymbol; - private Symbol _equitySymbol; - - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - SetStartDate(2013, 10, 07); //Set Start Date - SetEndDate(2013, 10, 11); //Set End Date - _equitySymbol = AddEquity("SPY").Symbol; - _customDataSymbol = AddData(_equitySymbol).Symbol; - } - - /// - /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - /// - /// Slice object keyed by symbol containing the stock data - public override void OnData(Slice slice) - { - var data = slice.Get(); - if (!data.IsNullOrEmpty()) - { - // based on the custom data property we will buy or short the underlying equity - if (data[_customDataSymbol].SomeCustomProperty == "buy") - { - SetHoldings(_equitySymbol, 1); - } - else if (data[_customDataSymbol].SomeCustomProperty == "sell") - { - SetHoldings(_equitySymbol, -1); - } - } - } - - /// - /// Order fill event handler. On an order fill update the resulting information is passed to this method. - /// - /// Order event details containing details of the events - public override void OnOrderEvent(OrderEvent orderEvent) - { - if (orderEvent.Status.IsFill()) - { - Debug($"Purchased Stock: {orderEvent.Symbol}"); - } - } - } -} diff --git a/Demonstration.py b/Demonstration.py deleted file mode 100644 index 2f55e0d..0000000 --- a/Demonstration.py +++ /dev/null @@ -1,47 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataAlgorithm(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.''' - - self.SetStartDate(2020, 10, 7) #Set Start Date - self.SetEndDate(2020, 10, 11) #Set End Date - self.equity_symbol = self.AddEquity("SPY", Resolution.Daily).Symbol - self.custom_data_symbol = self.AddData(MyCustomDataType, self.equity_symbol).Symbol - - def OnData(self, slice): - ''' OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - - :param Slice slice: Slice object keyed by symbol containing the stock data - ''' - data = slice.Get(MyCustomDataType) - if data: - custom_data = data[self.custom_data_symbol] - if custom_data.SomeCustomProperty == "buy": - self.SetHoldings(self.equitySymbol, 1) - elif custom_data.SomeCustomProperty == "sell": - self.SetHoldings(self.equitySymbol, -1) - - def OnOrderEvent(self, orderEvent): - ''' Order fill event handler. On an order fill update the resulting information is passed to this method. - - :param OrderEvent orderEvent: Order event details containing details of the events - ''' - if orderEvent.Status == OrderStatus.Fill: - self.Debug(f'Purchased Stock: {orderEvent.Symbol}') \ No newline at end of file diff --git a/DemonstrationUniverse.cs b/DemonstrationUniverse.cs deleted file mode 100644 index d8b962c..0000000 --- a/DemonstrationUniverse.cs +++ /dev/null @@ -1,66 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.DataSource; - -namespace QuantConnect.Algorithm.CSharp -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataUniverse : QCAlgorithm - { - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - // Data ADDED via universe selection is added with Daily resolution. - UniverseSettings.Resolution = Resolution.Daily; - - SetStartDate(2022, 2, 14); - SetEndDate(2022, 2, 18); - SetCash(100000); - - // add a custom universe data source (defaults to usa-equity) - AddUniverse("MyCustomDataUniverseType", Resolution.Daily, data => - { - foreach (var datum in data) - { - Log($"{datum.Symbol},{datum.SomeCustomProperty},{datum.SomeNumericProperty}"); - } - - // define our selection criteria - return from d in data - where d.SomeCustomProperty == "buy" - select d.Symbol; - }); - } - - /// - /// Event fired each time that we add/remove securities from the data feed - /// - /// Security additions/removals for this time step - public override void OnSecuritiesChanged(SecurityChanges changes) - { - Log(changes.ToString()); - } - } -} \ No newline at end of file diff --git a/DemonstrationUniverse.py b/DemonstrationUniverse.py deleted file mode 100644 index 80b3657..0000000 --- a/DemonstrationUniverse.py +++ /dev/null @@ -1,50 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataUniverse(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. ''' - - # Data ADDED via universe selection is added with Daily resolution. - self.UniverseSettings.Resolution = Resolution.Daily - - self.SetStartDate(2022, 2, 14) - self.SetEndDate(2022, 2, 18) - self.SetCash(100000) - - # add a custom universe data source (defaults to usa-equity) - self.AddUniverse(MyCustomDataUniverseType, "MyCustomDataUniverseType", Resolution.Daily, self.UniverseSelection) - - def UniverseSelection(self, data): - ''' Selected the securities - - :param List of MyCustomUniverseType data: List of MyCustomUniverseType - :return: List of Symbol objects ''' - - for datum in data: - self.Log(f"{datum.Symbol},{datum.Followers},{datum.DayPercentChange},{datum.WeekPercentChange}") - - # define our selection criteria - return [d.Symbol for d in data if d.SomeCustomProperty == 'buy'] - - def OnSecuritiesChanged(self, changes): - ''' Event fired each time that we add/remove securities from the data feed - - :param SecurityChanges changes: Security additions/removals for this time step - ''' - self.Log(changes.ToString()) \ No newline at end of file diff --git a/DropboxDownloader.py b/DropboxDownloader.py deleted file mode 100644 index 54b7e55..0000000 --- a/DropboxDownloader.py +++ /dev/null @@ -1,119 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to download files from a given dropbox directory. -# Files to be downloaded are filtered based on given date present in file name. - -# ARGUMENTS -# DROPBOX_API_KEY: Dropbox API KEY with read access. -# DROPBOX_SOURCE_DIRECTORY: path of the dropbox directory to search files within. -# DROPBOX_OUTPUT_DIRECTORY(optional): base path of the output directory to store to downloaded files. -# cmdline args expected in order: DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - -import requests -import json -import sys -import time -import os -from pathlib import Path - -DROPBOX_API_KEY = os.environ.get("DROPBOX_API_KEY") -DROPBOX_SOURCE_DIRECTORY = os.environ.get("DROPBOX_SOURCE_DIRECTORY") -QC_DATAFLEET_DEPLOYMENT_DATE = os.environ.get("QC_DATAFLEET_DEPLOYMENT_DATE") -DROPBOX_OUTPUT_DIRECTORY = os.environ.get("DROPBOX_OUTPUT_DIRECTORY", "/raw") - -def DownloadZipFile(filePath): - - print(f"Starting downloading file at: {filePath}") - - # defining the api-endpoint - API_ENDPOINT_DOWNLOAD = "https://content.dropboxapi.com/2/files/download" - - # data to be sent to api - data = {"path": filePath} - - headers = {"Authorization": f"Bearer {DROPBOX_API_KEY}", - "Dropbox-API-Arg": json.dumps(data)} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_DOWNLOAD, headers=headers) - - response.raise_for_status() # ensure we notice bad responses - - fileName = filePath.split("/")[-1] - outputPath = os.path.join(DROPBOX_OUTPUT_DIRECTORY, fileName) - - with open(outputPath, "wb") as f: - f.write(response.content) - print(f"Succesfully saved file at: {outputPath}") - -def GetFilePathsFromDate(targetLocation, dateString): - # defining the api-endpoint - API_ENDPOINT_FILEPATH = "https://api.dropboxapi.com/2/files/list_folder" - - headers = {"Content-Type": "application/json", - "Authorization": f"Bearer {DROPBOX_API_KEY}"} - - # data to be sent to api - data = {"path": targetLocation, - "recursive": False, - "include_media_info": False, - "include_deleted": False, - "include_has_explicit_shared_members": False, - "include_mounted_folders": True, - "include_non_downloadable_files": True} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_FILEPATH, headers=headers, data = json.dumps(data)) - - response.raise_for_status() # ensure we notice bad responses - - target_paths = [entry["path_display"] for entry in response.json()["entries"] if dateString in entry["path_display"]] - return target_paths - -def main(): - global DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - inputCount = len(sys.argv) - if inputCount > 1: - DROPBOX_API_KEY = sys.argv[1] - if inputCount > 2: - DROPBOX_SOURCE_DIRECTORY = sys.argv[2] - if inputCount > 3: - QC_DATAFLEET_DEPLOYMENT_DATE = sys.argv[3] - if inputCount > 4: - DROPBOX_OUTPUT_DIRECTORY = sys.argv[4] - - # make output path if doesn't exists - Path(DROPBOX_OUTPUT_DIRECTORY).mkdir(parents=True, exist_ok=True) - - target_paths = GetFilePathsFromDate(DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE) - print(f"Found {len(target_paths)} files with following paths {target_paths}") - - #download files - for path in target_paths: - count = 0 - maxTries = 3 - while True: - try: - DownloadZipFile(path) - break - except Exception as e: - count +=1 - if count > maxTries: - print(f"Error for file with path {path} --error message: {e}") - break - print(f"Error, sleep for 5 sec and retry download file with --path: {path}") - time.sleep(5) - -if __name__== "__main__": - main() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Lean.DataSource.AlphaVantage.sln b/Lean.DataSource.AlphaVantage.sln new file mode 100644 index 0000000..64e4214 --- /dev/null +++ b/Lean.DataSource.AlphaVantage.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.AlphaVantage", "QuantConnect.AlphaVantage\QuantConnect.AlphaVantage.csproj", "{0230D4D2-9D1B-42AF-A17C-6B7ADE5F6CC1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.AlphaVantage.Tests", "QuantConnect.AlphaVantage.Tests\QuantConnect.AlphaVantage.Tests.csproj", "{C7B6B18A-92D8-4CE7-86AE-9DB5A1BBA27D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0230D4D2-9D1B-42AF-A17C-6B7ADE5F6CC1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0230D4D2-9D1B-42AF-A17C-6B7ADE5F6CC1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0230D4D2-9D1B-42AF-A17C-6B7ADE5F6CC1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0230D4D2-9D1B-42AF-A17C-6B7ADE5F6CC1}.Release|Any CPU.Build.0 = Release|Any CPU + {C7B6B18A-92D8-4CE7-86AE-9DB5A1BBA27D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7B6B18A-92D8-4CE7-86AE-9DB5A1BBA27D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7B6B18A-92D8-4CE7-86AE-9DB5A1BBA27D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7B6B18A-92D8-4CE7-86AE-9DB5A1BBA27D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {3C3446A4-57C1-4201-95F3-C85256FB1C8D} + EndGlobalSection +EndGlobal diff --git a/MyCustomDataType.cs b/MyCustomDataType.cs deleted file mode 100644 index 8c8a025..0000000 --- a/MyCustomDataType.cs +++ /dev/null @@ -1,156 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataType : BaseData - { - /// - /// Some custom data property - /// - [ProtoMember(2000)] - public string SomeCustomProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - $"{config.Symbol.Value.ToLowerInvariant()}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); - return new MyCustomDataType - { - Symbol = config.Symbol, - SomeCustomProperty = csv[1], - Time = parsedDate - Period, - }; - } - - /// - /// Clones the data - /// - /// A clone of the object - public override BaseData Clone() - { - return new MyCustomDataType - { - Symbol = Symbol, - Time = Time, - EndTime = EndTime, - SomeCustomProperty = SomeCustomProperty, - }; - } - - /// - /// Indicates whether the data source is tied to an underlying symbol and requires that corporate events be applied to it as well, such as renames and delistings - /// - /// false - public override bool RequiresMapping() - { - return true; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {SomeCustomProperty}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} diff --git a/MyCustomDataUniverseType.cs b/MyCustomDataUniverseType.cs deleted file mode 100644 index 5db2f4b..0000000 --- a/MyCustomDataUniverseType.cs +++ /dev/null @@ -1,141 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; -using System.Globalization; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataUniverseType : BaseData - { - /// - /// Some custom data property - /// - public string SomeCustomProperty { get; set; } - - /// - /// Some custom data property - /// - public decimal SomeNumericProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - "universe", - $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var someNumericProperty = decimal.Parse(csv[2], NumberStyles.Any, CultureInfo.InvariantCulture); - - return new MyCustomDataUniverseType - { - Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]), - SomeNumericProperty = someNumericProperty, - SomeCustomProperty = csv[3], - Time = date - Period, - Value = someNumericProperty - }; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {Value}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} \ No newline at end of file diff --git a/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderAdditionalTests.cs b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderAdditionalTests.cs new file mode 100644 index 0000000..41714e5 --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderAdditionalTests.cs @@ -0,0 +1,44 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Configuration; + +namespace QuantConnect.AlphaVantage.Tests +{ + [TestFixture] + public class AlphaVantageDataDownloaderAdditionalTests + { + [TestCase("freee")] + [TestCase("Plan88")] + public void InvalidPricePlanThrowException(string pricePlan) + { + Config.Set("alpha-vantage-price-plan", pricePlan); + Assert.Throws(() => new AlphaVantageDataDownloader()); + } + + [TestCase("")] + [TestCase("free")] + [TestCase("Free")] + [TestCase("Plan30")] + [TestCase("plan30")] + public void ValidPricePlanDoesNotThrow(string pricePlan) + { + Config.Set("alpha-vantage-price-plan", pricePlan); + Assert.DoesNotThrow(() => new AlphaVantageDataDownloader()); + } + } +} diff --git a/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderMockTests.cs b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderMockTests.cs new file mode 100644 index 0000000..8fb2105 --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderMockTests.cs @@ -0,0 +1,313 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Moq; +using System; +using RestSharp; +using System.Net; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Data.Market; +using System.Collections.Generic; + +namespace QuantConnect.AlphaVantage.Tests +{ + [TestFixture] + public class AlphaVantageDataDownloaderMockTests + { + private const string API_KEY = "TESTKEY"; + private const string BASE_URL = "https://www.alphavantage.co/"; + + private Mock _avClient; + private AlphaVantageDataDownloader _downloader; + private readonly TradeBarComparer _tradeBarComparer = new TradeBarComparer(); + + [SetUp] + public void SetUp() + { + _avClient = new Mock(); + _avClient.SetupAllProperties(); + + _downloader = new AlphaVantageDataDownloader(_avClient.Object, API_KEY); + } + + [TearDown] + public void TearDown() + { + if (_downloader != null ) + { + _downloader.Dispose(); + } + } + + [Test] + public void GetDailyLessThan100DaysGetsCompactDailyData() + { + var ticker = "AAPL"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var resolution = Resolution.Daily; + var start = new DateTime(2021, 4, 4); + var end = new DateTime(2021, 7, 12); + + var expectedBars = new[] + { + new TradeBar(DateTime.Parse("2021-04-05"), symbol, 133.64m, 136.69m, 133.40m, 135.93m, 5471616), + new TradeBar(DateTime.Parse("2021-04-06"), symbol, 135.58m, 135.64m, 134.09m, 134.22m, 3620964), + }; + + IRestRequest request = null; + _avClient.Setup(m => m.Execute(It.IsAny(), It.IsAny())) + .Callback((IRestRequest r, Method m) => request = r) + .Returns(new RestResponse + { + StatusCode = HttpStatusCode.OK, + ContentType = "application/x-download", + Content = "timestamp,open,high,low,close,volume\n" + + "2021-04-06,135.5800,135.6400,134.0900,134.2200,3620964\n" + + "2021-04-05,133.6400,136.6900,133.4000,135.9300,5471616\n" + }) + .Verifiable(); + + var result = _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)); + + _avClient.Verify(); + var requestUrl = BuildUrl(request); + Assert.AreEqual(Method.GET, request.Method); + Assert.AreEqual($"{BASE_URL}query?symbol=AAPL&datatype=csv&function=TIME_SERIES_DAILY", requestUrl); + + Assert.IsInstanceOf>(result); + var bars = ((IEnumerable)result).ToList(); + Assert.AreEqual(2, bars.Count); + Assert.That(bars[0], Is.EqualTo(expectedBars[0]).Using(_tradeBarComparer)); + Assert.That(bars[1], Is.EqualTo(expectedBars[1]).Using(_tradeBarComparer)); + } + + [Test] + public void GetDailyGreaterThan100DaysGetsFullDailyData() + { + var ticker = "AAPL"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var resolution = Resolution.Daily; + var start = new DateTime(2021, 4, 4); + var end = new DateTime(2024, 1, 1); + + var expectedBars = new[] + { + new TradeBar(DateTime.Parse("2021-04-05"), symbol, 133.64m, 136.69m, 133.40m, 135.93m, 5471616), + new TradeBar(DateTime.Parse("2021-04-06"), symbol, 135.58m, 135.64m, 134.09m, 134.22m, 3620964), + }; + + IRestRequest request = null; + _avClient.Setup(m => m.Execute(It.IsAny(), It.IsAny())) + .Callback((IRestRequest r, Method m) => request = r) + .Returns(new RestResponse + { + StatusCode = HttpStatusCode.OK, + ContentType = "application/x-download", + Content = "timestamp,open,high,low,close,volume\n" + + "2021-04-06,135.5800,135.6400,134.0900,134.2200,3620964\n" + + "2021-04-05,133.6400,136.6900,133.4000,135.9300,5471616\n" + }) + .Verifiable(); + + var result = _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)); + + _avClient.Verify(); + var requestUrl = BuildUrl(request); + Assert.AreEqual(Method.GET, request.Method); + Assert.AreEqual($"{BASE_URL}query?symbol=AAPL&datatype=csv&function=TIME_SERIES_DAILY&outputsize=full", requestUrl); + + Assert.IsInstanceOf>(result); + var bars = ((IEnumerable)result).ToList(); + Assert.AreEqual(2, bars.Count); + Assert.That(bars[0], Is.EqualTo(expectedBars[0]).Using(_tradeBarComparer)); + Assert.That(bars[1], Is.EqualTo(expectedBars[1]).Using(_tradeBarComparer)); + } + + [TestCase(Resolution.Minute, "1min")] + [TestCase(Resolution.Hour, "60min")] + public void GetMinuteHourGetsIntradayData(Resolution resolution, string interval) + { + var ticker = "IBM"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var year = DateTime.UtcNow.Year - 1; + var start = new DateTime(year, 04, 05, 0, 0, 0); + var end = new DateTime(year, 05, 06, 15, 0, 0); + + var expectedBars = new[] + { + new TradeBar(start.AddHours(9.5), symbol, 133.71m, 133.72m, 133.62m, 133.62m, 1977), + new TradeBar(start.AddHours(10.5), symbol, 134.30m, 134.56m, 134.245m, 134.34m, 154723), + new TradeBar(end.AddHours(-5.5), symbol, 135.54m, 135.56m, 135.26m, 135.28m, 2315), + new TradeBar(end.AddHours(-4.5), symbol, 134.905m,134.949m, 134.65m, 134.65m, 101997), + }; + + var responses = new[] + { + "time,open,high,low,close,volume\n" + + $"{year}-04-05 10:30:00,134.3,134.56,134.245,134.34,154723\n" + + $"{year}-04-05 09:30:00,133.71,133.72,133.62,133.62,1977\n", + "time,open,high,low,close,volume\n" + + $"{year}-05-06 10:30:00,134.905,134.949,134.65,134.65,101997\n" + + $"{year}-05-06 09:30:00,135.54,135.56,135.26,135.28,2315\n", + }; + + var requestCount = 0; + var requestUrls = new List(); + _avClient.Setup(m => m.Execute(It.IsAny(), It.IsAny())) + .Callback((IRestRequest r, Method m) => requestUrls.Add(BuildUrl(r))) + .Returns(() => new RestResponse + { + StatusCode = HttpStatusCode.OK, + ContentType = "application/x-download", + Content = responses[requestCount++] + }) + .Verifiable(); + + var result = _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)).ToList(); + + _avClient.Verify(); + Assert.AreEqual(2, requestUrls.Count); + Assert.AreEqual($"{BASE_URL}query?symbol=IBM&datatype=csv&function=TIME_SERIES_INTRADAY&adjusted=false&outputsize=full&interval={interval}&month=2023-04", requestUrls[0]); + Assert.AreEqual($"{BASE_URL}query?symbol=IBM&datatype=csv&function=TIME_SERIES_INTRADAY&adjusted=false&outputsize=full&interval={interval}&month=2023-05", requestUrls[1]); + + var bars = result.Cast().ToList(); + Assert.AreEqual(4, bars.Count); + Assert.That(bars[0], Is.EqualTo(expectedBars[0]).Using(_tradeBarComparer)); + Assert.That(bars[1], Is.EqualTo(expectedBars[1]).Using(_tradeBarComparer)); + Assert.That(bars[2], Is.EqualTo(expectedBars[2]).Using(_tradeBarComparer)); + Assert.That(bars[3], Is.EqualTo(expectedBars[3]).Using(_tradeBarComparer)); + } + + [TestCase(Resolution.Tick)] + [TestCase(Resolution.Second)] + public void GetUnsupportedResolutionThrowsException(Resolution resolution) + { + var ticker = "IBM"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var start = DateTime.UtcNow.AddMonths(-2); + var end = DateTime.UtcNow; + + Assert.Throws(() => _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)).ToList()); + } + + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Daily)] + public void UnexpectedResponseContentTypeThrowsException(Resolution resolution) + { + var ticker = "IBM"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var start = DateTime.UtcNow.AddMonths(-2); + var end = DateTime.UtcNow; + + _avClient.Setup(m => m.Execute(It.IsAny(), It.IsAny())) + .Returns(() => new RestResponse + { + StatusCode = HttpStatusCode.OK, + ContentType = "application/json" + }) + .Verifiable(); + + Assert.Throws(() => _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)).ToList()); + } + + [Test] + public void GetIntradayDataGreaterThanTwoYears() + { + var ticker = "IBM"; + var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA); + var resolution = Resolution.Minute; + var start = new DateTime(2020, 4, 4); + var end = new DateTime(2020, 6, 5); + + var expectedBars = new[] + { + new TradeBar(DateTime.Parse("2020-04-05 7:34:00"), symbol, 133.64m, 136.69m, 133.40m, 135.93m, 5471616), + new TradeBar(DateTime.Parse("2020-04-05 7:35:00"), symbol, 135.58m, 135.64m, 134.09m, 134.22m, 3620964), + }; + + IRestRequest request = null; + _avClient.Setup(m => m.Execute(It.IsAny(), It.IsAny())) + .Callback((IRestRequest r, Method m) => request = r) + .Returns(new RestResponse + { + StatusCode = HttpStatusCode.OK, + ContentType = "application/x-download", + Content = "timestamp,open,high,low,close,volume\n" + + "2020-04-05 7:34:00,133.6400,136.6900,133.4000,135.9300,5471616\n" + + "2020-04-05 7:35:00,135.5800,135.6400,134.0900,134.2200,3620964\n" + + "2020-04-05 7:35:00,135.5800,135.6400,134.0900,134.2200,3620964\n" + + "2020-04-06 7:36:00,133.6400,136.6900,133.4000,135.9300,5471616\n" + + "2020-05-06 7:37:00,133.6400,136.6900,133.4000,135.9300,5471616\n" + + "2020-05-07 7:38:00,133.6400,136.6900,133.4000,135.9300,5471616\n" + + "2020-05-07 7:39:00,133.6400,136.6900,133.4000,135.9300,5471616\n" + }) + .Verifiable(); + + var result = _downloader.Get(new DataDownloaderGetParameters(symbol, resolution, start, end)).ToList(); + + _avClient.Verify(); + var requestUrl = BuildUrl(request); + Assert.That(request.Method, Is.EqualTo(Method.GET)); + Assert.That(requestUrl, Is.EqualTo($"{BASE_URL}query?symbol=IBM&datatype=csv&function=TIME_SERIES_INTRADAY&adjusted=false&outputsize=full&interval=1min&month=2020-06")); + + Assert.That(result.Count, Is.EqualTo(7 * 3)); // 6 mock result * 3 request per each month + Assert.That(result[0], Is.EqualTo(expectedBars[0]).Using(_tradeBarComparer)); + Assert.That(result[1], Is.EqualTo(expectedBars[1]).Using(_tradeBarComparer)); + } + + [Test] + public void AuthenticatorAddsApiKeyToRequest() + { + var request = new Mock(); + var authenticator = _avClient.Object.Authenticator; + + Assert.NotNull(authenticator); + authenticator.Authenticate(_avClient.Object, request.Object); + request.Verify(m => m.AddOrUpdateParameter("apikey", API_KEY)); + } + + private string BuildUrl(IRestRequest request) + { + if (request == null) + throw new ArgumentNullException(nameof(request)); + + var client = new RestClient(BASE_URL); + var uri = client.BuildUri(request); + + return uri.ToString(); + } + + private class TradeBarComparer : IEqualityComparer + { + public bool Equals(TradeBar x, TradeBar y) + { + if (x == null || y == null) + return false; + + return x.Symbol == y.Symbol && + x.Time == y.Time && + x.Open == y.Open && + x.High == y.High && + x.Low == y.Low && + x.Close == y.Close && + x.Volume == y.Volume; + } + + public int GetHashCode(TradeBar obj) => obj.GetHashCode(); + } + } +} diff --git a/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderTests.cs b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderTests.cs new file mode 100644 index 0000000..32e997e --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/AlphaVantageDataDownloaderTests.cs @@ -0,0 +1,137 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Util; +using QuantConnect.Securities; +using QuantConnect.Data.Market; +using System.Collections.Generic; + +namespace QuantConnect.AlphaVantage.Tests +{ + [TestFixture] + public class AlphaVantageDataDownloaderTests + { + private AlphaVantageDataDownloader _downloader; + private MarketHoursDatabase _marketHoursDatabase; + + [SetUp] + public void SetUp() + { + _downloader = new(); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } + + [TearDown] + public void TearDown() + { + if (_downloader != null) + { + _downloader.Dispose(); + } + } + + /// + /// Provides a collection of valid test case data for the Downloader method. + /// Each TestCaseData represents a valid set of data to be used in testing. + /// It is important to note that the test cases should not exceed 25 requests. (Because free api key gives only 25 request per day) + /// + /// + /// An IEnumerable of TestCaseData, each representing a valid set of data for the Downloader method. + /// + public static IEnumerable DownloaderValidCaseData + { + get + { + var symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA); + + yield return new TestCaseData(symbol, Resolution.Minute, new DateTime(2024, 1, 1, 5, 30, 0), new DateTime(2024, 2, 1, 20, 0, 0), TickType.Trade); + yield return new TestCaseData(symbol, Resolution.Minute, new DateTime(2024, 1, 8, 9, 30, 0), new DateTime(2024, 1, 12, 16, 0, 0), TickType.Trade); + yield return new TestCaseData(symbol, Resolution.Minute, new DateTime(2015, 2, 2, 9, 30, 0), new DateTime(2015, 3, 1, 16, 0, 0), TickType.Trade); + yield return new TestCaseData(symbol, Resolution.Hour, new DateTime(2023, 11, 8, 9, 30, 0), new DateTime(2024, 2, 2, 16, 0, 0), TickType.Trade); + yield return new TestCaseData(symbol, Resolution.Daily, new DateTime(2023, 1, 8, 9, 30, 0), new DateTime(2024, 2, 2, 16, 0, 0), TickType.Trade); + } + } + + [TestCaseSource(nameof(DownloaderValidCaseData))] + public void DownloadDataWithDifferentValidParameters(Symbol symbol, Resolution resolution, DateTime startUtc, DateTime endUtc, TickType tickType) + { + var downloadParameters = new DataDownloaderGetParameters(symbol, resolution, startUtc, endUtc, tickType); + + var baseData = _downloader.Get(downloadParameters).ToList(); + + Assert.IsNotEmpty(baseData); + Assert.IsTrue(baseData.First().Time >= ConvertUtcTimeToSymbolExchange(symbol, startUtc)); + Assert.IsTrue(baseData.Last().Time <= ConvertUtcTimeToSymbolExchange(symbol, endUtc)); + + foreach (var data in baseData) + { + Assert.IsTrue(data.DataType == MarketDataType.TradeBar); + var tradeBar = data as TradeBar; + Assert.Greater(tradeBar.Open, 0m); + Assert.Greater(tradeBar.High, 0m); + Assert.Greater(tradeBar.Low, 0m); + Assert.Greater(tradeBar.Close, 0m); + Assert.IsTrue(tradeBar.Period.ToHigherResolutionEquivalent(true) == resolution); + } + } + + public static IEnumerable DownloaderInvalidCaseData + { + get + { + var symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA); + + var startUtc = new DateTime(2024, 1, 1); + var endUtc = new DateTime(2024, 2, 1); + + yield return new TestCaseData(symbol, Resolution.Minute, startUtc, endUtc, TickType.Quote, false) + .SetDescription($"Not supported {nameof(TickType.Quote)} -> empty result"); + yield return new TestCaseData(symbol, Resolution.Minute, startUtc, endUtc, TickType.OpenInterest, false) + .SetDescription($"Not supported {nameof(TickType.OpenInterest)} -> empty result"); + yield return new TestCaseData(symbol, Resolution.Tick, startUtc, endUtc, TickType.Trade, true) + .SetDescription($"Not supported {nameof(Resolution.Tick)} -> throw Exception"); + yield return new TestCaseData(symbol, Resolution.Second, startUtc, endUtc, TickType.Trade, true) + .SetDescription($"Not supported {nameof(Resolution.Second)} -> throw Exception"); + yield return new TestCaseData(symbol, Resolution.Minute, endUtc, startUtc, TickType.Trade, false) + .SetDescription("startDateTime > endDateTime -> empty result"); + } + } + + [TestCaseSource(nameof(DownloaderInvalidCaseData))] + public void DownloadDataWithDifferentInvalidParameters(Symbol symbol, Resolution resolution, DateTime start, DateTime end, TickType tickType, bool isThrowException) + { + var downloadParameters = new DataDownloaderGetParameters(symbol, resolution, start, end, tickType); + + if (isThrowException) + { + Assert.Throws(() => _downloader.Get(downloadParameters).ToList()); + return; + } + + var baseData = _downloader.Get(downloadParameters).ToList(); + + Assert.IsEmpty(baseData); + } + + private DateTime ConvertUtcTimeToSymbolExchange(Symbol symbol, DateTime dateTimeUtc) + { + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + return dateTimeUtc.ConvertFromUtc(exchangeTimeZone); + } + } +} diff --git a/QuantConnect.AlphaVantage.Tests/QuantConnect.AlphaVantage.Tests.csproj b/QuantConnect.AlphaVantage.Tests/QuantConnect.AlphaVantage.Tests.csproj new file mode 100644 index 0000000..e533dcf --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/QuantConnect.AlphaVantage.Tests.csproj @@ -0,0 +1,39 @@ + + + + Release + AnyCPU + net6.0 + false + UnitTest + bin\$(Configuration)\ + QuantConnect.AlphaVantage.Tests + QuantConnect.AlphaVantage.Tests + QuantConnect.AlphaVantage.Tests + QuantConnect.AlphaVantage.Tests + false + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + PreserveNewest + + + + diff --git a/QuantConnect.AlphaVantage.Tests/TestSetup.cs b/QuantConnect.AlphaVantage.Tests/TestSetup.cs new file mode 100644 index 0000000..71297b0 --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/TestSetup.cs @@ -0,0 +1,66 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.IO; +using NUnit.Framework; +using System.Collections; +using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.AlphaVantage.Tests +{ + [SetUpFixture] + public class TestSetup + { + [OneTimeSetUp] + public void GlobalSetup() + { + // Log.DebuggingEnabled = true; + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); + + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) + { + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); + + if (envKey.StartsWith("QC_")) + { + var key = envKey.Substring(3).Replace("_", "-").ToLower(); + + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); + } + } + + // resets the version among other things + Globals.Reset(); + } + } +} diff --git a/QuantConnect.AlphaVantage.Tests/config.json b/QuantConnect.AlphaVantage.Tests/config.json new file mode 100644 index 0000000..a39ae2f --- /dev/null +++ b/QuantConnect.AlphaVantage.Tests/config.json @@ -0,0 +1,7 @@ +{ + "data-folder": "../../../../Lean/Data/", + "data-directory": "../../../../Lean/Data/", + + "alpha-vantage-api-key": "", + "alpha-vantage-price-plan": "" +} diff --git a/QuantConnect.AlphaVantage/AlphaVantageAuthenticator.cs b/QuantConnect.AlphaVantage/AlphaVantageAuthenticator.cs new file mode 100644 index 0000000..a0a3d10 --- /dev/null +++ b/QuantConnect.AlphaVantage/AlphaVantageAuthenticator.cs @@ -0,0 +1,51 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using RestSharp; +using RestSharp.Authenticators; + +namespace QuantConnect.AlphaVantage +{ + /// + /// Implements authentication for Alpha Vantage API + /// + internal class AlphaVantageAuthenticator : IAuthenticator + { + private readonly string _apiKey; + + /// + /// Construct authenticator + /// + /// API key + /// See https://www.alphavantage.co/support/#api-key to get a free key. + public AlphaVantageAuthenticator(string apiKey) + { + if (string.IsNullOrEmpty(apiKey)) + { + throw new ArgumentNullException(nameof(apiKey)); + } + + _apiKey = apiKey; + } + + /// + /// Authenticate request + /// + /// The + /// The + public void Authenticate(IRestClient client, IRestRequest request) + => request.AddOrUpdateParameter("apikey", _apiKey); + } +} diff --git a/QuantConnect.AlphaVantage/AlphaVantageDataDownloader.cs b/QuantConnect.AlphaVantage/AlphaVantageDataDownloader.cs new file mode 100644 index 0000000..383cbad --- /dev/null +++ b/QuantConnect.AlphaVantage/AlphaVantageDataDownloader.cs @@ -0,0 +1,484 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NodaTime; +using CsvHelper; +using RestSharp; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; +using QuantConnect.Data; +using QuantConnect.Util; +using System.Globalization; +using QuantConnect.Logging; +using Newtonsoft.Json.Linq; +using QuantConnect.Securities; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Collections.Concurrent; +using System.Net.NetworkInformation; +using QuantConnect.AlphaVantage.Enums; + +namespace QuantConnect.AlphaVantage +{ + /// + /// Alpha Vantage data downloader + /// + public class AlphaVantageDataDownloader : IDataDownloader, IDisposable + { + private readonly MarketHoursDatabase _marketHoursDatabase; + private readonly IRestClient _avClient; + private readonly RateGate _rateGate; + private bool _disposed; + + /// + /// Represents a mapping of symbols to their corresponding time zones for exchange information. + /// + private readonly ConcurrentDictionary _symbolExchangeTimeZones = new ConcurrentDictionary(); + + /// + /// A dictionary that associates each Alpha Vantage Premium Membership with its corresponding rate limit. + /// + public readonly Dictionary RateLimits = new() + { + { AlphaVantagePricePlan.Free, 25 }, + { AlphaVantagePricePlan.Plan30, 30 }, + { AlphaVantagePricePlan.Plan75, 75 }, + { AlphaVantagePricePlan.Plan150, 150 }, + { AlphaVantagePricePlan.Plan300, 300 }, + { AlphaVantagePricePlan.Plan600, 600 }, + { AlphaVantagePricePlan.Plan1200, 1200 } + }; + + /// + /// Initializes a new instance of the + /// getting the AlphaVantage API key from the configuration + /// + public AlphaVantageDataDownloader() : this(Config.Get("alpha-vantage-api-key")) + { + } + + /// + /// Construct AlphaVantageDataDownloader with default RestClient + /// + /// API key + public AlphaVantageDataDownloader(string apiKey) : this(new RestClient(), apiKey) + { + } + + /// + /// Dependency injection constructor + /// + /// The to use + /// API key + public AlphaVantageDataDownloader(IRestClient restClient, string apiKey) + { + _avClient = restClient; + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + _avClient.BaseUrl = new Uri("https://www.alphavantage.co/"); + _avClient.Authenticator = new AlphaVantageAuthenticator(apiKey); + + var pricePlan = GetAlphaVantagePricePlan(); + + _rateGate = new RateGate(RateLimits[pricePlan], Time.OneMinute); + + ValidateSubscription(); + } + + /// + /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). + /// + /// model class for passing in parameters for historical data + /// Enumerable of base data for this symbol + public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetParameters) + { + var symbol = dataDownloaderGetParameters.Symbol; + var resolution = dataDownloaderGetParameters.Resolution; + var startUtc = dataDownloaderGetParameters.StartUtc; + var endUtc = dataDownloaderGetParameters.EndUtc; + var tickType = dataDownloaderGetParameters.TickType; + + if (endUtc < startUtc) + { + Log.Error($"{nameof(AlphaVantageDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + return Enumerable.Empty(); + } + + if (tickType != TickType.Trade) + { + Log.Error($"{nameof(AlphaVantageDataDownloader)}.{nameof(Get)}: Not supported data type - {tickType}. " + + $"Currently available support only for historical of type - TradeBar"); + return Enumerable.Empty(); + } + + var request = new RestRequest("query", DataFormat.Json); + request.AddParameter("symbol", symbol.Value); + request.AddParameter("datatype", "csv"); + + IEnumerable data = null; + switch (resolution) + { + case Resolution.Minute: + case Resolution.Hour: + data = GetIntradayData(request, startUtc, endUtc, resolution); + break; + case Resolution.Daily: + data = GetDailyData(request, startUtc, endUtc, symbol); + break; + default: + throw new ArgumentOutOfRangeException(nameof(resolution), $"{resolution} resolution not supported by API."); + } + + var period = resolution.ToTimeSpan(); + var startBySymbolExchange = ConvertTickTimeBySymbol(symbol, startUtc); + var endBySymbolExchange = ConvertTickTimeBySymbol(symbol, endUtc); + + return data.Where(d => d.Time >= startBySymbolExchange && d.Time <= endBySymbolExchange).Select(d => new TradeBar(d.Time, symbol, d.Open, d.High, d.Low, d.Close, d.Volume, period)); + } + + /// + /// Get data from daily API + /// + /// Base request + /// Start time + /// End time + /// Symbol to download + /// + private IEnumerable GetDailyData(RestRequest request, DateTime startUtc, DateTime endUtc, Symbol symbol) + { + request.AddParameter("function", "TIME_SERIES_DAILY"); + + // The default output only includes 100 trading days of data. If we want need more, specify full output + if (GetBusinessDays(startUtc, endUtc, symbol) > 100) + { + request.AddParameter("outputsize", "full"); + } + + return GetTimeSeries(request); + } + + /// + /// This API returns current and 20+ years of historical intraday OHLCV time series of the equity specified + /// https://www.alphavantage.co/documentation/#intraday-extended + /// + /// The exchange uses Eastern Time for the US market + /// Base request + /// Start time + /// End time + /// Data resolution to request + /// + private IEnumerable GetIntradayData(RestRequest request, DateTime startUtc, DateTime endUtc, Resolution resolution) + { + request.AddParameter("function", "TIME_SERIES_INTRADAY"); + request.AddParameter("adjusted", "false"); + request.AddParameter("outputsize", "full"); + switch (resolution) + { + case Resolution.Minute: + request.AddParameter("interval", "1min"); + break; + case Resolution.Hour: + request.AddParameter("interval", "60min"); + break; + default: + throw new ArgumentOutOfRangeException($"{resolution} resolution not supported by intraday API."); + } + + var slices = GetSlices(startUtc, endUtc); + foreach (var slice in slices) + { + request.AddOrUpdateParameter("month", slice); + var data = GetTimeSeries(request); + foreach (var record in data) + { + yield return record; + } + } + } + + /// + /// Execute request and parse response. + /// + /// The request + /// data + private IEnumerable GetTimeSeries(RestRequest request) + { + if (_rateGate.IsRateLimited) + { + Log.Trace("Requests are limited to 5 per minute. Reduce the time between start and end times or simply wait, and this process will continue automatically."); + } + + _rateGate.WaitToProceed(); + //var url = _avClient.BuildUri(request); + Log.Trace("Downloading /{0}?{1}", request.Resource, string.Join("&", request.Parameters.Where(p => p.Name != "apikey"))); + var response = _avClient.Get(request); + + if (response.ContentType != "application/x-download") + { + throw new FormatException($"Unexpected content received from API.\n{response.Content}"); + } + + using (var reader = new StringReader(response.Content)) + { + using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture)) + { + return csv.GetRecords() + .OrderBy(t => t.Time) + .ToList(); // Execute query before readers are disposed. + } + } + } + + /// + /// Get slice names for date range. + /// See https://www.alphavantage.co/documentation/#intraday-extended + /// + /// Start date + /// End date + /// Slice names + private static IEnumerable GetSlices(DateTime startUtc, DateTime endUtc) + { + do + { + yield return startUtc.ToString("yyyy-MM"); + startUtc = startUtc.AddMonths(1); + } while (startUtc.Date <= endUtc.Date); + } + + /// + /// From https://stackoverflow.com/questions/1617049/calculate-the-number-of-business-days-between-two-dates + /// + private int GetBusinessDays(DateTime start, DateTime end, Symbol symbol) + { + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + + var current = start.Date; + var days = 0; + while (current < end) + { + if (exchangeHours.IsDateOpen(current)) + { + days++; + } + current = current.AddDays(1); + } + + return days; + } + + /// + /// Converts the provided tick timestamp, given in DateTime UTC Kind, to the exchange time zone associated with the specified Lean symbol. + /// + /// The Lean symbol for which the timestamp is associated. + /// The DateTime in Utc format Kind + /// A DateTime object representing the converted timestamp in the exchange time zone. + private DateTime ConvertTickTimeBySymbol(Symbol symbol, DateTime dateTimeUtc) + { + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out var exchangeTimeZone)) + { + // read the exchange time zone from market-hours-database + exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + _symbolExchangeTimeZones.TryAdd(symbol, exchangeTimeZone); + } + + return dateTimeUtc.ConvertFromUtc(exchangeTimeZone); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // dispose managed state (managed objects) + _rateGate.Dispose(); + } + + // free unmanaged resources (unmanaged objects) and override finalizer + _disposed = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// + /// Retrieves the Alpha Vantage Price Plan based on the configured settings. + /// If no plan is specified or an invalid plan is provided, it defaults to the Free Price Plan. + /// + /// The Alpha Vantage Price Plan. + /// Thrown when an invalid or empty Price Plan is encountered. + private AlphaVantagePricePlan GetAlphaVantagePricePlan() + { + if (Config.TryGetValue("alpha-vantage-price-plan", "Free", out var plan) && string.IsNullOrEmpty(plan)) + { + Log.Trace("Using the Free Price Plan by default. To change the Price Plan, please set the 'alpha-vantage-price-plan' configuration explicitly."); + plan = "Free"; + } + + if (!Enum.TryParse(plan, true, out var pricePlan)) + { + throw new ArgumentException($"Invalid Price Plan: `{plan}`. Please ensure that the Price Plan is set and not empty."); + } + + return pricePlan; + } + + private class ModulesReadLicenseRead : Api.RestResponse + { + [JsonProperty(PropertyName = "license")] + public string License; + + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; + } + + /// + /// Validate the user of this project has permission to be using it via our web API. + /// + private static void ValidateSubscription() + { + try + { + const int productId = 334; + var userId = Config.GetInt("job-user-id"); + var token = Config.Get("api-access-token"); + var organizationId = Config.Get("job-organization-id", null); + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) + { + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() + { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information + try + { + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) + { + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) + { + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); + } + } + information.Add("networkInterfaces", interfaceDictionary); + } + catch (Exception) + { + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); + } + var request = new RestRequest("modules/license/read", Method.POST) { RequestFormat = DataFormat.Json }; + request.AddParameter("application/json", JsonConvert.SerializeObject(information), ParameterType.RequestBody); + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) + { + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } + + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) + { + organizationId = result.OrganizationId; + } + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) + { + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); + } + } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); + } + } + catch (Exception e) + { + Log.Error($"{nameof(AlphaVantageDataDownloader)}.{nameof(ValidateSubscription)}: Failed during validation, shutting down. Error : {e.Message}"); + throw; + } + } + } +} diff --git a/QuantConnect.AlphaVantage/Enums/AlphaVantagePricePlan.cs b/QuantConnect.AlphaVantage/Enums/AlphaVantagePricePlan.cs new file mode 100644 index 0000000..9f877b7 --- /dev/null +++ b/QuantConnect.AlphaVantage/Enums/AlphaVantagePricePlan.cs @@ -0,0 +1,59 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.AlphaVantage.Enums +{ + /// + /// Represents different plans with associated request limits and monthly prices. + /// premium plans + /// + public enum AlphaVantagePricePlan + { + /// + /// 25 requests per day + free access + /// + Free = 0, + + /// + /// 30 API requests per minute + 15-minute delayed US market data + /// + Plan30 = 1, + + /// + /// 75 API requests per minute + 15-minute delayed US market data + /// + Plan75 = 2, + + /// + /// 150 API requests per minute + Realtime US market data + /// + Plan150 = 3, + + /// + /// 300 API requests per minute + Realtime US market data + /// + Plan300 = 4, + + /// + /// 600 API requests per minute + Realtime US market data + /// + Plan600 = 5, + + /// + /// 1200 API requests per minute + Realtime US market data + /// + Plan1200 = 6, + } +} diff --git a/QuantConnect.AlphaVantage/QuantConnect.AlphaVantage.csproj b/QuantConnect.AlphaVantage/QuantConnect.AlphaVantage.csproj new file mode 100644 index 0000000..d8a9649 --- /dev/null +++ b/QuantConnect.AlphaVantage/QuantConnect.AlphaVantage.csproj @@ -0,0 +1,39 @@ + + + + Release + AnyCPU + net6.0 + QuantConnect.AlphaVantage + QuantConnect.AlphaVantage + QuantConnect.AlphaVantage + QuantConnect.AlphaVantage + Library + bin\$(Configuration) + false + true + false + QuantConnect LEAN AlphaVantage Data Source: AlphaVantage Data Source plugin for Lean + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + + + + + diff --git a/QuantConnect.AlphaVantage/TimeSeries.cs b/QuantConnect.AlphaVantage/TimeSeries.cs new file mode 100644 index 0000000..39d06f7 --- /dev/null +++ b/QuantConnect.AlphaVantage/TimeSeries.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using CsvHelper.Configuration.Attributes; + +namespace QuantConnect.AlphaVantage +{ + /// + /// Alpha Vantage time series model + /// + internal class TimeSeries + { + [Name("time", "timestamp")] + public DateTime Time { get; set; } + + [Name("open")] + public decimal Open { get; set; } + + [Name("high")] + public decimal High { get; set; } + + [Name("low")] + public decimal Low { get; set; } + + [Name("close")] + public decimal Close { get; set; } + + [Name("volume")] + public decimal Volume { get; set; } + } +} diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj deleted file mode 100644 index f379576..0000000 --- a/QuantConnect.DataSource.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net6.0 - QuantConnect.DataSource - QuantConnect.DataSource.MyCustomDataType - bin\$(Configuration) - $(OutputPath)\QuantConnect.DataSource.MyCustomDataType.xml - - - - - - - - - - - - - - - - - - - - diff --git a/README.md b/README.md index 6dabf21..2f9622b 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,63 @@ ![LEAN Data Source SDK](https://github.com/QuantConnect/Lean.DataSource.AlphaVantage/assets/79997186/edaf8bbe-592a-4ac0-9b8a-cf98a591d7a3) -# Lean DataSource SDK +# Lean Alpha Vantage DataSource Plugin -[![Build Status](https://github.com/QuantConnect/LeanDataSdk/workflows/Build%20%26%20Test/badge.svg)](https://github.com/QuantConnect/LeanDataSdk/actions?query=workflow%3A%22Build%20%26%20Test%22) +[![Build Status](https://github.com/QuantConnect/LeanDataSdk/workflows/Build%20%26%20Test/badge.svg)](https://github.com/QuantConnect/LeanDataSdk/actions?query=workflow%3A%22Build%20%26%20Test%22) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ### Introduction -The Lean Data SDK is a cross-platform template repository for developing custom data types for Lean. -These data types will be consumed by [QuantConnect](https://www.quantconnect.com/) trading algorithms and research environment, locally or in the cloud. +Welcome to the Alpha Vantage API Connector Library for .NET 6. This open-source project provides a robust and efficient C# library designed to seamlessly connect with the Alpha Vantage API. The library facilitates easy integration with the QuantConnect [LEAN Algorithmic Trading Engine](https://github.com/quantConnect/Lean), offering a clear and straightforward way for users to incorporate Alpha Vantage's extensive financial datasets into their algorithmic trading strategies. -It is composed by example .Net solution for the data type and converter scripts. +### Alpha Vantage Overview +Alpha Vantage provides real-time and historical financial market data through a set of powerful and developer-friendly data APIs. From traditional asset classes (e.g., stocks, ETFs, mutual funds) to economic indicators, from foreign exchange rates to commodities, from fundamental data to technical indicators, Alpha Vantage is your one-stop-shop for enterprise-grade global market data delivered through cloud-based APIs. -### Prerequisites +### Features +- **Easy Integration:** Simple and intuitive functions to connect with the Alpha Vantage API, tailored for seamless use within the QuantConnect LEAN Algorithmic Trading Engine. -The solution targets dotnet 5, for installation instructions please follow [dotnet download](https://dotnet.microsoft.com/download). +- **Rich Financial Data:** Access a wide range of financial data, including stock quotes, technical indicators, historical prices, and more, provided by Alpha Vantage. -The data downloader and converter script can be developed in different ways: C# executable, Python script, Python Jupyter notebook or even a bash script. -- The python script should be compatible with python 3.6.8 -- Bash script will run on Ubuntu Bionic +- **Flexible Configuration:** The library supports various configuration options, including API key, allowing users to tailor their requests to specific needs. -Specifically, the enviroment where these scripts will be run is [quantconnect/research](https://hub.docker.com/repository/docker/quantconnect/research) based on [quantconnect/lean:foundation](https://hub.docker.com/repository/docker/quantconnect/lean). +- **Symbol SecurityType Support:** The library currently supports the following symbol security types: + - [x] Equity + - [ ] Option + - [ ] Commodity + - [ ] Forex + - [ ] Future + - [ ] Crypto + - [ ] Index -### Installation - -The "Use this template" feature should be used for each unique data source which requires its own data processing. Once it is cloned locally, you should be able to successfully build the solution, run all tests and execute the downloader and/or conveter scripts. The final version should pass all CI tests of GitHub Actions. - -Once ready, please contact support@quantconnect.com and we will create a listing in the QuantConnect Data Market for your company and link to your public repository and commit hash. - -### Datasets Vendor Requirements +- **Backtesting and Research:** Test your algorithm in backtest and research modes within [QuantConnect.Lean CLI](https://www.quantconnect.com/docs/v2/lean-cli), leveraging the Alpha Vantage API data to refine and optimize your trading strategies. -Key requirements for new vendors include: +### Contribute to the Project +Contributions to this open-source project are welcome! If you find any issues, have suggestions for improvements, or want to add new features, please open an issue or submit a pull request. - - A well-defined dataset with a clear and static vision for the data to minimize churn or changes as people will be building systems from it. This is easiest with "raw" data (e.g. sunshine hours vs a sentiment algorithm) - - Robust ticker and security links to ensure the tickers are tracked well through time, or accurately point in time. ISIN, FIGI, or point in time ticker supported - - Robust funding to ensure viable for at least 1 year - - Robust API to ensure reliable up-time. No dead links on site or and 502 servers while using API - - Consistent delivery schedule, on time and in time for market trading - - Consistent data format with notifications and lead time on data format updates - - At least 1 year of historical point in time data - - Survivorship bias free data - - Good documentation for the dataset - - -### Tutorials +### Installation - - See [Tutorials](https://www.quantconnect.com/docs/v2/our-platform/datasets/contributing-datasets) for a step by step guide for creating a new LEAN Data Source. +To contribute to the Alpha Vantage API Connector Library for .NET 6 within QuantConnect LEAN, follow these steps: +1. **Obtain API Key:** Sign up for a free Alpha Vantage API key [here](https://www.alphavantage.co/) if you don't have one. +2. **Fork the Project:** Fork the repository by clicking the "Fork" button at the top right of the GitHub page. +3. Clone Your Forked Repository: +``` +git clone https://github.com/your-username/alpha-vantage-connector-dotnet.git +``` +4. **Configuration:** + - Set the `alpha-vantage-api-key` in your QuantConnect configuration (config.json or environment variables). + - [optional] Set the `alpha-vantage-price-plan` (by default: Free) +``` +{ + "alpha-vantage-api-key": "", + "alpha-vantage-price-plan": "", +} +``` + +### Price Plan +For detailed information on Alpha Vantage's pricing plans, please refer to the [Alpha Vantage Pricing](https://www.alphavantage.co/premium/) page. + +### Documentation +Refer to the [documentation](https://www.quantconnect.com/docs/v2/lean-cli/datasets/alphavantage) for detailed information on the library's functions, parameters, and usage examples. + +### License +This project is licensed under the MIT License - see the [LICENSE](#) file for details. + +Happy coding and algorithmic trading! diff --git a/examples.md b/examples.md deleted file mode 100644 index 9086ae3..0000000 --- a/examples.md +++ /dev/null @@ -1 +0,0 @@ -https://github.com/QuantConnect?q=Lean.DataSource&type=&language=&sort= \ No newline at end of file diff --git a/output/alternative/mycustomdatatype/spy.csv b/output/alternative/mycustomdatatype/spy.csv deleted file mode 100644 index e450a4d..0000000 --- a/output/alternative/mycustomdatatype/spy.csv +++ /dev/null @@ -1,6 +0,0 @@ -20131001,buy -20131003,buy -20131006,buy -20131007,sell -20131009,buy -20131011,sell \ No newline at end of file diff --git a/renameDataset.sh b/renameDataset.sh deleted file mode 100644 index 51ccd6d..0000000 --- a/renameDataset.sh +++ /dev/null @@ -1,57 +0,0 @@ -# Get {vendorNameDatasetName} -vendorNameDatasetName=${PWD##*.} -vendorNameDatasetNameUniverse=${vendorNameDatasetName}Universe - -# Rename the MyCustomDataType.cs file to {vendorNameDatasetName}.cs -mv MyCustomDataType.cs ${vendorNameDatasetName}.cs -mv MyCustomDataUniverseType.cs ${vendorNameDatasetNameUniverse}.cs - -# In the QuantConnect.DataSource.csproj file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" QuantConnect.DataSource.csproj -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" QuantConnect.DataSource.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" QuantConnect.DataSource.csproj - -# In the {vendorNameDatasetName}.cs file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" ${vendorNameDatasetName}.cs - -# In the {vendorNameDatasetNameUniverse}.cs file, rename the MyCustomDataUniverseType class to {vendorNameDatasetNameUniverse} -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" ${vendorNameDatasetNameUniverse}.cs - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the MyCustomDataType class to to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.cs -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.py - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetName}Algorithm -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.cs -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.py - -# In the {vendorNameDatasetName}UniverseSelectionAlgorithm.cs file, rename the MyCustomDataUniverseType class to to {vendorNameDatasetName}Universe -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.cs -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.py - -# In the {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetNameUniverse}SelectionAlgorithm -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.cs -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.py - -# Rename the Lean.DataSource.vendorNameDatasetName/Demonstration.cs/py file to {vendorNameDatasetName}Algorithm.cs/py -mv Demonstration.cs ${vendorNameDatasetName}Algorithm.cs -mv Demonstration.py ${vendorNameDatasetName}Algorithm.py - -# Rename the Lean.DataSource.vendorNameDatasetName/DemonstrationUniverseSelectionAlgorithm.cs/py file to {vendorNameDatasetName}UniverseSelectionAlgorithm.cs/py -mv DemonstrationUniverse.cs ${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -mv DemonstrationUniverse.py ${vendorNameDatasetNameUniverse}SelectionAlgorithm.py - -# Rename the tests/MyCustomDataTypeTests.cs file to tests/{vendorNameDatasetName}Tests.cs -sed -i "s/MyCustomDataType/${vendorNameDatasetName}/g" tests/MyCustomDataTypeTests.cs -mv tests/MyCustomDataTypeTests.cs tests/${vendorNameDatasetName}Tests.cs - -# In tests/Tests.csproj, rename the Demonstration.cs and DemonstrationUniverse.cs to {vendorNameDatasetName}Algorithm.cs and {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" tests/Tests.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" tests/Tests.csproj - -# In the MyCustomDataDownloader.cs and Program.cs files, rename the MyCustomDataDownloader to {vendorNameDatasetNameUniverse}DataDownloader -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/Program.cs -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/MyCustomDataDownloader.cs - -# Rename the DataProcessing/MyCustomDataDownloader.cs file to DataProcessing/{vendorNameDatasetName}DataDownloader.cs -mv DataProcessing/MyCustomDataDownloader.cs DataProcessing/${vendorNameDatasetName}DataDownloader.cs \ No newline at end of file diff --git a/tests/MyCustomDataTypeTests.cs b/tests/MyCustomDataTypeTests.cs deleted file mode 100644 index c0b907c..0000000 --- a/tests/MyCustomDataTypeTests.cs +++ /dev/null @@ -1,99 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using ProtoBuf; -using System.IO; -using System.Linq; -using ProtoBuf.Meta; -using Newtonsoft.Json; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - [TestFixture] - public class MyCustomDataTypeTests - { - [Test] - public void JsonRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - var serialized = JsonConvert.SerializeObject(expected); - var result = JsonConvert.DeserializeObject(serialized, type); - - AssertAreEqual(expected, result); - } - - [Test] - public void ProtobufRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - - RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2000, type); - - using (var stream = new MemoryStream()) - { - Serializer.Serialize(stream, expected); - - stream.Position = 0; - - var result = Serializer.Deserialize(type, stream); - - AssertAreEqual(expected, result, filterByCustomAttributes: true); - } - } - - [Test] - public void Clone() - { - var expected = CreateNewInstance(); - var result = expected.Clone(); - - AssertAreEqual(expected, result); - } - - private void AssertAreEqual(object expected, object result, bool filterByCustomAttributes = false) - { - foreach (var propertyInfo in expected.GetType().GetProperties()) - { - // we skip Symbol which isn't protobuffed - if (filterByCustomAttributes && propertyInfo.CustomAttributes.Count() != 0) - { - Assert.AreEqual(propertyInfo.GetValue(expected), propertyInfo.GetValue(result)); - } - } - foreach (var fieldInfo in expected.GetType().GetFields()) - { - Assert.AreEqual(fieldInfo.GetValue(expected), fieldInfo.GetValue(result)); - } - } - - private BaseData CreateNewInstance() - { - return new MyCustomDataType - { - Symbol = Symbol.Empty, - Time = DateTime.Today, - DataType = MarketDataType.Base, - SomeCustomProperty = "This is some market related information" - }; - } - } -} \ No newline at end of file diff --git a/tests/Tests.csproj b/tests/Tests.csproj deleted file mode 100644 index 8e308d0..0000000 --- a/tests/Tests.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - net6.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - - all - - - - - - - - -