From 674f19ac7714dbe9ffb13c539a56603e66fb2914 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 25 Jan 2024 00:07:34 +0200 Subject: [PATCH 1/2] remove: ToolBox\IEX --- Launcher/config.json | 3 - .../DataFeeds/IEXDataQueueHandlerTests.cs | 429 -------------- ToolBox/IEX/IEXDataDownloader.cs | 83 --- ToolBox/IEX/IEXDataQueueHandler.cs | 524 ------------------ ToolBox/IEX/IEXDownloaderProgram.cs | 81 --- ToolBox/IEX/IEXEventSourceCollection.cs | 213 ------- .../IEX/Response/StreamResponseStocksUS.cs | 315 ----------- ToolBox/Program.cs | 5 - ToolBox/QuantConnect.ToolBox.csproj | 1 - ToolBox/README.md | 1 - 10 files changed, 1655 deletions(-) delete mode 100644 Tests/Engine/DataFeeds/IEXDataQueueHandlerTests.cs delete mode 100644 ToolBox/IEX/IEXDataDownloader.cs delete mode 100644 ToolBox/IEX/IEXDataQueueHandler.cs delete mode 100644 ToolBox/IEX/IEXDownloaderProgram.cs delete mode 100644 ToolBox/IEX/IEXEventSourceCollection.cs delete mode 100644 ToolBox/IEX/Response/StreamResponseStocksUS.cs diff --git a/Launcher/config.json b/Launcher/config.json index 326597e5066a..cf2c9183d5f7 100644 --- a/Launcher/config.json +++ b/Launcher/config.json @@ -225,9 +225,6 @@ // To get your access token go to https://www.eia.gov/opendata "us-energy-information-auth-token": "", - // Required for IEX history requests - "iex-cloud-api-key": "", - // Required for market data from Coin API "coinapi-api-key": "", "coinapi-product": "free", // free, startup, streamer, professional, enterprise diff --git a/Tests/Engine/DataFeeds/IEXDataQueueHandlerTests.cs b/Tests/Engine/DataFeeds/IEXDataQueueHandlerTests.cs deleted file mode 100644 index c6fda48bfe74..000000000000 --- a/Tests/Engine/DataFeeds/IEXDataQueueHandlerTests.cs +++ /dev/null @@ -1,429 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Accord.Math; -using LaunchDarkly.EventSource; -using NUnit.Framework; -using QuantConnect.Configuration; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Logging; -using QuantConnect.Securities; -using QuantConnect.ToolBox.IEX; -using QuantConnect.Util; - -namespace QuantConnect.Tests.Engine.DataFeeds -{ - [TestFixture] - [Explicit("Tests are dependent on network and are long")] - public class IEXDataQueueHandlerTests - { - private readonly string _apiKey = Config.Get("iex-cloud-api-key"); - - private static readonly string[] HardCodedSymbolsSNP = { - "AAPL", "MSFT", "AMZN", "FB", "GOOGL", "GOOG", "BRK.B", "JNJ", "PG", "NVDA", "V", "JPM", "HD", "UNH", "MA", "VZ", - "PYPL", "NFLX", "ADBE", "DIS", "CRM", "CMCSA", "PFE", "WMT", "MRK", "T", "INTC", "TMO", "ABT", "KO", "PEP", "BAC", - "COST", "MCD", "NKE", "CSCO", "DHR", "NEE", "QCOM", "ABBV", "AVGO", "XOM", "ACN", "MDT", "TXN", "CVX", "BMY", - "AMGN", "LOW", "UNP", "HON", "LIN", "UPS", "PM", "ORCL", "LLY", "SBUX", "AMT", "NOW", "IBM", "AMD", "MMM", "WFC", - "C", "LMT", "CHTR", "BLK", "INTU", "CAT", "RTX", "ISRG", "BA", "SPGI", "FIS", "TGT", "ZTS", "MDLZ", "PLD", "GILD", - "CVS", "DE", "ANTM", "MS", "MO", "ADP", "D", "DUK", "BDX", "SYK", "BKNG", "CCI", "EQIX", "CL", "GS", "FDX", "GE", - "TMUS", "TJX", "SO", "APD", "ATVI", "CB", "CI", "SCHW", "CSX", "AXP", "REGN", "SHW", "ITW", "TFC", "MU", "AMAT", - "VRTX", "ICE", "CME", "ADSK", "FISV", "PGR", "DG", "NSC", "HUM", "USB", "MMC", "LRCX", "EL", "NEM", "BSX", "GPN", - "PNC", "ECL", "ILMN", "EW", "NOC", "KMB", "AEP", "GM", "ADI", "AON", "DD", "MCO", "WM", "ETN", "TWTR", "DLR", - "BAX", "EXC", "BIIB", "CTSH", "EMR", "ROP", "IDXX", "XEL", "SRE", "EA", "GIS", "LHX", "PSA", "CMG", "DOW", "CNC", - "APH", "COF", "SNPS", "HCA", "SBAC", "EBAY", "ORLY", "CMI", "DXCM", "TEL", "TT", "JCI", "A", "WEC", "KLAC", "COP", - "ALGN", "TRV", "ROST", "CDNS", "F", "GD", "PPG", "INFO", "XLNX", "PEG", "ES", "TROW", "IQV", "PCAR", "BLL", "VRSK", - "MSCI", "MET", "YUM", "MNST", "SYY", "CTAS", "BK", "ADM", "CARR", "STZ", "ALL", "MSI", "ZBH", "AWK", "ROK", "AIG", - "MCHP", "PH", "APTV", "ANSS", "ED", "AZO", "SWK", "PAYX", "CLX", "ALXN", "TDG", "BBY", "RMD", "FCX", "KR", "PRU", - "MAR", "OTIS", "FAST", "GLW", "HPQ", "SWKS", "CTVA", "WBA", "MTD", "HLT", "WLTW", "DTE", "LUV", "KMI", "MCK", "WMB", - "WELL", "CPRT", "AFL", "DHI", "MKC", "AME", "VFC", "DLTR", "CERN", "EIX", "CHD", "PPL", "FRC", "WY", "FTV", "MKTX", - "STT", "HSY", "WST", "ETR", "PSX", "O", "SLB", "AEE", "EOG", "LEN", "DAL", "DFS", "AJG", "KEYS", "KHC", "SPG", - "AMP", "VRSN", "LH", "AVB", "VMC", "MXIM", "MPC", "LYB", "FLT", "RSG", "PAYC", "HOLX", "TTWO", "ODFL", "CMS", - "ARE", "CDW", "IP", "FE", "CAG", "CBRE", "TSN", "EFX", "AMCR", "KSU", "FITB", "MLM", "DGX", "NTRS", "INCY", "DOV", - "FTNT", "VIAC", "COO", "EQR", "ETSY", "BR", "GWW", "LVS", "K", "VAR", "ZBRA", "XYL", "TYL", "AKAM", "TSCO", "VLO", - "EXR", "STE", "TFX", "EXPD", "DPZ", "PEAK", "QRVO", "VTR", "TER", "CTLT", "SIVB", "GRMN", "KMX", "NUE", "POOL", - "PKI", "MAS", "CTXS", "WAT", "TIF", "NDAQ", "NVR", "HIG", "DRE", "ABC", "SYF", "HRL", "OKE", "PXD", "CE", "FMC", - "CAH", "LNT", "GPC", "IR", "EXPE", "ESS", "AES", "MAA", "MTB", "RF", "BF.B", "EVRG", "SJM", "IEX", "KEY", "URI", - "J", "NLOK", "BIO", "CHRW", "DRI", "CNP", "WHR", "AVY", "ULTA", "ABMD", "CFG", "TDY", "JKHY", "FBHS", "EMN", "WDC", - "PHM", "HPE", "ANET", "ATO", "IFF", "LDOS", "PKG", "CINF", "HAS", "STX", "WAB", "IT", "HBAN", "HAL", "JBHT", - "HES", "BXP", "AAP", "PFG", "ALB", "OMC", "NTAP", "XRAY", "UAL", "RCL", "WRK", "CPB", "LW", "RJF", "PNW", "BKR", - "ALLE", "HSIC", "LKQ", "FOXA", "UDR", "MGM", "BWA", "PWR", "CBOE", "WU", "WRB", "SNA", "NI", "CXO", "PNR", - "ROL", "LUMN", "UHS", "L", "RE", "FFIV", "TXT", "GL", "NRG", "OXY", "AIZ", "IRM", "HST", "MYL", "LB", "COG", - "AOS", "IPG", "LYV", "WYNN", "CCL", "HWM", "IPGP", "JNPR", "DVA", "NWL", "MOS", "TPR", "DISH", "SEE", "TAP", - "LNC", "CMA", "HII", "PRGO", "HBI", "CF", "RHI", "REG", "MHK", "AAL", "DISCK", "LEG", "ZION", "IVZ", "BEN", "NWSA", - "NLSN", "FRT", "VNO", "DXC", "FLIR", "AIV", "PBCT", "ALK", "PVH", "KIM", "FANG", "FOX", "NCLH", "GPS", "VNT", "FLS", - "UNM", "RL", "XRX", "DVN", "MRO", "DISCA", "NOV", "APA", "SLG", "HFC", "UAA", "UA", "FTI", "NWS" - }; - - [SetUp] - public void Setup() - { - Log.DebuggingEnabled = Config.GetBool("debug-mode"); - } - - private void ProcessFeed(IEnumerator enumerator, Action callback = null) - { - Task.Run(() => - { - try - { - while (enumerator.MoveNext()) - { - BaseData tick = enumerator.Current; - callback?.Invoke(tick); - } - } - catch (AssertionException) - { - throw; - } - catch (Exception err) - { - Log.Error(err.Message); - } - }); - } - - /// - /// Subscribe to multiple symbols in a series of calls - /// - [Test] - public void IEXCouldSubscribeManyTimes() - { - var iex = new IEXDataQueueHandler(); - - var configs = new[] { - GetSubscriptionDataConfig(Symbol.Create("GOOG", SecurityType.Equity, Market.USA), Resolution.Second), - GetSubscriptionDataConfig(Symbol.Create("FB", SecurityType.Equity, Market.USA), Resolution.Second), - GetSubscriptionDataConfig(Symbol.Create("AAPL", SecurityType.Equity, Market.USA), Resolution.Second), - GetSubscriptionDataConfig(Symbol.Create("MSFT", SecurityType.Equity, Market.USA), Resolution.Second) - }; - - Array.ForEach(configs, (c) => - { - ProcessFeed( - iex.Subscribe(c, (s, e) => { }), - tick => - { - if (tick != null) - { - Log.Trace(tick.ToString()); - } - }); - }); - - Thread.Sleep(20000); - - Log.Trace("Unsubscribing from all except AAPL"); - - Array.ForEach(configs, (c) => - { - if (!string.Equals(c.Symbol.Value, "AAPL")) - { - iex.Unsubscribe(c); - } - }); - - Thread.Sleep(20000); - - iex.Dispose(); - } - - [Test] - public void IEXCouldSubscribeAndUnsubscribe() - { - // MBLY is the most liquid IEX instrument - var iex = new IEXDataQueueHandler(); - var unsubscribed = false; - Action callback = (tick) => - { - if (tick == null) - return; - - Log.Trace(tick.ToString()); - if (unsubscribed && tick.Symbol.Value == "MBLY") - { - Assert.Fail("Should not receive data for unsubscribed symbol"); - } - }; - - var configs = new[] { - GetSubscriptionDataConfig(Symbol.Create("MBLY", SecurityType.Equity, Market.USA), Resolution.Second), - GetSubscriptionDataConfig(Symbol.Create("USO", SecurityType.Equity, Market.USA), Resolution.Second) - }; - - Array.ForEach(configs, (c) => - { - ProcessFeed( - iex.Subscribe(c, (s, e) => { }), - callback); - }); - - Thread.Sleep(20000); - - iex.Unsubscribe(Enumerable.First(configs, c => string.Equals(c.Symbol.Value, "MBLY"))); - - Log.Trace("Unsubscribing"); - Thread.Sleep(2000); - // some messages could be inflight, but after a pause all MBLY messages must have beed consumed by ProcessFeed - unsubscribed = true; - - Thread.Sleep(20000); - iex.Dispose(); - } - - [Test] - public void IEXCouldSubscribeMoreThan100Symbols() - { - var configs = HardCodedSymbolsSNP.Select(s => - GetSubscriptionDataConfig(Symbol.Create(s, SecurityType.Equity, Market.USA), - Resolution.Second)).ToArray(); - - using (var iex = new IEXDataQueueHandler()) - { - Array.ForEach(configs, dataConfig => iex.Subscribe(dataConfig, (s, e) => { })); - Thread.Sleep(20000); - Assert.IsTrue(iex.IsConnected); - } - } - - [Test] - public void IEXEventSourceCollectionCanSubscribeManyTimes() - { - // Send few consecutive requests to subscribe to a large amount of symbols (after the first request to change the subscription) - // and make sure no exception will be thrown - if event-source-collection can't subscribe to all it throws after timeout - Assert.DoesNotThrow(() => - { - using (var events = new IEXEventSourceCollection((o, args) => { }, _apiKey)) - { - var rnd = new Random(); - for (var i = 0; i < 5; i++) - { - // Shuffle and select first random amount of symbol - // in range from 300 to 500 (snp symbols count) - var shuffled = HardCodedSymbolsSNP - .OrderBy(n => Guid.NewGuid()) - .ToArray(); - - var selected = shuffled - .Take(rnd.Next(300, shuffled.Length)) - .ToArray(); - - events.UpdateSubscription(selected); - } - } - }); - } - - [Test] - public void IEXEventSourceCollectionSubscriptionThoroughTest() - { - using (var mockedEventsSource = new MockedIEXEventSourceCollection((o, args) => { }, _apiKey)) - { - // -- 1 -- SUBSCRIBE FOR THE FIRST TIME -- - var amount1 = 210; - var symbols1 = HardCodedSymbolsSNP.Take(amount1).ToArray(); - - mockedEventsSource.UpdateSubscription(symbols1); - - var subscribed = mockedEventsSource.GetAllSubscribedSymbols().ToArray(); - Assert.AreEqual(amount1, subscribed.Count()); - Assert.IsTrue(subscribed.All(i => symbols1.Contains(i))); - - Assert.AreEqual(5, mockedEventsSource.CreateNewSubscriptionCalledTimes); - Assert.AreEqual(0, mockedEventsSource.RemoveOldClientCalledTimes); - - // Signal can be obtained - mockedEventsSource.TestCounter.Signal(); - Assert.AreEqual(0, mockedEventsSource.TestCounter.CurrentCount); - mockedEventsSource.TestCounter.Reset(1); - - // -- 2 -- SUBSCRIBE FOR THE SECOND TIME -- UPDATES SUBSCRIPTION -- - mockedEventsSource.Reset(); - - // Remove 3 elements with a step 50 - var symbols2 = symbols1.RemoveAt(51).RemoveAt(101).RemoveAt(151); - mockedEventsSource.UpdateSubscription(symbols2); - - // We removed three symbols, so three subscription became invalid - Assert.AreEqual(3, mockedEventsSource.RemoveOldClientCalledTimes); - // Two of old subscriptions were good, 3 new ones to be initiated to replace removed - Assert.AreEqual(3, mockedEventsSource.CreateNewSubscriptionCalledTimes); - - subscribed = mockedEventsSource.GetAllSubscribedSymbols().ToArray(); - Assert.AreEqual(symbols2.Length, subscribed.Length); - Assert.IsTrue(subscribed.All(i => symbols2.Contains(i))); - - mockedEventsSource.TestCounter.Signal(); - Assert.AreEqual(0, mockedEventsSource.TestCounter.CurrentCount); - mockedEventsSource.TestCounter.Reset(1); - - // Removed subscriptions had a symbol that was not in current subscription - mockedEventsSource.RemovedClientSymbols.DoForEach(removed => - { - Assert.IsFalse(removed.All(s => symbols2.Contains(s))); - }); - } - } - - private SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) - { - return new SubscriptionDataConfig( - typeof(T), - symbol, - resolution, - TimeZones.Utc, - TimeZones.Utc, - true, - true, - false); - } - - public class MockedIEXEventSourceCollection : IEXEventSourceCollection - { - public CountdownEvent TestCounter => Counter; - public int CreateNewSubscriptionCalledTimes; - public int RemoveOldClientCalledTimes; - public List RemovedClientSymbols = new List(); - - public MockedIEXEventSourceCollection(EventHandler messageAction, string apiKey) - : base(messageAction, apiKey) - { - } - - protected override EventSource CreateNewSubscription(string[] symbols) - { - CreateNewSubscriptionCalledTimes++; - // Decrement the counter - Counter.Signal(); - return CreateNewClient(symbols); - } - - protected override void RemoveOldClient(EventSource oldClient) - { - RemoveOldClientCalledTimes++; - RemovedClientSymbols.Add(ClientSymbolsDictionary[oldClient]); - // Call base class to remove from inner dictionary and dispose - base.RemoveOldClient(oldClient); - } - - public IEnumerable GetAllSubscribedSymbols() - { - return ClientSymbolsDictionary.Values.SelectMany(x => x); - } - - public void Reset() - { - CreateNewSubscriptionCalledTimes = 0; - RemoveOldClientCalledTimes = 0; - RemovedClientSymbols.Clear(); - } - } - - - #region History provider tests - - public static TestCaseData[] TestParameters - { - get - { - return new[] - { - // valid parameters - new TestCaseData(Symbols.SPY, Resolution.Daily, typeof(TradeBar), TimeSpan.FromDays(15), true, false), - new TestCaseData(Symbols.SPY, Resolution.Minute, typeof(TradeBar), TimeSpan.FromDays(5), true, false), - - // invalid resolution == empty result. - new TestCaseData(Symbols.SPY, Resolution.Tick, typeof(TradeBar), TimeSpan.FromSeconds(15), false, false), - new TestCaseData(Symbols.SPY, Resolution.Second, typeof(TradeBar), Time.OneMinute, false, false), - new TestCaseData(Symbols.SPY, Resolution.Hour, typeof(TradeBar), Time.OneDay, false, false), - - // invalid period == empty result - new TestCaseData(Symbols.SPY, Resolution.Minute, typeof(TradeBar), TimeSpan.FromDays(45), false, false), // beyond 30 days - new TestCaseData(Symbols.SPY, Resolution.Daily, typeof(TradeBar), TimeSpan.FromDays(-15), false, false), // date in future - - // invalid data type = empty result - new TestCaseData(Symbols.SPY, Resolution.Daily, typeof(QuoteBar), TimeSpan.FromDays(15), false, false), - - // invalid symbol: XYZ -> not found Exception - new TestCaseData(Symbol.Create("XYZ", SecurityType.Equity, Market.FXCM), Resolution.Daily, typeof(TradeBar), TimeSpan.FromDays(15), false, true), - - // invalid security type, no exception, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, typeof(TradeBar), TimeSpan.FromDays(15), false, false) - }; - } - } - - [Test, TestCaseSource(nameof(TestParameters))] - public void IEXCouldGetHistory(Symbol symbol, Resolution resolution, Type dataType, TimeSpan period, bool received, bool throwsException) - { - TestDelegate test = () => - { - var historyProvider = new IEXDataQueueHandler(); - var now = DateTime.UtcNow; - - var requests = new[] - { - new HistoryRequest(now.Add(-period), - now, - dataType, - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - TimeZones.NewYork, - resolution, - true, - false, - DataNormalizationMode.Raw, - TickType.Trade) - }; - - var slices = historyProvider.GetHistory(requests, TimeZones.Utc).ToArray(); - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); - - if (received) - { - // Slices not empty - Assert.IsNotEmpty(slices); - - // And are ordered by time - Assert.That(slices, Is.Ordered.By("Time")); - } - else - { - Assert.IsEmpty(slices); - } - }; - - if (throwsException) - { - Assert.That(test, Throws.Exception); - } - else - { - Assert.DoesNotThrow(test); - } - } - - #endregion - - } -} \ No newline at end of file diff --git a/ToolBox/IEX/IEXDataDownloader.cs b/ToolBox/IEX/IEXDataDownloader.cs deleted file mode 100644 index 7049bd0d9b78..000000000000 --- a/ToolBox/IEX/IEXDataDownloader.cs +++ /dev/null @@ -1,83 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Securities; -using System; -using System.Collections.Generic; - -namespace QuantConnect.ToolBox.IEX -{ - public class IEXDataDownloader : IDataDownloader, IDisposable - { - private readonly IEXDataQueueHandler _handler; - - public IEXDataDownloader() - { - _handler = new IEXDataQueueHandler(); - } - - public void Dispose() - { - _handler.Dispose(); - } - - /// - /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). - /// - /// model class for passing in parameters for historical data - /// Enumerable of base data for this symbol - public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetParameters) - { - var symbol = dataDownloaderGetParameters.Symbol; - var resolution = dataDownloaderGetParameters.Resolution; - var startUtc = dataDownloaderGetParameters.StartUtc; - var endUtc = dataDownloaderGetParameters.EndUtc; - var tickType = dataDownloaderGetParameters.TickType; - - if (tickType != TickType.Trade) - { - yield break; - } - - if (!(resolution == Resolution.Daily || resolution == Resolution.Minute)) - throw new NotSupportedException("Resolution not available: " + resolution); - - if (endUtc < startUtc) - throw new ArgumentException("The end date must be greater or equal than the start date."); - - var historyRequests = new[] { - new HistoryRequest(startUtc, - endUtc, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), - TimeZones.NewYork, - resolution, - true, - false, - DataNormalizationMode.Raw, - TickType.Trade) - }; - - foreach (var slice in _handler.GetHistory(historyRequests, TimeZones.EasternStandard)) - { - yield return slice[symbol]; - } - } - } -} diff --git a/ToolBox/IEX/IEXDataQueueHandler.cs b/ToolBox/IEX/IEXDataQueueHandler.cs deleted file mode 100644 index a93b3c80020e..000000000000 --- a/ToolBox/IEX/IEXDataQueueHandler.cs +++ /dev/null @@ -1,524 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Generic; -using QuantConnect.Data; -using QuantConnect.Configuration; -using QuantConnect.Packets; -using QuantConnect.Logging; -using Newtonsoft.Json.Linq; -using QuantConnect.Data.Market; -using System.Collections.Concurrent; -using System.Runtime.CompilerServices; -using System.Threading; -using QuantConnect.Interfaces; -using NodaTime; -using System.Globalization; -using static QuantConnect.StringExtensions; -using QuantConnect.Util; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using Newtonsoft.Json; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.ToolBox.IEX.Response; - -namespace QuantConnect.ToolBox.IEX -{ - /// - /// IEX live data handler. - /// See more at https://iexcloud.io/docs/api/ - /// - public class IEXDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler - { - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified); - private static readonly TimeSpan SubscribeDelay = TimeSpan.FromMilliseconds(1500); - private static bool _invalidHistDataTypeWarningFired; - - private readonly IEXEventSourceCollection _clients; - private readonly ManualResetEvent _refreshEvent = new ManualResetEvent(false); - private readonly string _apiKey = Config.Get("iex-cloud-api-key"); - - private readonly ConcurrentDictionary _symbols = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); - private readonly ConcurrentDictionary _iexLastTradeTime = new ConcurrentDictionary(); - - private readonly CancellationTokenSource _cts = new CancellationTokenSource(); - private int _dataPointCount; - - private readonly IDataAggregator _aggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")); - private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; - - public bool IsConnected => _clients.IsConnected; - - - /// - /// Initializes a new instance of the class. - /// - public IEXDataQueueHandler() - { - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); - - _subscriptionManager.SubscribeImpl += (symbols, t) => - { - symbols.DoForEach(symbol => - { - if (!_symbols.TryAdd(symbol.Value, symbol)) - { - throw new InvalidOperationException($"Invalid logic, SubscriptionManager tried to subscribe to existing symbol : {symbol.Value}"); - } - }); - - Refresh(); - return true; - }; - - _subscriptionManager.UnsubscribeImpl += (symbols, t) => - { - symbols.DoForEach(symbol => - { - Symbol tmp; - _symbols.TryRemove(symbol.Value, out tmp); - }); - - Refresh(); - return true; - }; - - if (string.IsNullOrWhiteSpace(_apiKey)) - { - throw new ArgumentException("Could not read IEX API key from config.json. " + - "Please make sure to add \"iex-cloud-api-key\" to your configuration file"); - } - - // Set the sse-clients collection - _clients = new IEXEventSourceCollection(((o, args) => - { - var message = args.Message.Data; - ProcessJsonObject(message); - - }), _apiKey); - - // In this thread, we check at each interval whether the client needs to be updated - // Subscription renewal requests may come in dozens and all at relatively same time - we cannot update them one by one when work with SSE - var clientUpdateThread = new Thread(() => - { - while (!_cts.Token.IsCancellationRequested) - { - _refreshEvent.WaitOne(); - Thread.Sleep(SubscribeDelay); - - _refreshEvent.Reset(); - - try - { - _clients.UpdateSubscription(_symbols.Keys.ToArray()); - } - catch (Exception e) - { - Log.Error(e); - throw; - } - } - - }) - { IsBackground = true }; - clientUpdateThread.Start(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ProcessJsonObject(string json) - { - try - { - var dataList = JsonConvert.DeserializeObject>(json); - - foreach (var item in dataList) - { - var symbolString = item.Symbol; - Symbol symbol; - if (!_symbols.TryGetValue(symbolString, out symbol)) - { - // Symbol is no loner in dictionary, it may be the stream not has been updated yet, - // and the old client is still sending messages for unsubscribed symbols - - // so there can be residual messages for the symbol, which we must skip - continue; - } - - var lastPrice = item.IexRealtimePrice ?? 0; - var lastSize = item.IexRealtimeSize ?? 0; - - // Refers to the last update time of iexRealtimePrice in milliseconds since midnight Jan 1, 1970 UTC or -1 or 0. - // If the value is -1 or 0, IEX has not quoted the symbol in the trading day. - var lastUpdateMillis = item.IexLastUpdated ?? 0; - - if (lastUpdateMillis <= 0) - { - continue; - } - - // (!) Epoch timestamp in milliseconds of the last market hours trade excluding the closing auction trade. - var lastTradeMillis = item.LastTradeTime ?? 0; - - if (lastTradeMillis <= 0) - { - continue; - } - - // If there is a last trade time but no last price or size - this is an error - if (lastPrice == 0 || lastSize == 0) - { - throw new InvalidOperationException("ProcessJsonObject(): Invalid price & size."); - } - - // Check if there is a kvp entry for a symbol - long value; - var isInDictionary = _iexLastTradeTime.TryGetValue(symbolString, out value); - - // We should update with trade-tick if: - // - there exist an entry for a symbol and new trade time is different from time in dictionary - // - not in dictionary, means the first trade-tick case - if (isInDictionary && value != lastTradeMillis || !isInDictionary) - { - var lastTradeDateTime = UnixEpoch.AddMilliseconds(lastTradeMillis); - var lastTradeTimeNewYork = lastTradeDateTime.ConvertFromUtc(TimeZones.NewYork); - - var tradeTick = new Tick() - { - Symbol = symbol, - Time = lastTradeTimeNewYork, - TickType = TickType.Trade, - Value = lastPrice, - Quantity = lastSize - }; - - _aggregator.Update(tradeTick); - - _iexLastTradeTime[symbolString] = lastTradeMillis; - } - } - } - catch (Exception err) - { - Log.Error("IEXDataQueueHandler.ProcessJsonObject(): " + err.Message); - } - } - - /// - /// Subscribe to the specified configuration - /// - /// defines the parameters to subscribe to a data feed - /// handler to be fired on new data available - /// The new enumerator for this subscription request - public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) - { - if (dataConfig.ExtendedMarketHours) - { - Log.Error("IEXDataQueueHandler.Subscribe(): Unfortunately no updates could be received from IEX outside the regular exchange open hours. " + - "Please be aware that only regular hours updates will be submitted to an algorithm."); - } - - if (dataConfig.Resolution < Resolution.Second) - { - Log.Error($"IEXDataQueueHandler.Subscribe(): Selected data resolution ({dataConfig.Resolution}) " + - "is not supported by current implementation of IEXDataQueueHandler. Sorry. Please try the higher resolution."); - } - - if (!CanSubscribe(dataConfig.Symbol)) - { - return null; - } - - var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); - - return enumerator; - } - - /// - /// Sets the job we're subscribing for - /// - /// Job we're subscribing for - public void SetJob(LiveNodePacket job) - { - } - - /// - /// Checks if this brokerage supports the specified symbol - /// - /// The symbol - /// returns true if brokerage supports the specified symbol; otherwise false - private static bool CanSubscribe(Symbol symbol) - { - return symbol.SecurityType == SecurityType.Equity; - } - - private void Refresh() - { - _refreshEvent.Set(); - } - - /// - /// Removes the specified configuration - /// - /// Subscription config to be removed - public void Unsubscribe(SubscriptionDataConfig dataConfig) - { - _subscriptionManager.Unsubscribe(dataConfig); - _aggregator.Remove(dataConfig); - } - - /// - /// Dispose connection to IEX - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) - { - _aggregator.DisposeSafely(); - _cts.Cancel(); - - _clients.Dispose(); - - Log.Trace("IEXDataQueueHandler.Dispose(): Disconnected from IEX data provider"); - } - - ~IEXDataQueueHandler() - { - Dispose(false); - } - - #region IHistoryProvider implementation - - /// - /// Gets the total number of data points emitted by this history provider - /// - public override int DataPointCount => _dataPointCount; - - /// - /// Initializes this history provider to work for the specified job - /// - /// The initialization parameters - public override void Initialize(HistoryProviderInitializeParameters parameters) - { - } - - /// - /// Gets the history for the requested securities - /// - /// The historical data requests - /// The time zone used when time stamping the slice instances - /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - if (string.IsNullOrWhiteSpace(_apiKey)) - { - Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX require an API key."); - return Enumerable.Empty(); - } - - // Create subscription objects from the configs - var subscriptions = new List(); - requests.DoForEach(request => - { - // IEX does return historical TradeBar - give one time warning if inconsistent data type was requested - if (request.DataType != typeof(TradeBar) && !_invalidHistDataTypeWarningFired) - { - Log.Error($"IEXDataQueueHandler.GetHistory(): Not supported data type - {request.DataType.Name}. " + - "Currently available support only for historical of type - TradeBar"); - _invalidHistDataTypeWarningFired = true; - return; - } - - var history = ProcessHistoryRequests(request); - var subscription = CreateSubscription(request, history); - subscriptions.Add(subscription); - }); - - var result = subscriptions.Any() ? CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone) : Enumerable.Empty(); - return result; - } - - /// - /// Populate request data - /// - private IEnumerable ProcessHistoryRequests(Data.HistoryRequest request) - { - var ticker = request.Symbol.ID.Symbol; - var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork); - var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork); - - if (request.Resolution == Resolution.Minute && start <= DateTime.Today.AddDays(-30)) - { - Log.Error("IEXDataQueueHandler.GetHistory(): History calls with minute resolution for IEX available only for trailing 30 calendar days."); - yield break; - } - - if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute) - { - Log.Error("IEXDataQueueHandler.GetHistory(): History calls for IEX only support daily & minute resolution."); - yield break; - } - - Log.Trace("IEXDataQueueHandler.ProcessHistoryRequests(): Submitting request: " + - Invariant($"{request.Symbol.SecurityType}-{ticker}: {request.Resolution} {start}->{end}") + - ". Please wait.."); - - const string baseUrl = "https://cloud.iexapis.com/stable/stock"; - var now = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork); - var span = now - start; - var urls = new List(); - - switch (request.Resolution) - { - case Resolution.Minute: - { - var begin = start; - while (begin < end) - { - var url = - $"{baseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}"; - urls.Add(url); - begin = begin.AddDays(1); - } - - break; - } - case Resolution.Daily: - { - string suffix; - if (span.Days < 30) - { - suffix = "1m"; - } - else if (span.Days < 3 * 30) - { - suffix = "3m"; - } - else if (span.Days < 6 * 30) - { - suffix = "6m"; - } - else if (span.Days < 12 * 30) - { - suffix = "1y"; - } - else if (span.Days < 24 * 30) - { - suffix = "2y"; - } - else if (span.Days < 60 * 30) - { - suffix = "5y"; - } - else - { - suffix = "max"; // max is 15 years - } - - var url = - $"{baseUrl}/{ticker}/chart/{suffix}?token={_apiKey}"; - urls.Add(url); - - break; - } - } - - // Download and parse data - var requests = new List>(); - - urls.DoForEach(url => - { - requests.Add(Task.Run(async () => - { - using (var client = new WebClient()) - { - return await client.DownloadStringTaskAsync(new Uri(url)).ConfigureAwait(false); - } - })); - }); - - var responses = Task.WhenAll(requests).Result; - - foreach (var response in responses) - { - var parsedResponse = JArray.Parse(response); - - // Parse - foreach (var item in parsedResponse.Children()) - { - DateTime date; - TimeSpan period; - if (item["minute"] != null) - { - date = DateTime.ParseExact(item["date"].Value(), "yyyy-MM-dd", CultureInfo.InvariantCulture); - var minutes = TimeSpan.ParseExact(item["minute"].Value(), "hh\\:mm", CultureInfo.InvariantCulture); - date += minutes; - period = TimeSpan.FromMinutes(1); - } - else - { - date = Parse.DateTime(item["date"].Value()); - period = TimeSpan.FromDays(1); - } - - if (date < start || date > end) - { - continue; - } - - Interlocked.Increment(ref _dataPointCount); - - if (item["open"].Type == JTokenType.Null) - { - continue; - } - - decimal open, high, low, close, volume; - - if (request.Resolution == Resolution.Daily && - request.DataNormalizationMode == DataNormalizationMode.Raw) - { - open = item["uOpen"].Value(); - high = item["uHigh"].Value(); - low = item["uLow"].Value(); - close = item["uClose"].Value(); - volume = item["uVolume"].Value(); - } - else - { - open = item["open"].Value(); - high = item["high"].Value(); - low = item["low"].Value(); - close = item["close"].Value(); - volume = item["volume"].Value(); - } - - var tradeBar = new TradeBar(date, request.Symbol, open, high, low, close, volume, period); - - yield return tradeBar; - } - } - } - - #endregion - } -} diff --git a/ToolBox/IEX/IEXDownloaderProgram.cs b/ToolBox/IEX/IEXDownloaderProgram.cs deleted file mode 100644 index ea8d2a64237d..000000000000 --- a/ToolBox/IEX/IEXDownloaderProgram.cs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Linq; -using QuantConnect.Util; -using QuantConnect.Data; -using QuantConnect.Logging; -using System.Collections.Generic; -using QuantConnect.Configuration; - -namespace QuantConnect.ToolBox.IEX -{ - public static class IEXDownloaderProgram - { - /// - /// Primary entry point to the program. - /// - public static void IEXDownloader(IList tickers, string resolution, DateTime fromDate, DateTime toDate) - { - if (resolution.IsNullOrEmpty() || tickers.IsNullOrEmpty()) - { - Console.WriteLine("IEXDownloader ERROR: '--tickers=' or '--resolution=' parameter is missing"); - Console.WriteLine("--tickers=eg SPY,AAPL"); - Console.WriteLine("--resolution=Minute/Daily"); - Environment.Exit(1); - } - - try - { - // Load settings from command line - var castResolution = (Resolution)Enum.Parse(typeof(Resolution), resolution); - - // Load settings from config.json - var dataDirectory = Globals.DataFolder; - var startDate = fromDate.ConvertToUtc(TimeZones.NewYork); - var endDate = toDate.ConvertToUtc(TimeZones.NewYork); - - // Create an instance of the downloader - const string market = Market.USA; - var securityType = SecurityType.Equity; - - using (var downloader = new IEXDataDownloader()) - { - foreach (var ticker in tickers) - { - // Download the data - var symbolObject = Symbol.Create(ticker, securityType, market); - var data = downloader.Get(new DataDownloaderGetParameters(symbolObject, castResolution, startDate, endDate)).ToArray(); - - if (data.Length == 0) - { - continue; - } - - // Save the data - var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory); - writer.Write(data); - } - } - } - catch (Exception err) - { - Log.Error(err); - } - Console.ReadLine(); - } - } -} diff --git a/ToolBox/IEX/IEXEventSourceCollection.cs b/ToolBox/IEX/IEXEventSourceCollection.cs deleted file mode 100644 index faf306d603a6..000000000000 --- a/ToolBox/IEX/IEXEventSourceCollection.cs +++ /dev/null @@ -1,213 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using LaunchDarkly.EventSource; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.ToolBox.IEX -{ - /// - /// Class wraps a collection of clients for getting data on SSE. - /// SSE endpoints are limited to 50 symbols per connection. To consume more than 50 symbols we need multiple connections . - /// - public class IEXEventSourceCollection : IDisposable - { - private static readonly TimeSpan TimeoutToUpdate = TimeSpan.FromSeconds(30); - private const int SymbolsPerConnectionLimit = 50; - private readonly string _apiKey; - private readonly EventHandler _messageAction; - protected readonly ConcurrentDictionary ClientSymbolsDictionary = new ConcurrentDictionary(); - protected readonly CountdownEvent Counter = new CountdownEvent(1); - - // IEX API documentation says: - // "We limit requests to 100 per second per IP measured in milliseconds, so no more than 1 request per 10 milliseconds." - private readonly RateGate _rateGate = new RateGate(1, TimeSpan.FromMilliseconds(10)); - - /// - /// Indicates whether a client is connected - i.e delivers any data. - /// - public bool IsConnected { get; private set; } - - /// - /// Creates a new instance of - /// - public IEXEventSourceCollection(EventHandler messageAction, string apiKey) - { - _messageAction = messageAction; - _apiKey = apiKey; - } - - /// - /// Updates the data subscription to reflect the current user-subscribed symbols set. - /// - /// Symbols that user is currently subscribed to - /// - public void UpdateSubscription(string[] symbols) - { - Log.Debug("IEXEventSourceCollection.UpdateSubscription(): Subscription update started"); - - var remainingSymbols = new List(symbols); - var clientsToRemove = new List(); - - foreach (var kvp in ClientSymbolsDictionary) - { - var clientSymbols = kvp.Value; - - // Need to perform no changes if all client symbols are relevant - // and subscription is fully loaded (otherwise it is also a replacement) - if (clientSymbols.All(symbols.Contains) && clientSymbols.Length == SymbolsPerConnectionLimit) - { - Log.Debug($"IEXEventSourceCollection.UpdateSubscription(): Leave unchanged subscription for: {string.Join(",", clientSymbols)}"); - - // Just remove symbols from collection of remaining - remainingSymbols.RemoveAll(i => clientSymbols.Contains(i)); - continue; - } - - clientsToRemove.Add(kvp.Key); - } - - Log.Debug($"IEXEventSourceCollection.UpdateSubscription(): {clientsToRemove.Count} old clients to remove"); - - if (!remainingSymbols.Any()) - { - throw new Exception("IEXEventSourceCollection.UpdateSubscription(): Invalid logic, remaining symbols can't be an empty set."); - } - - // Group all remaining symbols in a smaller packages to comply with per-connection-limits - var packagedSymbolsList = new List(); - do - { - if (remainingSymbols.Count > SymbolsPerConnectionLimit) - { - var firstFifty = remainingSymbols.Take(SymbolsPerConnectionLimit).ToArray(); - remainingSymbols.RemoveAll(i => firstFifty.Contains(i)); - packagedSymbolsList.Add(firstFifty); - } - else - { - // Add all remaining symbols as a last package - packagedSymbolsList.Add(remainingSymbols.ToArray()); - break; - } - } - while (remainingSymbols.Any()); - - // Create new client for every package (make sure that we do not exceed the rate-gate-limit while creating) - packagedSymbolsList.DoForEach(package => - { - Log.Debug($"IEXEventSourceCollection.CreateNewSubscription(): Creating new subscription for: {string.Join(",", package)}"); - - var client = CreateNewSubscription(package); - - // Add to the dictionary - ClientSymbolsDictionary.TryAdd(client, package); - }); - - Counter.Signal(); - if (!Counter.Wait(TimeoutToUpdate)) - { - throw new Exception("IEXEventSourceCollection.UpdateSubscription(): Could not update subscription within a timeout"); - } - - clientsToRemove.DoForEach(RemoveOldClient); - - // Reset counter - Log.Debug($"IEXEventSourceCollection.CreateNewSubscription(): Updated successfully. Resetting the counter."); - Counter.Reset(1); - - IsConnected = true; - } - - protected virtual EventSource CreateNewSubscription(string[] symbols) - { - Counter.AddCount(); // Increment - _rateGate.WaitToProceed(); - - var client = CreateNewClient(symbols); - - // Set up the handlers - client.Opened += (sender, args) => - { - Log.Debug($"ClientOnOpened(): Sender's hashcode is {sender.GetHashCode()}"); - - Counter.Signal(); // Decrement - - Log.Debug($"ClientOnOpened(): Counter count after decrement: {Counter.CurrentCount}"); - }; - - client.MessageReceived += _messageAction; - - client.Error += (sender, args) => - { - var exception = args.Exception; - Log.Debug($"ClientOnError(): EventSource Error Occurred. Details: {exception.Message} " + - $"ErrorType: {exception.GetType().FullName}"); - }; - - client.Closed += (sender, args) => - { - Log.Debug("ClientOnClosed(): Closing a client"); - }; - - // Client start call will block until Stop() is called (!) - runs continuously in a background - Task.Run(async () => await client.StartAsync().ConfigureAwait(false)); - - return client; - } - - protected EventSource CreateNewClient(string[] symbols) - { - var url = BuildUrlString(symbols); - var client = new EventSource(LaunchDarkly.EventSource.Configuration.Builder(new Uri(url)).Build()); - return client; - } - - protected virtual void RemoveOldClient(EventSource client) - { - Log.Debug($"IEXEventSourceCollection.UpdateSubscription(): Remove subscription for: {string.Join(",", ClientSymbolsDictionary[client])}"); - - string[] stub; - ClientSymbolsDictionary.TryRemove(client, out stub); - - client.DisposeSafely(); - } - - private string BuildUrlString(IEnumerable symbols) - { - var url = "https://cloud-sse.iexapis.com/stable/stocksUSNoUTP1Second?token=" + _apiKey; - url += "&symbols=" + string.Join(",", symbols); - return url; - } - - public void Dispose() - { - foreach (var client in ClientSymbolsDictionary.Keys) - { - client.Close(); - client.DisposeSafely(); - } - - IsConnected = false; - } - } -} diff --git a/ToolBox/IEX/Response/StreamResponseStocksUS.cs b/ToolBox/IEX/Response/StreamResponseStocksUS.cs deleted file mode 100644 index 799f7e34318e..000000000000 --- a/ToolBox/IEX/Response/StreamResponseStocksUS.cs +++ /dev/null @@ -1,315 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -namespace QuantConnect.ToolBox.IEX.Response -{ - public class StreamResponseStocksUS - { - /// - /// Refers to the stock ticker. - /// - public string Symbol { get; set; } - - /// - /// Refers to the company name. - /// - public string CompanyName { get; set; } - - /// - /// Refers to the primary listing exchange for the symbol. - /// - public string PrimaryExchange { get; set; } - - /// - /// Refers to the source of the latest price. Possible values are tops, sip, previousclose, close, or iexlasttrade. - /// The iexlastrade value indicates that the latest price is the price of the last trade on IEX rather than the SIP closing price. - /// iexlasttrade is provided for Nasdaq-listed symbols between 4:00 p.m.and 8 p.m.E.T. if you do not have UTP authorization. - /// - public string CalculationPrice { get; set; } - - /// - /// Refers to the official open price from the SIP. 15 minute delayed (can be null after 00:00 ET, before 9:45 and weekends) - /// - public decimal? Open { get; set; } - - /// - /// Refers to the official listing exchange time for the open from the SIP. 15 minute delayed - /// - public long? OpenTime { get; set; } - - /// - /// This will represent a human readable description of the source of open. - /// - public string OpenSource { get; set; } - - /// - /// Refers to the 15-minute delayed official close price from the SIP. For Nasdaq-listed stocks, if you do not have UTP authorization, - /// between 4:00 p.m. and 8 p.m. E.T. this field will return the price of the last trade on IEX rather than the SIP closing price. - /// - public decimal? Close { get; set; } - - /// - /// Refers to the official listing exchange time for the close from the SIP. 15 minute delayed - /// - public long? CloseTime { get; set; } - - /// - /// This will represent a human readable description of the source of close. - /// - public string CloseSource { get; set; } - - /// - /// Refers to the market-wide highest price from the SIP. 15 minute delayed during normal market hours 9:30 - 16:00 (null before 9:45 and weekends). - /// - public decimal High { get; set; } - - /// - /// Refers to the official listing exchange time for the high from the SIP. 15 minute delayed - /// - public long? HighTime { get; set; } - - /// - /// This will represent a human readable description of the source of high. - /// - public string HighSource { get; set; } - - /// - /// Refers to the market-wide lowest price from the SIP. 15 minute delayed during normal market hours 9:30 - 16:00 (null before 9:45 and weekends). - /// - public decimal Low { get; set; } - - /// - /// Refers to the official listing exchange time for the low from the SIP. 15 minute delayed - /// - public long LowTime { get; set; } - - /// - /// This will represent a human readable description of the source of low. - /// - public string LowSource { get; set; } - - /// - /// Refers to the latest relevant price of the security which is derived from multiple sources. We first look for an IEX real - /// time price. If an IEX real time price is older than 15 minutes, 15 minute delayed market price is used. If a 15 minute - /// delayed price is not available, we will use the current day close price. If a current day close price is not available, - /// we will use the last available closing price (listed below as previousClose) IEX real time price represents trades - /// on IEX only. Trades occur across over a dozen exchanges, so the last IEX price can be used to indicate the overall market price. - /// This will not included pre or post market prices - /// - public decimal LatestPrice { get; set; } - - /// - /// This will represent a human readable description of the source of latestPrice. - /// Possible values are IEX real time price, 15 minute delayed price, Close or Previous close - /// - public string LatestSource { get; set; } - - /// - /// Refers to a human readable time/date of when latestPrice was last updated. The format will vary based on latestSource is - /// intended to be displayed to a user. Use latestUpdate for machine readable timestamp. - /// - public string LatestTime { get; set; } - - /// - /// Refers to the machine readable epoch timestamp of when latestPrice was last updated. Represented in milliseconds since midnight Jan 1, 1970. - /// - public long LatestUpdate { get; set; } - - /// - /// Refers to the latest total market volume of the stock across all markets. This will be the most recent volume of the stock during trading hours, - /// or it will be the total volume of the last available trading day. - /// - public long LatestVolume { get; set; } - - /// - /// Refers to the price of the last trade on IEX. - /// - public decimal? IexRealtimePrice { get; set; } - - /// - /// Refers to the size of the last trade on IEX. - /// - public int? IexRealtimeSize { get; set; } - - /// - /// Refers to the last update time of iexRealtimePrice in milliseconds since midnight Jan 1, 1970 UTC or -1 or 0. - /// If the value is -1 or 0, IEX has not quoted the symbol in the trading day. - /// - public long? IexLastUpdated { get; set; } - - /// - /// Refers to the 15 minute delayed market price from the SIP during normal market hours 9:30 - 16:00 ET. - /// - public decimal? DelayedPrice { get; set; } - - /// - /// Refers to the last update time of the delayed market price during normal market hours 9:30 - 16:00 ET. - /// - public long? DelayedPriceTime { get; set; } - - /// - /// Refers to the 15 minute delayed odd Lot trade price from the SIP during normal market hours 9:30 - 16:00 ET. - /// - public decimal? OddLotDelayedPrice { get; set; } - - /// - /// Refers to the last update time of the odd Lot trade price during normal market hours 9:30 - 16:00 ET. - /// - public long? OddLotDelayedPriceTime { get; set; } - - /// - /// Refers to the 15 minute delayed price outside normal market hours 0400 - 0930 ET and 1600 - 2000 ET. - /// This provides pre market and post market price. This is purposefully separate from latestPrice so users can display the two prices separately. - /// - public decimal? ExtendedPrice { get; set; } - - /// - /// Refers to the price change between extendedPrice and latestPrice - /// - public decimal? ExtendedChange { get; set; } - - /// - /// Refers to the price change percent between extendedPrice and latestPrice - /// - public decimal? ExtendedChangePercent { get; set; } - - /// - /// Refers to the last update time of extendedPrice - /// - public long? ExtendedPriceTime { get; set; } - - /// - /// Refers to the previous trading day closing price. - /// - public decimal PreviousClose { get; set; } - - /// - /// Refers to the previous trading day volume. - /// - public long PreviousVolume { get; set; } - - /// - /// Refers to the change in price between latestPrice and previousClose - /// - public decimal Change { get; set; } - - /// - /// Refers to the percent change in price between latestPrice and previousClose. For example, a 5% change would be represented as 0.05. - /// You can use the query string parameter displayPercent to return this field multiplied by 100. So, 5% change would be represented as 5. - /// - public decimal ChangePercent { get; set; } - - /// - /// Total volume for the stock, but only updated after market open. To get premarket volume, use latestVolume - /// - public long Volume { get; set; } - - /// - /// Refers to IEX�s percentage of the market in the stock. - /// - public decimal? IexMarketPercent { get; set; } - - /// - /// Refers to shares traded in the stock on IEX. - /// - public int? IexVolume { get; set; } - - /// - /// Refers to the 30 day average volume. - /// - public int AvgTotalVolume { get; set; } - - /// - /// Refers to the best bid price on IEX. - /// - public decimal? IexBidPrice { get; set; } - - /// - /// Refers to amount of shares on the bid on IEX. - /// - public int? IexBidSize { get; set; } - - /// - /// Refers to the best ask price on IEX. - /// - public decimal? IexAskPrice { get; set; } - - /// - /// Refers to amount of shares on the ask on IEX. - /// - public int? IexAskSize { get; set; } - - /// - /// Refers to IEX previous day open price - /// - public decimal? IexOpen { get; set; } - - /// - /// Refers to IEX previous day open time - /// - public long? IexOpenTime { get; set; } - - /// - /// Refers to IEX previous day close price - /// - public decimal? IexClose { get; set; } - - /// - /// Refers to IEX previous day close time - /// - public long? IexCloseTime { get; set; } - - /// - /// is calculated in real time using latestPrice. - /// - public long MarketCap { get; set; } - - /// - /// Refers to the price-to-earnings ratio for the company. - /// - public decimal? PeRatio { get; set; } - - /// - /// Refers to the adjusted 52 week high. - /// - public decimal Week52High { get; set; } - - /// - /// Refers to the adjusted 52 week low. - /// - public decimal Week52Low { get; set; } - - /// - /// Refers to the price change percentage from start of year to previous close. - /// - public decimal YtdChange { get; set; } - - /// - /// Epoch timestamp in milliseconds of the last market hours trade excluding the closing auction trade. - /// - public long? LastTradeTime { get; set; } - - /// - /// Returns a string that represents the current object. - /// - /// - public override string ToString() - { - return $"{Symbol},{LatestTime},{LatestPrice},{LatestVolume}"; - } - - } -} diff --git a/ToolBox/Program.cs b/ToolBox/Program.cs index b6c76f0b7079..ad0bef0c7065 100644 --- a/ToolBox/Program.cs +++ b/ToolBox/Program.cs @@ -21,7 +21,6 @@ using QuantConnect.ToolBox.CoinApiDataConverter; using QuantConnect.ToolBox.CryptoiqDownloader; using QuantConnect.ToolBox.DukascopyDownloader; -using QuantConnect.ToolBox.IEX; using QuantConnect.ToolBox.IVolatilityEquityConverter; using QuantConnect.ToolBox.KaikoDataConverter; using QuantConnect.ToolBox.KrakenDownloader; @@ -86,10 +85,6 @@ var factorFileProvider case "dukascopydownloader": DukascopyDownloaderProgram.DukascopyDownloader(tickers, resolution, fromDate, toDate); break; - case "iexdl": - case "iexdownloader": - IEXDownloaderProgram.IEXDownloader(tickers, resolution, fromDate, toDate); - break; case "kdl": case "krakendownloader": KrakenDownloaderProgram.KrakenDownloader(tickers, resolution, fromDate, toDate); diff --git a/ToolBox/QuantConnect.ToolBox.csproj b/ToolBox/QuantConnect.ToolBox.csproj index 15922803b8ec..5549b27eab35 100644 --- a/ToolBox/QuantConnect.ToolBox.csproj +++ b/ToolBox/QuantConnect.ToolBox.csproj @@ -43,7 +43,6 @@ - diff --git a/ToolBox/README.md b/ToolBox/README.md index 17bad1c2ce2f..3e9d7c4ce55c 100644 --- a/ToolBox/README.md +++ b/ToolBox/README.md @@ -32,7 +32,6 @@ Example: --app=YahooDownloader --tickers=SPY,AAPL --resolution=Daily --from-date - OandaDownloader or ODL - QuandlBitfinexDownloader or QBDL - YahooDownloader or YDL - - IEXDownloader or IEXDL - BitfinexDownloader or BFXDL - **'--from-date=yyyyMMdd-HH:mm:ss'** required - **'--tickers=SPY,AAPL,etc'** required, except for QuandlBitfinexDownloader (QBDL) From e8af441cff1a884d1f242dd6ebcd45c42042526a Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 30 Jan 2024 01:09:42 +0200 Subject: [PATCH 2/2] revert: IEX config in config.json remove: IEX downloader param in toolboxParse --- Configuration/ToolboxArgumentParser.cs | 4 ++-- Launcher/config.json | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Configuration/ToolboxArgumentParser.cs b/Configuration/ToolboxArgumentParser.cs index 6c143a1dc090..0e16b7a17d79 100644 --- a/Configuration/ToolboxArgumentParser.cs +++ b/Configuration/ToolboxArgumentParser.cs @@ -35,7 +35,7 @@ public static class ToolboxArgumentParser { new CommandLineOption("app", CommandOptionType.SingleValue, "[REQUIRED] Target tool, CASE INSENSITIVE: GDAXDownloader or GDAXDL/CryptoiqDownloader or CDL" - + "/DukascopyDownloader or DDL/IEXDownloader or IEXDL" + + "/DukascopyDownloader" + "/FxcmDownloader or FDL/FxcmVolumeDownload or FVDL/GoogleDownloader or GDL/IBDownloader or IBDL" + "/KrakenDownloader or KDL/OandaDownloader or ODL/QuandlBitfinexDownloader or QBDL" + "/YahooDownloader or YDL/AlgoSeekFuturesConverter or ASFC" @@ -55,7 +55,7 @@ public static class ToolboxArgumentParser new CommandLineOption("to-date", CommandOptionType.SingleValue, "[OPTIONAL for downloaders] If not provided 'DateTime.UtcNow' will " + "be used. --to-date=yyyyMMdd-HH:mm:ss"), new CommandLineOption("exchange", CommandOptionType.SingleValue, "[REQUIRED for CryptoiqDownloader] [Optional for KaikoDataConverter] The exchange to process, if not defined, all exchanges will be processed."), - new CommandLineOption("api-key", CommandOptionType.SingleValue, "[REQUIRED for QuandlBitfinexDownloader, IEXDownloader]"), + new CommandLineOption("api-key", CommandOptionType.SingleValue, "[REQUIRED for QuandlBitfinexDownloader]"), new CommandLineOption("date", CommandOptionType.SingleValue, "[REQUIRED for AlgoSeekFuturesConverter, AlgoSeekOptionsConverter, KaikoDataConverter]" + "Date for the option bz files: --date=yyyyMMdd"), new CommandLineOption("source-dir", CommandOptionType.SingleValue, "[REQUIRED for IVolatilityEquityConverter, KaikoDataConverter," diff --git a/Launcher/config.json b/Launcher/config.json index cf2c9183d5f7..326597e5066a 100644 --- a/Launcher/config.json +++ b/Launcher/config.json @@ -225,6 +225,9 @@ // To get your access token go to https://www.eia.gov/opendata "us-energy-information-auth-token": "", + // Required for IEX history requests + "iex-cloud-api-key": "", + // Required for market data from Coin API "coinapi-api-key": "", "coinapi-product": "free", // free, startup, streamer, professional, enterprise