Skip to content

Commit

Permalink
Merge pull request #19 from jhonabreul/feature-null-history-for-unsup…
Browse files Browse the repository at this point in the history 
…ported-securities

Return null on unsupported history and data download requests
  • Loading branch information
@jhonabreul
jhonabreul authored Feb 27, 2024
2 parents cb7b0ba + 951526c commit 285d5fc
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 84 deletions.
3 changes: 3 additions & 0 deletions QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageDataQueueHandlerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Microsoft.CodeAnalysis;
using QuantConnect.Data.Market;
using System.Collections.Generic;
using QuantConnect.Tests;

namespace QuantConnect.CoinbaseBrokerage.Tests
{
Expand All @@ -39,6 +40,8 @@ private static IEnumerable<TestCaseData> TestParameters
{
get
{
TestGlobals.Initialize();

yield return new TestCaseData(BTCUSDC, Resolution.Tick);
yield return new TestCaseData(BTCUSDC, Resolution.Second);
yield return new TestCaseData(BTCUSDC, Resolution.Minute);
Expand Down
83 changes: 37 additions & 46 deletions QuantConnect.CoinbaseBrokerage.Tests/CoinbaseBrokerageHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,68 +25,55 @@
using System.Collections.Generic;
using QuantConnect.Configuration;
using QuantConnect.Lean.Engine.DataFeeds;
using QuantConnect.Lean.Engine.HistoricalData;

namespace QuantConnect.CoinbaseBrokerage.Tests
{
[TestFixture]
public class CoinbaseBrokerageHistoryProviderTests
{
[Test, TestCaseSource(nameof(TestParameters))]
public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool shouldBeEmpty)
public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool unsupported)
{
var aggregator = new AggregationManager();

var brokerage = new CoinbaseBrokerage(
Config.Get("coinbase-url", "wss://advanced-trade-ws.coinbase.com"),
Config.Get("coinbase-api-key"),
Config.Get("coinbase-api-secret"),
Config.Get("coinbase-rest-api", "https://api.coinbase.com"),
null,
aggregator,
new AggregationManager(),
null);

var historyProvider = new BrokerageHistoryProvider();
historyProvider.SetBrokerage(brokerage);
historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), null));

var now = DateTime.UtcNow;

var requests = new[]
{
new HistoryRequest(now.Add(-period),
now,
typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
DateTimeZone.Utc,
resolution,
false,
false,
DataNormalizationMode.Adjusted,
tickType)
};

var history = historyProvider.GetHistory(requests, TimeZones.Utc).ToList();

foreach (var slice in history)
var request = new HistoryRequest(now.Add(-period),
now,
typeof(TradeBar),
symbol,
resolution,
SecurityExchangeHours.AlwaysOpen(TimeZones.Utc),
DateTimeZone.Utc,
resolution,
false,
false,
DataNormalizationMode.Adjusted,
tickType);

var history = brokerage.GetHistory(request)?.ToList();

if (unsupported)
{
var bar = slice.Bars[symbol];

Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}");
Assert.IsNull(history);
return;
}

if (shouldBeEmpty)
{
Assert.IsTrue(history.Count == 0);
}
else
Assert.IsNotNull(history);
Assert.IsNotEmpty(history);

foreach (var bar in history.Cast<TradeBar>())
{
Assert.IsTrue(history.Count > 0);
Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}");
}

Log.Trace("Data points retrieved: " + historyProvider.DataPointCount);
Log.Trace("Data points retrieved: " + history.Count);
}

private static IEnumerable<TestCaseData> TestParameters
Expand All @@ -106,21 +93,25 @@ private static IEnumerable<TestCaseData> TestParameters
yield return new TestCaseData(BTCUSDC, Resolution.Minute, TickType.Trade, Time.OneHour, false);
yield return new TestCaseData(BTCUSDC, Resolution.Hour, TickType.Trade, Time.OneDay, false);

// quote tick type, no error, empty result
// invalid period
yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(-15), true);

// quote tick type, null result
yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Quote, TimeSpan.FromDays(15), true);
yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.OpenInterest, TimeSpan.FromDays(15), true);

// invalid resolution, no error, empty result
// invalid resolution, null result
yield return new TestCaseData(BTCUSD, Resolution.Tick, TickType.Trade, TimeSpan.FromSeconds(15), true);
yield return new TestCaseData(BTCUSD, Resolution.Second, TickType.Trade, Time.OneMinute, true);

// invalid period, no error, empty result
yield return new TestCaseData(BTCUSD, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(-15), true);

// invalid symbol, no error, empty result
// invalid symbol, null result
yield return new TestCaseData(Symbol.Create("ABCXYZ", SecurityType.Crypto, Market.Coinbase), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true);

// invalid security type, no error, empty result
// invalid security type, null result
yield return new TestCaseData(Symbols.EURGBP, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true);

// invalid market, null result
yield return new TestCaseData(Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Binance), Resolution.Daily, TickType.Trade, TimeSpan.FromDays(15), true);
}
}
}
Expand Down
12 changes: 3 additions & 9 deletions QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,8 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

if (tickType != TickType.Trade)
{
return Enumerable.Empty<BaseData>();
}

var type = default(Type);
if(resolution == Resolution.Tick)
Type type;
if (resolution == Resolution.Tick)
{
type = typeof(Tick);
}
Expand Down Expand Up @@ -77,8 +72,7 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
tickType);

var brokerage = CreateBrokerage();
var data = brokerage.GetHistory(historyRequest);
return data;
return brokerage.GetHistory(historyRequest);
}

/// <summary>
Expand Down
4 changes: 4 additions & 0 deletions QuantConnect.CoinbaseBrokerage.ToolBox/CoinbaseDownloaderProgram.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ public static void CoinbaseDownloader(IList<string> tickers, string resolution,
// Download the data
var symbolObject = Symbol.Create(ticker, SecurityType.Crypto, market);
var data = downloader.Get(new DataDownloaderGetParameters(symbolObject, castResolution, fromDate, toDate));
if (data == null)
{
continue;
}

// Save the data
var writer = new LeanDataWriter(castResolution, symbolObject, dataDirectory, TickType.Trade);
Expand Down
61 changes: 33 additions & 28 deletions QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.HistoryProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@ public partial class CoinbaseBrokerage
/// <summary>
/// Prevent spam to external source
/// </summary>
private bool _loggedCoinbaseSupportsOnlyTradeBars = false;
private bool _loggedCoinbaseSupportsOnlyTradeBars;
private bool _loggedUnsupportedAssetForHistory;
private bool _loggedUnsupportedResolutionForHistory;
private bool _loggedInvalidTimeRangeForHistory;

/// <summary>
/// Gets the history for the requested security
Expand All @@ -40,6 +43,16 @@ public partial class CoinbaseBrokerage
/// <returns>An enumerable of bars covering the span specified in the request</returns>
public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
{
if (!CanSubscribe(request.Symbol))
{
if (!_loggedUnsupportedAssetForHistory)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "UnsupportedAsset",
$"Unsupported asset: {request.Symbol.Value}, no history returned"));
}
return null;
}

// Coinbase API only allows us to support history requests for TickType.Trade
if (request.TickType != TickType.Trade)
{
Expand All @@ -49,50 +62,41 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
_algorithm?.Debug($"Warning.{nameof(CoinbaseBrokerage)}: history provider only supports trade information, does not support quotes.");
Log.Error($"{nameof(CoinbaseBrokerage)}.{nameof(GetHistory)}(): only supports TradeBars");
}
yield break;
return null;
}

if (!_symbolMapper.IsKnownLeanSymbol(request.Symbol))
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol",
$"Unknown symbol: {request.Symbol.Value}, no history returned"));
yield break;
}

if (request.Symbol.SecurityType != SecurityType.Crypto)
if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType",
$"{request.Symbol.SecurityType} security type not supported, no history returned"));
yield break;
if (!_loggedUnsupportedResolutionForHistory)
{
_loggedUnsupportedResolutionForHistory = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution not supported, no history returned"));
}
return null;
}

if (request.StartTimeUtc >= request.EndTimeUtc)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange",
"The history request start date must precede the end date, no history returned"));
yield break;
}

if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
{
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
$"{request.Resolution} resolution not supported, no history returned"));
yield break;
if (!_loggedInvalidTimeRangeForHistory)
{
_loggedInvalidTimeRangeForHistory = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange",
"The history request start date must precede the end date, no history returned"));
}
return null;
}

Log.Debug($"{nameof(CoinbaseBrokerage)}.{nameof(GetHistory)}: Submitting request: {request.Symbol.Value}: {request.Resolution} {request.StartTimeUtc} UTC -> {request.EndTimeUtc} UTC");

foreach (var tradeBar in GetHistoryFromCandles(request))
{
yield return tradeBar;
}
return GetHistoryFromCandles(request);
}

/// <summary>
/// Returns TradeBars from Coinbase candles (only for Minute/Hour/Daily resolutions)
/// </summary>
/// <param name="request">The history request instance</param>
private IEnumerable<TradeBar> GetHistoryFromCandles(HistoryRequest request)
private IEnumerable<BaseData> GetHistoryFromCandles(HistoryRequest request)
{
var productId = _symbolMapper.GetBrokerageSymbol(request.Symbol);
var resolutionTimeSpan = request.Resolution.ToTimeSpan();
Expand All @@ -107,6 +111,7 @@ private IEnumerable<TradeBar> GetHistoryFromCandles(HistoryRequest request)
Resolution.Minute => CandleGranularity.OneMinute,
Resolution.Hour => CandleGranularity.OneHour,
Resolution.Daily => CandleGranularity.OneDay,
// This should never happen if the right checks are in place in the caller
_ => throw new NotSupportedException($"The resolution {request.Resolution} is not supported.")
};

Expand Down
3 changes: 2 additions & 1 deletion QuantConnect.CoinbaseBrokerage/CoinbaseBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -370,7 +370,8 @@ protected virtual bool CanSubscribe(Symbol symbol)
{
return !symbol.Value.Contains("UNIVERSE") &&
symbol.SecurityType == SecurityType.Crypto &&
symbol.ID.Market == MarketName;
symbol.ID.Market == MarketName &&
_symbolMapper.IsKnownLeanSymbol(symbol);
}

#endregion
Expand Down

0 comments on commit 285d5fc

Please sign in to comment.