diff --git a/pdr_backend/lake/fetch_ohlcv.py b/pdr_backend/lake/fetch_ohlcv.py index ab543c82c..da558a0e4 100644 --- a/pdr_backend/lake/fetch_ohlcv.py +++ b/pdr_backend/lake/fetch_ohlcv.py @@ -3,6 +3,7 @@ from enforce_typing import enforce_types import numpy as np +import requests from pdr_backend.cli.arg_feed import ArgFeed from pdr_backend.cli.arg_timeframe import ArgTimeframe @@ -55,6 +56,53 @@ def safe_fetch_ohlcv_ccxt( return None +@enforce_types +def safe_fetch_ohlcv_dydx( + exch, + symbol: str, + timeframe: str, + since: UnixTimeMs, + limit: int, +) -> Union[List[tuple], None]: + """ + @description + calls ccxt.exchange.fetch_ohlcv() but if there's an error it + emits a warning and returns None, vs crashing everything + + @arguments + exch -- eg dydx + symbol -- eg "BTC-USD" + timeframe -- eg "1HOUR", "5MINS" + since -- timestamp of first candle. In unix time (in ms) + limit -- max is 100 candles to retrieve, + + @return + raw_tohlcv_data -- [a TOHLCV tuple, for each timestamp]. + where row 0 is oldest + and TOHLCV = {unix time (in ms), Open, High, Low, Close, Volume} + """ + + try: + if exch == "dydx": + sinceIso = since.to_iso_timestr() + headers = {"Accept": "application/json"} + response = requests.get( + f"https://indexer.dydx.trade/v4/candles/perpetualMarkets/{symbol}?resolution={timeframe}&fromISO={sinceIso}&limit={limit}", + headers=headers, + ) + response = requests.get( + f"https://indexer.dydx.trade/v4/candles/perpetualMarkets/BTC-USD?resolution=5MINS&fromISO=2024-02-27T00:00:00.000Z&limit=1", + headers=headers, + ) + data = response.json() + return data + else: + return None + except Exception as e: + logger.warning("exchange: %s", e) + return None + + @enforce_types def clean_raw_ohlcv( raw_tohlcv_data: Union[list, None], diff --git a/pdr_backend/lake/test/test_fetch_ohlcv.py b/pdr_backend/lake/test/test_fetch_ohlcv.py index 6a26fec1d..2531bd305 100644 --- a/pdr_backend/lake/test/test_fetch_ohlcv.py +++ b/pdr_backend/lake/test/test_fetch_ohlcv.py @@ -1,6 +1,7 @@ import ccxt import pytest from enforce_typing import enforce_types +import requests_mock import polars as pl @@ -11,6 +12,7 @@ _ohlcv_to_uts, clean_raw_ohlcv, safe_fetch_ohlcv_ccxt, + safe_fetch_ohlcv_dydx, ) from pdr_backend.util.time_types import UnixTimeMs from pdr_backend.lake.constants import TOHLCV_SCHEMA_PL @@ -25,6 +27,22 @@ RAW7 = [T7, 0.5, 10, 0.10, 3.3, 7.0] RAW8 = [T8, 0.5, 9, 0.09, 4.4, 7.0] +mock_dydx_response = { + "candles": [ + { + "startedAt": "2024-02-28T16:50:00.000Z", + "open": "61840", + "high": "61848", + "low": "61687", + "close": "61800", + "baseTokenVolume": "23.6064", + "usdVolume": "1458183.4133", + "trades": 284, + "startingOpenInterest": "504.4262", + } + ] +} + @enforce_types def test_clean_raw_ohlcv(): @@ -63,11 +81,11 @@ def test_clean_raw_ohlcv(): @enforce_types @pytest.mark.parametrize("exch", [ccxt.binanceus(), ccxt.kraken()]) -def test_safe_fetch_ohlcv(exch): +def test_safe_fetch_ohlcv_ccxt(exch): since = UnixTimeMs.from_timestr("2023-06-18") symbol, timeframe, limit = "ETH/USDT", "5m", 1000 - # happy path + # happy path ccxt raw_tohlc_data = safe_fetch_ohlcv_ccxt(exch, symbol, timeframe, since, limit) assert_raw_tohlc_data_ok(raw_tohlc_data) @@ -97,6 +115,43 @@ def test_safe_fetch_ohlcv(exch): assert v is None +@enforce_types +def test_safe_fetch_ohlcv_dydx(): + with requests_mock.Mocker() as m: + m.register_uri( + "GET", + "https://indexer.dydx.trade/v4/candles/perpetualMarkets/BTC-USD?resolution=5MINS&fromISO=2024-02-27T00:00:00.000Z&limit=1", + json=mock_dydx_response, + ) + # happy path dydx + exch, symbol, timeframe, since, limit = ( + "dydx", + "BTC-USD", + "5MINS", + UnixTimeMs.from_timestr("2024-02-27"), + 1, + ) + result = safe_fetch_ohlcv_dydx(exch, symbol, timeframe, since, limit) + + # check the result is a list called 'candles' with data for only one 5min candle (because limit=1) + assert result is not None + assert list(result.keys())[0] == "candles" and len(result) == 1 + + # check the candle's data + assert ( + list(result["candles"][0].keys())[0] == "startedAt" + and result["candles"][0]["startedAt"] == "2024-02-28T16:50:00.000Z" + ) + assert ( + list(result["candles"][0].keys())[2] == "high" + and result["candles"][0]["high"] == "61848" + ) + assert ( + list(result["candles"][0].keys())[5] == "baseTokenVolume" + and result["candles"][0]["baseTokenVolume"] == "23.6064" + ) + + @enforce_types def assert_raw_tohlc_data_ok(raw_tohlc_data): assert raw_tohlc_data, raw_tohlc_data diff --git a/pdr_backend/util/test_noganache/test_time_types.py b/pdr_backend/util/test_noganache/test_time_types.py index 12cbc4f60..b74aaec32 100644 --- a/pdr_backend/util/test_noganache/test_time_types.py +++ b/pdr_backend/util/test_noganache/test_time_types.py @@ -80,6 +80,29 @@ def test_ut_to_timestr(): assert UnixTimeMs(1648576512345).to_timestr() == "2022-03-29_17:55:12.345" +@enforce_types +def test_ut_to_iso_timestr(): + # ensure it returns a str + assert isinstance(UnixTimeMs(0).to_iso_timestr(), str) + assert isinstance(UnixTimeMs(1).to_iso_timestr(), str) + assert isinstance(UnixTimeMs(1648576500000).to_iso_timestr(), str) + assert isinstance(UnixTimeMs(1648576500001).to_iso_timestr(), str) + + # unix start time (Jan 1 1970), with increasing levels of precision + assert UnixTimeMs(0).to_iso_timestr() == "1970-01-01T00:00:00.000Z" + assert UnixTimeMs(1).to_iso_timestr() == "1970-01-01T00:00:00.001Z" + assert UnixTimeMs(123).to_iso_timestr() == "1970-01-01T00:00:00.123Z" + assert UnixTimeMs(1000).to_iso_timestr() == "1970-01-01T00:00:01.000Z" + assert UnixTimeMs(1234).to_iso_timestr() == "1970-01-01T00:00:01.234Z" + assert UnixTimeMs(10002).to_iso_timestr() == "1970-01-01T00:00:10.002Z" + assert UnixTimeMs(12345).to_iso_timestr() == "1970-01-01T00:00:12.345Z" + + # modern times + assert UnixTimeMs(1648512000000).to_iso_timestr() == "2022-03-29T00:00:00.000Z" + assert UnixTimeMs(1648576500000).to_iso_timestr() == "2022-03-29T17:55:00.000Z" + assert UnixTimeMs(1648576512345).to_iso_timestr() == "2022-03-29T17:55:12.345Z" + + @enforce_types def test_dt_to_ut_and_back(): dt = datetime.datetime.strptime("2022-03-29_17:55", "%Y-%m-%d_%H:%M") diff --git a/pdr_backend/util/time_types.py b/pdr_backend/util/time_types.py index 8099958e0..14f731196 100644 --- a/pdr_backend/util/time_types.py +++ b/pdr_backend/util/time_types.py @@ -78,5 +78,10 @@ def to_timestr(self) -> str: return dt.strftime("%Y-%m-%d_%H:%M:%S.%f")[:-3] + def to_iso_timestr(self) -> str: + dt: datetime = self.to_dt() + + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" # tack on timezone + def pretty_timestr(self) -> str: return f"timestamp={self}, dt={self.to_timestr()}" diff --git a/setup.py b/setup.py index e19796635..54affcf78 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ "pytest-env", "pyyaml", "requests", + "requests-mock", "scikit-learn", "statsmodels", "types-pyYAML",