Add safe fetch ohlcv dydx method (#686)

* Added a new method safe_fetch_ohlcv_dydx() to fetch_ohlcv.py. Added happy path unit test with mocked dydx response.

* Split the test_safe_fetch_ohlcv() function into two: one for ccxt, one for dydx, so that the ccxt parametrization does not affect the dydx test.

* Added new utility time function to_iso_timestr() to time_types.py and a corresponding unit test for it.

pdr_backend/lake/fetch_ohlcv.py

@@ -3,6 +3,7 @@
 
 from enforce_typing import enforce_types
 import numpy as np
+import requests
 
 from pdr_backend.cli.arg_feed import ArgFeed
 from pdr_backend.cli.arg_timeframe import ArgTimeframe
@@ -55,6 +56,53 @@ def safe_fetch_ohlcv_ccxt(
         return None
 
 
+@enforce_types
+def safe_fetch_ohlcv_dydx(
+    exch,
+    symbol: str,
+    timeframe: str,
+    since: UnixTimeMs,
+    limit: int,
+) -> Union[List[tuple], None]:
+    """
+    @description
+      calls ccxt.exchange.fetch_ohlcv() but if there's an error it
+      emits a warning and returns None, vs crashing everything
+
+    @arguments
+      exch -- eg dydx
+      symbol -- eg "BTC-USD"
+      timeframe -- eg "1HOUR", "5MINS"
+      since -- timestamp of first candle. In unix time (in ms)
+      limit -- max is 100 candles to retrieve,
+
+    @return
+      raw_tohlcv_data -- [a TOHLCV tuple, for each timestamp].
+        where row 0 is oldest
+        and TOHLCV = {unix time (in ms), Open, High, Low, Close, Volume}
+    """
+
+    try:
+        if exch == "dydx":
+            sinceIso = since.to_iso_timestr()
+            headers = {"Accept": "application/json"}
+            response = requests.get(
+                f"https://indexer.dydx.trade/v4/candles/perpetualMarkets/{symbol}?resolution={timeframe}&fromISO={sinceIso}&limit={limit}",
+                headers=headers,
+            )
+            response = requests.get(
+                f"https://indexer.dydx.trade/v4/candles/perpetualMarkets/BTC-USD?resolution=5MINS&fromISO=2024-02-27T00:00:00.000Z&limit=1",
+                headers=headers,
+            )
+            data = response.json()
+            return data
+        else:
+            return None
+    except Exception as e:
+        logger.warning("exchange: %s", e)
+        return None
+
+
 @enforce_types
 def clean_raw_ohlcv(
     raw_tohlcv_data: Union[list, None],

pdr_backend/lake/test/test_fetch_ohlcv.py

@@ -1,6 +1,7 @@
 import ccxt
 import pytest
 from enforce_typing import enforce_types
+import requests_mock
 
 import polars as pl
 
@@ -11,6 +12,7 @@
     _ohlcv_to_uts,
     clean_raw_ohlcv,
     safe_fetch_ohlcv_ccxt,
+    safe_fetch_ohlcv_dydx,
 )
 from pdr_backend.util.time_types import UnixTimeMs
 from pdr_backend.lake.constants import TOHLCV_SCHEMA_PL
@@ -25,6 +27,22 @@
 RAW7 = [T7, 0.5, 10, 0.10, 3.3, 7.0]
 RAW8 = [T8, 0.5, 9, 0.09, 4.4, 7.0]
 
+mock_dydx_response = {
+    "candles": [
+        {
+            "startedAt": "2024-02-28T16:50:00.000Z",
+            "open": "61840",
+            "high": "61848",
+            "low": "61687",
+            "close": "61800",
+            "baseTokenVolume": "23.6064",
+            "usdVolume": "1458183.4133",
+            "trades": 284,
+            "startingOpenInterest": "504.4262",
+        }
+    ]
+}
+
 
 @enforce_types
 def test_clean_raw_ohlcv():
@@ -63,11 +81,11 @@ def test_clean_raw_ohlcv():
 
 @enforce_types
 @pytest.mark.parametrize("exch", [ccxt.binanceus(), ccxt.kraken()])
-def test_safe_fetch_ohlcv(exch):
+def test_safe_fetch_ohlcv_ccxt(exch):
     since = UnixTimeMs.from_timestr("2023-06-18")
     symbol, timeframe, limit = "ETH/USDT", "5m", 1000
 
-    # happy path
+    # happy path ccxt
     raw_tohlc_data = safe_fetch_ohlcv_ccxt(exch, symbol, timeframe, since, limit)
     assert_raw_tohlc_data_ok(raw_tohlc_data)
 
@@ -97,6 +115,43 @@ def test_safe_fetch_ohlcv(exch):
     assert v is None
 
 
+@enforce_types
+def test_safe_fetch_ohlcv_dydx():
+    with requests_mock.Mocker() as m:
+        m.register_uri(
+            "GET",
+            "https://indexer.dydx.trade/v4/candles/perpetualMarkets/BTC-USD?resolution=5MINS&fromISO=2024-02-27T00:00:00.000Z&limit=1",
+            json=mock_dydx_response,
+        )
+        # happy path dydx
+        exch, symbol, timeframe, since, limit = (
+            "dydx",
+            "BTC-USD",
+            "5MINS",
+            UnixTimeMs.from_timestr("2024-02-27"),
+            1,
+        )
+        result = safe_fetch_ohlcv_dydx(exch, symbol, timeframe, since, limit)
+
+        # check the result is a list called 'candles' with data for only one 5min candle (because limit=1)
+        assert result is not None
+        assert list(result.keys())[0] == "candles" and len(result) == 1
+
+        # check the candle's data
+        assert (
+            list(result["candles"][0].keys())[0] == "startedAt"
+            and result["candles"][0]["startedAt"] == "2024-02-28T16:50:00.000Z"
+        )
+        assert (
+            list(result["candles"][0].keys())[2] == "high"
+            and result["candles"][0]["high"] == "61848"
+        )
+        assert (
+            list(result["candles"][0].keys())[5] == "baseTokenVolume"
+            and result["candles"][0]["baseTokenVolume"] == "23.6064"
+        )
+
+
 @enforce_types
 def assert_raw_tohlc_data_ok(raw_tohlc_data):
     assert raw_tohlc_data, raw_tohlc_data

pdr_backend/util/test_noganache/test_time_types.py

@@ -80,6 +80,29 @@ def test_ut_to_timestr():
     assert UnixTimeMs(1648576512345).to_timestr() == "2022-03-29_17:55:12.345"
 
 
+@enforce_types
+def test_ut_to_iso_timestr():
+    # ensure it returns a str
+    assert isinstance(UnixTimeMs(0).to_iso_timestr(), str)
+    assert isinstance(UnixTimeMs(1).to_iso_timestr(), str)
+    assert isinstance(UnixTimeMs(1648576500000).to_iso_timestr(), str)
+    assert isinstance(UnixTimeMs(1648576500001).to_iso_timestr(), str)
+
+    # unix start time (Jan 1 1970), with increasing levels of precision
+    assert UnixTimeMs(0).to_iso_timestr() == "1970-01-01T00:00:00.000Z"
+    assert UnixTimeMs(1).to_iso_timestr() == "1970-01-01T00:00:00.001Z"
+    assert UnixTimeMs(123).to_iso_timestr() == "1970-01-01T00:00:00.123Z"
+    assert UnixTimeMs(1000).to_iso_timestr() == "1970-01-01T00:00:01.000Z"
+    assert UnixTimeMs(1234).to_iso_timestr() == "1970-01-01T00:00:01.234Z"
+    assert UnixTimeMs(10002).to_iso_timestr() == "1970-01-01T00:00:10.002Z"
+    assert UnixTimeMs(12345).to_iso_timestr() == "1970-01-01T00:00:12.345Z"
+
+    # modern times
+    assert UnixTimeMs(1648512000000).to_iso_timestr() == "2022-03-29T00:00:00.000Z"
+    assert UnixTimeMs(1648576500000).to_iso_timestr() == "2022-03-29T17:55:00.000Z"
+    assert UnixTimeMs(1648576512345).to_iso_timestr() == "2022-03-29T17:55:12.345Z"
+
+
 @enforce_types
 def test_dt_to_ut_and_back():
     dt = datetime.datetime.strptime("2022-03-29_17:55", "%Y-%m-%d_%H:%M")

pdr_backend/util/time_types.py

@@ -78,5 +78,10 @@ def to_timestr(self) -> str:
 
         return dt.strftime("%Y-%m-%d_%H:%M:%S.%f")[:-3]
 
+    def to_iso_timestr(self) -> str:
+        dt: datetime = self.to_dt()
+
+        return dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"  # tack on timezone
+
     def pretty_timestr(self) -> str:
         return f"timestamp={self}, dt={self.to_timestr()}"

setup.py

@@ -31,6 +31,7 @@
     "pytest-env",
     "pyyaml",
     "requests",
+    "requests-mock",
     "scikit-learn",
     "statsmodels",
     "types-pyYAML",