Merge branch 'main' into techspike-dash

READMEs/predictoor.md

@@ -138,12 +138,13 @@ pdr deploy_pred_submitter_mgr my_ppss.yaml sapphire-testnet
 
 #### Update YAML config with the contract address
 
-Next, update `my_ppss.yaml` and input the contract address in place of `predictoor_ss.pred_submitter_mgr`:
+Next, update `my_ppss.yaml` and input the contract address in place of `predictoor_ss.bot_only.pred_submitter_mgr`:
 
 ```
 predictoor_ss:
   ...
-  pred_submitter_mgr: "CONTRACT_ADDRESS"
+  bot_only:
+    pred_submitter_mgr: "CONTRACT_ADDRESS"
   ...
 ```
 

pdr_backend/exchange/constants.py

@@ -3,3 +3,4 @@
 # all possible resolution values are listed at:
 # https://docs.dydx.exchange/developers/indexer/indexer_api#enumerated-values
 TIMEFRAME_TO_DYDX_RESOLUTION = {"5m": "5MINS", "1h": "1HOUR"}
+TIMEFRAME_TO_DYDX_RESOLUTION_SECONDS = {"5m": 300, "1h": 3600}

pdr_backend/exchange/fetch_ohlcv_dydx.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import timedelta
 from typing import List, Optional, Union
 
 from enforce_typing import enforce_types
@@ -9,6 +10,7 @@
 from pdr_backend.exchange.constants import (
     BASE_URL_DYDX,
     TIMEFRAME_TO_DYDX_RESOLUTION,
+    TIMEFRAME_TO_DYDX_RESOLUTION_SECONDS,
 )
 from pdr_backend.util.time_types import UnixTimeMs
 
@@ -54,11 +56,13 @@ def fetch_ohlcv_dydx(
     ticker = _dydx_ticker(pair_str)
     resolution = _dydx_resolution(timeframe)
     fromISO = since.to_iso_timestr()
+    toISO_dt = since.to_dt() + _time_delta_from_timeframe(timeframe, limit)
+    toISO = UnixTimeMs(toISO_dt.timestamp() * 1e3).to_iso_timestr()
     headers = {"Accept": "application/json"}
     try:
         s = (
             f"{baseURL}/candles/perpetualMarkets/{ticker}"
-            f"?resolution={resolution}&fromISO={fromISO}&limit={limit}"
+            f"?resolution={resolution}&fromISO={fromISO}&toISO={toISO}&limit={limit}"
         )
         response = requests.get(s, headers=headers, timeout=20)
 
@@ -125,3 +129,11 @@ def _dydx_resolution(timeframe: str):
 @enforce_types
 def _float_or_none(x: Optional[str]) -> Optional[float]:
     return None if x is None else float(x)
+
+
+def _time_delta_from_timeframe(timeframe: str, limit: int) -> timedelta:
+    # Convert timeframe and limit to a timedelta.
+    if timeframe not in TIMEFRAME_TO_DYDX_RESOLUTION_SECONDS:
+        raise ValueError(f"Don't currently support timeframe={timeframe}")
+    second_duration = TIMEFRAME_TO_DYDX_RESOLUTION_SECONDS[timeframe]
+    return timedelta(seconds=second_duration * limit)

pdr_backend/exchange/test/test_fetch_ohlcv_dydx.py

@@ -1,4 +1,6 @@
+from datetime import datetime, timedelta
 from enforce_typing import enforce_types
+
 import pytest
 import requests_mock
 
@@ -8,6 +10,7 @@
     _dydx_ticker,
     _dydx_resolution,
     _float_or_none,
+    _time_delta_from_timeframe,
 )
 from pdr_backend.util.time_types import UnixTimeMs
 
@@ -76,9 +79,8 @@ def test_dydx__real_response__basic():
     assert len(tohlcv) == 6
 
 
-@pytest.mark.skip(reason="Unskip once #879 is fixed")
 @enforce_types
-def test_dydx__real_response__fromISO_issue_879():
+def test_dydx__real_response__fromISO():
     # setup problem: 'tsince'
     tsince_iso_str = "2024-02-27_00:00:00.000"
     tsince_UnixTimeMs = UnixTimeMs.from_timestr(tsince_iso_str)
@@ -88,22 +90,34 @@ def test_dydx__real_response__fromISO_issue_879():
     # setup problem: the rest
     symbol = "BTC/USD"
     timeframe = "5m"
-    limit = 1
+    limit = 10
 
     # get result
     raw_tohlcv_data = fetch_ohlcv_dydx(symbol, timeframe, tsince_UnixTimeMs, limit)
-    tohlcv = raw_tohlcv_data[0]
 
-    # dydx api doesn't properly address fromISO. We must fix this, see #879
-    t = tohlcv[0]
-    t_UnixTimeMs = UnixTimeMs(t)
-    t_iso_str = t_UnixTimeMs.to_iso_timestr()  # bad eg '2024-04-16T00:25:00.000Z'
-    assert t_iso_str == tsince_iso_str
-    assert t_UnixTimeMs == tsince_UnixTimeMs
+    assert len(raw_tohlcv_data) == 10, "Length must be 10, limit is 10"
+
+    # First timestamp is expected to be:
+    # 2024-02-27T00:00:00.000Z
+    dt = datetime.fromisoformat("2024-02-27T00:00:00.000")
+    unix_ms = dt.timestamp() * 1e3
+    assert (
+        raw_tohlcv_data[-1][0] == unix_ms
+    ), f"Expected {unix_ms}, got {raw_tohlcv_data[-1][0]}"
+
+    # Last timestamp is expected to be:
+    # 2024-02-27T00:45:00.000Z
+    dt = datetime.fromisoformat("2024-02-27T00:45:00.000")
+    unix_ms = dt.timestamp() * 1e3
+    assert (
+        raw_tohlcv_data[0][0] == unix_ms
+    ), f"Expected {unix_ms}, got {raw_tohlcv_data[0][0]}"
 
-    # when #879 fixed, add proper vals here
-    # ohlcv = tohlcv[1:]
-    # assert ohlcv == (fix me val, ..)
+    # Price checks
+    assert raw_tohlcv_data[-1][1] == 54541.0
+    assert raw_tohlcv_data[-1][2] == 54661.0
+    assert raw_tohlcv_data[0][3] == 54545.0
+    assert raw_tohlcv_data[9][4] == 54645.0
 
 
 @enforce_types
@@ -168,3 +182,26 @@ def test_fetch_ohlcv_float_or_none():
         _ = _float_or_none(3)
     with pytest.raises(ValueError):
         _ = _float_or_none("foo")
+
+
+def test_time_delta_from_valid_timeframes():
+    test_cases = [
+        ("5m", 1, timedelta(seconds=300 * 1)),
+        ("5m", 10, timedelta(seconds=300 * 10)),
+        ("5m", 25, timedelta(seconds=300 * 25)),
+        ("5m", 500, timedelta(seconds=300 * 500)),
+        ("1h", 1, timedelta(seconds=3600 * 1)),
+        ("1h", 5, timedelta(seconds=3600 * 5)),
+        ("1h", 42, timedelta(seconds=3600 * 42)),
+    ]
+
+    for timeframe, limit, expected in test_cases:
+        assert (
+            _time_delta_from_timeframe(timeframe, limit) == expected
+        ), f"Failed for timeframe={timeframe}"
+
+
+def test_time_delta_from_invalid_timeframe():
+    with pytest.raises(ValueError) as excinfo:
+        _time_delta_from_timeframe("1hh", 1)
+    assert "Don't currently support timeframe=1hh" in str(excinfo.value)

pdr_backend/ppss/predictoor_ss.py

@@ -88,7 +88,7 @@ def s_until_epoch_end(self) -> int:
 
     @property
     def pred_submitter_mgr(self) -> str:
-        return self.d["pred_submitter_mgr"]
+        return self.d["bot_only"]["pred_submitter_mgr"]
 
     # --------------------------------
     # setters (add as needed)
@@ -180,13 +180,13 @@ def predictoor_ss_test_dict(
         "predict_train_feedsets": feedset_list,
         "approach": 1,
         "stake_amount": 1,
-        "pred_submitter_mgr": pred_submitter_mgr,
         "sim_only": {
             "others_stake": 2313,
             "others_accuracy": 0.50001,
             "revenue": 0.93007,
         },
         "bot_only": {
+            "pred_submitter_mgr": pred_submitter_mgr,
             "s_until_epoch_end": 60,
             "s_start_payouts": 0,
         },

pdr_backend/predictoor/test/test_predictoor_agent.py

@@ -75,7 +75,9 @@ def _test_predictoor_agent_main(
             monkeypatch,
         )
         assert ppss.predictoor_ss.approach == approach
-        ppss.predictoor_ss.d["pred_submitter_mgr"] = pred_submitter_mgr.contract_address
+        ppss.predictoor_ss.d["bot_only"][
+            "pred_submitter_mgr"
+        ] = pred_submitter_mgr.contract_address
         feed_contracts = ppss.web3_pp.query_feed_contracts()
         web3_config = ppss.web3_pp.web3_config
         w3 = ppss.web3_pp.w3
@@ -349,7 +351,7 @@ def test_calc_stakes_across_feeds(tmpdir, monkeypatch):
     )
     feed_contracts = ppss.web3_pp.query_feed_contracts()
     predictoor_ss = ppss.predictoor_ss
-    predictoor_ss.d["pred_submitter_mgr"] = "0x1"
+    predictoor_ss.d["bot_only"]["pred_submitter_mgr"] = "0x1"
     predictoor_ss.d["stake_amount"] = 1.0
     predictoor_ss.d["bot_only"]["s_until_epoch_end"] = 50000
     web3_config = ppss.web3_pp.web3_config

ppss.yaml

@@ -15,13 +15,13 @@ predictoor_ss:
       - binance BTC/USDT ETH/USDT c 5m
   approach: 2 # 1->50/50; 2->two-sided model-based; 3-> one-sided model-based
   stake_amount: 100 # How much your bot stakes. In OCEAN per epoch, per feed
-  pred_submitter_mgr: "DeployNewMgr" # DeployNewMgr | <address of deployed mgr>  
   sim_only:
     others_stake: 10000 # How much all others' bots stake. In OCEAN per epoch, per feed. Calcs: stake volume = 10e6 $/day/20_feeds = 34700 $/epoch/20_feeds (bc 288 5m epochs/day) = 1735 $/epoch/feed = 2313 OCEAN/epoch/feed (at 1 OCEAN=$0.75)
     others_accuracy: 0.50001 # What percent of others' bots stake is correct. Value in range 0.0-1.0. 0.50001 means mostly 50-50 stake and some small fry predicting a bit accurately
     revenue: 0.93006 # Sales revenue going towards predictoors. In OCEAN per epoch, per feed. Calcs: 37500 OCEAN/week/20_feeds (Predictoor DF rewards) = 18.6012 OCEAN/epoch/20_feeds (bc 2016 5m epochs/week) = 0.93006 OCEAN/epoch/feed
 
   bot_only:
+    pred_submitter_mgr: "DeployNewMgr" # DeployNewMgr | <address of deployed mgr>  
     s_start_payouts: 220 # in s. Run payout if > this time left. 0 to disable
     s_until_epoch_end: 60 # in s. Start predicting if > this time left