From 0208cfbde41623d30aadd0566d9a207376aef88e Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:01:10 +0200 Subject: [PATCH 1/6] hotfix(price_pusher_edge): Fixed some price pusher edge cases --- pragma-oracle | 2 +- .../pragma_sdk/common/fetchers/fetchers/bitstamp.py | 3 ++- .../pragma_sdk/common/fetchers/fetchers/defillama.py | 3 ++- pragma-sdk/pragma_sdk/common/fetchers/fetchers/huobi.py | 3 ++- pragma-sdk/pragma_sdk/common/fetchers/fetchers/kucoin.py | 4 +++- pragma-sdk/pragma_sdk/common/fetchers/fetchers/okx.py | 3 ++- .../pragma_sdk/common/fetchers/fetchers/starknetamm.py | 5 ++--- .../common/fetchers/future_fetchers/binance.py | 5 +++-- .../pragma_sdk/common/fetchers/future_fetchers/bybit.py | 4 ++-- .../pragma_sdk/common/fetchers/future_fetchers/okx.py | 5 +++-- price-pusher/config/config.example.yaml | 9 +-------- price-pusher/price_pusher/configs/fetchers.py | 2 ++ price-pusher/price_pusher/core/request_handlers/api.py | 1 - 13 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pragma-oracle b/pragma-oracle index 7f7c636a..00b95204 160000 --- a/pragma-oracle +++ b/pragma-oracle @@ -1 +1 @@ -Subproject commit 7f7c636adf6eabb08626820f5fbe3d6540cbe7e3 +Subproject commit 00b95204f764af0f7c6147a4699af51f2d4e19c6 diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/bitstamp.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/bitstamp.py index 54f0ce5a..86ed5c32 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/bitstamp.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/bitstamp.py @@ -1,4 +1,5 @@ import asyncio +import time from typing import Any, List from aiohttp import ClientSession @@ -39,7 +40,7 @@ def format_url(self, pair: Pair) -> str: return url def _construct(self, pair: Pair, result: Any) -> SpotEntry: - timestamp = int(result["timestamp"]) + timestamp = int(time.time()) price = float(result["last"]) price_int = int(price * (10 ** pair.decimals())) diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/defillama.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/defillama.py index b7983693..6e44e907 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/defillama.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/defillama.py @@ -1,4 +1,5 @@ import asyncio +import time from typing import Any, List, Optional from aiohttp import ClientSession @@ -89,7 +90,7 @@ def _construct( self, pair: Pair, result: Any, hop_result: Optional[Any] = None ) -> SpotEntry: base_id = AssetConfig.get_coingecko_id_from_ticker(pair.base_currency.id) - timestamp = int(result["coins"][f"coingecko:{base_id}"]["timestamp"]) + timestamp = int(time.time()) decimals = pair.decimals() if hop_result is not None: quote_id = AssetConfig.get_coingecko_id_from_ticker(pair.quote_currency.id) diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/huobi.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/huobi.py index 5a16c227..69df50a8 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/huobi.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/huobi.py @@ -1,4 +1,5 @@ import asyncio +import time from typing import List, Optional, Any from aiohttp import ClientSession @@ -104,7 +105,7 @@ def _construct( hop_ask = float(hop_result["tick"]["ask"][0]) hop_price = (hop_bid + hop_ask) / 2 price = hop_price / price - timestamp = int(result["ts"] / 1000) + timestamp = int(time.time()) price_int = int(price * (10 ** pair.decimals())) volume = float(result["tick"]["vol"]) if hop_result is None else 0 logger.debug("Fetched price %d for %s from Huobi", price_int, pair) diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/kucoin.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/kucoin.py index 15bf35c9..253b1d63 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/kucoin.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/kucoin.py @@ -1,4 +1,6 @@ import asyncio +import time + from typing import List, Optional, Any from aiohttp import ClientSession @@ -98,7 +100,7 @@ def _construct( if hop_result is not None: hop_price = float(hop_result["data"]["price"]) price = hop_price / price - timestamp = int(result["data"]["time"] / 1000) + timestamp = int(time.time()) price_int = int(price * (10 ** pair.decimals())) logger.debug("Fetched price %d for %s from Kucoin", price_int, pair) diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/okx.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/okx.py index cf854770..88e21c3a 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/okx.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/okx.py @@ -1,4 +1,5 @@ import asyncio +import time import json from typing import Any, List @@ -67,7 +68,7 @@ def format_url(self, pair: Pair) -> str: def _construct(self, pair: Pair, result: Any, usdt_price: float = 1) -> SpotEntry: data = result["data"][0] - timestamp = int(int(data["ts"]) / 1000) + timestamp = int(time.time()) price = float(data["last"]) / usdt_price price_int = int(price * (10 ** pair.decimals())) volume = float(data["volCcy24h"]) diff --git a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/starknetamm.py b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/starknetamm.py index fadbba91..104c3b3b 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/fetchers/starknetamm.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/fetchers/starknetamm.py @@ -62,9 +62,8 @@ async def fetch_pair( def format_url(self, pair: Pair, timestamp: Optional[int] = None) -> str: new_pair = self.hop_handler.get_hop_pair(pair) or pair if timestamp: - return f"{self.EKUBO_PUBLIC_API}/price/{pair.base_currency.id}/{new_pair}?atTime={timestamp}&period=3600" - - return f"{self.EKUBO_PUBLIC_API}/price/{pair.base_currency.id}/{new_pair}?period=3600" + return f"{self.EKUBO_PUBLIC_API}/price/{new_pair}?atTime={timestamp}&period=3600" + return f"{self.EKUBO_PUBLIC_API}/price/{new_pair}?period=3600" async def fetch( self, session: ClientSession diff --git a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/binance.py b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/binance.py index 725a74b6..64246a09 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/binance.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/binance.py @@ -1,4 +1,6 @@ import json +import time + from datetime import datetime, timezone from typing import Any, List, Optional, Tuple @@ -92,7 +94,6 @@ def _construct( result_arr = [] decimals = pair.decimals() for data in result: - timestamp = int(data["time"]) price = float(data["markPrice"]) price_int = int(price * (10**decimals)) volume = float(self._retrieve_volume(data["symbol"], volume_arr)) / ( @@ -116,7 +117,7 @@ def _construct( pair_id=pair.id, price=price_int, volume=int(volume), - timestamp=int(timestamp / 1000), + timestamp=int(time.time()), source=self.SOURCE, publisher=self.publisher, expiry_timestamp=expiry_timestamp * 1000, diff --git a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/bybit.py b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/bybit.py index d7487337..9a9dec97 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/bybit.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/bybit.py @@ -1,5 +1,6 @@ import asyncio import json +import time from typing import Any, List from aiohttp import ClientSession @@ -59,7 +60,6 @@ def format_url(self, pair: Pair) -> str: def _construct(self, pair: Pair, result: Any) -> FutureEntry: data = result["result"]["list"][0] decimals = pair.decimals() - timestamp = int(int(result["time"]) / 1000) price = float(data["lastPrice"]) price_int = int(price * (10**decimals)) @@ -73,7 +73,7 @@ def _construct(self, pair: Pair, result: Any) -> FutureEntry: pair_id=pair.id, price=price_int, volume=volume, - timestamp=timestamp, + timestamp=int(time.time()), source=self.SOURCE, publisher=self.publisher, expiry_timestamp=expiry_timestamp, diff --git a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/okx.py b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/okx.py index c6fb71ab..99edc2ef 100644 --- a/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/okx.py +++ b/pragma-sdk/pragma_sdk/common/fetchers/future_fetchers/okx.py @@ -1,4 +1,6 @@ import json +import time + from typing import Any, List, Union from aiohttp import ClientSession @@ -85,7 +87,6 @@ def format_url(self, pair: Pair) -> str: return url def _construct(self, pair: Pair, data: Any, expiry_timestamp: int) -> FutureEntry: - timestamp = int(int(data["ts"]) / 1000) decimals = pair.decimals() price = float(data["last"]) price_int = int(price * (10**decimals)) @@ -97,7 +98,7 @@ def _construct(self, pair: Pair, data: Any, expiry_timestamp: int) -> FutureEntr pair_id=pair.id, price=price_int, volume=volume, - timestamp=timestamp, + timestamp=int(time.time()), source=self.SOURCE, publisher=self.publisher, expiry_timestamp=int(expiry_timestamp), diff --git a/price-pusher/config/config.example.yaml b/price-pusher/config/config.example.yaml index 183620e4..09bb3680 100644 --- a/price-pusher/config/config.example.yaml +++ b/price-pusher/config/config.example.yaml @@ -1,15 +1,8 @@ - pairs: spot: + - STRK/USD - BTC/USD - ETH/USD - - DAI/USD - LORDS/USD time_difference: 120 price_deviation: 0.0025 - -- pairs: - future: - - BTC/USD - - ETH/USD - time_difference: 600 - price_deviation: 0.05 \ No newline at end of file diff --git a/price-pusher/price_pusher/configs/fetchers.py b/price-pusher/price_pusher/configs/fetchers.py index 4660a525..98f81859 100644 --- a/price-pusher/price_pusher/configs/fetchers.py +++ b/price-pusher/price_pusher/configs/fetchers.py @@ -13,6 +13,7 @@ OkxFetcher, BinanceFetcher, PropellerFetcher, + StarknetAMMFetcher, ) from pragma_sdk.common.fetchers.future_fetchers import BinanceFutureFetcher, ByBitFutureFetcher @@ -27,6 +28,7 @@ BybitFetcher, BinanceFetcher, PropellerFetcher, + StarknetAMMFetcher, ] ALL_FUTURE_FETCHERS: List[FetcherInterfaceT] = [BinanceFutureFetcher, ByBitFutureFetcher] diff --git a/price-pusher/price_pusher/core/request_handlers/api.py b/price-pusher/price_pusher/core/request_handlers/api.py index 83d6c73c..1515a361 100644 --- a/price-pusher/price_pusher/core/request_handlers/api.py +++ b/price-pusher/price_pusher/core/request_handlers/api.py @@ -42,7 +42,6 @@ async def _fetch_latest_spot_entry(self, pair: Pair) -> List[Entry]: ) if entry_result is None: return [] - entry = SpotEntry( pair_id=entry_result.pair_id, price=int(entry_result.data, 16), From 79992b8eeef8995053afc2c52cbaacede21791f7 Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:06:27 +0200 Subject: [PATCH 2/6] hotfix(price_pusher_edge): Don't notify for deviation == deviation max --- price-pusher/price_pusher/core/listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/price-pusher/price_pusher/core/listener.py b/price-pusher/price_pusher/core/listener.py index b3879763..08693fc7 100644 --- a/price-pusher/price_pusher/core/listener.py +++ b/price-pusher/price_pusher/core/listener.py @@ -337,7 +337,7 @@ def _new_price_is_deviating(self, pair_id: str, new_price: int, oracle_price: in """ max_deviation = self.price_config.price_deviation * oracle_price deviation = abs(new_price - oracle_price) - is_deviating = deviation >= max_deviation + is_deviating = deviation > max_deviation if is_deviating: deviation_percentage = (deviation / oracle_price) * 100 logger.info( From 0af12afe4258000e435b2277ae700df1ef866bea Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:10:23 +0200 Subject: [PATCH 3/6] hotfix(price_pusher_edge): new deviation :) --- price-pusher/tests/unit/core/test_listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/price-pusher/tests/unit/core/test_listener.py b/price-pusher/tests/unit/core/test_listener.py index dac0efaf..903fa1a2 100644 --- a/price-pusher/tests/unit/core/test_listener.py +++ b/price-pusher/tests/unit/core/test_listener.py @@ -128,7 +128,7 @@ async def test_oracle_needs_update_because_deviating(caplog, price_listener): caplog.set_level(logging.INFO) orchestrator_entry = SpotEntry( pair_id="BTC/USD", - price=110, + price=111, timestamp=1000000000, source="source_1", publisher="publisher_1", From 83059b7ba04c44bb2ee333cc86380e1f2c15cd9c Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:20:01 +0200 Subject: [PATCH 4/6] hotfix(price_pusher_edge): Checkpoints now pass the callback --- pragma-sdk/pragma_sdk/onchain/mixins/oracle.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pragma-sdk/pragma_sdk/onchain/mixins/oracle.py b/pragma-sdk/pragma_sdk/onchain/mixins/oracle.py index 27d00790..d817986f 100644 --- a/pragma-sdk/pragma_sdk/onchain/mixins/oracle.py +++ b/pragma-sdk/pragma_sdk/onchain/mixins/oracle.py @@ -397,6 +397,7 @@ async def set_future_checkpoints( ], aggregation_mode.serialize(), max_fee=self.execution_config.max_fee, + callback=self.track_nonce, ) index += pagination logger.info( @@ -412,6 +413,7 @@ async def set_future_checkpoints( ], aggregation_mode.serialize(), max_fee=self.execution_config.max_fee, + callback=self.track_nonce, ) return invocation @@ -448,6 +450,7 @@ async def set_checkpoints( ], aggregation_mode.serialize(), max_fee=self.execution_config.max_fee, + callback=self.track_nonce, ) index += pagination logger.info( @@ -463,6 +466,7 @@ async def set_checkpoints( ], aggregation_mode.serialize(), max_fee=self.execution_config.max_fee, + callback=self.track_nonce, ) return invocation From 522e71211bb0824205e5ff027ada75adb1bbee2e Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:23:30 +0200 Subject: [PATCH 5/6] =?UTF-8?q?hotfix(price=5Fpusher=5Fedge):=20Fixed=20fe?= =?UTF-8?q?tchers=20test=20=F0=9F=98=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration/fetchers/fetcher_configs.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pragma-sdk/tests/integration/fetchers/fetcher_configs.py b/pragma-sdk/tests/integration/fetchers/fetcher_configs.py index fa865760..2db9be2e 100644 --- a/pragma-sdk/tests/integration/fetchers/fetcher_configs.py +++ b/pragma-sdk/tests/integration/fetchers/fetcher_configs.py @@ -24,9 +24,9 @@ "name": "Defillama", "expected_result": [ SpotEntry( - "BTC/USD", 2604800000000, 1692779346, "DEFILLAMA", PUBLISHER_NAME + "BTC/USD", 2604800000000, 12345, "DEFILLAMA", PUBLISHER_NAME ), - SpotEntry("ETH/USD", 164507000000, 1692779707, "DEFILLAMA", PUBLISHER_NAME), + SpotEntry("ETH/USD", 164507000000, 12345, "DEFILLAMA", PUBLISHER_NAME), ], }, "BitstampFetcher": { @@ -34,8 +34,8 @@ "fetcher_class": BitstampFetcher, "name": "Bitstamp", "expected_result": [ - SpotEntry("BTC/USD", 2602100000000, 1692781034, "BITSTAMP", PUBLISHER_NAME), - SpotEntry("ETH/USD", 164250000000, 1692780986, "BITSTAMP", PUBLISHER_NAME), + SpotEntry("BTC/USD", 2602100000000, 12345, "BITSTAMP", PUBLISHER_NAME), + SpotEntry("ETH/USD", 164250000000, 12345, "BITSTAMP", PUBLISHER_NAME), ], }, "CoinbaseFetcher": { @@ -55,7 +55,7 @@ SpotEntry( "BTC/USD", 2640240000000, - 1692829724, + 12345, "OKX", PUBLISHER_NAME, volume=18382.3898, @@ -63,7 +63,7 @@ SpotEntry( "ETH/USD", 167372000000, - 1692829751, + 12345, "OKX", PUBLISHER_NAME, volume=185341.3646, @@ -145,7 +145,7 @@ FutureEntry( "BTC/USD", 2589900000000, - 1692982428, + 12345, "BYBIT", PUBLISHER_NAME, 0, @@ -154,7 +154,7 @@ FutureEntry( "ETH/USD", 164025000000, - 1692982480, + 12345, "BYBIT", PUBLISHER_NAME, 0, @@ -196,7 +196,7 @@ FutureEntry( pair_id="BTC/USD", price=2664490000000, - timestamp=1695293953, + timestamp=12345, source="OKX", publisher=PUBLISHER_NAME, volume=274, @@ -205,7 +205,7 @@ FutureEntry( pair_id="BTC/USD", price=2666120000000, - timestamp=1695293953, + timestamp=12345, source="OKX", publisher=PUBLISHER_NAME, volume=1020, @@ -214,7 +214,7 @@ FutureEntry( pair_id="ETH/USD", price=159390000000, - timestamp=1695293986, + timestamp=12345, source="OKX", publisher=PUBLISHER_NAME, volume=2276, @@ -223,7 +223,7 @@ FutureEntry( pair_id="ETH/USD", price=159092000000, - timestamp=1695293987, + timestamp=12345, source="OKX", publisher=PUBLISHER_NAME, volume=6178, From 455e86daace8d3d1dc98eedb487c4df51e3c24b2 Mon Sep 17 00:00:00 2001 From: akhercha Date: Tue, 30 Jul 2024 17:24:56 +0200 Subject: [PATCH 6/6] =?UTF-8?q?hotfix(price=5Fpusher=5Fedge):=20Format=20?= =?UTF-8?q?=F0=9F=98=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pragma-sdk/tests/integration/fetchers/fetcher_configs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pragma-sdk/tests/integration/fetchers/fetcher_configs.py b/pragma-sdk/tests/integration/fetchers/fetcher_configs.py index 2db9be2e..1b96d511 100644 --- a/pragma-sdk/tests/integration/fetchers/fetcher_configs.py +++ b/pragma-sdk/tests/integration/fetchers/fetcher_configs.py @@ -23,9 +23,7 @@ "fetcher_class": DefillamaFetcher, "name": "Defillama", "expected_result": [ - SpotEntry( - "BTC/USD", 2604800000000, 12345, "DEFILLAMA", PUBLISHER_NAME - ), + SpotEntry("BTC/USD", 2604800000000, 12345, "DEFILLAMA", PUBLISHER_NAME), SpotEntry("ETH/USD", 164507000000, 12345, "DEFILLAMA", PUBLISHER_NAME), ], },