From f15a4395f9dc004cf43338cde00a9eeee8ae28de Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Fri, 29 Sep 2023 20:55:26 -0300 Subject: [PATCH 1/4] DIPDUP_DEBUG env --- src/dipdup/cli.py | 4 +++- src/dipdup/env.py | 35 ++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 48f50ca0a..a2a32ce8a 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -18,6 +18,7 @@ import asyncclick as click from dipdup import __version__ +from dipdup import env from dipdup.install import EPILOG from dipdup.install import WELCOME_ASCII from dipdup.performance import metrics @@ -216,6 +217,8 @@ async def cli(ctx: click.Context, config: list[str], env_file: list[str]) -> Non from dipdup.sys import set_up_logging set_up_logging() + if env.DEBUG: + logging.getLogger('dipdup').setLevel(logging.DEBUG) env_file_paths = [Path(file) for file in env_file] config_paths = [Path(file) for file in config] @@ -232,7 +235,6 @@ async def cli(ctx: click.Context, config: list[str], env_file: list[str]) -> Non logging.getLogger('dipdup').setLevel(logging.INFO) return - from dipdup import env from dipdup.config import DipDupConfig from dipdup.exceptions import InitializationRequiredError from dipdup.package import DipDupPackage diff --git a/src/dipdup/env.py b/src/dipdup/env.py index ffa1fffc2..90731e2ef 100644 --- a/src/dipdup/env.py +++ b/src/dipdup/env.py @@ -62,17 +62,30 @@ def get_path(key: str) -> Path | None: def set_test() -> None: global TEST, REPLAY_PATH TEST = True - REPLAY_PATH = str(Path(__file__).parent.parent.parent / 'tests' / 'replays') - env['DIPDUP_REPLAY_PATH'] = REPLAY_PATH + REPLAY_PATH = Path(__file__).parent.parent.parent / 'tests' / 'replays' -if get('CI') == 'true': - env['DIPDUP_CI'] = '1' -if platform.system() == 'Linux' and Path('/.dockerenv').exists(): - env['DIPDUP_DOCKER'] = '1' +CI: bool +DEBUG: bool +DOCKER: bool +NEXT: bool +REPLAY_PATH: Path | None +TEST: bool -CI = get_bool('DIPDUP_CI') -DOCKER = get_bool('DIPDUP_DOCKER') -NEXT = get_bool('DIPDUP_NEXT') -REPLAY_PATH = get_path('DIPDUP_REPLAY_PATH') -TEST = get_bool('DIPDUP_TEST') + +def read() -> None: + global CI, DEBUG, DOCKER, NEXT, REPLAY_PATH, TEST + CI = get_bool('DIPDUP_CI') + DEBUG = get_bool('DIPDUP_DEBUG') + DOCKER = get_bool('DIPDUP_DOCKER') + NEXT = get_bool('DIPDUP_NEXT') + REPLAY_PATH = get_path('DIPDUP_REPLAY_PATH') + TEST = get_bool('DIPDUP_TEST') + + if get('CI') == 'true': + CI = True + if platform.system() == 'Linux' and Path('/.dockerenv').exists(): + DOCKER = True + + +read() From a441f59f5a2b40e2cde1ef6c2d519c46d6f847d0 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Fri, 29 Sep 2023 21:25:03 -0300 Subject: [PATCH 2/4] Fixed issue with determining the last level when syncing with node --- CHANGELOG.md | 5 +++ src/dipdup/config/evm_subsquid_events.py | 2 ++ src/dipdup/http.py | 3 +- .../indexes/evm_subsquid_events/index.py | 8 +++-- tests/configs/demo_evm_events_node.yml | 36 +++++++++++++++++++ tests/test_demos.py | 1 + 6 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/configs/demo_evm_events_node.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 47349e796..38380eaae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,15 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic ## [Unreleased] +### Added + +- env: Added `DIPDUP_DEBUG` environment variable to enable debug logging. + ### Fixed - demos: Fixed decimal overflow in `demo_uniswap` project. - evm.node: Fixed incorrect log request parameters. +- evm.subsquid.events: Fixed issue with determining the last level when syncing with node. ## [7.0.0] - 2023-09-25 diff --git a/src/dipdup/config/evm_subsquid_events.py b/src/dipdup/config/evm_subsquid_events.py index 8c99cddb0..55f2ac254 100644 --- a/src/dipdup/config/evm_subsquid_events.py +++ b/src/dipdup/config/evm_subsquid_events.py @@ -58,6 +58,7 @@ class SubsquidEventsIndexConfig(IndexConfig): :param datasource: Subsquid datasource :param handlers: Event handlers :param abi: One or more `evm.abi` datasource(s) for the same network + :param node_only: Don't use Subsquid Archives API (dev only) :param first_level: Level to start indexing from :param last_level: Level to stop indexing and disable this index """ @@ -66,6 +67,7 @@ class SubsquidEventsIndexConfig(IndexConfig): datasource: SubsquidDatasourceConfig handlers: tuple[SubsquidEventsHandlerConfig, ...] = field(default_factory=tuple) abi: AbiDatasourceConfig | tuple[AbiDatasourceConfig, ...] | None = None + node_only: bool = False first_level: int = 0 last_level: int = 0 diff --git a/src/dipdup/http.py b/src/dipdup/http.py index 8f499edb8..7e7a7fa82 100644 --- a/src/dipdup/http.py +++ b/src/dipdup/http.py @@ -23,6 +23,7 @@ from aiolimiter import AsyncLimiter from dipdup import __version__ +from dipdup import env from dipdup.config import ResolvedHttpConfig from dipdup.exceptions import FrameworkException from dipdup.exceptions import InvalidRequestError @@ -143,7 +144,7 @@ async def _retry_request( """Retry a request in case of failure sleeping according to config""" attempt = 1 retry_sleep = self._config.retry_sleep - retry_count = self._config.retry_count + retry_count = 0 if env.TEST else self._config.retry_count retry_count_str = 'inf' if retry_count is sys.maxsize else str(retry_count) while True: diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py index b40484ed7..f98477584 100644 --- a/src/dipdup/indexes/evm_subsquid_events/index.py +++ b/src/dipdup/indexes/evm_subsquid_events/index.py @@ -147,16 +147,20 @@ async def _synchronize(self, sync_level: int) -> None: self._logger.info('Subsquid is %s levels behind; %s available', subsquid_lag, subsquid_available) if subsquid_available < NODE_SYNC_LIMIT: use_node = True + elif self._config.node_only: + self._logger.debug('Using node anyway') + use_node = True # NOTE: Fetch last blocks from node if there are not enough realtime messages in queue if use_node and self.node_datasources: - sync_level = node_sync_level + sync_level = min(sync_level, node_sync_level) + self._logger.debug('Using node datasource; sync level: %s', sync_level) topics = set() for handler in self._config.handlers: typename = handler.contract.module_name topics.add(self.topics[typename][handler.name]) # FIXME: This is terribly inefficient (but okay for the last mile); see advanced example in web3.py docs. - for level in range(first_level, sync_level): + for level in range(first_level, sync_level + 1): # NOTE: Get random one every time level_logs = await self.random_node.get_logs( { diff --git a/tests/configs/demo_evm_events_node.yml b/tests/configs/demo_evm_events_node.yml new file mode 100644 index 000000000..51b83c648 --- /dev/null +++ b/tests/configs/demo_evm_events_node.yml @@ -0,0 +1,36 @@ +spec_version: 2.0 +package: demo_evm_events + +datasources: + ethscan: + kind: abi.etherscan + + mainnet_node: + kind: evm.node + url: https://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} + ws_url: wss://eth-mainnet.g.alchemy.com/v2/${ALCHEMY_KEY:-''} + + mainnet_subsquid: + kind: evm.subsquid + url: ${ARCHIVE_URL:-https://v2.archive.subsquid.io/network/ethereum-mainnet} + node: mainnet_node + http: + replay_path: ${DIPDUP_REPLAY_PATH:-} + +contracts: + eth_usdt: + kind: evm + address: 0xdac17f958d2ee523a2206206994597c13d831ec7 + typename: eth_usdt + +indexes: + eth_usdt_events: + kind: evm.subsquid.events + datasource: mainnet_subsquid + handlers: + - callback: on_transfer + contract: eth_usdt + name: Transfer + first_level: 18077421 + last_level: 18077421 + node_only: true diff --git a/tests/test_demos.py b/tests/test_demos.py index 765482c8c..57ea16d67 100644 --- a/tests/test_demos.py +++ b/tests/test_demos.py @@ -241,6 +241,7 @@ async def assert_run_dao() -> None: ('demo_raw.yml', 'demo_raw', 'init', partial(assert_init, 'demo_raw')), ('demo_evm_events.yml', 'demo_evm_events', 'run', assert_run_evm_events), ('demo_evm_events.yml', 'demo_evm_events', 'init', partial(assert_init, 'demo_evm_events')), + ('demo_evm_events_node.yml', 'demo_evm_events', 'run', assert_run_evm_events), ) From b1823cda3f6ecdb553a1f6352b54650292e09e3b Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Fri, 29 Sep 2023 21:45:59 -0300 Subject: [PATCH 3/4] Remove outdated check --- src/dipdup/indexes/evm_subsquid_events/index.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py index f98477584..47b59990b 100644 --- a/src/dipdup/indexes/evm_subsquid_events/index.py +++ b/src/dipdup/indexes/evm_subsquid_events/index.py @@ -99,13 +99,6 @@ async def _process_queue(self) -> None: break for message_level, level_logs in logs_by_level.items(): - # NOTE: If it's not a next block - resync with Subsquid - if message_level != self.state.level + 1: - self._logger.info('Not enough messages in queue; resyncing to %s', message_level) - self._queue.clear() - self.datasource.set_sync_level(None, message_level) - return - await self._process_level_events(tuple(level_logs), self.topics, message_level) def get_sync_level(self) -> int: From a76bad6aa9d3c1691d088cdc0e6110e9393c028f Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Fri, 29 Sep 2023 22:29:42 -0300 Subject: [PATCH 4/4] Log realtime messages --- src/dipdup/datasources/evm_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py index 5a600f1d4..b0af3869d 100644 --- a/src/dipdup/datasources/evm_node.py +++ b/src/dipdup/datasources/evm_node.py @@ -267,7 +267,7 @@ async def _on_message(self, message: Message) -> None: if subscription_id not in self._subscription_ids: raise FrameworkException(f'{self.name}: Unknown subscription ID: {subscription_id}') subscription = self._subscription_ids[subscription_id] - self._logger.debug('Received subscription for channel %s', subscription_id) + self._logger.info('Received a message from channel %s', subscription_id) await self._handle_subscription(subscription, data['params']['result']) else: raise DatasourceError(f'Unknown method: {data["method"]}', self.name)