From c38ca0e199eb6737fa6a36447c67b72009303e56 Mon Sep 17 00:00:00 2001 From: Michael Zaikin Date: Thu, 21 Nov 2024 19:32:49 +0000 Subject: [PATCH] Handle starknet event keys/data correctly --- pyproject.toml | 63 +++++++++---------- requirements.txt | 2 +- src/dipdup/abi/cairo.py | 20 +++++- src/dipdup/indexes/starknet_events/matcher.py | 15 +++-- 4 files changed, 58 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eebbb1a0e..6c519995b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,38 +48,37 @@ classifiers = [ "Typing :: Typed", ] -dependencies = [ - "aiohttp~=3.10", - "aiolimiter~=1.1", - "anyio~=4.4", - "appdirs~=1.4", - "APScheduler~=3.10", - "async-lru~=2.0", - "asyncpg~=0.29", - "click~=8.1", - "datamodel-code-generator~=0.26", - "eth-abi~=5.0", - "lru-dict~=1.3", - "orjson~=3.10", - "prometheus-client~=0.20", - "pycryptodome~=3.20", - "pydantic~=2.9", - "pyhumps~=3.8", - "pysignalr~=1.0", - "python-dotenv~=1.0", - "python-json-logger~=2.0", - "ruamel.yaml~=0.18.6", - "sentry-sdk~=2.16", - "sqlparse~=0.5", - "starknet-py==0.24.0", - "strict-rfc3339~=0.7", - "survey~=5.4", - "tabulate~=0.9", - # NOTE: Heavily patched; don't update without testing. - "tortoise-orm==0.21.7", - "uvloop~=0.20", - "web3~=7.2", -] +[tool.poetry.dependencies] +aiohttp = "~3.10" +aiolimiter = "~1.1" +anyio = "~4.4" +appdirs = "~1.4" +APScheduler = "~3.10" +async-lru = "~2.0" +asyncpg = "~0.29" +click = "~8.1" +datamodel-code-generator = "~0.26" +eth-abi = "~5.0" +lru-dict = "~1.3" +orjson = "~3.10" +prometheus-client = "~0.20" +pycryptodome = "~3.20" +pydantic = "~2.9" +pyhumps = "~3.8" +pysignalr = "~1.0" +python-dotenv = "~1.0" +python-json-logger = "~2.0" +ruamel.yaml = "~0.18.6" +sentry-sdk = "~2.16" +sqlparse = "~0.5" +starknet-py = { git = "https://github.com/m-kus/starknet.py", rev = "8d76ee0af683062bf63c6b32a5fd49608e6b731e" } +strict-rfc3339 = "~0.7" +survey = "~5.4" +tabulate = "~0.9" +# NOTE: Heavily patched; don't update without testing. +tortoise-orm = "0.21.7" +uvloop = "~0.20" +web3 = "~7.2" [project.optional-dependencies] migrations = [ diff --git a/requirements.txt b/requirements.txt index fbedafd6b..f1edf1ca7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -83,7 +83,7 @@ sentry-sdk==2.16.0 six==1.16.0 sniffio==1.3.1 sqlparse==0.5.1 -starknet-py==0.24.0 +git+https://github.com/m-kus/starknet.py@8d76ee0af683062bf63c6b32a5fd49608e6b731e#egg=starknet-py strict-rfc3339==0.7 survey==5.4.0 sympy==1.11.1 diff --git a/src/dipdup/abi/cairo.py b/src/dipdup/abi/cairo.py index 49b636924..7da31f718 100644 --- a/src/dipdup/abi/cairo.py +++ b/src/dipdup/abi/cairo.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections import OrderedDict from functools import cache from typing import TYPE_CHECKING from typing import Any @@ -16,7 +17,7 @@ from starknet_py.abi.v2 import Abi # type: ignore[import-untyped] from starknet_py.cairo.data_types import CairoType # type: ignore[import-untyped] from starknet_py.cairo.data_types import EventType - from starknet_py.serialization import PayloadSerializer # type: ignore[import-untyped] + from starknet_py.serialization import CairoDataSerializer # type: ignore[import-untyped] from dipdup.package import DipDupPackage @@ -25,7 +26,7 @@ class CairoEventAbi(TypedDict): name: str event_identifier: str members: dict[str, CairoType] - serializer: PayloadSerializer + sorted_serializers: OrderedDict[str, CairoDataSerializer] class CairoAbi(TypedDict): @@ -93,12 +94,25 @@ def convert_abi(package: DipDupPackage) -> dict[str, CairoAbi]: for name, event_type in parsed_abi.events.items(): if name in converted_abi['events']: raise NotImplementedError('Multiple events with the same name are not supported') + + serializers = serializer_for_event(event_type).serializers + + # Event payload is returned from RPC in two arrays: keys (including event selector) and data. + # Since any event field can be marked as key, the original ordering might be broken. + # + # We need to reorder deserializers so that the keys remain in the beginning and + # the rest of the fields are moved towards the end (preserving their inner ordering). + # + # That way we can apply the deserializers to the concatenation of keys (without first element) + data. + sorted_members = event_type.keys + [name for name in serializers if name not in event_type.keys] + sorted_serializers = OrderedDict((name, serializers[name]) for name in sorted_members) + converted_abi['events'].append( CairoEventAbi( name=name, event_identifier=sn_keccak(name), members=event_type.types, - serializer=serializer_for_event(event_type), + sorted_serializers=sorted_serializers, ) ) abi_by_typename[contract_typename] = converted_abi diff --git a/src/dipdup/indexes/starknet_events/matcher.py b/src/dipdup/indexes/starknet_events/matcher.py index 5a6cf7758..b647c02f9 100644 --- a/src/dipdup/indexes/starknet_events/matcher.py +++ b/src/dipdup/indexes/starknet_events/matcher.py @@ -1,5 +1,5 @@ import logging -from collections import deque +from collections import deque, OrderedDict from collections.abc import Iterable from typing import Any @@ -69,15 +69,18 @@ def prepare_event_handler_args( name=snake_to_pascal(handler_config.name) + 'Payload', ) - serializer = package._cairo_abis.get_event_abi( + event_abi = package._cairo_abis.get_event_abi( typename=typename, name=handler_config.name, - )['serializer'] - data = [int(s, 16) for s in matched_event.data] + ) + + # Skipping first key which is the event selector + # Note that some fields might be encoded with more than one felt (complex types) + raw_data = [int(x, 16) for x in matched_event.keys[1:] + matched_event.data] # holding context for error building - with DeserializationContext.create(data) as context: - data_dict = deserialize_to_dict(serializer.serializers, context) + with DeserializationContext.create(raw_data) as context: + data_dict = deserialize_to_dict(event_abi['sorted_serializers'], context) typed_payload = parse_object(type_=type_, data=data_dict) return StarknetEvent(