Support complex types in Starknet ABI (#1171)

* Support complex types in abi

* Add array type & fix config validation

* Fixups

* Fix missing node var & add special handling for u256/bytearray

* Temporary workaround for event parsing

* Add test for config validation fix

---------

Co-authored-by: Igor Sereda <[email protected]>

src/dipdup/abi/cairo.py

@@ -32,23 +32,42 @@ class CairoAbi(TypedDict):
     events: list[CairoEventAbi]
 
 
-def _convert_type(type_: CairoType) -> str:
+def _convert_type(type_: CairoType) -> dict[str, Any]:
     # TODO: Support all types
-    return {
+    if type_.__class__.__name__ in {'EventType', 'StructType'}:
+        if type_.name == 'Uint256':
+            return {'type': 'integer'}
+        if type_.name == 'core::byte_array::ByteArray':
+            return {'type': 'string'}
+        return {
+            'type': 'object',
+            'properties': {
+                key: _convert_type(value)
+                for key, value in type_.types.items()
+            },
+            'required': tuple(type_.types.keys()),
+            'additionalProperties': False,
+        }
+
+    if type_.__class__.__name__ == 'ArrayType':
+        return {
+            'type': 'array',
+            'items': _convert_type(type_.inner_type),
+        }
+
+    simple_type = {
         'FeltType': 'integer',
         'UintType': 'integer',
         'BoolType': 'boolean',
     }[type_.__class__.__name__]
+    return {'type': simple_type}
 
 
 def _jsonschema_from_event(event: EventType) -> dict[str, Any]:
     # TODO: Unpack nested types (starknet.py could do that)
     return {
         '$schema': 'http://json-schema.org/draft/2019-09/schema#',
-        'type': 'object',
-        'properties': {key: {'type': _convert_type(value)} for key, value in event.types.items()},
-        'required': tuple(event.types.keys()),
-        'additionalProperties': False,
+        **_convert_type(event)
     }
 
 

src/dipdup/config/starknet.py

@@ -4,6 +4,7 @@
 from abc import ABC
 from typing import Annotated
 from typing import Literal
+from typing import TypeAlias
 
 from pydantic import AfterValidator
 from pydantic import ConfigDict
@@ -17,7 +18,7 @@
 from dipdup.config.starknet_subsquid import StarknetSubsquidDatasourceConfig
 from dipdup.exceptions import ConfigurationError
 
-StarknetDatasourceConfigU = StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig
+StarknetDatasourceConfigU: TypeAlias = StarknetSubsquidDatasourceConfig | StarknetNodeDatasourceConfig
 
 _HEX_ADDRESS_REGEXP = re.compile(r'(0x)?[0-9a-f]{1,64}', re.IGNORECASE | re.ASCII)
 

src/dipdup/config/starknet_events.py

@@ -9,6 +9,7 @@
 from dipdup.config import Alias
 from dipdup.config import HandlerConfig
 from dipdup.config.starknet import StarknetContractConfig
+from dipdup.config.starknet import StarknetDatasourceConfigU
 from dipdup.config.starknet import StarknetIndexConfig
 from dipdup.models.starknet import StarknetSubscription
 from dipdup.subscriptions import Subscription
@@ -61,6 +62,7 @@ class StarknetEventsIndexConfig(StarknetIndexConfig):
     """
 
     kind: Literal['starknet.events']
+    datasources: tuple[Alias[StarknetDatasourceConfigU], ...]
     handlers: tuple[StarknetEventsHandlerConfig, ...]
 
     def get_subscriptions(self) -> set[Subscription]:

src/dipdup/datasources/starknet_node.py

@@ -12,6 +12,8 @@
 
 
 class StarknetNodeDatasource(IndexDatasource[StarknetNodeDatasourceConfig]):
+    NODE_LAST_MILE = 128
+
     _default_http_config = HttpConfig(
         batch_size=1000,
     )

src/dipdup/indexes/starknet_events/matcher.py

@@ -73,7 +73,7 @@ def prepare_event_handler_args(
         typename=typename,
         name=handler_config.name,
     )['serializer']
-    data = [int(s, 16) for s in matched_event.data]
+    data = [int(x, 16) for x in matched_event.keys[1:] + matched_event.data]
 
     # holding context for error building
     with DeserializationContext.create(data) as context:

tests/configs/demo_starknet_events.yml

@@ -0,0 +1,27 @@
+spec_version: 3.0
+package: demo_starknet_events
+
+datasources:
+  subsquid:
+    kind: starknet.subsquid
+    url: https://v2.archive.subsquid.io/network/starknet-mainnet
+  node:
+    kind: starknet.node
+    url: https://starknet-mainnet.g.alchemy.com/v2
+
+contracts:
+  stark_usdt:
+    kind: starknet
+    address: '0x68f5c6a61780768455de69077e07e89787839bf8166decfbf92b645209c0fb8'
+    typename: stark_usdt
+
+indexes:
+  starknet_usdt_events:
+    kind: starknet.events
+    datasources:
+      - subsquid
+      - node
+    handlers:
+      - callback: on_transfer
+        contract: stark_usdt
+        name: Transfer

tests/test_config/test_config.py

@@ -1,7 +1,9 @@
 import tempfile
+from collections.abc import Generator
 from pathlib import Path
 
 import pytest
+from _pytest.fixtures import SubRequest
 from pydantic import ValidationError
 
 from dipdup.config import DipDupConfig
@@ -19,9 +21,16 @@
 from dipdup.models.tezos_tzkt import TransactionSubscription
 from dipdup.yaml import DipDupYAMLConfig
 
+TEST_CONFIGS = Path(__file__).parent.parent / 'configs'
+
+
+@pytest.fixture(params=list(Path.glob(TEST_CONFIGS, 'demo_*.yml')))
+def demo_dipdup_config(request: SubRequest) -> Generator[Path, None, None]:
+    yield request.param
+
 
 def create_config(merge_subs: bool = False, origs: bool = False) -> DipDupConfig:
-    path = Path(__file__).parent.parent / 'configs' / 'dipdup.yaml'
+    path = TEST_CONFIGS / 'dipdup.yaml'
     config = DipDupConfig.load([path])
     if origs:
         config.indexes['hen_mainnet'].types += (TezosOperationType.origination,)  # type: ignore
@@ -88,12 +97,10 @@ async def test_reserved_keywords() -> None:
     )
 
     # FIXME: Can't use `from_` field alias in dataclasses
-    raw_config, _ = DipDupYAMLConfig.load(
-        paths=[Path(__file__).parent.parent / 'configs' / 'demo_tezos_token_transfers_4.yml']
-    )
+    raw_config, _ = DipDupYAMLConfig.load(paths=[TEST_CONFIGS / 'demo_tezos_token_transfers_4.yml'])
     assert raw_config['indexes']['tzbtc_holders_mainnet']['handlers'][1]['from_'] == 'tzbtc_mainnet'
 
-    config = DipDupConfig.load([Path(__file__).parent.parent / 'configs' / 'demo_tezos_token_transfers_4.yml'])
+    config = DipDupConfig.load([TEST_CONFIGS / 'demo_tezos_token_transfers_4.yml'])
     assert config.indexes['tzbtc_holders_mainnet'].handlers[1].from_ == 'tzbtc_mainnet'  # type: ignore[misc,union-attr]
 
 
@@ -148,5 +155,5 @@ async def test_http_config() -> None:
     )
 
 
-# async def test_evm() -> None:
-#     DipDupConfig.load([Path(__file__).parent.parent / 'configs' / 'evm_subsquid.yml'])
+async def test_load_demo_config(demo_dipdup_config: Path) -> None:
+    DipDupConfig.load([demo_dipdup_config])