balances datasource, demo, doc, changelog, models

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic
 
 ### Added
 
+- tezos.tzkt: Added new index token_balances
 - env: Added `DIPDUP_DEBUG` environment variable to enable debug logging.
 
 ### Fixed

docs/1.getting-started/7.indexes.md

@@ -18,6 +18,7 @@ Multiple indexes are available for different workloads. Every index is linked to
 | [tezos.tzkt.operations](../2.indexes/5.tezos_tzkt_operations.md)                       | Tezos          | TzKT       | typed operations            |
 | [tezos.tzkt.operations_unfiltered](../2.indexes/6.tezos_tzkt_operations_unfiltered.md) | Tezos          | TzKT       | untyped operations          |
 | [tezos.tzkt.token_transfers](../2.indexes/7.tezos_tzkt_token_transfers.md)             | Tezos          | TzKT       | TZIP-12/16 token transfers  |
+| [tezos.tzkt.token_balances](../2.indexes/8.tezos_tzkt_token_balances.md)               | Tezos          | TzKT       | TZIP-12/16 token balances   |
 
 Indexes can join multiple contracts considered as a single application. Also, contracts can be used by multiple indexes of any kind, but make sure that they are independent of each other and that indexed data don't overlap.
 

docs/2.indexes/8.tezos_tzkt_token_balances.md

@@ -0,0 +1,19 @@
+---
+title: "Token balances"
+description: "This index allows indexing token balances of contracts compatible with FA1.2 or FA2 standards."
+network: "tezos"
+---
+
+# `tezos.tzkt.token_balances` index
+
+This index allows indexing token balances of contracts compatible with [FA1.2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-7/README.md) or [FA2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-12/tzip-12.md) standards.
+
+```yaml [dipdup.yaml]
+{{ #include ../src/demo_token_balances/dipdup.yaml }}
+```
+
+Callback receives `TzktTokenBalanceData` model that optionally contains the owner, token, and balance values
+
+```python
+{{ #include ../src/demo_token_balances/handlers/on_balance_update.py }}
+```

src/dipdup/datasources/tezos_tzkt.py

@@ -33,7 +33,7 @@
 from dipdup.models import Head
 from dipdup.models import MessageType
 from dipdup.models import ReindexingReason
-from dipdup.models.tezos_tzkt import HeadSubscription
+from dipdup.models.tezos_tzkt import HeadSubscription, TzktTokenBalanceData
 from dipdup.models.tezos_tzkt import TzktBigMapData
 from dipdup.models.tezos_tzkt import TzktBlockData
 from dipdup.models.tezos_tzkt import TzktEventData
@@ -111,6 +111,18 @@
     'originationId',
     'migrationId',
 )
+TOKEN_BALANCE_FIELDS = (
+    'id',
+    'transfersCount',
+    'firstLevel',
+    'firstTime',
+    'lastLevel',
+    'lastTime',
+    'account',
+    'token',
+    'balance',
+    'balanceValue',
+)
 EVENT_FIELDS = (
     'id',
     'level',
@@ -893,6 +905,48 @@ async def iter_token_transfers(
         ):
             yield batch
 
+    async def get_token_balances(
+        self,
+        token_addresses: set[str],
+        token_ids: set[int],
+        first_level: int,
+        last_level: int,
+        offset: int | None = None,
+        limit: int | None = None,
+    ) -> tuple[TzktTokenBalanceData, ...]:
+        params = self._get_request_params(
+            first_level,
+            last_level,
+            offset=offset or 0,
+            limit=limit,
+            select=TOKEN_BALANCE_FIELDS,
+            values=True,
+            cursor=True,
+            **{
+                'token.contract.in': ','.join(token_addresses),
+                'token.id.in': ','.join(str(token_id) for token_id in token_ids),
+            },
+        )
+        raw_token_balances = await self._request_values_dict('get', url='v1/tokens/balances', params=params)
+        return tuple(TzktTokenBalanceData.from_json(item) for item in raw_token_balances)
+
+    async def iter_token_balances(
+        self,
+        token_addresses: set[str],
+        token_ids: set[int],
+        first_level: int,
+        last_level: int,
+    ) -> AsyncIterator[tuple[TzktTokenBalanceData, ...]]:
+        async for batch in self._iter_batches(
+            self.get_token_balances,
+            token_addresses,
+            token_ids,
+            first_level,
+            last_level,
+            cursor=True,
+        ):
+            yield batch
+
     async def get_events(
         self,
         addresses: set[str],

src/dipdup/models/tezos_tzkt.py

@@ -535,6 +535,63 @@ def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenTransferDat
             tzkt_origination_id=token_transfer_json.get('originationId'),
             tzkt_migration_id=token_transfer_json.get('migrationId'),
         )
+
+
+@dataclass(frozen=True)
+class TzktTokenBalanceData:
+    """Basic structure for token transver received from TzKT SignalR API"""
+
+    id: int
+    transfers_count: int
+    first_level: int
+    first_time: datetime
+    last_level: int
+    last_time: datetime
+    # owner account
+    account_address: str | None = None
+    account_alias: str | None = None
+    # token object
+    tzkt_token_id: int | None = None
+    contract_address: str | None = None
+    contract_alias: str | None = None
+    token_id: int | None = None
+    standard: TzktTokenStandard | None = None
+    metadata: dict[str, Any] | None = None
+
+    balance: str | None = None
+    balance_value: float | None = None
+
+
+    @classmethod
+    def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenBalanceData':
+        """Convert raw token transfer message from REST or WS into dataclass"""
+        token_json = token_transfer_json.get('token') or {}
+        standard = token_json.get('standard')
+        metadata = token_json.get('metadata')
+        contract_json = token_json.get('contract') or {}
+
+
+        return TzktTokenBalanceData(
+            id=token_transfer_json['id'],
+            transfers_count=token_transfer_json['transfersCount'],
+            first_level=token_transfer_json['firstLevel'],
+            first_time=_parse_timestamp(token_transfer_json['firstTime']),
+            last_level=token_transfer_json['lastLevel'],
+            last_time=_parse_timestamp(token_transfer_json['lastTime']),
+
+            account_address=token_transfer_json.get('account', {}).get('address'),
+            account_alias=token_transfer_json.get('account', {}).get('alias'),
+
+            tzkt_token_id=token_json['id'],
+            contract_address=contract_json.get('address'),
+            contract_alias=contract_json.get('alias'),
+            token_id=token_json.get('tokenId'),
+            standard=TzktTokenStandard(standard) if standard else None,
+            metadata=metadata if isinstance(metadata, dict) else {},
+
+            balance=token_transfer_json.get('balance'),
+            balance_value=token_transfer_json.get('balanceValue'),
+        )
 
 
 @dataclass(frozen=True)

src/dipdup/projects/demo_token_balances/dipdup.yaml.j2

@@ -0,0 +1,21 @@
+spec_version: 2.0
+package: {{ project.package }}
+
+contracts:
+  tzbtc_mainnet:
+    kind: tezos
+    address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn
+    typename: tzbtc
+
+datasources:
+  tzkt:
+    kind: tezos.tzkt
+    url: https://api.tzkt.io
+
+indexes:
+  tzbtc_holders_mainnet:
+    kind: tezos.tzkt.token_balances
+    datasource: tzkt
+    handlers:
+      - callback: on_balance_update
+        contract: tzbtc_mainnet

src/dipdup/projects/demo_token_balances/models/__init__.py.j2

@@ -0,0 +1,8 @@
+from dipdup import fields
+
+from dipdup.models import Model
+
+
+class Holder(Model):
+    address = fields.TextField(pk=True)
+    balance = fields.DecimalField(decimal_places=8, max_digits=20, default=0)

src/dipdup/projects/demo_token_balances/replay.yaml

@@ -0,0 +1,5 @@
+spec_version: 2.0
+replay:
+  description: TzBTC FA1.2 token balances
+  package: demo_token_balances
+  template: demo_token_balances