diff --git a/README.md b/README.md index c73d1595e..a31e49430 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,11 @@ indexes: handlers: - callback: on_mint pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: mint_OBJKT - - destination: HEN_objkts + - type: transaction + destination: HEN_objkts entrypoint: mint ``` @@ -131,13 +133,13 @@ from demo_hic_et_nunc.types.hen_minter.parameter.mint_objkt import MintOBJKTPara from demo_hic_et_nunc.types.hen_minter.storage import HenMinterStorage from demo_hic_et_nunc.types.hen_objkts.parameter.mint import MintParameter from demo_hic_et_nunc.types.hen_objkts.storage import HenObjktsStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import TransactionContext, OperationHandlerContext async def on_mint( ctx: OperationHandlerContext, - mint_objkt: OperationContext[MintOBJKTParameter, HenMinterStorage], - mint: OperationContext[MintParameter, HenObjktsStorage], + mint_objkt: TransactionContext[MintOBJKTParameter, HenMinterStorage], + mint: TransactionContext[MintParameter, HenObjktsStorage], ) -> None: holder, _ = await models.Holder.get_or_create(address=mint.parameter.address) token = models.Token( @@ -201,15 +203,19 @@ templates: handlers: - callback: on_fa12_token_to_tez pattern: - - destination: + - type: transaction + destination: entrypoint: tokenToTezPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_tez_to_token pattern: - - destination: + - type: transaction + destination: entrypoint: tezToTokenPayment - - destination: + - type: transaction + destination: entrypoint: transfer indexes: diff --git a/src/demo_hic_et_nunc/dipdup.yml b/src/demo_hic_et_nunc/dipdup.yml index 1b3b1464b..d9f85ad3a 100644 --- a/src/demo_hic_et_nunc/dipdup.yml +++ b/src/demo_hic_et_nunc/dipdup.yml @@ -27,19 +27,24 @@ indexes: handlers: - callback: on_mint pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: mint_OBJKT - - destination: HEN_objkts + - type: transaction + destination: HEN_objkts entrypoint: mint - callback: on_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: swap - callback: on_cancel_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: cancel_swap - callback: on_collect pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: collect diff --git a/src/demo_hic_et_nunc/handlers/on_cancel_swap.py b/src/demo_hic_et_nunc/handlers/on_cancel_swap.py index c7836ae54..b454c55b2 100644 --- a/src/demo_hic_et_nunc/handlers/on_cancel_swap.py +++ b/src/demo_hic_et_nunc/handlers/on_cancel_swap.py @@ -1,12 +1,12 @@ import demo_hic_et_nunc.models as models from demo_hic_et_nunc.types.hen_minter.parameter.cancel_swap import CancelSwapParameter from demo_hic_et_nunc.types.hen_minter.storage import HenMinterStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_cancel_swap( ctx: OperationHandlerContext, - cancel_swap: OperationContext[CancelSwapParameter, HenMinterStorage], + cancel_swap: TransactionContext[CancelSwapParameter, HenMinterStorage], ) -> None: swap = await models.Swap.filter(id=int(cancel_swap.parameter.__root__)).get() swap.status = models.SwapStatus.CANCELED diff --git a/src/demo_hic_et_nunc/handlers/on_collect.py b/src/demo_hic_et_nunc/handlers/on_collect.py index 288ecb1dc..bdf7428ec 100644 --- a/src/demo_hic_et_nunc/handlers/on_collect.py +++ b/src/demo_hic_et_nunc/handlers/on_collect.py @@ -1,12 +1,12 @@ import demo_hic_et_nunc.models as models from demo_hic_et_nunc.types.hen_minter.parameter.collect import CollectParameter from demo_hic_et_nunc.types.hen_minter.storage import HenMinterStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_collect( ctx: OperationHandlerContext, - collect: OperationContext[CollectParameter, HenMinterStorage], + collect: TransactionContext[CollectParameter, HenMinterStorage], ) -> None: swap = await models.Swap.filter(id=collect.parameter.swap_id).get() seller = await swap.creator diff --git a/src/demo_hic_et_nunc/handlers/on_mint.py b/src/demo_hic_et_nunc/handlers/on_mint.py index e7409091c..5639d131b 100644 --- a/src/demo_hic_et_nunc/handlers/on_mint.py +++ b/src/demo_hic_et_nunc/handlers/on_mint.py @@ -3,13 +3,13 @@ from demo_hic_et_nunc.types.hen_minter.storage import HenMinterStorage from demo_hic_et_nunc.types.hen_objkts.parameter.mint import MintParameter from demo_hic_et_nunc.types.hen_objkts.storage import HenObjktsStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_mint( ctx: OperationHandlerContext, - mint_objkt: OperationContext[MintOBJKTParameter, HenMinterStorage], - mint: OperationContext[MintParameter, HenObjktsStorage], + mint_objkt: TransactionContext[MintOBJKTParameter, HenMinterStorage], + mint: TransactionContext[MintParameter, HenObjktsStorage], ) -> None: holder, _ = await models.Holder.get_or_create(address=mint.parameter.address) token = models.Token( diff --git a/src/demo_hic_et_nunc/handlers/on_swap.py b/src/demo_hic_et_nunc/handlers/on_swap.py index 7fac41a14..95698c52a 100644 --- a/src/demo_hic_et_nunc/handlers/on_swap.py +++ b/src/demo_hic_et_nunc/handlers/on_swap.py @@ -1,12 +1,12 @@ import demo_hic_et_nunc.models as models from demo_hic_et_nunc.types.hen_minter.parameter.swap import SwapParameter from demo_hic_et_nunc.types.hen_minter.storage import HenMinterStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_swap( ctx: OperationHandlerContext, - swap: OperationContext[SwapParameter, HenMinterStorage], + swap: TransactionContext[SwapParameter, HenMinterStorage], ) -> None: holder, _ = await models.Holder.get_or_create(address=swap.data.sender_address) swap_model = models.Swap( diff --git a/src/demo_quipuswap/dipdup.yml b/src/demo_quipuswap/dipdup.yml index f8e398780..98a0a82e0 100644 --- a/src/demo_quipuswap/dipdup.yml +++ b/src/demo_quipuswap/dipdup.yml @@ -39,31 +39,40 @@ templates: handlers: - callback: on_fa12_token_to_tez pattern: - - destination: + - type: transaction + destination: entrypoint: tokenToTezPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_tez_to_token pattern: - - destination: + - type: transaction + destination: entrypoint: tezToTokenPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_invest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: investLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_divest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: divestLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_withdraw_profit pattern: - - destination: + - type: transaction + destination: entrypoint: withdrawProfit quipuswap_fa2: @@ -74,31 +83,40 @@ templates: handlers: - callback: on_fa2_token_to_tez pattern: - - destination: + - type: transaction + destination: entrypoint: tokenToTezPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa2_tez_to_token pattern: - - destination: + - type: transaction + destination: entrypoint: tezToTokenPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_invest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: investLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_divest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: divestLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_withdraw_profit pattern: - - destination: + - type: transaction + destination: entrypoint: withdrawProfit indexes: diff --git a/src/demo_quipuswap/handlers/on_fa12_divest_liquidity.py b/src/demo_quipuswap/handlers/on_fa12_divest_liquidity.py index 0a25bc59b..9d62237cf 100644 --- a/src/demo_quipuswap/handlers/on_fa12_divest_liquidity.py +++ b/src/demo_quipuswap/handlers/on_fa12_divest_liquidity.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa12_token.storage import Fa12TokenStorage from demo_quipuswap.types.quipu_fa12.parameter.divest_liquidity import DivestLiquidityParameter from demo_quipuswap.types.quipu_fa12.storage import QuipuFa12Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa12_divest_liquidity( ctx: OperationHandlerContext, - divest_liquidity: OperationContext[DivestLiquidityParameter, QuipuFa12Storage], - transfer: OperationContext[TransferParameter, Fa12TokenStorage], + divest_liquidity: TransactionContext[DivestLiquidityParameter, QuipuFa12Storage], + transfer: TransactionContext[TransferParameter, Fa12TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -25,6 +25,7 @@ async def on_fa12_divest_liquidity( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None tez_qty = Decimal(transaction.amount) / (10 ** 6) token_qty = Decimal(transfer.parameter.value) / (10 ** decimals) shares_qty = int(divest_liquidity.parameter.shares) diff --git a/src/demo_quipuswap/handlers/on_fa12_invest_liquidity.py b/src/demo_quipuswap/handlers/on_fa12_invest_liquidity.py index c0b5c95f9..a977706c8 100644 --- a/src/demo_quipuswap/handlers/on_fa12_invest_liquidity.py +++ b/src/demo_quipuswap/handlers/on_fa12_invest_liquidity.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa12_token.storage import Fa12TokenStorage from demo_quipuswap.types.quipu_fa12.parameter.invest_liquidity import InvestLiquidityParameter from demo_quipuswap.types.quipu_fa12.storage import QuipuFa12Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa12_invest_liquidity( ctx: OperationHandlerContext, - invest_liquidity: OperationContext[InvestLiquidityParameter, QuipuFa12Storage], - transfer: OperationContext[TransferParameter, Fa12TokenStorage], + invest_liquidity: TransactionContext[InvestLiquidityParameter, QuipuFa12Storage], + transfer: TransactionContext[TransferParameter, Fa12TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -24,6 +24,7 @@ async def on_fa12_invest_liquidity( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) + assert invest_liquidity.data.amount is not None tez_qty = Decimal(invest_liquidity.data.amount) / (10 ** 6) token_qty = Decimal(transfer.parameter.value) / (10 ** decimals) new_shares_qty = int(storage.storage.ledger[trader].balance) + int(storage.storage.ledger[trader].frozen_balance) # type: ignore diff --git a/src/demo_quipuswap/handlers/on_fa12_tez_to_token.py b/src/demo_quipuswap/handlers/on_fa12_tez_to_token.py index 86ed3b395..4060acdc6 100644 --- a/src/demo_quipuswap/handlers/on_fa12_tez_to_token.py +++ b/src/demo_quipuswap/handlers/on_fa12_tez_to_token.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa12_token.storage import Fa12TokenStorage from demo_quipuswap.types.quipu_fa12.parameter.tez_to_token_payment import TezToTokenPaymentParameter from demo_quipuswap.types.quipu_fa12.storage import QuipuFa12Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa12_tez_to_token( ctx: OperationHandlerContext, - tez_to_token_payment: OperationContext[TezToTokenPaymentParameter, QuipuFa12Storage], - transfer: OperationContext[TransferParameter, Fa12TokenStorage], + tez_to_token_payment: TransactionContext[TezToTokenPaymentParameter, QuipuFa12Storage], + transfer: TransactionContext[TransferParameter, Fa12TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -22,6 +22,7 @@ async def on_fa12_tez_to_token( min_token_quantity = Decimal(tez_to_token_payment.parameter.min_out) / (10 ** decimals) token_quantity = Decimal(transfer.parameter.value) / (10 ** decimals) + assert tez_to_token_payment.data.amount is not None tez_quantity = Decimal(tez_to_token_payment.data.amount) / (10 ** 6) assert min_token_quantity <= token_quantity, tez_to_token_payment.data.hash diff --git a/src/demo_quipuswap/handlers/on_fa12_token_to_tez.py b/src/demo_quipuswap/handlers/on_fa12_token_to_tez.py index ff92ad1d8..217aabb13 100644 --- a/src/demo_quipuswap/handlers/on_fa12_token_to_tez.py +++ b/src/demo_quipuswap/handlers/on_fa12_token_to_tez.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa12_token.storage import Fa12TokenStorage from demo_quipuswap.types.quipu_fa12.parameter.token_to_tez_payment import TokenToTezPaymentParameter from demo_quipuswap.types.quipu_fa12.storage import QuipuFa12Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa12_token_to_tez( ctx: OperationHandlerContext, - token_to_tez_payment: OperationContext[TokenToTezPaymentParameter, QuipuFa12Storage], - transfer: OperationContext[TransferParameter, Fa12TokenStorage], + token_to_tez_payment: TransactionContext[TokenToTezPaymentParameter, QuipuFa12Storage], + transfer: TransactionContext[TransferParameter, Fa12TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -23,6 +23,7 @@ async def on_fa12_token_to_tez( min_tez_quantity = Decimal(token_to_tez_payment.parameter.min_out) / (10 ** 6) token_quantity = Decimal(token_to_tez_payment.parameter.amount) / (10 ** decimals) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None tez_quantity = Decimal(transaction.amount) / (10 ** 6) assert min_tez_quantity <= tez_quantity, token_to_tez_payment.data.hash diff --git a/src/demo_quipuswap/handlers/on_fa12_withdraw_profit.py b/src/demo_quipuswap/handlers/on_fa12_withdraw_profit.py index 417f2be28..80ecdf0ef 100644 --- a/src/demo_quipuswap/handlers/on_fa12_withdraw_profit.py +++ b/src/demo_quipuswap/handlers/on_fa12_withdraw_profit.py @@ -3,12 +3,12 @@ import demo_quipuswap.models as models from demo_quipuswap.types.quipu_fa12.parameter.withdraw_profit import WithdrawProfitParameter from demo_quipuswap.types.quipu_fa12.storage import QuipuFa12Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa12_withdraw_profit( ctx: OperationHandlerContext, - withdraw_profit: OperationContext[WithdrawProfitParameter, QuipuFa12Storage], + withdraw_profit: TransactionContext[WithdrawProfitParameter, QuipuFa12Storage], ) -> None: if ctx.template_values is None: @@ -20,6 +20,7 @@ async def on_fa12_withdraw_profit( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None position.realized_pl += Decimal(transaction.amount) / (10 ** 6) # type: ignore await position.save() diff --git a/src/demo_quipuswap/handlers/on_fa20_divest_liquidity.py b/src/demo_quipuswap/handlers/on_fa20_divest_liquidity.py index 266499c93..81a9001d4 100644 --- a/src/demo_quipuswap/handlers/on_fa20_divest_liquidity.py +++ b/src/demo_quipuswap/handlers/on_fa20_divest_liquidity.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa2_token.storage import Fa2TokenStorage from demo_quipuswap.types.quipu_fa2.parameter.divest_liquidity import DivestLiquidityParameter from demo_quipuswap.types.quipu_fa2.storage import QuipuFa2Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa20_divest_liquidity( ctx: OperationHandlerContext, - divest_liquidity: OperationContext[DivestLiquidityParameter, QuipuFa2Storage], - transfer: OperationContext[TransferParameter, Fa2TokenStorage], + divest_liquidity: TransactionContext[DivestLiquidityParameter, QuipuFa2Storage], + transfer: TransactionContext[TransferParameter, Fa2TokenStorage], ) -> None: if ctx.template_values is None: @@ -26,6 +26,7 @@ async def on_fa20_divest_liquidity( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None tez_qty = Decimal(transaction.amount) / (10 ** 6) token_qty = sum(Decimal(tx.amount) for tx in transfer.parameter.__root__[0].txs) / (10 ** decimals) shares_qty = int(divest_liquidity.parameter.shares) diff --git a/src/demo_quipuswap/handlers/on_fa20_invest_liquidity.py b/src/demo_quipuswap/handlers/on_fa20_invest_liquidity.py index 2f82320a3..e46472634 100644 --- a/src/demo_quipuswap/handlers/on_fa20_invest_liquidity.py +++ b/src/demo_quipuswap/handlers/on_fa20_invest_liquidity.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa2_token.storage import Fa2TokenStorage from demo_quipuswap.types.quipu_fa2.parameter.invest_liquidity import InvestLiquidityParameter from demo_quipuswap.types.quipu_fa2.storage import QuipuFa2Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa20_invest_liquidity( ctx: OperationHandlerContext, - invest_liquidity: OperationContext[InvestLiquidityParameter, QuipuFa2Storage], - transfer: OperationContext[TransferParameter, Fa2TokenStorage], + invest_liquidity: TransactionContext[InvestLiquidityParameter, QuipuFa2Storage], + transfer: TransactionContext[TransferParameter, Fa2TokenStorage], ) -> None: if ctx.template_values is None: @@ -25,6 +25,7 @@ async def on_fa20_invest_liquidity( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) + assert invest_liquidity.data.amount is not None tez_qty = Decimal(invest_liquidity.data.amount) / (10 ** 6) token_qty = sum(Decimal(tx.amount) for tx in transfer.parameter.__root__[0].txs) / (10 ** decimals) new_shares_qty = int(storage.storage.ledger[trader].balance) + int(storage.storage.ledger[trader].frozen_balance) # type: ignore diff --git a/src/demo_quipuswap/handlers/on_fa20_withdraw_profit.py b/src/demo_quipuswap/handlers/on_fa20_withdraw_profit.py index 2f2aa0151..e29b68615 100644 --- a/src/demo_quipuswap/handlers/on_fa20_withdraw_profit.py +++ b/src/demo_quipuswap/handlers/on_fa20_withdraw_profit.py @@ -3,12 +3,12 @@ import demo_quipuswap.models as models from demo_quipuswap.types.quipu_fa2.parameter.withdraw_profit import WithdrawProfitParameter from demo_quipuswap.types.quipu_fa2.storage import QuipuFa2Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa20_withdraw_profit( ctx: OperationHandlerContext, - withdraw_profit: OperationContext[WithdrawProfitParameter, QuipuFa2Storage], + withdraw_profit: TransactionContext[WithdrawProfitParameter, QuipuFa2Storage], ) -> None: if ctx.template_values is None: @@ -20,6 +20,7 @@ async def on_fa20_withdraw_profit( position, _ = await models.Position.get_or_create(trader=trader, symbol=symbol) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None position.realized_pl += Decimal(transaction.amount) / (10 ** 6) # type: ignore await position.save() diff --git a/src/demo_quipuswap/handlers/on_fa2_tez_to_token.py b/src/demo_quipuswap/handlers/on_fa2_tez_to_token.py index 2eb9a50cf..152a84389 100644 --- a/src/demo_quipuswap/handlers/on_fa2_tez_to_token.py +++ b/src/demo_quipuswap/handlers/on_fa2_tez_to_token.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa2_token.storage import Fa2TokenStorage from demo_quipuswap.types.quipu_fa2.parameter.tez_to_token_payment import TezToTokenPaymentParameter from demo_quipuswap.types.quipu_fa2.storage import QuipuFa2Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa2_tez_to_token( ctx: OperationHandlerContext, - tez_to_token_payment: OperationContext[TezToTokenPaymentParameter, QuipuFa2Storage], - transfer: OperationContext[TransferParameter, Fa2TokenStorage], + tez_to_token_payment: TransactionContext[TezToTokenPaymentParameter, QuipuFa2Storage], + transfer: TransactionContext[TransferParameter, Fa2TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -21,6 +21,7 @@ async def on_fa2_tez_to_token( trader = tez_to_token_payment.data.sender_address min_token_quantity = Decimal(tez_to_token_payment.parameter.min_out) / (10 ** decimals) + assert tez_to_token_payment.data.amount is not None token_quantity = sum(Decimal(tx.amount) for tx in transfer.parameter.__root__[0].txs) / (10 ** decimals) tez_quantity = Decimal(tez_to_token_payment.data.amount) / (10 ** 6) assert min_token_quantity <= token_quantity, tez_to_token_payment.data.hash diff --git a/src/demo_quipuswap/handlers/on_fa2_token_to_tez.py b/src/demo_quipuswap/handlers/on_fa2_token_to_tez.py index ac691540c..4da646bb6 100644 --- a/src/demo_quipuswap/handlers/on_fa2_token_to_tez.py +++ b/src/demo_quipuswap/handlers/on_fa2_token_to_tez.py @@ -5,13 +5,13 @@ from demo_quipuswap.types.fa2_token.storage import Fa2TokenStorage from demo_quipuswap.types.quipu_fa2.parameter.token_to_tez_payment import TokenToTezPaymentParameter from demo_quipuswap.types.quipu_fa2.storage import QuipuFa2Storage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_fa2_token_to_tez( ctx: OperationHandlerContext, - token_to_tez_payment: OperationContext[TokenToTezPaymentParameter, QuipuFa2Storage], - transfer: OperationContext[TransferParameter, Fa2TokenStorage], + token_to_tez_payment: TransactionContext[TokenToTezPaymentParameter, QuipuFa2Storage], + transfer: TransactionContext[TransferParameter, Fa2TokenStorage], ) -> None: if ctx.template_values is None: raise Exception('This index must be templated') @@ -23,6 +23,7 @@ async def on_fa2_token_to_tez( min_tez_quantity = Decimal(token_to_tez_payment.parameter.min_out) / (10 ** decimals) token_quantity = Decimal(token_to_tez_payment.parameter.amount) / (10 ** decimals) transaction = next(op for op in ctx.operations if op.amount) + assert transaction.amount is not None tez_quantity = Decimal(transaction.amount) / (10 ** 6) assert min_tez_quantity <= tez_quantity, token_to_tez_payment.data.hash diff --git a/src/demo_registrydao/dipdup.yml b/src/demo_registrydao/dipdup.yml index 5250e9bd7..9e774ca98 100644 --- a/src/demo_registrydao/dipdup.yml +++ b/src/demo_registrydao/dipdup.yml @@ -20,13 +20,21 @@ templates: registry_dao: kind: operation datasource: tzkt + types: + - transaction + - origination contracts: - handlers: - callback: on_propose pattern: - - destination: + - type: transaction + destination: entrypoint: propose + - callback: on_origination + pattern: + - type: origination + originated_contract: indexes: registry: diff --git a/src/demo_registrydao/handlers/on_origination.py b/src/demo_registrydao/handlers/on_origination.py new file mode 100644 index 000000000..db450dfd1 --- /dev/null +++ b/src/demo_registrydao/handlers/on_origination.py @@ -0,0 +1,10 @@ +import demo_registrydao.models as models +from demo_registrydao.types.registry.storage import RegistryStorage +from dipdup.models import OperationHandlerContext, OriginationContext, TransactionContext + + +async def on_origination( + ctx: OperationHandlerContext, + registry_origination: OriginationContext[RegistryStorage], +) -> None: + ... diff --git a/src/demo_registrydao/handlers/on_propose.py b/src/demo_registrydao/handlers/on_propose.py index fee75dc01..68d226f26 100644 --- a/src/demo_registrydao/handlers/on_propose.py +++ b/src/demo_registrydao/handlers/on_propose.py @@ -1,11 +1,11 @@ import demo_registrydao.models as models from demo_registrydao.types.registry.parameter.propose import ProposeParameter from demo_registrydao.types.registry.storage import RegistryStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, OriginationContext, TransactionContext async def on_propose( ctx: OperationHandlerContext, - propose: OperationContext[ProposeParameter, RegistryStorage], + propose: TransactionContext[ProposeParameter, RegistryStorage], ) -> None: - print(propose.storage) + ... diff --git a/src/demo_tezos_domains/dipdup.yml b/src/demo_tezos_domains/dipdup.yml index e260ace92..667621542 100644 --- a/src/demo_tezos_domains/dipdup.yml +++ b/src/demo_tezos_domains/dipdup.yml @@ -24,11 +24,13 @@ templates: handlers: - callback: on_admin_update pattern: - - destination: + - type: transaction + destination: entrypoint: admin_update - callback: on_execute pattern: - - destination: + - type: transaction + destination: entrypoint: execute indexes: diff --git a/src/demo_tezos_domains/handlers/on_admin_update.py b/src/demo_tezos_domains/handlers/on_admin_update.py index 2d0e438ad..5405b6af6 100644 --- a/src/demo_tezos_domains/handlers/on_admin_update.py +++ b/src/demo_tezos_domains/handlers/on_admin_update.py @@ -2,12 +2,12 @@ from demo_tezos_domains.handlers.on_storage_diff import on_storage_diff from demo_tezos_domains.types.name_registry.parameter.admin_update import AdminUpdateParameter from demo_tezos_domains.types.name_registry.storage import NameRegistryStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_admin_update( ctx: OperationHandlerContext, - admin_update: OperationContext[AdminUpdateParameter, NameRegistryStorage], + admin_update: TransactionContext[AdminUpdateParameter, NameRegistryStorage], ) -> None: storage = admin_update.storage await on_storage_diff(storage) diff --git a/src/demo_tezos_domains/handlers/on_execute.py b/src/demo_tezos_domains/handlers/on_execute.py index c036e0bcb..20e97d1d6 100644 --- a/src/demo_tezos_domains/handlers/on_execute.py +++ b/src/demo_tezos_domains/handlers/on_execute.py @@ -2,12 +2,12 @@ from demo_tezos_domains.handlers.on_storage_diff import on_storage_diff from demo_tezos_domains.types.name_registry.parameter.execute import ExecuteParameter from demo_tezos_domains.types.name_registry.storage import NameRegistryStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_execute( ctx: OperationHandlerContext, - execute: OperationContext[ExecuteParameter, NameRegistryStorage], + execute: TransactionContext[ExecuteParameter, NameRegistryStorage], ) -> None: storage = execute.storage await on_storage_diff(storage) diff --git a/src/demo_tzcolors/dipdup.yml b/src/demo_tzcolors/dipdup.yml index 529ece913..8c8c62afd 100644 --- a/src/demo_tzcolors/dipdup.yml +++ b/src/demo_tzcolors/dipdup.yml @@ -28,15 +28,18 @@ templates: handlers: - callback: on_create_auction pattern: - - destination: + - type: transaction + destination: entrypoint: create_auction - callback: on_bid pattern: - - destination: + - type: transaction + destination: entrypoint: bid - callback: on_withdraw pattern: - - destination: + - type: transaction + destination: entrypoint: withdraw indexes: diff --git a/src/demo_tzcolors/handlers/on_bid.py b/src/demo_tzcolors/handlers/on_bid.py index 8c436f3a2..c49a4ede7 100644 --- a/src/demo_tzcolors/handlers/on_bid.py +++ b/src/demo_tzcolors/handlers/on_bid.py @@ -1,12 +1,12 @@ import demo_tzcolors.models as models from demo_tzcolors.types.tzcolors_auction.parameter.bid import BidParameter from demo_tzcolors.types.tzcolors_auction.storage import TzcolorsAuctionStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_bid( ctx: OperationHandlerContext, - bid: OperationContext[BidParameter, TzcolorsAuctionStorage], + bid: TransactionContext[BidParameter, TzcolorsAuctionStorage], ) -> None: auction = await models.Auction.filter( id=bid.parameter.__root__, diff --git a/src/demo_tzcolors/handlers/on_create_auction.py b/src/demo_tzcolors/handlers/on_create_auction.py index aa2af316d..1fc874f61 100644 --- a/src/demo_tzcolors/handlers/on_create_auction.py +++ b/src/demo_tzcolors/handlers/on_create_auction.py @@ -1,12 +1,12 @@ import demo_tzcolors.models as models from demo_tzcolors.types.tzcolors_auction.parameter.create_auction import CreateAuctionParameter from demo_tzcolors.types.tzcolors_auction.storage import TzcolorsAuctionStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_create_auction( ctx: OperationHandlerContext, - create_auction: OperationContext[CreateAuctionParameter, TzcolorsAuctionStorage], + create_auction: TransactionContext[CreateAuctionParameter, TzcolorsAuctionStorage], ) -> None: holder, _ = await models.Address.get_or_create(address=create_auction.data.sender_address) diff --git a/src/demo_tzcolors/handlers/on_withdraw.py b/src/demo_tzcolors/handlers/on_withdraw.py index 1510676c7..4ec68c6b5 100644 --- a/src/demo_tzcolors/handlers/on_withdraw.py +++ b/src/demo_tzcolors/handlers/on_withdraw.py @@ -1,12 +1,12 @@ import demo_tzcolors.models as models from demo_tzcolors.types.tzcolors_auction.parameter.withdraw import WithdrawParameter from demo_tzcolors.types.tzcolors_auction.storage import TzcolorsAuctionStorage -from dipdup.models import OperationContext, OperationHandlerContext +from dipdup.models import OperationHandlerContext, TransactionContext async def on_withdraw( ctx: OperationHandlerContext, - withdraw: OperationContext[WithdrawParameter, TzcolorsAuctionStorage], + withdraw: TransactionContext[WithdrawParameter, TzcolorsAuctionStorage], ) -> None: auction = await models.Auction.filter( id=withdraw.parameter.__root__, diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index 7ed9007c5..a8cfed9c9 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -17,6 +17,7 @@ DipDupConfig, IndexTemplateConfig, OperationHandlerConfig, + OperationHandlerTransactionPatternConfig, OperationIndexConfig, TzktDatasourceConfig, ) @@ -104,6 +105,9 @@ async def fetch_schemas(config: DipDupConfig): with open(storage_schema_path, 'w') as file: file.write(json.dumps(storage_schema, indent=4, sort_keys=True)) + if not isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): + continue + parameter_schemas_path = join(contract_schemas_path, 'parameter') with suppress(FileExistsError): mkdir(parameter_schemas_path) @@ -128,8 +132,6 @@ async def fetch_schemas(config: DipDupConfig): with open(entrypoint_schema_path, 'r') as file: existing_schema = json.loads(file.read()) if entrypoint_schema != existing_schema: - # FIXME: Different field order counts as different schema - # raise ValueError(f'Contract "{contract.address}" falsely claims to be a "{contract.typename}"') _logger.warning('Contract "%s" falsely claims to be a "%s"', contract_config.address, contract_config.typename) elif isinstance(index_config, BigMapIndexConfig): diff --git a/src/dipdup/config.py b/src/dipdup/config.py index 8bf007bc7..e46cf9176 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -7,6 +7,7 @@ import sys from collections import defaultdict from dataclasses import field +from enum import Enum from os import environ as env from os.path import dirname from typing import Any, Callable, Dict, List, Optional, Type, Union, cast @@ -29,6 +30,11 @@ _logger = logging.getLogger(__name__) +class OperationType(Enum): + transaction = 'transaction' + origination = 'origination' + + @dataclass class SqliteDatabaseConfig: """ @@ -142,13 +148,14 @@ def valid_url(cls, v): @dataclass -class OperationHandlerPatternConfig: +class OperationHandlerTransactionPatternConfig: """Operation handler pattern config :param destination: Alias of the contract to match :param entrypoint: Contract entrypoint """ + type: Literal['transaction'] destination: Union[str, ContractConfig] entrypoint: str @@ -162,7 +169,7 @@ def contract_config(self) -> ContractConfig: return self.destination @property - def parameter_type_cls(self) -> Type: + def parameter_type_cls(self) -> Optional[Type]: if self._parameter_type_cls is None: raise RuntimeError('Config is not initialized') return self._parameter_type_cls @@ -181,6 +188,54 @@ def storage_type_cls(self) -> Type: def storage_type_cls(self, typ: Type) -> None: self._storage_type_cls = typ + def get_handler_imports(self, package: str) -> str: + return '\n'.join( + [ + f'from {package}.types.{self.contract_config.module_name}.parameter.{camel_to_snake(self.entrypoint)} import {snake_to_camel(self.entrypoint)}Parameter', + f'from {package}.types.{self.contract_config.module_name}.storage import {snake_to_camel(self.contract_config.module_name)}Storage', + ] + ) + + def get_handler_argument(self) -> str: + return f'{camel_to_snake(self.entrypoint)}: TransactionContext[{snake_to_camel(self.entrypoint)}Parameter, {snake_to_camel(self.contract_config.module_name)}Storage],' + + +@dataclass +class OperationHandlerOriginationPatternConfig: + type: Literal['origination'] + originated_contract: Union[str, ContractConfig] + + def __post_init_post_parse__(self): + self._storage_type_cls = None + + @property + def parameter_type_cls(self) -> Optional[Type]: + return None + + @property + def contract_config(self) -> ContractConfig: + assert isinstance(self.originated_contract, ContractConfig) + return self.originated_contract + + @property + def storage_type_cls(self) -> Type: + if self._storage_type_cls is None: + raise RuntimeError('Config is not initialized') + return self._storage_type_cls + + @storage_type_cls.setter + def storage_type_cls(self, typ: Type) -> None: + self._storage_type_cls = typ + + def get_handler_imports(self, package: str) -> str: + return f'from {package}.types.{self.contract_config.module_name}.storage import {snake_to_camel(self.contract_config.module_name)}Storage' + + def get_handler_argument(self) -> str: + return f'{self.contract_config.module_name}_origination: OriginationContext[{snake_to_camel(self.contract_config.module_name)}Storage],' + + +OperationHandlerPatternConfig = Union[OperationHandlerOriginationPatternConfig, OperationHandlerTransactionPatternConfig] + @dataclass class HandlerConfig: @@ -266,6 +321,7 @@ class OperationIndexConfig(IndexConfig): kind: Literal["operation"] contracts: List[Union[str, ContractConfig]] handlers: List[OperationHandlerConfig] + types: Optional[List[OperationType]] = None first_block: int = 0 last_block: int = 0 @@ -425,14 +481,25 @@ def __post_init_post_parse__(self): except KeyError as e: raise ConfigurationError(f'Contract `{contract}` not found in `contracts` config section') from e - for handler in index_config.handlers: - callback_patterns[handler.callback].append(handler.pattern) - for pattern in handler.pattern: - if isinstance(pattern.destination, str): - try: - pattern.destination = self.contracts[pattern.destination] - except KeyError as e: - raise ConfigurationError(f'Contract `{pattern.destination}` not found in `contracts` config section') from e + for handler_config in index_config.handlers: + callback_patterns[handler_config.callback].append(handler_config.pattern) + for pattern_config in handler_config.pattern: + if isinstance(pattern_config, OperationHandlerTransactionPatternConfig): + if isinstance(pattern_config.destination, str): + try: + pattern_config.destination = self.contracts[pattern_config.destination] + except KeyError as e: + raise ConfigurationError( + f'Contract `{pattern_config.destination}` not found in `contracts` config section' + ) from e + elif isinstance(pattern_config, OperationHandlerOriginationPatternConfig): + if isinstance(pattern_config.originated_contract, str): + try: + pattern_config.originated_contract = self.contracts[pattern_config.originated_contract] + except KeyError as e: + raise ConfigurationError( + f'Contract `{pattern_config.originated_contract}` not found in `contracts` config section' + ) from e elif isinstance(index_config, BigMapIndexConfig): if isinstance(index_config.datasource, str): @@ -548,18 +615,19 @@ async def initialize(self) -> None: await self._initialize_handler_callback(operation_handler_config) for operation_pattern_config in operation_handler_config.pattern: - _logger.info('Registering parameter type for entrypoint `%s`', operation_pattern_config.entrypoint) - parameter_type_module = importlib.import_module( - f'{self.package}' - f'.types' - f'.{operation_pattern_config.contract_config.module_name}' - f'.parameter' - f'.{camel_to_snake(operation_pattern_config.entrypoint)}' - ) - parameter_type_cls = getattr( - parameter_type_module, snake_to_camel(operation_pattern_config.entrypoint) + 'Parameter' - ) - operation_pattern_config.parameter_type_cls = parameter_type_cls + if isinstance(operation_pattern_config, OperationHandlerTransactionPatternConfig): + _logger.info('Registering parameter type for entrypoint `%s`', operation_pattern_config.entrypoint) + parameter_type_module = importlib.import_module( + f'{self.package}' + f'.types' + f'.{operation_pattern_config.contract_config.module_name}' + f'.parameter' + f'.{camel_to_snake(operation_pattern_config.entrypoint)}' + ) + parameter_type_cls = getattr( + parameter_type_module, snake_to_camel(operation_pattern_config.entrypoint) + 'Parameter' + ) + operation_pattern_config.parameter_type_cls = parameter_type_cls _logger.info('Registering storage type') storage_type_module = importlib.import_module( diff --git a/src/dipdup/datasources/tzkt/cache.py b/src/dipdup/datasources/tzkt/cache.py index 27223cf58..c6ff40023 100644 --- a/src/dipdup/datasources/tzkt/cache.py +++ b/src/dipdup/datasources/tzkt/cache.py @@ -8,7 +8,9 @@ BigMapHandlerPatternConfig, BigMapIndexConfig, OperationHandlerConfig, + OperationHandlerOriginationPatternConfig, OperationHandlerPatternConfig, + OperationHandlerTransactionPatternConfig, OperationIndexConfig, ) from dipdup.models import BigMapData, OperationData @@ -48,14 +50,16 @@ async def add(self, operation: OperationData): self._operations[key].append(operation) def match_operation(self, pattern_config: OperationHandlerPatternConfig, operation: OperationData) -> bool: - self._logger.debug('pattern: %s, %s', pattern_config.entrypoint, pattern_config.contract_config.address) - self._logger.debug('operation: %s, %s', operation.entrypoint, operation.target_address) - if pattern_config.entrypoint != operation.entrypoint: - return False - if pattern_config.contract_config.address != operation.target_address: - return False - self._logger.debug('Match!') - return True + if isinstance(pattern_config, OperationHandlerTransactionPatternConfig): + return all( + [ + pattern_config.entrypoint == operation.entrypoint, + pattern_config.contract_config.address == operation.target_address, + ] + ) + if isinstance(pattern_config, OperationHandlerOriginationPatternConfig): + return pattern_config.contract_config.address == operation.originated_contract_address + raise NotImplementedError async def process( self, diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py index a74e9928f..b1f60d509 100644 --- a/src/dipdup/datasources/tzkt/datasource.py +++ b/src/dipdup/datasources/tzkt/datasource.py @@ -1,7 +1,7 @@ import asyncio import logging -from functools import partial -from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union +from enum import Enum +from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from aiosignalrcore.hub.base_hub_connection import BaseHubConnection # type: ignore from aiosignalrcore.hub_connection_builder import HubConnectionBuilder # type: ignore @@ -16,7 +16,10 @@ BigMapIndexConfig, BlockIndexConfig, OperationHandlerConfig, + OperationHandlerOriginationPatternConfig, + OperationHandlerTransactionPatternConfig, OperationIndexConfig, + OperationType, ) from dipdup.datasources.tzkt.cache import BigMapCache, OperationCache from dipdup.datasources.tzkt.enums import TzktMessageType @@ -26,47 +29,199 @@ BigMapContext, BigMapData, BigMapHandlerContext, - OperationContext, OperationData, OperationHandlerContext, + OriginationContext, + TransactionContext, ) -from dipdup.utils import http_request TZKT_HTTP_REQUEST_LIMIT = 10000 -TZKT_HTTP_REQUEST_SLEEP = 1 OPERATION_FIELDS = ( "type", "id", "level", "timestamp", - # "block", "hash", "counter", - "initiator", "sender", "nonce", - # "gasLimit", - # "gasUsed", - # "storageLimit", - # "storageUsed", - # "bakerFee", - # "storageFee", - # "allocationFee", "target", "amount", - "parameter", "storage", "status", - # "errors", "hasInternals", - # "quote", - "diffs,", + "diffs", +) +ORIGINATION_OPERATION_FIELDS = ( + *OPERATION_FIELDS, + "originatedContract", ) +TRANSACTION_OPERATION_FIELDS = ( + *OPERATION_FIELDS, + "parameter", + "hasInternals", +) + IndexName = str Address = str Path = str -OperationType = str + + +class OperationFetcherChannel(Enum): + sender_transactions = 'sender_transactions' + target_transactions = 'target_transactions' + originations = 'originations' + + +class OperationFetcher: + def __init__( + self, + url: str, + proxy: TzktRequestProxy, + first_level: int, + last_level: int, + addresses: List[str], + operation_subscriptions: Dict[Address, List[OperationType]], + ) -> None: + self._url = url + self._proxy = proxy + self._first_level = first_level + self._last_level = last_level + self._origination_addresses = [ + address for address, types in operation_subscriptions.items() if address in addresses and OperationType.origination in types + ] + self._transaction_addresses = [ + address for address, types in operation_subscriptions.items() if address in addresses and OperationType.transaction in types + ] + self._logger = logging.getLogger(__name__) + self._head: int = 0 + self._heads: Dict[OperationFetcherChannel, int] = {} + self._offsets: Dict[OperationFetcherChannel, int] = {} + self._fetched: Dict[OperationFetcherChannel, bool] = {} + self._operations: Dict[int, List[Dict[str, Any]]] = {} + + def _get_head(self, operations: List[Dict[str, Any]]): + for i in range(len(operations) - 1)[::-1]: + if operations[i]['level'] != operations[i + 1]['level']: + return operations[i]['level'] + return operations[0]['level'] + + async def _fetch_originations(self) -> None: + key = OperationFetcherChannel.originations + if not self._origination_addresses: + self._fetched[key] = True + self._heads[key] = self._last_level + if self._fetched[key]: + return + + self._logger.debug('Fetching originations of %s', self._origination_addresses) + + originations = await self._proxy.http_request( + 'get', + url=f'{self._url}/v1/operations/originations', + params={ + "originatedContract.in": ','.join(self._origination_addresses), + "offset": self._offsets[key], + "limit": TZKT_HTTP_REQUEST_LIMIT, + "level.gt": self._first_level, + "level.le": self._last_level, + "select": ','.join(ORIGINATION_OPERATION_FIELDS), + "status": "applied", + }, + ) + + for op in originations: + # NOTE: type needs to be set manually when requesting operations by specific type + op['type'] = 'origination' + level = op['level'] + if level not in self._operations: + self._operations[level] = [] + self._operations[level].append(op) + + self._logger.debug('Got %s', len(originations)) + + if len(originations) < TZKT_HTTP_REQUEST_LIMIT: + self._fetched[key] = True + self._heads[key] = self._last_level + else: + self._offsets[key] += TZKT_HTTP_REQUEST_LIMIT + self._heads[key] = self._get_head(originations) + + async def _fetch_transactions(self, field: str): + key = getattr(OperationFetcherChannel, field + '_transactions') + if not self._transaction_addresses: + self._fetched[key] = True + self._heads[key] = self._last_level + if self._fetched[key]: + return + + self._logger.debug('Fetching %s transactions of %s', field, self._transaction_addresses) + + transactions = await self._proxy.http_request( + 'get', + url=f'{self._url}/v1/operations/transactions', + params={ + f"{field}.in": ','.join(self._transaction_addresses), + "offset": self._offsets[key], + "limit": TZKT_HTTP_REQUEST_LIMIT, + "level.gt": self._first_level, + "level.le": self._last_level, + "select": ','.join(TRANSACTION_OPERATION_FIELDS), + "status": "applied", + }, + ) + + for op in transactions: + # NOTE: type needs to be set manually when requesting operations by specific type + op['type'] = 'transaction' + level = op['level'] + if level not in self._operations: + self._operations[level] = [] + self._operations[level].append(op) + + self._logger.debug('Got %s', len(transactions)) + + if len(transactions) < TZKT_HTTP_REQUEST_LIMIT: + self._fetched[key] = True + self._heads[key] = self._last_level + else: + self._offsets[key] += TZKT_HTTP_REQUEST_LIMIT + self._heads[key] = self._get_head(transactions) + + async def fetch_operations_by_level(self): + for type_ in ( + OperationFetcherChannel.sender_transactions, + OperationFetcherChannel.target_transactions, + OperationFetcherChannel.originations, + ): + self._heads[type_] = 0 + self._offsets[type_] = 0 + self._fetched[type_] = False + + while True: + min_head = sorted(self._heads.items(), key=lambda x: x[1])[0][0] + if min_head == OperationFetcherChannel.originations: + await self._fetch_originations() + elif min_head == OperationFetcherChannel.target_transactions: + await self._fetch_transactions('target') + elif min_head == OperationFetcherChannel.sender_transactions: + await self._fetch_transactions('sender') + else: + raise RuntimeError + + head = min(self._heads.values()) + while self._head <= head: + if self._head in self._operations: + operations = self._operations.pop(self._head) + operations = sorted(list(({op['id']: op for op in operations}).values()), key=lambda op: op['id']) + yield self._head, operations + self._head += 1 + + if all(list(self._fetched.values())): + break + + assert not self._operations class TzktDatasource: @@ -130,40 +285,38 @@ async def start(self): self._logger.info('Starting datasource') rest_only = False - self._logger.info('Initial synchronizing operation indexes') for operation_index_config in self._operation_index_by_name.values(): + for contract in operation_index_config.contracts: + await self.add_operation_subscription(contract.address, operation_index_config.types) + + for big_map_index_config in self._big_map_index_by_name.values(): + for handler_config in big_map_index_config.handlers: + for pattern_config in handler_config.pattern: + await self.add_big_map_subscription(pattern_config.contract_config.address, pattern_config.path) + latest_block = await self.get_latest_block() + + self._logger.info('Initial synchronizing operation indexes') + for index_config_name, operation_index_config in self._operation_index_by_name.items(): + self._logger.info('Synchronizing `%s`', index_config_name) if operation_index_config.last_block: - await self.fetch_operations(operation_index_config.last_block, initial=True) + current_level = operation_index_config.last_block rest_only = True - continue - - for contract in operation_index_config.contracts: - await self.add_operation_subscription(contract.address) + else: + current_level = latest_block['level'] - latest_block = await self.get_latest_block() - current_level = latest_block['level'] - state_level = operation_index_config.state.level - if current_level != state_level: - await self.fetch_operations(current_level, initial=True) + await self.fetch_operations(operation_index_config, current_level) self._logger.info('Initial synchronizing big map indexes') - for big_map_index_config in self._big_map_index_by_name.values(): - + for index_config_name, big_map_index_config in self._big_map_index_by_name.items(): + self._logger.info('Synchronizing `%s`', index_config_name) if big_map_index_config.last_block: - await self.fetch_big_maps(big_map_index_config.last_block, initial=True) + current_level = big_map_index_config.last_block rest_only = True - continue - - for handler_config in big_map_index_config.handlers: - for pattern_config in handler_config.pattern: - await self.add_big_map_subscription(pattern_config.contract_config.address, pattern_config.path) + else: + current_level = latest_block['level'] - latest_block = await self.get_latest_block() - current_level = latest_block['level'] - state_level = big_map_index_config.state.level - if current_level != state_level: - await self.fetch_big_maps(current_level, initial=True) + await self.fetch_big_maps(big_map_index_config, current_level) if not rest_only: self._logger.info('Starting websocket client') @@ -183,7 +336,7 @@ async def on_connect(self): def on_error(self, message: CompletionMessage): raise Exception(message.error) - async def subscribe_to_operations(self, address: str, types: List[str]) -> None: + async def subscribe_to_operations(self, address: str, types: List[OperationType]) -> None: self._logger.info('Subscribing to %s, %s', address, types) while self._get_client().transport.state != ConnectionState.connected: @@ -194,7 +347,7 @@ async def subscribe_to_operations(self, address: str, types: List[str]) -> None: [ { 'address': address, - 'types': ','.join(types), + 'types': ','.join([t.value for t in types]), } ], ) @@ -202,51 +355,23 @@ async def subscribe_to_operations(self, address: str, types: List[str]) -> None: async def subscribe_to_big_maps(self, address: Address, path: Path) -> None: self._logger.info('Subscribing to %s, %s', address, path) - async def _fetch_operations(self, addresses: List[str], offset: int, first_level: int, last_level: int) -> List[Dict[str, Any]]: - self._logger.info('Fetching levels %s-%s with offset %s', first_level, last_level, offset) + async def fetch_operations(self, index_config: OperationIndexConfig, last_level: int) -> None: + self._logger.info('Fetching operations prior to level %s', last_level) - operations = await self._proxy.http_request( - 'get', - url=f'{self._url}/v1/operations/transactions', - params={ - "sender.in": ','.join(addresses), - "offset": offset, - "limit": TZKT_HTTP_REQUEST_LIMIT, - "level.gt": first_level, - "level.le": last_level, - "select": ','.join(OPERATION_FIELDS), - "status": "applied", - }, - ) + first_level = index_config.state.level + addresses = [c.address for c in index_config.contract_configs] - target_operations = await self._proxy.http_request( - 'get', - url=f'{self._url}/v1/operations/transactions', - params={ - "target.in": ','.join(addresses), - "offset": offset, - "limit": TZKT_HTTP_REQUEST_LIMIT, - "level.gt": first_level, - "level.le": last_level, - "select": ','.join(OPERATION_FIELDS), - "status": "applied", - }, + fetcher = OperationFetcher( + url=self._url, + proxy=self._proxy, + first_level=first_level, + last_level=last_level, + addresses=addresses, + operation_subscriptions=self._operation_subscriptions, ) - sender_operation_keys = {op['id'] for op in operations} - for op in target_operations: - if op['id'] not in sender_operation_keys: - operations.append(op) - - operations = sorted(operations, key=lambda op: op['id']) - - self._logger.info('%s operations fetched', len(operations)) - self._logger.debug(operations) - return operations - - async def fetch_operations(self, last_level: int, initial: bool = False) -> None: - async def _process_level_operations(operations) -> None: - self._logger.info('Processing %s operations of level %s', len(operations), operations[0]['level']) + async for level, operations in fetcher.fetch_operations_by_level(): + self._logger.info('Processing %s operations of level %s', len(operations), level) await self.on_operation_message( message=[ { @@ -257,41 +382,6 @@ async def _process_level_operations(operations) -> None: sync=True, ) - self._logger.info('Fetching operations prior to level %s', last_level) - for index_config in self._operation_index_by_name.values(): - - level = index_config.state.level - - operations = [] - offset = 0 - addresses = [c.address for c in index_config.contract_configs] - - while True: - fetched_operations = await self._fetch_operations(addresses, offset, level, last_level) - operations += fetched_operations - - while True: - for i in range(len(operations) - 1): - if operations[i]['level'] != operations[i + 1]['level']: - await _process_level_operations(operations[: i + 1]) - operations = operations[i + 1 :] - break - else: - break - - if len(fetched_operations) < TZKT_HTTP_REQUEST_LIMIT: - break - - offset += TZKT_HTTP_REQUEST_LIMIT - self._logger.info('Sleeping %s seconds before fetching next batch', TZKT_HTTP_REQUEST_SLEEP) - await asyncio.sleep(TZKT_HTTP_REQUEST_SLEEP) - - if operations: - await _process_level_operations(operations) - - if not initial: - self._operations_synchronized.set() - async def _fetch_big_maps( self, addresses: List[Address], paths: List[Path], offset: int, first_level: int, last_level: int ) -> List[Dict[str, Any]]: @@ -314,7 +404,7 @@ async def _fetch_big_maps( self._logger.debug(big_maps) return big_maps - async def fetch_big_maps(self, last_level: int, initial: bool = False) -> None: + async def fetch_big_maps(self, index_config: BigMapIndexConfig, last_level: int) -> None: async def _process_level_big_maps(big_maps): self._logger.info('Processing %s big map updates of level %s', len(big_maps), big_maps[0]['level']) await self.on_big_map_message( @@ -328,43 +418,36 @@ async def _process_level_big_maps(big_maps): ) self._logger.info('Fetching big map updates prior to level %s', last_level) - for index_config in self._big_map_index_by_name.values(): + level = index_config.state.level - level = index_config.state.level + big_maps = [] + offset = 0 + addresses, paths = set(), set() + for handler_config in index_config.handlers: + for pattern_config in handler_config.pattern: + addresses.add(pattern_config.contract_config.address) + paths.add(pattern_config.path) - big_maps = [] - offset = 0 - addresses, paths = set(), set() - for handler_config in index_config.handlers: - for pattern_config in handler_config.pattern: - addresses.add(pattern_config.contract_config.address) - paths.add(pattern_config.path) + while True: + fetched_big_maps = await self._fetch_big_maps(list(addresses), list(paths), offset, level, last_level) + big_maps += fetched_big_maps while True: - fetched_big_maps = await self._fetch_big_maps(list(addresses), list(paths), offset, level, last_level) - big_maps += fetched_big_maps - - while True: - for i in range(len(big_maps) - 1): - if big_maps[i]['level'] != big_maps[i + 1]['level']: - await _process_level_big_maps(big_maps[: i + 1]) - big_maps = big_maps[i + 1 :] - break - else: + for i in range(len(big_maps) - 1): + if big_maps[i]['level'] != big_maps[i + 1]['level']: + await _process_level_big_maps(big_maps[: i + 1]) + big_maps = big_maps[i + 1 :] break - - if len(fetched_big_maps) < TZKT_HTTP_REQUEST_LIMIT: + else: break - offset += TZKT_HTTP_REQUEST_LIMIT - self._logger.info('Sleeping %s seconds before fetching next batch', TZKT_HTTP_REQUEST_SLEEP) - await asyncio.sleep(TZKT_HTTP_REQUEST_SLEEP) + if len(fetched_big_maps) < TZKT_HTTP_REQUEST_LIMIT: + break - if big_maps: - await _process_level_big_maps(big_maps) + offset += TZKT_HTTP_REQUEST_LIMIT - if not initial: - self._big_maps_synchronized.set() + if big_maps: + await _process_level_big_maps(big_maps) async def fetch_jsonschemas(self, address: str) -> Dict[str, Any]: self._logger.info('Fetching jsonschemas for address `%s', address) @@ -387,9 +470,12 @@ async def on_operation_message( message_type = TzktMessageType(item['type']) if message_type == TzktMessageType.STATE: - level = item['state'] - self._logger.info('Got state message, current level %s, index level %s', level, self._operation_cache.level) - await self.fetch_operations(level) + last_level = item['state'] + for index_config in self._operation_index_by_name.values(): + first_level = index_config.state.level + self._logger.info('Got state message, current level %s, index level %s', last_level, first_level) + await self.fetch_operations(index_config, last_level) + self._operations_synchronized.set() elif message_type == TzktMessageType.DATA: if not sync and not self._operations_synchronized.is_set(): @@ -401,8 +487,6 @@ async def on_operation_message( async with self._callback_lock: for operation_json in item['data']: operation = self.convert_operation(operation_json) - if operation.type != 'transaction': - continue if operation.status != 'applied': continue await self._operation_cache.add(operation) @@ -434,9 +518,12 @@ async def on_big_map_message( message_type = TzktMessageType(item['type']) if message_type == TzktMessageType.STATE: - level = item['state'] - self._logger.info('Got state message, current level %s, index level %s', level, self._operation_cache.level) - await self.fetch_big_maps(level) + last_level = item['state'] + for index_config in self._big_map_index_by_name.values(): + first_level = self._operation_cache.level + self._logger.info('Got state message, current level %s, index level %s', first_level, first_level) + await self.fetch_big_maps(index_config, last_level) + self._big_maps_synchronized.set() elif message_type == TzktMessageType.DATA: if not sync and not self._big_maps_synchronized.is_set(): @@ -465,9 +552,9 @@ async def on_big_map_message( else: self._logger.warning('%s is not supported', message_type) - async def add_operation_subscription(self, address: str, types: Optional[List[str]] = None) -> None: + async def add_operation_subscription(self, address: str, types: Optional[List[OperationType]] = None) -> None: if types is None: - types = ['transaction'] + types = [OperationType.transaction] if address not in self._operation_subscriptions: self._operation_subscriptions[address] = types @@ -487,21 +574,35 @@ async def on_operation_match( operations=operations, template_values=index_config.template_values, ) - args: List[Union[OperationHandlerContext, OperationContext]] = [handler_context] + args: List[Union[OperationHandlerContext, TransactionContext, OriginationContext]] = [handler_context] for pattern_config, operation in zip(handler_config.pattern, matched_operations): - parameter_type = pattern_config.parameter_type_cls - parameter = parameter_type.parse_obj(operation.parameter_json) + if isinstance(pattern_config, OperationHandlerTransactionPatternConfig): + parameter_type = pattern_config.parameter_type_cls + parameter = parameter_type.parse_obj(operation.parameter_json) if parameter_type else None - storage_type = pattern_config.storage_type_cls - storage = operation.get_merged_storage(storage_type) + storage_type = pattern_config.storage_type_cls + storage = operation.get_merged_storage(storage_type) - operation_context = OperationContext( - data=operation, - parameter=parameter, - storage=storage, - ) - args.append(operation_context) + transaction_context = TransactionContext( + data=operation, + parameter=parameter, + storage=storage, + ) + args.append(transaction_context) + + elif isinstance(pattern_config, OperationHandlerOriginationPatternConfig): + storage_type = pattern_config.storage_type_cls + storage = operation.get_merged_storage(storage_type) + + origination_context = OriginationContext( + data=operation, + storage=storage, + ) + args.append(origination_context) + + else: + raise NotImplementedError await handler_config.callback_fn(*args) @@ -548,31 +649,31 @@ async def on_big_map_match( @classmethod def convert_operation(cls, operation_json: Dict[str, Any]) -> OperationData: storage = operation_json.get('storage') - # FIXME: KT1CpeSQKdkhWi4pinYcseCFKmDhs5M74BkU + # FIXME: Plain storage, has issues in codegen: KT1CpeSQKdkhWi4pinYcseCFKmDhs5M74BkU if not isinstance(storage, Dict): storage = {} return OperationData( - # FIXME: type is null - type=operation_json['type'] or 'transaction', + type=operation_json['type'], id=operation_json['id'], level=operation_json['level'], timestamp=operation_json['timestamp'], block=operation_json.get('block'), hash=operation_json['hash'], counter=operation_json['counter'], - sender_address=operation_json['sender']['address'], - target_address=operation_json['target']['address'], + sender_address=operation_json['sender']['address'] if operation_json.get('sender') else None, + target_address=operation_json['target']['address'] if operation_json.get('target') else None, amount=operation_json['amount'], status=operation_json['status'], has_internals=operation_json['hasInternals'], sender_alias=operation_json['sender'].get('alias'), nonce=operation_json.get('nonce'), - target_alias=operation_json['target'].get('alias'), + target_alias=operation_json['target'].get('alias') if operation_json.get('target') else None, entrypoint=operation_json['parameter']['entrypoint'] if operation_json.get('parameter') else None, parameter_json=operation_json['parameter']['value'] if operation_json.get('parameter') else None, - initiator_address=operation_json['initiator']['address'] if operation_json.get('initiator') else None, - parameter=operation_json.get('parameters'), + originated_contract_address=operation_json['originatedContract']['address'] + if operation_json.get('originatedContract') + else None, storage=storage, diffs=operation_json.get('diffs'), ) diff --git a/src/dipdup/models.py b/src/dipdup/models.py index d3751c8ac..acbbb6258 100644 --- a/src/dipdup/models.py +++ b/src/dipdup/models.py @@ -36,18 +36,17 @@ class Meta: @dataclass class OperationData: - # FIXME: Bug in TzKT, shouldn't be optional - type: Optional[str] + type: str id: int level: int timestamp: datetime hash: str counter: int sender_address: str - target_address: str - amount: int + target_address: Optional[str] + amount: Optional[int] status: str - has_internals: bool + has_internals: Optional[bool] storage: Dict[str, Any] block: Optional[str] = None sender_alias: Optional[str] = None @@ -55,8 +54,7 @@ class OperationData: target_alias: Optional[str] = None entrypoint: Optional[str] = None parameter_json: Optional[Any] = None - initiator_address: Optional[str] = None - parameter: Optional[str] = None + originated_contract_address: Optional[str] = None diffs: Optional[List[Dict[str, Any]]] = None def _merge_bigmapdiffs(self, storage_dict: Dict[str, Any], bigmap_name: str, array: bool) -> None: @@ -119,12 +117,18 @@ def get_merged_storage(self, storage_type: Type[StorageType]) -> StorageType: @dataclass -class OperationContext(Generic[ParameterType, StorageType]): +class TransactionContext(Generic[ParameterType, StorageType]): data: OperationData parameter: ParameterType storage: StorageType +@dataclass +class OriginationContext(Generic[StorageType]): + data: OperationData + storage: StorageType + + class BigMapAction(Enum): ADD = 'add_key' UPDATE = 'update_key' diff --git a/src/dipdup/templates/operation_handler.py.j2 b/src/dipdup/templates/operation_handler.py.j2 index 61153892b..95fe54662 100644 --- a/src/dipdup/templates/operation_handler.py.j2 +++ b/src/dipdup/templates/operation_handler.py.j2 @@ -1,16 +1,15 @@ -from dipdup.models import OperationHandlerContext, OperationContext +from dipdup.models import OperationHandlerContext, TransactionContext, OriginationContext import {{ package }}.models as models {% for pattern in patterns %} -from {{ package }}.types.{{ pattern.destination.module_name }}.parameter.{{ camel_to_snake(pattern.entrypoint) }} import {{ snake_to_camel(pattern.entrypoint) }}Parameter -from {{ package }}.types.{{ pattern.destination.module_name }}.storage import {{ snake_to_camel(pattern.destination.module_name) }}Storage +{{ pattern.get_handler_imports(package) }} {%- endfor %} async def {{ handler }}( ctx: OperationHandlerContext, {%- for pattern in patterns %} - {{ camel_to_snake(pattern.entrypoint) }}: OperationContext[{{ snake_to_camel(pattern.entrypoint) }}Parameter, {{ snake_to_camel(pattern.destination.module_name) }}Storage], + {{ pattern.get_handler_argument() }} {%- endfor %} ) -> None: ... diff --git a/tests/integration_tests/hic_et_nunc.yml b/tests/integration_tests/hic_et_nunc.yml index 42a13e245..8ff08acda 100644 --- a/tests/integration_tests/hic_et_nunc.yml +++ b/tests/integration_tests/hic_et_nunc.yml @@ -1,4 +1,4 @@ -spec_version: 0.0.1 +spec_version: 0.1 package: demo_hic_et_nunc database: @@ -27,21 +27,26 @@ indexes: handlers: - callback: on_mint pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: mint_OBJKT - - destination: HEN_objkts + - type: transaction + destination: HEN_objkts entrypoint: mint - callback: on_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: swap - callback: on_cancel_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: cancel_swap - callback: on_collect pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: collect first_block: 1365000 last_block: 1366000 \ No newline at end of file diff --git a/tests/integration_tests/quipuswap.yml b/tests/integration_tests/quipuswap.yml index f01637619..efc48efa1 100644 --- a/tests/integration_tests/quipuswap.yml +++ b/tests/integration_tests/quipuswap.yml @@ -1,4 +1,4 @@ -spec_version: 0.0.1 +spec_version: 0.1 package: demo_quipuswap database: @@ -39,27 +39,35 @@ templates: handlers: - callback: on_fa12_token_to_tez pattern: - - destination: + - type: transaction + destination: entrypoint: tokenToTezPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_tez_to_token pattern: - - destination: + - type: transaction + destination: entrypoint: tezToTokenPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_invest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: investLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa12_divest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: divestLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer first_block: 1407528 last_block: 1408934 @@ -72,30 +80,38 @@ templates: handlers: - callback: on_fa2_token_to_tez pattern: - - destination: + - type: transaction + destination: entrypoint: tokenToTezPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa2_tez_to_token pattern: - - destination: + - type: transaction + destination: entrypoint: tezToTokenPayment - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_invest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: investLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer - callback: on_fa20_divest_liquidity pattern: - - destination: + - type: transaction + destination: entrypoint: divestLiquidity - - destination: + - type: transaction + destination: entrypoint: transfer first_block: 1407528 - last_block: 1407728 + last_block: 1408928 indexes: kusd_mainnet: diff --git a/tests/integration_tests/registrydao.yml b/tests/integration_tests/registrydao.yml new file mode 100644 index 000000000..9e774ca98 --- /dev/null +++ b/tests/integration_tests/registrydao.yml @@ -0,0 +1,43 @@ +spec_version: 0.1 +package: demo_registrydao + +database: + kind: sqlite + path: registrydao.sqlite3 + +contracts: + registry: + address: KT1QMdCTqzmY4QKHntV1nZEinLPU1GbxUFQu + typename: registry + +datasources: + tzkt: + kind: tzkt + url: ${TZKT_URL:-https://staging.api.edo2net.tzkt.io} + +templates: + + registry_dao: + kind: operation + datasource: tzkt + types: + - transaction + - origination + contracts: + - + handlers: + - callback: on_propose + pattern: + - type: transaction + destination: + entrypoint: propose + - callback: on_origination + pattern: + - type: origination + originated_contract: + +indexes: + registry: + template: registry_dao + values: + contract: registry diff --git a/tests/integration_tests/test_codegen.py b/tests/integration_tests/test_codegen.py index dce540d3e..5c4572dcd 100644 --- a/tests/integration_tests/test_codegen.py +++ b/tests/integration_tests/test_codegen.py @@ -30,7 +30,7 @@ def import_submodules(package, recursive=True): class CodegenTest(IsolatedAsyncioTestCase): async def test_codegen(self): - for name in ['hic_et_nunc.yml', 'quipuswap.yml', 'tzcolors.yml']: + for name in ['hic_et_nunc.yml', 'quipuswap.yml', 'tzcolors.yml', 'tezos_domains_big_map.yml', 'registrydao.yml']: with self.subTest(name): config_path = join(dirname(__file__), name) config = DipDupConfig.load([config_path]) diff --git a/tests/integration_tests/test_demos.py b/tests/integration_tests/test_demos.py index c93e746d9..6f00258c6 100644 --- a/tests/integration_tests/test_demos.py +++ b/tests/integration_tests/test_demos.py @@ -1,9 +1,12 @@ import subprocess +from contextlib import suppress from os import mkdir from os.path import dirname, join from shutil import rmtree from unittest import IsolatedAsyncioTestCase +from tortoise.transactions import in_transaction + import demo_hic_et_nunc.models import demo_quipuswap.models import demo_tezos_domains.models @@ -13,12 +16,12 @@ class DemosTest(IsolatedAsyncioTestCase): + # TODO: store cache in xdg_cache_home, keep databases and logs after last run def setUp(self): + with suppress(FileNotFoundError): + rmtree('/tmp/dipdup') mkdir('/tmp/dipdup') - def tearDown(self): - rmtree('/tmp/dipdup') - def run_dipdup(self, config: str): subprocess.run( [ @@ -53,8 +56,11 @@ async def test_quipuswap(self): async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_quipuswap.models'): trades = await demo_quipuswap.models.Trade.filter().count() positions = await demo_quipuswap.models.Position.filter().count() - self.assertEqual(94, trades) - self.assertEqual(56, positions) + async with in_transaction() as conn: + symbols = (await conn.execute_query('select count(distinct(symbol)) from trade group by symbol;'))[0] + self.assertEqual(2, symbols) + self.assertEqual(93, trades) + self.assertEqual(54, positions) async def test_tzcolors(self): self.run_dipdup('tzcolors.yml') diff --git a/tests/integration_tests/tezos_domains.yml b/tests/integration_tests/tezos_domains.yml index a1da95d41..2b535cd7b 100644 --- a/tests/integration_tests/tezos_domains.yml +++ b/tests/integration_tests/tezos_domains.yml @@ -24,11 +24,13 @@ templates: handlers: - callback: on_admin_update pattern: - - destination: + - type: transaction + destination: entrypoint: admin_update - callback: on_execute pattern: - - destination: + - type: transaction + destination: entrypoint: execute last_block: 55363 diff --git a/tests/integration_tests/tzcolors.yml b/tests/integration_tests/tzcolors.yml index 20ae4e48a..e8e779dde 100644 --- a/tests/integration_tests/tzcolors.yml +++ b/tests/integration_tests/tzcolors.yml @@ -1,4 +1,4 @@ -spec_version: 0.0.1 +spec_version: 0.1 package: demo_tzcolors database: @@ -28,15 +28,18 @@ templates: handlers: - callback: on_create_auction pattern: - - destination: + - type: transaction + destination: entrypoint: create_auction - callback: on_bid pattern: - - destination: + - type: transaction + destination: entrypoint: bid - callback: on_withdraw pattern: - - destination: + - type: transaction + destination: entrypoint: withdraw first_block: 1335654 last_block: 1340654 diff --git a/tests/test_dipdup/dipdup.yml b/tests/test_dipdup/dipdup.yml index a898b4d58..e8ca1c390 100644 --- a/tests/test_dipdup/dipdup.yml +++ b/tests/test_dipdup/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.0.1 +spec_version: 0.1 package: demo_hic_et_nunc database: @@ -27,19 +27,24 @@ indexes: handlers: - callback: on_mint pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: mint_OBJKT - - destination: HEN_objkts + - type: transaction + destination: HEN_objkts entrypoint: mint - callback: on_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: swap - callback: on_cancel_swap pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: cancel_swap - callback: on_collect pattern: - - destination: HEN_minter + - type: transaction + destination: HEN_minter entrypoint: collect \ No newline at end of file diff --git a/tests/test_dipdup/test_datasources/test_tzkt/operations-storage.json b/tests/test_dipdup/test_datasources/test_tzkt/operations-storage.json index f0a54c7e6..f02ae98c5 100644 --- a/tests/test_dipdup/test_datasources/test_tzkt/operations-storage.json +++ b/tests/test_dipdup/test_datasources/test_tzkt/operations-storage.json @@ -8,7 +8,6 @@ "timestamp": "2021-03-23T15:48:41Z", "hash": "oon18UdUrsyFK5F1seSf7RnhTWciKJoJSPcbrqXZYxiitGcxeJD", "counter": 69209, - "initiator": null, "sender": { "address": "tz1RKPcdraL3D3SQitGbvUZmBoqefepxRW1x" }, diff --git a/tests/test_dipdup/test_datasources/test_tzkt/operations.json b/tests/test_dipdup/test_datasources/test_tzkt/operations.json index 9d0feb23a..bcedd0772 100644 --- a/tests/test_dipdup/test_datasources/test_tzkt/operations.json +++ b/tests/test_dipdup/test_datasources/test_tzkt/operations.json @@ -57,9 +57,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opGZHyGpDt6c8x2mKexrhc8btiMkuyF1EHeL3hQvaNtTxsyzUGu", "counter": 7057537, - "initiator": { - "address": "tz1ZfxjoBeGkRG7zLgyH6gnYVNAL9KQwRnCB" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -87,9 +84,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opGZHyGpDt6c8x2mKexrhc8btiMkuyF1EHeL3hQvaNtTxsyzUGu", "counter": 7057537, - "initiator": { - "address": "tz1ZfxjoBeGkRG7zLgyH6gnYVNAL9KQwRnCB" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -117,9 +111,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opGZHyGpDt6c8x2mKexrhc8btiMkuyF1EHeL3hQvaNtTxsyzUGu", "counter": 7057537, - "initiator": { - "address": "tz1ZfxjoBeGkRG7zLgyH6gnYVNAL9KQwRnCB" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -147,9 +138,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opGZHyGpDt6c8x2mKexrhc8btiMkuyF1EHeL3hQvaNtTxsyzUGu", "counter": 7057537, - "initiator": { - "address": "tz1ZfxjoBeGkRG7zLgyH6gnYVNAL9KQwRnCB" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -205,9 +193,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opGZHyGpDt6c8x2mKexrhc8btiMkuyF1EHeL3hQvaNtTxsyzUGu", "counter": 7057537, - "initiator": { - "address": "tz1ZfxjoBeGkRG7zLgyH6gnYVNAL9KQwRnCB" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -310,10 +295,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opDvkDvtCoKhefWZSz7nQ46tUYxyScqa5Ex5HbiLsbEn2BiKMis", "counter": 8895059, - "initiator": { - "alias": "Django Bits", - "address": "tz1YRG68NdqtAcsFEwTUw6FsSsiBb5kagEDo" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -342,10 +323,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opDvkDvtCoKhefWZSz7nQ46tUYxyScqa5Ex5HbiLsbEn2BiKMis", "counter": 8895059, - "initiator": { - "alias": "Django Bits", - "address": "tz1YRG68NdqtAcsFEwTUw6FsSsiBb5kagEDo" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -373,10 +350,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opDvkDvtCoKhefWZSz7nQ46tUYxyScqa5Ex5HbiLsbEn2BiKMis", "counter": 8895059, - "initiator": { - "alias": "Django Bits", - "address": "tz1YRG68NdqtAcsFEwTUw6FsSsiBb5kagEDo" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -405,10 +378,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opDvkDvtCoKhefWZSz7nQ46tUYxyScqa5Ex5HbiLsbEn2BiKMis", "counter": 8895059, - "initiator": { - "alias": "Django Bits", - "address": "tz1YRG68NdqtAcsFEwTUw6FsSsiBb5kagEDo" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -464,10 +433,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opDvkDvtCoKhefWZSz7nQ46tUYxyScqa5Ex5HbiLsbEn2BiKMis", "counter": 8895059, - "initiator": { - "alias": "Django Bits", - "address": "tz1YRG68NdqtAcsFEwTUw6FsSsiBb5kagEDo" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -566,9 +531,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "ooXs2CrjtpPNGf6GLdW5i3rw7Vd3cvUiC4DTfWKiU9KgckPYb4S", "counter": 11987839, - "initiator": { - "address": "tz1SGLEydAbC5ZqQY2NDPNFsFQ63w4XidaYk" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" @@ -670,9 +632,6 @@ "block": "BMZunRxN8A3tsNHhEFtN2Yb5ArX7Ls8KGbGTy1svoPbLNQRDA8H", "hash": "opNB2Hggf3617i9A4TLzZ8QkWd2P4JwzRVzGex9EkueCHJLG4Mk", "counter": 12123298, - "initiator": { - "address": "tz1QzteR6agXmyGwrGvvXAqZmoRr75ANLHJ4" - }, "sender": { "alias": "Hic et nunc Minter", "address": "KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9" diff --git a/tests/test_dipdup/test_datasources/test_tzkt/test_cache.py b/tests/test_dipdup/test_datasources/test_tzkt/test_cache.py index 668f39acd..915283285 100644 --- a/tests/test_dipdup/test_datasources/test_tzkt/test_cache.py +++ b/tests/test_dipdup/test_datasources/test_tzkt/test_cache.py @@ -3,7 +3,7 @@ from unittest.async_case import IsolatedAsyncioTestCase # type: ignore from unittest.mock import ANY, AsyncMock, MagicMock # type: ignore -from dipdup.config import ContractConfig, OperationHandlerConfig, OperationHandlerPatternConfig, OperationIndexConfig +from dipdup.config import ContractConfig, OperationHandlerConfig, OperationHandlerTransactionPatternConfig, OperationIndexConfig from dipdup.datasources.tzkt.cache import OperationCache, OperationGroup from dipdup.datasources.tzkt.datasource import TzktDatasource from dipdup.models import OperationData, State @@ -19,8 +19,10 @@ async def asyncSetUp(self): OperationHandlerConfig( callback='', pattern=[ - OperationHandlerPatternConfig( - destination=ContractConfig(address='KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW'), entrypoint='hDAO_batch' + OperationHandlerTransactionPatternConfig( + type='transaction', + destination=ContractConfig(address='KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW'), + entrypoint='hDAO_batch', ) ], ) @@ -62,8 +64,10 @@ async def test_process(self): OperationHandlerConfig( callback='', pattern=[ - OperationHandlerPatternConfig( - destination=ContractConfig(address='KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW'), entrypoint='hDAO_batch' + OperationHandlerTransactionPatternConfig( + type='transaction', + destination=ContractConfig(address='KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW'), + entrypoint='hDAO_batch', ) ], ), diff --git a/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py b/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py index 7105ceaad..32796489c 100644 --- a/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py +++ b/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py @@ -10,9 +10,15 @@ from demo_hic_et_nunc.types.hen_minter.parameter.collect import CollectParameter from demo_registrydao.types.registry.parameter.propose import ProposeParameter from demo_registrydao.types.registry.storage import Proposals, RegistryStorage -from dipdup.config import ContractConfig, OperationHandlerConfig, OperationHandlerPatternConfig, OperationIndexConfig +from dipdup.config import ( + ContractConfig, + OperationHandlerConfig, + OperationHandlerTransactionPatternConfig, + OperationIndexConfig, + OperationType, +) from dipdup.datasources.tzkt.datasource import TzktDatasource -from dipdup.models import IndexType, OperationContext, OperationData, OperationHandlerContext, State +from dipdup.models import IndexType, OperationData, OperationHandlerContext, State, TransactionContext from dipdup.utils import tortoise_wrapper @@ -26,8 +32,10 @@ async def asyncSetUp(self): OperationHandlerConfig( callback='', pattern=[ - OperationHandlerPatternConfig( - destination=ContractConfig(address='KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9'), entrypoint='collect' + OperationHandlerTransactionPatternConfig( + type='transaction', + destination=ContractConfig(address='KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9'), + entrypoint='collect', ) ], ) @@ -64,8 +72,8 @@ async def test_start(self): with patch('aiohttp.ClientSession.get', get_mock): await self.datasource.start() - fetch_operations_mock.assert_awaited_with(1337, initial=True) - self.assertEqual({self.index_config.contracts[0].address: ['transaction']}, self.datasource._operation_subscriptions) + fetch_operations_mock.assert_awaited_with(self.index_config, 1337) + self.assertEqual({self.index_config.contracts[0].address: [OperationType.transaction]}, self.datasource._operation_subscriptions) client.start.assert_awaited() async def test_on_connect_subscribe_to_operations(self): @@ -74,7 +82,7 @@ async def test_on_connect_subscribe_to_operations(self): client.send = send_mock client.transport.state = ConnectionState.connected self.datasource._operation_subscriptions = { - self.index_config.contracts[0].address: ['transaction'], + self.index_config.contracts[0].address: [OperationType.transaction], } await self.datasource.on_connect() @@ -87,10 +95,11 @@ async def test_on_connect_subscribe_to_operations(self): self.assertEqual(2, len(client.handlers)) async def test_on_fetch_operations(self): - self.datasource._operation_subscriptions = {self.index_config.contracts[0].address: ['transaction']} + self.datasource._operation_subscriptions = {self.index_config.contracts[0].address: [OperationType.transaction]} with open(join(dirname(__file__), 'operations.json')) as file: operations_message = json.load(file) del operations_message['state'] + stripped_operations_message = operations_message['data'] on_operation_message_mock = AsyncMock() @@ -100,7 +109,7 @@ async def test_on_fetch_operations(self): self.datasource.on_operation_message = on_operation_message_mock with patch('aiohttp.ClientSession.get', get_mock): - await self.datasource.fetch_operations(1337) + await self.datasource.fetch_operations(self.index_config, 9999999) on_operation_message_mock.assert_awaited_with( message=[operations_message], @@ -112,7 +121,7 @@ async def test_on_operation_message_state(self): self.datasource.fetch_operations = fetch_operations_mock await self.datasource.on_operation_message([{'type': 0, 'state': 123}], self.index_config) - fetch_operations_mock.assert_awaited_with(123) + fetch_operations_mock.assert_awaited_with(self.index_config, 123) async def test_on_operation_message_data(self): with open(join(dirname(__file__), 'operations.json')) as file: @@ -154,7 +163,7 @@ async def test_on_operation_match(self): await self.datasource.on_operation_match(self.index_config, self.index_config.handlers[0], [matched_operation], operations) self.assertIsInstance(callback_mock.await_args[0][0], OperationHandlerContext) - self.assertIsInstance(callback_mock.await_args[0][1], OperationContext) + self.assertIsInstance(callback_mock.await_args[0][1], TransactionContext) self.assertIsInstance(callback_mock.await_args[0][1].parameter, CollectParameter) self.assertIsInstance(callback_mock.await_args[0][1].data, OperationData) @@ -163,6 +172,8 @@ async def test_on_operation_match_with_storage(self): operations_message = json.load(file) self.index_config.handlers[0].pattern[0].parameter_type_cls = ProposeParameter + for op in operations_message['data']: + op['type'] = 'transaction' operations = [TzktDatasource.convert_operation(op) for op in operations_message['data']] matched_operation = operations[0] diff --git a/tests/test_dipdup/test_models.py b/tests/test_dipdup/test_models.py index 41526feed..99cc275ec 100644 --- a/tests/test_dipdup/test_models.py +++ b/tests/test_dipdup/test_models.py @@ -74,7 +74,7 @@ def test_merged_storage(self): operation_data = OperationData( storage=storage, diffs=diffs, - type=None, + type='transaction', id=0, level=0, timestamp=0,