Merge pull request #571 from lidofinance/feat/account-cl-pending-depo…

…sits-validators

feat: account cl pending deposits validators

src/constants.py

@@ -1,6 +1,6 @@
-# https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc
 from src.types import Gwei
 
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc
 FAR_FUTURE_EPOCH = 2 ** 64 - 1
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters-1
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2 ** 8
@@ -12,7 +12,7 @@
 PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#gwei-values
 EFFECTIVE_BALANCE_INCREMENT = 2 ** 0 * 10 ** 9
-MAX_EFFECTIVE_BALANCE = 32 * 10 ** 9
+MAX_EFFECTIVE_BALANCE = Gwei(32 * 10 ** 9)
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#execution
 MAX_WITHDRAWALS_PER_PAYLOAD = 2 ** 4
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#withdrawal-prefixes
@@ -29,6 +29,8 @@
 MIN_ACTIVATION_BALANCE = Gwei(2 ** 5 * 10 ** 9)
 MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2 ** 11 * 10 ** 9)
 
+LIDO_DEPOSIT_AMOUNT = MIN_ACTIVATION_BALANCE
+
 # Local constants
 GWEI_TO_WEI = 10 ** 9
 SHARE_RATE_PRECISION_E27 = 10 ** 27

src/modules/accounting/accounting.py

@@ -216,11 +216,13 @@ def get_updated_modules_stats(
     def _get_consensus_lido_state(self, blockstamp: ReferenceBlockStamp) -> tuple[ValidatorsCount, ValidatorsBalance]:
         lido_validators = self.w3.lido_validators.get_lido_validators(blockstamp)
 
-        count = len(lido_validators)
-        total_balance = Gwei(sum(int(validator.balance) for validator in lido_validators))
+        validators_count = len(lido_validators)
+        active_balance = sum(int(validator.balance) for validator in lido_validators)
+        pending_deposits = self.w3.lido_validators.calculate_pending_deposits_sum(lido_validators)
+        total_balance = Gwei(active_balance + pending_deposits)
 
-        logger.info({'msg': 'Calculate lido state on CL. (Validators count, Total balance in gwei)', 'value': (count, total_balance)})
-        return ValidatorsCount(count), ValidatorsBalance(total_balance)
+        logger.info({'msg': f'Calculate Lido state on CL. {validators_count=}, {active_balance=}, {pending_deposits=}, {total_balance=} (Gwei)'})
+        return ValidatorsCount(validators_count), ValidatorsBalance(total_balance)
 
     def _get_finalization_data(self, blockstamp: ReferenceBlockStamp) -> tuple[FinalizationShareRate, FinalizationBatches]:
         simulation = self.simulate_full_rebase(blockstamp)

src/services/bunker_cases/abnormal_cl_rebase.py

@@ -216,10 +216,11 @@ def _get_lido_validators_balance_with_vault(
         Get Lido validator balance with withdrawals vault balance
         """
         real_cl_balance = AbnormalClRebase.calculate_validators_balance_sum(lido_validators)
+        pending_deposits_sum = LidoValidatorsProvider.calculate_pending_deposits_sum(lido_validators)
         withdrawals_vault_balance = int(
             self.w3.from_wei(self.w3.lido_contracts.get_withdrawal_balance_no_cache(blockstamp), "gwei")
         )
-        return Gwei(real_cl_balance + withdrawals_vault_balance)
+        return Gwei(real_cl_balance + pending_deposits_sum + withdrawals_vault_balance)
 
     def _get_withdrawn_from_vault_between_blocks(
         self, prev_blockstamp: BlockStamp, ref_blockstamp: ReferenceBlockStamp

src/web3py/extensions/lido_validators.py

@@ -6,6 +6,7 @@
 from eth_typing import ChecksumAddress
 from web3.module import Module
 
+from src.constants import FAR_FUTURE_EPOCH, LIDO_DEPOSIT_AMOUNT
 from src.providers.consensus.types import Validator
 from src.providers.keys.types import LidoKey
 from src.types import BlockStamp, StakingModuleId, NodeOperatorId, NodeOperatorGlobalIndex, StakingModuleAddress
@@ -172,6 +173,16 @@ def merge_validators_with_keys(keys: list[LidoKey], validators: list[Validator])
 
         return lido_validators
 
+    @staticmethod
+    def calculate_pending_deposits_sum(lido_validators: list[LidoValidator]) -> int:
+        # NOTE: Using 32 ETH as a default validator pending balance is OK for the current protocol implementation.
+        #       It must be changed in case of validators consolidation feature implementation.
+        return sum(
+            LIDO_DEPOSIT_AMOUNT
+            for validator in lido_validators
+            if int(validator.balance) == 0 and int(validator.validator.activation_epoch) == FAR_FUTURE_EPOCH
+        )
+
     @lru_cache(maxsize=1)
     def get_lido_validators_by_node_operators(self, blockstamp: BlockStamp) -> ValidatorsByNodeOperator:
         merged_validators = self.get_lido_validators(blockstamp)

tests/factory/no_registry.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 from faker import Faker
+from hexbytes import HexBytes
 from pydantic_factories import Use
 
 from src.constants import FAR_FUTURE_EPOCH
@@ -19,10 +20,29 @@ class ValidatorStateFactory(Web3Factory):
 
     exit_epoch = FAR_FUTURE_EPOCH
 
+    @classmethod
+    def build(cls, **kwargs: Any):
+        if 'pubkey' not in kwargs:
+            kwargs['pubkey'] = HexBytes(faker.binary(length=48)).hex()
+        return super().build(**kwargs)
+
 
 class ValidatorFactory(Web3Factory):
     __model__ = Validator
 
+    @classmethod
+    def build_pending_deposit_vals(cls, **kwargs: Any):
+        return cls.build(
+            balance=str(0),
+            validator=ValidatorStateFactory.build(
+                activation_eligibility_epoch=str(FAR_FUTURE_EPOCH),
+                activation_epoch=str(FAR_FUTURE_EPOCH),
+                exit_epoch=str(FAR_FUTURE_EPOCH),
+                effective_balance=str(0),
+            ),
+            **kwargs
+        )
+
 
 class LidoKeyFactory(Web3Factory):
     __model__ = LidoKey
@@ -53,6 +73,19 @@ def build_with_activation_epoch_bound(cls, max_value: int, **kwargs: Any):
             validator=ValidatorStateFactory.build(activation_epoch=str(faker.pyint(max_value=max_value - 1))), **kwargs
         )
 
+    @classmethod
+    def build_pending_deposit_vals(cls, **kwargs: Any):
+        return cls.build(
+            balance=str(0),
+            validator=ValidatorStateFactory.build(
+                activation_eligibility_epoch=str(FAR_FUTURE_EPOCH),
+                activation_epoch=str(FAR_FUTURE_EPOCH),
+                exit_epoch=str(FAR_FUTURE_EPOCH),
+                effective_balance=str(0),
+            ),
+            **kwargs
+        )
+
     @classmethod
     def build_not_active_vals(cls, epoch, **kwargs: Any):
         return cls.build(

tests/modules/accounting/bunker/conftest.py

@@ -9,6 +9,7 @@
 from src.services.bunker_cases.abnormal_cl_rebase import AbnormalClRebase
 from src.services.bunker_cases.types import BunkerConfig
 from src.types import BlockNumber, BlockStamp, ReferenceBlockStamp
+from tests.modules.ejector.test_exit_order_state_service import FAR_FUTURE_EPOCH
 
 
 def simple_ref_blockstamp(block_number: int) -> ReferenceBlockStamp:
@@ -25,7 +26,9 @@ def simple_key(pubkey: str) -> LidoKey:
     return key
 
 
-def simple_validator(index, pubkey, balance, slashed=False, withdrawable_epoch='', exit_epoch='100500') -> Validator:
+def simple_validator(
+    index, pubkey, balance, slashed=False, withdrawable_epoch='', exit_epoch='100500', activation_epoch="0"
+) -> Validator:
     return Validator(
         index=str(index),
         balance=str(balance),
@@ -36,7 +39,7 @@ def simple_validator(index, pubkey, balance, slashed=False, withdrawable_epoch='
             effective_balance=str(32 * 10**9),
             slashed=slashed,
             activation_eligibility_epoch='',
-            activation_epoch='0',
+            activation_epoch=activation_epoch,
             exit_epoch=exit_epoch,
             withdrawable_epoch=withdrawable_epoch,
         ),
@@ -134,6 +137,7 @@ def _get_withdrawal_vault_balance(blockstamp: BlockStamp):
             31: 2 * 10**18,
             33: 2 * 10**18,
             40: 2 * 10**18,
+            50: 2 * 10**18,
         }
         return balance[blockstamp.block_number]
 
@@ -199,6 +203,14 @@ def _get_validators(state: ReferenceBlockStamp, _=None):
                 simple_validator(4, '0x04', 32 * 10**9),
                 simple_validator(5, '0x05', (32 * 10**9) + 824112),
             ],
+            50: [
+                simple_validator(4, '0x00', balance=0, activation_epoch=FAR_FUTURE_EPOCH),
+                simple_validator(1, '0x01', 32 * 10**9),
+                simple_validator(2, '0x02', 32 * 10**9),
+                simple_validator(3, '0x03', (32 * 10**9) + 333333),
+                simple_validator(4, '0x04', balance=0, activation_epoch=FAR_FUTURE_EPOCH),
+                simple_validator(5, '0x05', (32 * 10**9) + 824112),
+            ],
             1000: [
                 simple_validator(0, '0x00', 32 * 10**9),
                 simple_validator(1, '0x01', 32 * 10**9),

tests/modules/accounting/bunker/test_bunker_abnormal_cl_rebase.py

@@ -90,6 +90,7 @@ def test_is_abnormal_cl_rebase(
 @pytest.mark.parametrize(
     ("blockstamp", "expected_rebase"),
     [
+        (simple_ref_blockstamp(50), 512000000),
         (simple_ref_blockstamp(40), 420650924),
         (simple_ref_blockstamp(20), 140216974),
         (simple_ref_blockstamp(123), 1120376622),
@@ -234,6 +235,7 @@ def test_calculate_cl_rebase_between_blocks(
 @pytest.mark.parametrize(
     ("blockstamp", "expected_result"),
     [
+        (simple_ref_blockstamp(50), 98001157445),
         (simple_ref_blockstamp(40), 98001157445),
         (simple_ref_blockstamp(20), 77999899300),
     ],

tests/modules/accounting/test_accounting_module.py

@@ -6,6 +6,7 @@
 from web3.types import Wei
 
 from src import variables
+from src.constants import LIDO_DEPOSIT_AMOUNT
 from src.modules.accounting import accounting as accounting_module
 from src.modules.accounting.accounting import Accounting
 from src.modules.accounting.accounting import logger as accounting_logger
@@ -21,7 +22,6 @@
 from tests.factory.configs import ChainConfigFactory, FrameConfigFactory
 from tests.factory.contract_responses import LidoReportRebaseFactory
 from tests.factory.no_registry import LidoValidatorFactory, StakingModuleFactory
-from tests.web3_extentions.test_lido_validators import blockstamp
 
 
 @pytest.fixture(autouse=True)
@@ -101,13 +101,18 @@ def test_get_updated_modules_stats(accounting: Accounting):
 @pytest.mark.usefixtures("lido_validators")
 def test_get_consensus_lido_state(accounting: Accounting):
     bs = ReferenceBlockStampFactory.build()
-    validators = LidoValidatorFactory.batch(10)
+    validators = [
+        *[LidoValidatorFactory.build_pending_deposit_vals() for _ in range(3)],
+        *[LidoValidatorFactory.build_not_active_vals(bs.ref_epoch) for _ in range(3)],
+        *[LidoValidatorFactory.build_active_vals(bs.ref_epoch) for _ in range(2)],
+        *[LidoValidatorFactory.build_exit_vals(bs.ref_epoch) for _ in range(2)],
+    ]
     accounting.w3.lido_validators.get_lido_validators = Mock(return_value=validators)
 
     count, balance = accounting._get_consensus_lido_state(bs)
 
     assert count == 10
-    assert balance == sum((int(val.balance) for val in validators))
+    assert balance == sum((int(val.balance) for val in validators)) + 3 * LIDO_DEPOSIT_AMOUNT
 
 
 @pytest.mark.unit

tests/web3_extentions/test_lido_validators.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from src.constants import LIDO_DEPOSIT_AMOUNT
 from src.modules.accounting.types import BeaconStat
 from src.web3py.extensions.lido_validators import CountOfKeysDiffersException
 from tests.factory.blockstamp import ReferenceBlockStampFactory
@@ -38,6 +39,16 @@ def test_get_lido_validators(web3, lido_validators, contracts):
         assert v.lido_id.key == v.validator.pubkey
 
 
+@pytest.mark.unit
+def test_calc_pending_deposits_sum(web3, lido_validators, contracts):
+    lido_validators = LidoValidatorFactory.batch(30)
+    lido_validators.extend(LidoValidatorFactory.build_pending_deposit_vals() for _ in range(5))
+
+    pending_deposits_sum = web3.lido_validators.calculate_pending_deposits_sum(lido_validators)
+
+    assert pending_deposits_sum == 5 * LIDO_DEPOSIT_AMOUNT
+
+
 @pytest.mark.unit
 def test_kapi_has_lesser_keys_than_deposited_validators_count(web3, lido_validators, contracts):
     validators = ValidatorFactory.batch(10)