From 62be7f54fc3a009209420b64a98625f6cf9e1fc7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 19 Feb 2019 06:36:54 -0600 Subject: [PATCH] Properly do #645 on top of #587 --- specs/core/1_shard-data-chains.md | 389 +++++++++++++++++------------- 1 file changed, 219 insertions(+), 170 deletions(-) diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index c3b781323f..5b46028c1e 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -71,6 +71,8 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md | `SHARD_CHUNK_SIZE` | 2**5 (= 32) | bytes | | `SHARD_BLOCK_SIZE` | 2**14 (= 16,384) | bytes | | `MINOR_REWARD_QUOTIENT` | 2**8 (= 256) | | +| `MAX_POC_RESPONSE_DEPTH` | 5 | | +| `VALIDATOR_NULL` | 2**64 - 1 | | #### Time parameters @@ -84,19 +86,39 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md #### Max operations per block -| Name | Value | -|-------------------------------|---------------| -| `MAX_BRANCH_CHALLENGES` | 2**2 (= 4) | -| `MAX_BRANCH_RESPONSES` | 2**4 (= 16) | -| `MAX_EARLY_SUBKEY_REVEALS` | 2**4 (= 16) | +| Name | Value | +|----------------------------------------------------|---------------| +| `MAX_BRANCH_CHALLENGES` | 2**2 (= 4) | +| `MAX_BRANCH_RESPONSES` | 2**4 (= 16) | +| `MAX_EARLY_SUBKEY_REVEALS` | 2**4 (= 16) | +| `MAX_INTERACTIVE_CUSTODY_CHALLENGE_INITIATIONS` | 2 | +| `MAX_INTERACTIVE_CUSTODY_CHALLENGE_RESPONSES` | 16 | +| `MAX_INTERACTIVE_CUSTODY_CHALLENGE_CONTINUTATIONS` | 16 | #### Signature domains -| Name | Value | -|------------------------|-----------------| -| `DOMAIN_SHARD_PROPOSER`| 129 | -| `DOMAIN_SHARD_ATTESTER`| 130 | -| `DOMAIN_CUSTODY_SUBKEY`| 131 | +| Name | Value | +|------------------------------|-----------------| +| `DOMAIN_SHARD_PROPOSER` | 129 | +| `DOMAIN_SHARD_ATTESTER` | 130 | +| `DOMAIN_CUSTODY_SUBKEY` | 131 | +| `DOMAIN_CUSTODY_INTERACTIVE` | 132 | + +`EMPTY_CHALLENGE_DATA` is defined as: + +```python +InteractiveCustodyChallengeData( + challenger=VALIDATOR_NULL, + data_root=ZERO_HASH, + custody_bit=False, + responder_subkey=EMPTY_SIGNATURE, + current_custody_tree_node=ZERO_HASH, + depth=0, + offset=0, + max_depth=0, + deadline=0 +) +``` ## Helper functions @@ -303,6 +325,8 @@ Add member values to the end of the `Validator` object: 'open_branch_challenges': [BranchChallengeRecord], 'next_subkey_to_reveal': 'uint64', 'reveal_max_periods_late': 'uint64', + 'interactive_custody_challenge_data': InteractiveCustodyChallengeData, + 'now_challenging': 'uint64', ``` And the initializers: @@ -311,6 +335,8 @@ And the initializers: 'open_branch_challenges': [], 'next_subkey_to_reveal': get_current_custody_period(state), 'reveal_max_periods_late': 0, + 'interactive_custody_challenge_data': EMPTY_CHALLENGE_DATA + 'now_challenging': VALIDATOR_NULL ``` ### `BeaconBlockBody` @@ -321,6 +347,10 @@ Add member values to the `BeaconBlockBody` structure: 'branch_challenges': [BranchChallenge], 'branch_responses': [BranchResponse], 'subkey_reveals': [SubkeyReveal], + 'interactive_custody_challenge_initiations': [InteractiveCustodyChallengeInitiation], + 'interactive_custody_challenge_responses': [InteractiveCustodyChallengeResponse], + 'interactive_custody_challenge_continuations': [InteractiveCustodyChallengeContinuation], + ``` And initialize to the following: @@ -385,6 +415,65 @@ Define a `SubkeyReveal` as follows: } ``` +### `InteractiveCustodyChallengeData` + +```python +{ + # Who initiated the challenge + 'challenger': 'uint64', + # Initial data root + 'data_root': 'bytes32', + # Initial custody bit + 'custody_bit': 'bool', + # Responder subkey + 'responder_subkey': 'bytes96', + # The hash in the PoC tree in the position that we are currently at + 'current_custody_tree_node': 'bytes32', + # The position in the tree, in terms of depth and position offset + 'depth': 'uint64', + 'offset': 'uint64', + # Max depth of the branch + 'max_depth': 'uint64', + # Deadline to respond (as an epoch) + 'deadline': 'uint64', +} +``` + +### `InteractiveCustodyChallengeInitiation` + +```python +{ + 'attestation': SlashableAttestation, + 'responder_index': 'uint64', + 'challenger_index': 'uint64', + 'responder_subkey': 'bytes96', + 'signature': 'bytes96' +} +``` + +### `InteractiveCustodyChallengeResponse` + +```python +{ + 'responder_index': 'uint64', + 'hashes': ['bytes32'], + 'signature': 'bytes96', +} +``` + +### `InteractiveCustodyChallengeContinuation` + +```python +{ + 'challenger_index: 'uint64', + 'responder_index': 'uint64', + 'sub_index': 'uint64', + 'new_custody_tree_node': 'bytes32', + 'proof': ['bytes32'], + 'signature': 'bytes96' +} +``` + ## Helpers ### `get_attestation_merkle_depth` @@ -510,11 +599,64 @@ Verify that `len(block.body.branch_responses) <= MAX_BRANCH_RESPONSES`. For each `response` in `block.body.branch_responses`: -* Find the `BranchChallengeRecord` in `state.validator_registry[response.responder_index].open_branch_challenges` whose (`root`, `data_index`) match the (`root`, `data_index`) of the `response`. Verify that one such record exists (it is not possible for there to be more than one), call it `record`. -* Verify that `verify_merkle_branch(leaf=response.data, branch=response.branch, depth=record.depth, index=record.data_index, root=record.root)` is True. -* Verify that `get_current_epoch(state) >= record.inclusion_epoch + ENTRY_EXIT_DELAY`. -* Remove the `record` from `state.validator_registry[response.responder_index].open_branch_challenges` -* Determine the proposer `proposer_index = get_beacon_proposer_index(state, state.slot)` and set `state.validator_balances[proposer_index] += base_reward(state, index) // MINOR_REWARD_QUOTIENT`. +* Find the `BranchChallengeRecord` in `state.validator_registry[response.responder_index].open_branch_challenges` whose (`root`, `data_index`) match the (`root`, `data_index`) of the `response`. Verify that one of the following two cases is true: + * Such a record exists (it is not possible for there to be more than one). In this case, run `process_branch_exploration_response(response, state)` + * Such a record does not exist, but `state.validator_registry[response.responder_index].interactive_custody_challenge_data.challenger != VALIDATOR_NULL`. In this case, run `process_branch_custody_response(response, state)`. + +Here is `process_branch_exploration_response`: + +```python +def process_branch_exploration_response(response: BranchResponse, + state: BeaconState): + record = [ + x for x in state.validator_registry[response.responder_index].open_branch_challenges + if x.root == response.root and x.data_index == response.data_index + ][0] + assert verify_merkle_branch( + leaf=response.data, + branch=response.branch, + depth=record.depth, + index=record.data_index, + root=record.root + ) + # Must wait at least ENTRY_EXIT_DELAY before responding to a branch challenge + assert get_current_epoch(state) >= record.inclusion_epoch + ENTRY_EXIT_DELAY + state.validator_registry[response.responder_index].open_branch_challenges = [ + x for x in state.validator_registry[response.responder_index].open_branch_challenges + if x.root != response.root or x.data_index != response.data_index + ] + # Reward the proposer + proposer_index = get_beacon_proposer_index(state, state.slot) + state.validator_balances[proposer_index] += base_reward(state, index) // MINOR_REWARD_QUOTIENT +``` + +Here is `process_branch_custody_response`: + +```python +def process_branch_custody_response(response: BranchResponse, + state: BeaconState): + responder = state.validator_registry[response.responder_index] + challenge_data = responder.interactive_custody_challenge_data + assert challenge_data.depth == challenge_data.max_depth + # Verify we're not too late + assert get_current_epoch(state) < responder.withdrawable_epoch + # Verify the Merkle branch *of the data tree* + assert verify_merkle_branch( + leaf=response.data, + branch=response.branch, + depth=challenge_data.max_depth, + index=challenge_data.offset, + root=challenge_data.data_root + ) + # Responder wins + if hash(challenge_data.responder_subkey + response.data) == challenge_data.current_custody_tree_node: + penalize_validator(state, challenge_data.challenger_index, response.responder_index) + responder.interactive_custody_challenge_data = EMPTY_CHALLENGE_DATA + # Challenger wins + else: + penalize_validator(state, response.responder_index, challenge_data.challenger_index) + state.validator_registry[challenge_data.challenger_index].now_challenging = VALIDATOR_NULL +``` #### Subkey reveals @@ -540,118 +682,11 @@ In case (ii): * Set `state.validator_registry[reveal.validator_index].next_subkey_to_reveal += 1` * Set `state.validator_registry[reveal.validator_index].reveal_max_periods_late = max(state.validator_registry[reveal.validator_index].reveal_max_periods_late, get_current_period(state) - reveal.period)`. -## Per-epoch processing +#### Interactive custody challenge initiations -Add the following loop immediately below the `process_ejections` loop: +Verify that `len(block.body.interactive_custody_challenge_initiations) <= MAX_INTERACTIVE_CUSTODY_CHALLENGE_INITIATIONS`. -```python -def process_challenge_absences(state: BeaconState) -> None: - """ - Iterate through the validator registry - and penalize validators with balance that did not answer challenges. - """ - for index, validator in enumerate(state.validator_registry): - if len(validator.open_branch_challenges) > 0 and get_current_epoch(state) > validator.open_branch_challenges[0].inclusion_epoch + CHALLENGE_RESPONSE_DEADLINE: - penalize_validator(state, index, validator.open_branch_challenges[0].challenger_index) - if validator.challenge_data.challenger != VALIDATOR_NULL and get_current_epoch(state) > validator.challenge.deadline: - penalize_validator(state, index, validator.challenge_data.challenger_index) - if get_current_epoch(state) >= state.validator_registry[validator.now_challenging].withdrawal_epoch: - penalize_validator(state, index, validator.now_challenging) - penalize_validator(state, index, validator.challenge_data.challenger_index) -``` - -In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): - -```python -def eligible(index): - validator = state.validator_registry[index] - # Cannot exit if there are still open branch challenges - if len(validator.open_branch_challenges) > 0: - return False - # Cannot exit if you have not revealed all of your subkeys - elif validator.next_subkey_to_reveal <= epoch_to_custody_period(validator.exit_epoch): - return False - # Cannot exit if you already have - elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: - return False - # Return minimum time - else: - return current_epoch >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWAL_EPOCHS -``` - -## One-time phase 1 initiation transition - -Run the following on the fork block after per-slot processing and before per-block and per-epoch processing. - -For all `validator` in `ValidatorRegistry`, update it to the new format and fill the new member values with: - -```python - 'open_branch_challenges': [], - 'next_subkey_to_reveal': get_current_custody_period(state), - 'reveal_max_periods_late': 0, -``` - -# Proof of custody interactive game - -### Constants - -| Constant | Value | Unit | Approximation | -|--------------------------------------------|------------------|---------|---------------| -| `MAX_POC_RESPONSE_DEPTH` | 5 | layers | | -| `DOMAIN_CUSTODY_INTERACTIVE` | 132 | | | -| `VALIDATOR_NULL` | 2**64 - 1 | | | -| `MAX_INTERACTIVE_CHALLENGE_INITIATIONS` | 2 | | | -| `MAX_INTERACTIVE_CHALLENGE_RESPONSES` | 16 | | | -| `MAX_INTERACTIVE_CHALLENGE_CONTINUTATIONS` | 16 | | | - -### Data structures and verification - -Add the following data structure to the `Validator` record: - -```python - interactive_custody_challenge_data: InteractiveCustodyChallengeData, - now_challenging: 'uint64', -``` - -Where `InteractiveCustodyChallengeData` is defined as follows: - -```python -{ - # Who initiated the challenge - 'challenger': 'uint64', - # Initial data root - 'data_root': 'bytes32', - # Initial custody bit - 'custody_bit': 'bool', - # Responder subkey - 'responder_subkey': 'bytes96', - # The hash in the PoC tree in the position that we are currently at - 'current_custody_tree_node': 'bytes32', - # The position in the tree, in terms of depth and position offset - 'depth': 'uint64', - 'offset': 'uint64', - # Max depth of the branch - 'max_depth': 'uint64', - # Deadline to respond (as an epoch) - 'deadline': 'uint64', -} -``` - -The initial value is `EMPTY_CHALLENGE_DATA = InteractiveCustodyChallengeData(challenger=VALIDATOR_NULL, data_root=ZERO_HASH, custody_bit=False, responder_subkey=EMPTY_SIGNATURE, current_custody_tree_node=ZERO_HASH, depth=0, offset=0, max_depth=0, deadline=0)` - -We define an `InteractiveCustodyChallengeInitiation` as follows: - -```python -{ - 'attestation': SlashableAttestation, - 'responder_index': 'uint64', - 'challenger_index': 'uint64', - 'responder_subkey': 'bytes96', - 'signature': 'bytes96' -} -``` - -Here's the function for validating and processing an initiation: +For each `initiation` in `block.body.interactive_custody_challenge_initiations`, use the following function to process it: ```python def process_initiation(initiation: InteractiveCustodyChallengeInitiation, @@ -699,19 +734,13 @@ def process_initiation(initiation: InteractiveCustodyChallengeInitiation, challenger.now_challenging = responder_index ``` -We define an `InteractiveCustodyChallengeResponse` as follows: - -```python -{ - 'responder_index': 'uint64', - 'hashes': ['bytes32'], - 'signature': 'bytes96', -} -``` +#### Interactive custody challenge responses A response provides 32 hashes that are under current known proof of custody tree node. Note that at the beginning the tree node is just one bit of the custody root, so we ask the responder to sign to commit to the top 5 levels of the tree and therefore the root hash; at all other stages in the game responses are self-verifying. -Here's the function for verifying and processing a response: +Verify that `len(block.body.interactive_custody_challenge_responses) <= MAX_INTERACTIVE_CUSTODY_CHALLENGE_RESPONSES`. + +For each `response` in `block.body.interactive_custody_challenge_responses`, use the following function to process it: ```python def process_response(response: InteractiveCustodyChallengeResponse, @@ -743,20 +772,15 @@ def process_response(response: InteractiveCustodyChallengeResponse, responder.withdrawable_epoch = get_current_epoch(state) + MAX_POC_RESPONSE_DEPTH ``` -Once a response provides 32 hashes, the challenger has the right to choose any one of them that they feel is constructed incorrectly to continue the game. Note that eventually, the game will get to the point where the `new_custody_tree_node` is a leaf node. We define an `InteractiveCustodyChallengeContinuation` object as follows: +Note that once the `new_custody_tree_node` reaches the leaves of the tree, the responder can no longer provide a valid `InteractiveCustodyChallengeResponse`; instead, the responder or the challenger must provide a `BranchResponse` that provides a branch of the original data tree, at which point the custody leaf equation can be checked and either side of the custody game can "conclusively win". -```python -{ - 'challenger_index: 'uint64', - 'responder_index': 'uint64', - 'sub_index': 'uint64', - 'new_custody_tree_node': 'bytes32', - 'proof': ['bytes32'], - 'signature': 'bytes96' -} -``` +#### Interactive custody challenge continuations -Here's the function for verifying and processing a continuation challenge: +Once a response provides 32 hashes, the challenger has the right to choose any one of them that they feel is constructed incorrectly to continue the game. Note that eventually, the game will get to the point where the `new_custody_tree_node` is a leaf node. + +Verify that `len(block.body.interactive_custody_challenge_continuations) <= MAX_INTERACTIVE_CUSTODY_CHALLENGE_CONTINUATIONS`. + +For each `continuation` in `block.body.interactive_custody_challenge_continuations`, use the following function to process it: ```python def process_continuation(continuation: InteractiveCustodyChallengeContinuation, @@ -788,30 +812,55 @@ def process_continuation(continuation: InteractiveCustodyChallengeContinuation, challenge_data.offset = challenger_data.offset * 2**expected_depth + sub_index ``` -Once the `new_custody_tree_node` reaches the leaves of the tree, the responder can no longer provide a valid `InteractiveCustodyChallengeResponse`; instead, the responder or the challenger must provide a branch response that provides a branch of the original data tree, at which point the custody leaf equation can be checked and either side of the custody game can "conclusively win". +## Per-epoch processing + +Add the following loop immediately below the `process_ejections` loop: ```python -def process_branch_response(response: BranchResponse, - state: State): - responder = state.validator_registry[response.responder_index] - challenge_data = responder.interactive_custody_challenge_data - assert challenge_data.depth == challenge_data.max_depth - # Verify we're not too late - assert get_current_epoch(state) < responder.withdrawable_epoch - # Verify the Merkle branch *of the data tree* - assert verify_merkle_branch( - leaf=response.data, - branch=response.branch, - depth=challenge_data.max_depth, - index=challenge_data.offset, - root=challenge_data.data_root - ) - # Responder wins - if hash(challenge_data.responder_subkey + response.data) == challenge_data.current_custody_tree_node: - penalize_validator(state, challenge_data.challenger_index, response.responder_index) - responder.interactive_custody_challenge_data = EMPTY_CHALLENGE_DATA - # Challenger wins +def process_challenge_absences(state: BeaconState) -> None: + """ + Iterate through the validator registry + and penalize validators with balance that did not answer challenges. + """ + for index, validator in enumerate(state.validator_registry): + if len(validator.open_branch_challenges) > 0 and get_current_epoch(state) > validator.open_branch_challenges[0].inclusion_epoch + CHALLENGE_RESPONSE_DEADLINE: + penalize_validator(state, index, validator.open_branch_challenges[0].challenger_index) + if validator.challenge_data.challenger != VALIDATOR_NULL and get_current_epoch(state) > validator.challenge.deadline: + penalize_validator(state, index, validator.challenge_data.challenger_index) + if get_current_epoch(state) >= state.validator_registry[validator.now_challenging].withdrawal_epoch: + penalize_validator(state, index, validator.now_challenging) + penalize_validator(state, index, validator.challenge_data.challenger_index) +``` + +In `process_penalties_and_exits`, change the definition of `eligible` to the following (note that it is not a pure function because `state` is declared in the surrounding scope): + +```python +def eligible(index): + validator = state.validator_registry[index] + # Cannot exit if there are still open branch challenges + if len(validator.open_branch_challenges) > 0: + return False + # Cannot exit if you have not revealed all of your subkeys + elif validator.next_subkey_to_reveal <= epoch_to_custody_period(validator.exit_epoch): + return False + # Cannot exit if you already have + elif validator.withdrawable_epoch < FAR_FUTURE_EPOCH: + return False + # Return minimum time else: - penalize_validator(state, response.responder_index, challenge_data.challenger_index) - state.validator_registry[challenge_data.challenger_index].now_challenging = VALIDATOR_NULL + return current_epoch >= validator.exit_epoch + MIN_VALIDATOR_WITHDRAWAL_EPOCHS +``` + +## One-time phase 1 initiation transition + +Run the following on the fork block after per-slot processing and before per-block and per-epoch processing. + +For all `validator` in `ValidatorRegistry`, update it to the new format and fill the new member values with: + +```python + 'open_branch_challenges': [], + 'next_subkey_to_reveal': get_current_custody_period(state), + 'reveal_max_periods_late': 0, + 'interactive_custody_challenge_data': EMPTY_CHALLENGE_DATA, + 'new_challenging': VALIDATOR_NULL, ```