spec updates

beacon_chain/datatypes.nim

@@ -21,6 +21,36 @@ import milagro_crypto
   #  - Signature                   (48 bytes - 384-bit)
   #  - VerKey (public key)         (192 bytes)
 
+const
+  SHARD_COUNT*                              = 1024 # a constant referring to the number of shards
+  DEPOSIT_SIZE*                             = 2^5  # You need to deposit 32 ETH to be a validator in Casper
+  MIN_ONLINE_DEPOSIT_SIZE*                  = 2^4  # ETH
+  GWEI_PER_ETH*                             = 10^9 # Gwei/ETH
+  TARGET_COMMITTEE_SIZE*                    = 2^8  # validators
+  SLOT_DURATION*                            = 6    # seconds
+  CYCLE_LENGTH*                             = 64   # slots (~ 6 minutes)
+  MIN_VALIDATOR_SET_CHANGE_INTERVAL*        = 2^8  # slots (~25 minutes)
+  SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^17 # slots (~9 days)
+  MIN_ATTESTATION_INCLUSION_DELAY*          = 2^2  # slots (~25 minutes)
+  SQRT_E_DROP_TIME*                         = 2^16 # slots (~12 days); amount of time it takes for the
+                                                   # quadratic leak to cut deposits of non-participating
+                                                   # validators by ~39.4%
+  WITHDRAWALS_PER_CYCLE*                    = 2^2  # validators (5.2m ETH in ~6 months)
+  MIN_WITHDRAWAL_PERIOD*                    = 2^13 # slots (~14 hours)
+  DELETION_PERIOD*                          = 2^22 # slots (~290 days)
+  COLLECTIVE_PENALTY_CALCULATION_PERIOD*    = 2^20 # slots (~2.4 months)
+  SLASHING_WHISTLEBLOWER_REWARD_DENOMINATOR* = 2^9 # ?
+  BASE_REWARD_QUOTIENT*                     = 2^15 # per-slot interest rate assuming all validators are
+                                                   # participating, assuming total deposits of 1 ETH. It
+                                                   # corresponds to ~3.88% annual interest assuming 10
+                                                   # million participating ETH.
+  MAX_VALIDATOR_CHURN_QUOTIENT*             = 2^5  # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the
+                                                   # validators can change during each validator set
+                                                   # change.
+  POW_HASH_VOTING_PERIOD*                   = 2^10 # ?
+  POW_CONTRACT_MERKLE_TREE_DEPTH*           = 2^5  #
+  INITIAL_FORK_VERSION*                     = 0    # currently behaves like a constant
+
 type
   # Alias
   BLSPublicKey* = VerKey
@@ -98,8 +128,10 @@ type
     last_finalized_slot*: uint64                           # Last finalized slot
     last_justified_slot*: uint64                           # Last justified slot
     justified_streak*: uint64                              # Number of consecutive justified slots
-    shard_and_committee_for_slots*: seq[ShardAndCommittee] # Committee members and their assigned shard, per slot
-    persistent_committees*: Uint24                         # Persistent shard committees
+    shard_and_committee_for_slots*: array[2 * CYCLE_LENGTH, seq[ShardAndCommittee]] ## \
+    ## Committee members and their assigned shard, per slot, covers 2 cycles
+    ## worth of assignments
+    persistent_committees*: seq[seq[ValidatorRecord]]      # Persistent shard committees
     persistent_committee_reassignments*: seq[ShardReassignmentRecord]
     next_shuffling_seed*: Blake2_256_Digest                # Randao seed used for next shuffling
     deposits_penalized_in_period*: uint32                  # Total deposits penalized in the given withdrawal period
@@ -127,6 +159,13 @@ type
     exit_slot*: uint64                            # Slot when validator exited (or 0)
     exit_seq*: uint64                             # Sequence number when validator exited (or 0)
 
+  InitialValidator* = object
+    pubkey*: BLSPublicKey
+    proof_of_possession*: seq[byte]
+    withdrawal_shard*: uint16
+    withdrawal_address*: EthAddress
+    randao_commitment*: Blake2_256_Digest
+
   ValidatorStatusCodes* {.pure.} = enum
     PendingActivation = 0
     Active = 1
@@ -157,31 +196,3 @@ type
     #   with room to spare.
     #
     #   Also, IntSets uses machine int size while we require int64 even on 32-bit platform.
-
-
-const
-  SHARD_COUNT*                              = 1024 # a constant referring to the number of shards
-  DEPOSIT_SIZE*                             = 2^5  # You need to deposit 32 ETH to be a validator in Casper
-  SLOT_DURATION*                            = 16   # seconds
-  CYCLE_LENGTH*                             = 64   # slots
-  MIN_COMMITTEE_SIZE*                       = 2^7  # validators; 2018-11-05 version of spec also says:
-                                                   # See a recommended `MIN_COMMITTEE_SIZE`  of 111 here
-                                                   # https://vitalik.ca/files/Ithaca201807_Sharding.pdf).
-  SQRT_E_DROP_TIME*                         = 2^16 # slots (~12 days); amount of time it takes for the
-                                                   # quadratic leak to cut deposits of non-participating
-                                                   # validators by ~39.4%
-  BASE_REWARD_QUOTIENT*                     = 2^15 # per-slot interest rate assuming all validators are
-                                                   # participating, assuming total deposits of 1 ETH. It
-                                                   # corresponds to ~3.88% annual interest assuming 10
-                                                   # million participating ETH.
-  MIN_BALANCE*                              = 2^4  # ETH
-  MIN_ONLINE_DEPOSIT_SIZE*                  = 2^4  # ETH
-  GWEI_PER_ETH*                             = 10^9 # Gwei/ETH
-  MIN_VALIDATOR_SET_CHANGE_INTERVAL*        = 2^8  # slots (~1.1 hours)
-  RANDAO_SLOTS_PER_LAYER*                   = 2^12 # slots (~18 hours)
-  WITHDRAWAL_PERIOD*                        = 2^19 # slots (~97 days)
-  SHARD_PERSISTENT_COMMITTEE_CHANGE_PERIOD* = 2^16 # slots (~12 days)
-  MAX_VALIDATOR_CHURN_QUOTIENT*             = 2^5  # At most `1/MAX_VALIDATOR_CHURN_QUOTIENT` of the
-                                                   # validators can change during each validator set
-                                                   # change.
-  INITIAL_FORK_VERSION*                     = 0    # currently behaves like a constant

beacon_chain/private/helpers.nim

@@ -8,102 +8,75 @@
 # Helper functions
 import ../datatypes, sequtils, nimcrypto, math
 
-func get_active_validator_indices(validators: seq[ValidatorRecord]): seq[Uint24] =
-  ## Select the active validators
-  result = @[]
-  for idx, val in validators:
-    if val.status == ACTIVE:
-      result.add idx.Uint24
-
-func shuffle(values: seq[Uint24], seed: Blake2_256_Digest): seq[Uint24] {.noInit.}=
+func shuffle*[T](values: seq[T], seed: Blake2_256_Digest): seq[T] =
   ## Returns the shuffled ``values`` with seed as entropy.
   ## TODO: this calls out for tests, but I odn't particularly trust spec
   ## right now.
 
   let values_count = values.len
 
-  # Entropy is consumed from the seed in 3-byte (24 bit) chunks
-  const rand_bytes = 3
-  let rand_max = 2^(rand_bytes * 8) - 1
+  const
+    # Entropy is consumed from the seed in 3-byte (24 bit) chunks.
+    rand_bytes = 3
+    # The highest possible result of the RNG.
+    rand_max = 2^(rand_bytes * 8) - 1
 
   # The range of the RNG places an upper-bound on the size of the list that
   # may be shuffled. It is a logic error to supply an oversized list.
   assert values_count < rand_max
 
-  deepCopy(result, values)
-  var source = seed
+  result = values
+  var
+    source = seed
+    index = 0
+  while index < values_count - 1:
+    # Re-hash the `source` to obtain a new pattern of bytes.
 
-  var i = 0
-  while i < values.len - 1:
-    # Re-hash the `source` to obtain a new pattern of bytes
-    source = blake2_256.digest source.data
-    # Iterate through the `source` bytes in 3-byte chunks
+    # XXX "attempting to call undeclared routine init"
+    # source = blake2_256.digest source.data
+
+    # Iterate through the `source` bytes in 3-byte chunks.
     for pos in countup(0, 29, 3):
-      let remaining = values_count - i
+      let remaining = values_count - index
       if remaining == 1:
         break
 
       # Read 3-bytes of `source` as a 24-bit big-endian integer.
-      let sample_from_source = source.data[pos].Uint24 shl 16 or source.data[pos+1].Uint24 shl 8 or source.data[pos+2].Uint24
+      let sample_from_source =
+        source.data[pos].Uint24 shl 16 or
+        source.data[pos+1].Uint24 shl 8 or
+        source.data[pos+2].Uint24
 
       # Sample values greater than or equal to `sample_max` will cause
       # modulo bias when mapped into the `remaining` range.
       let sample_max = rand_max - rand_max mod remaining
 
       # Perform a swap if the consumed entropy will not cause modulo bias.
       if sample_from_source < sample_max:
-        let replacement_position = sample_from_source mod remaining + i
-        swap result[i], result[replacement_position]
-        inc i
-
-func split[T](lst: seq[T], N: Positive): seq[seq[T]] =
+        # Select a replacement index for the current index.
+        let replacement_position = sample_from_source mod remaining + index
+        swap result[index], result[replacement_position]
+        inc index
+
+func split*[T](lst: openArray[T], N: Positive): seq[seq[T]] =
+  ## split lst in N pieces, with each piece having `len(lst) div N` or
+  ## `len(lst) div N + 1` pieces
   # TODO: implement as an iterator
   result = newSeq[seq[T]](N)
   for i in 0 ..< N:
     result[i] = lst[lst.len * i div N ..< lst.len * (i+1) div N] # TODO: avoid alloc via toOpenArray
 
-func get_new_shuffling*(seed: Blake2_256_Digest, validators: seq[ValidatorRecord],
-    dynasty: int64, crosslinking_start_shard: int16): seq[seq[ShardAndCommittee]] {.noInit.} =
-  ## Split up validators into groups at the start of every epoch,
-  ## determining at what height they can make attestations and what shard they are making crosslinks for
-  ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png
-
-  let avs = get_active_validator_indices(validators)
-  var committees_per_slot, slots_per_committee: uint16
-
-  if avs.len >= CYCLE_LENGTH * MIN_COMMITTEE_SIZE:
-    committees_per_slot = uint16 avs.len div CYCLE_LENGTH div (MIN_COMMITTEE_SIZE * 2) + 1
-    slots_per_committee = 1
-  else:
-    committees_per_slot = 1
-    slots_per_committee = 1
-    while avs.len.uint16 * slots_per_committee < CYCLE_LENGTH * MIN_COMMITTEE_SIZE and
-        slots_per_committee < CYCLE_LENGTH:
-      slots_per_committee *= 2
-
-  result = @[]
-  for slot, slot_indices in shuffle(avs, seed).split(CYCLE_LENGTH):
-    let shard_indices = slot_indices.split(committees_per_slot)
-    let shard_id_start = crosslinking_start_shard.uint16 +
-                            slot.uint16 * committees_per_slot div slots_per_committee
-
-    var committees = newSeq[ShardAndCommittee](shard_indices.len)
-    for j, indices in shard_indices:
-      committees[j].shard_id = (shard_id_start + j.uint16) mod SHARD_COUNT
-      committees[j].committee = indices
-
-    result.add committees
-
 func get_shards_and_committees_for_slot*(state: BeaconState,
-        slot: uint64): seq[ShardAndCommittee] =
+                                         slot: uint64
+                                         ): seq[ShardAndCommittee] =
   # TODO: Spec why is active_state an argument?
   # TODO: this returns a scalar, not vector, but its return type in spec is a seq/list?
 
   let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH
   assert earliest_slot_in_array <= slot
   assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2
 
-  return @[state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]]
+  return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]
   # TODO, slot is a uint64; will be an issue on int32 arch.
   #       Clarify with EF if light clients will need the beacon chain
 
@@ -114,15 +87,19 @@ func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int):
 
   return state.recent_block_hashes[slot - earliest_slot_in_array]
 
-func get_new_recent_block_hashes*(
-  old_block_hashes: seq[Blake2_256_Digest],
-  parent_slot, current_slot: int64,
-  parent_hash: Blake2_256_Digest
-  ): seq[Blake2_256_Digest] =
+func get_new_recent_block_hashes*(old_block_hashes: seq[Blake2_256_Digest],
+                                  parent_slot, current_slot: int64,
+                                  parent_hash: Blake2_256_Digest
+                                  ): seq[Blake2_256_Digest] =
 
   # Should throw for `current_slot - CYCLE_LENGTH * 2 - 1` according to spec comment
   let d = current_slot - parent_slot
   result = old_block_hashes[d .. ^1]
   for _ in 0 ..< min(d, old_block_hashes.len):
     result.add parent_hash
 
+func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord =
+  let
+    first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee
+    index = first_committee[(slot mod len(first_committee).uint64).int]
+  state.validators[index]

beacon_chain/validator.nim

@@ -0,0 +1,94 @@
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+# Helpers and functions pertaining to managing the validator set
+
+import
+  options,
+  eth_common,
+  ./datatypes, ./private/helpers
+
+func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64): Option[int] =
+  for i, v in validators:
+      if v.status == WITHDRAWN and v.exit_slot + DELETION_PERIOD.uint64 <= current_slot:
+          return some(i)
+
+func add_validator*(validators: var seq[ValidatorRecord],
+                    pubkey: BLSPublicKey,
+                    proof_of_possession: seq[byte],
+                    withdrawal_shard: uint16,
+                    withdrawal_address: EthAddress,
+                    randao_commitment: Blake2_256_Digest,
+                    status: ValidatorStatusCodes,
+                    current_slot: uint64
+                    ): int =
+  # Check that validator really did register
+  # let signed_message = as_bytes32(pubkey) + as_bytes2(withdrawal_shard) + withdrawal_address + randao_commitment
+  # assert BLSVerify(pub=pubkey,
+  #                  msg=hash(signed_message),
+  #                  sig=proof_of_possession)
+
+  # Pubkey uniqueness
+  # assert pubkey not in [v.pubkey for v in validators]
+  let
+    rec = ValidatorRecord(
+      pubkey: pubkey,
+      withdrawal_shard: withdrawal_shard,
+      withdrawal_address: withdrawal_address,
+      randao_commitment: randao_commitment,
+      randao_last_change: current_slot,
+      balance: DEPOSIT_SIZE * GWEI_PER_ETH,
+      status: status,
+      exit_slot: 0,
+      exit_seq: 0
+    )
+
+  let index = min_empty_validator(validators, current_slot)
+  if index.isNone:
+      validators.add(rec)
+      return len(validators) - 1
+  else:
+      validators[index.get()] = rec
+      return index.get()
+
+func get_active_validator_indices(validators: openArray[ValidatorRecord]): seq[Uint24] =
+  ## Select the active validators
+  result = @[]
+  for idx, val in validators:
+    if val.status == ACTIVE:
+      result.add idx.Uint24
+
+func get_new_shuffling*(seed: Blake2_256_Digest,
+                        validators: openArray[ValidatorRecord],
+                        crosslinking_start_shard: int
+                        ): array[CYCLE_LENGTH, seq[ShardAndCommittee]] =
+  ## Split up validators into groups at the start of every epoch,
+  ## determining at what height they can make attestations and what shard they are making crosslinks for
+  ## Implementation should do the following: http://vitalik.ca/files/ShuffleAndAssign.png
+
+  let
+    active_validators = get_active_validator_indices(validators)
+    committees_per_slot = clamp(
+      len(active_validators) div CYCLE_LENGTH div TARGET_COMMITTEE_SIZE,
+      1, SHARD_COUNT div CYCLE_LENGTH)
+    # Shuffle with seed
+    shuffled_active_validator_indices = shuffle(active_validators, seed)
+    # Split the shuffled list into cycle_length pieces
+    validators_per_slot = split(shuffled_active_validator_indices, CYCLE_LENGTH)
+
+  assert validators_per_slot.len() == CYCLE_LENGTH # what split should do..
+
+  for slot, slot_indices in validators_per_slot:
+    let
+      shard_indices = split(slot_indices, committees_per_slot)
+      shard_id_start = crosslinking_start_shard + slot * committees_per_slot
+
+    var committees = newSeq[ShardAndCommittee](shard_indices.len)
+    for shard_position, indices in shard_indices:
+      committees[shard_position].shard_id = (shard_id_start + shard_position).uint16 mod SHARD_COUNT
+      committees[shard_position].committee = indices
+
+    result[slot] = committees

tests/all_tests.nim

@@ -6,5 +6,6 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
+  ./test_block_processing,
   ./test_ssz,
-  ./test_block_processing
+  ./test_validator

tests/test_validator.nim

@@ -0,0 +1,30 @@
+# Copyright (c) 2018 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at http://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at http://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  math, nimcrypto, unittest, sequtils,
+  ../beacon_chain/[datatypes, validator]
+
+func sumCommittees(v: openArray[seq[ShardAndCommittee]]): int =
+  for x in v:
+    for y in x:
+      inc result, y.committee.len
+
+suite "Validators":
+  ## For now just test that we can compile and execute block processing with mock data.
+
+  test "Smoke validator shuffling":
+    let
+      validators = repeat(
+        ValidatorRecord(
+          status: ACTIVE
+        ), 1024)
+
+    # XXX the shuffling looks really odd, probably buggy
+    let s = get_new_shuffling(Blake2_256_Digest(), validators, 0)
+    check:
+      s.len == CYCLE_LENGTH
+      sumCommittees(s) == validators.len() # all validators accounted for