Skip to content

Commit

Permalink
fix(sync)!: validate checkpoint merkle root (#1108)
Browse files Browse the repository at this point in the history 
Description
---
fix(sync): validate checkpoint merkle root
Rename NumPreshard variants
(#1092 (comment))

Motivation and Context
---
Validates the checkpoint blocks merkle root against the synced state so
that the validator can be sure they have the correct state. Note that
block and QC signatures are not currently validated pending a refactor
in the validations.

How Has This Been Tested?
---
Manually, submitting transactions, mining to next epoch, submitting more
transactions, deleting data of one VN and resyncing.

What process can a PR reviewer use to test or verify this change?
---
As above

Breaking Changes
---

- [ ] None
- [ ] Requires data directory to be deleted
- [x] Other - Please specify

BREAKING CHANGE: the merkle root construction differs slightly from
current development branch
  • Loading branch information
@sdbondi
sdbondi authored Aug 6, 2024
1 parent 847682e commit 6c1dbc6
Show file tree
Hide file tree
Showing 41 changed files with 698 additions and 428 deletions.
2 changes: 1 addition & 1 deletion applications/tari_dan_app_utilities/src/consensus_constants.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ impl ConsensusConstants {
committee_size: 7,
max_base_layer_blocks_ahead: 5,
max_base_layer_blocks_behind: 5,
num_preshards: NumPreshards::SixtyFour,
num_preshards: NumPreshards::P64,
pacemaker_max_base_time: Duration::from_secs(10),
}
}
Expand Down
19 changes: 4 additions & 15 deletions applications/tari_validator_node/src/p2p/rpc/service_impl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,26 +343,15 @@ impl ValidatorNodeRpcService for ValidatorNodeRpcServiceImpl {
return Err(RpcStatus::not_found("Cannot generate checkpoint for genesis epoch"));
}

let Some(local_committee_info) = self
.epoch_manager
.get_local_committee_info(prev_epoch)
.await
.optional()
.map_err(RpcStatus::log_internal_error(LOG_TARGET))?
else {
return Err(RpcStatus::bad_request(format!(
"This validator node is not registered for the previous epoch {prev_epoch}"
)));
};

let checkpoint = self
.shard_state_store
.with_read_tx(|tx| EpochCheckpoint::generate(tx, prev_epoch, local_committee_info.shard_group()))
.with_read_tx(|tx| EpochCheckpoint::get(tx, prev_epoch))
.optional()
.map_err(RpcStatus::log_internal_error(LOG_TARGET))?;
.map_err(RpcStatus::log_internal_error(LOG_TARGET))?
.ok_or_else(|| RpcStatus::not_found("Epoch checkpoint not found"))?;

Ok(Response::new(GetCheckpointResponse {
checkpoint: checkpoint.map(Into::into),
checkpoint: Some(checkpoint.into()),
}))
}

Expand Down
4 changes: 0 additions & 4 deletions dan_layer/common_types/src/hashing.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ pub fn block_hasher() -> TariHasher {
dan_hasher("Block")
}

pub fn state_root_hasher() -> TariHasher {
dan_hasher("JmtStateRoots")
}

pub fn quorum_certificate_hasher() -> TariHasher {
dan_hasher("QuorumCertificate")
}
Expand Down
40 changes: 20 additions & 20 deletions dan_layer/common_types/src/num_preshards.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,26 @@ use serde::{Deserialize, Serialize};
)]
#[derive(Clone, Debug, Copy, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum NumPreshards {
One = 1,
Two = 2,
Four = 4,
Eight = 8,
Sixteen = 16,
ThirtyTwo = 32,
SixtyFour = 64,
OneTwentyEight = 128,
TwoFiftySix = 256,
P1 = 1,
P2 = 2,
P4 = 4,
P8 = 8,
P16 = 16,
P32 = 32,
P64 = 64,
P128 = 128,
P256 = 256,
}

impl NumPreshards {
pub const MAX: Self = Self::TwoFiftySix;
pub const MAX: Self = Self::P256;

pub fn as_u32(self) -> u32 {
self as u32
}

pub fn is_one(self) -> bool {
self == Self::One
self == Self::P1
}
}

Expand All @@ -40,15 +40,15 @@ impl TryFrom<u32> for NumPreshards {

fn try_from(value: u32) -> Result<Self, Self::Error> {
match value {
1 => Ok(Self::One),
2 => Ok(Self::Two),
4 => Ok(Self::Four),
8 => Ok(Self::Eight),
16 => Ok(Self::Sixteen),
32 => Ok(Self::ThirtyTwo),
64 => Ok(Self::SixtyFour),
128 => Ok(Self::OneTwentyEight),
256 => Ok(Self::TwoFiftySix),
1 => Ok(Self::P1),
2 => Ok(Self::P2),
4 => Ok(Self::P4),
8 => Ok(Self::P8),
16 => Ok(Self::P16),
32 => Ok(Self::P32),
64 => Ok(Self::P64),
128 => Ok(Self::P128),
256 => Ok(Self::P256),
_ => Err(InvalidNumPreshards(value)),
}
}
Expand Down
2 changes: 1 addition & 1 deletion dan_layer/common_types/src/shard_group.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ mod tests {
#[test]
fn to_substate_address_range() {
let sg = ShardGroup::new(0, 63);
let range = sg.to_substate_address_range(NumPreshards::SixtyFour);
let range = sg.to_substate_address_range(NumPreshards::P64);
assert_eq!(*range.start(), SubstateAddress::zero());
assert_eq!(*range.end(), SubstateAddress::max());
}
Expand Down
110 changes: 55 additions & 55 deletions dan_layer/common_types/src/substate_address.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,20 +305,20 @@ mod tests {
#[test]
fn to_committee_shard_and_shard_range_match() {
let address = address_at(1, 8);
let shard = address.to_shard(NumPreshards::Eight);
let shard = address.to_shard(NumPreshards::P8);
assert_eq!(shard, 1);

let range = Shard::from(0).to_substate_address_range(NumPreshards::Two);
let range = Shard::from(0).to_substate_address_range(NumPreshards::P2);
assert_range(range, SubstateAddress::zero()..address_at(1, 2));
let range = Shard::from(1).to_substate_address_range(NumPreshards::Two);
let range = Shard::from(1).to_substate_address_range(NumPreshards::P2);
assert_range(range, address_at(1, 2)..=SubstateAddress::max());

for n in 0..7 {
let range = Shard::from(n).to_substate_address_range(NumPreshards::Eight);
let range = Shard::from(n).to_substate_address_range(NumPreshards::P8);
assert_range(range, address_at(n, 8)..address_at(n + 1, 8));
}

let range = Shard::from(7).to_substate_address_range(NumPreshards::Eight);
let range = Shard::from(7).to_substate_address_range(NumPreshards::P8);
assert_range(range, address_at(7, 8)..=address_at(8, 8));
}

Expand Down Expand Up @@ -381,28 +381,28 @@ mod tests {

#[test]
fn to_shard() {
let shard = SubstateAddress::zero().to_shard(NumPreshards::Two);
let shard = SubstateAddress::zero().to_shard(NumPreshards::P2);
assert_eq!(shard, 0);
let shard = address_at(1, 2).to_shard(NumPreshards::Two);
let shard = address_at(1, 2).to_shard(NumPreshards::P2);
assert_eq!(shard, 1);
let shard = plus_one(address_at(1, 2)).to_shard(NumPreshards::Two);
let shard = plus_one(address_at(1, 2)).to_shard(NumPreshards::P2);
assert_eq!(shard, 1);
let shard = SubstateAddress::max().to_shard(NumPreshards::Two);
let shard = SubstateAddress::max().to_shard(NumPreshards::P2);
assert_eq!(shard, 1);

for i in 0..=32 {
let shard = divide_shard_space(i, 32).to_shard(NumPreshards::One);
let shard = divide_shard_space(i, 32).to_shard(NumPreshards::P1);
assert_eq!(shard, 0);
}

// 2 shards, exactly half of the physical shard space
for i in 0..=8 {
let shard = divide_shard_space(i, 16).to_shard(NumPreshards::Two);
let shard = divide_shard_space(i, 16).to_shard(NumPreshards::P2);
assert_eq!(shard, 0, "{shard} is not 0 for i: {i}");
}

for i in 9..16 {
let shard = divide_shard_space(i, 16).to_shard(NumPreshards::Two);
let shard = divide_shard_space(i, 16).to_shard(NumPreshards::P2);
assert_eq!(shard, 1, "{shard} is not 1 for i: {i}");
}

Expand All @@ -423,7 +423,7 @@ mod tests {
}
}

let shard = divide_floor(SubstateAddress::max(), 2).to_shard(NumPreshards::TwoFiftySix);
let shard = divide_floor(SubstateAddress::max(), 2).to_shard(NumPreshards::P256);
assert_eq!(shard, 128);
}

Expand Down Expand Up @@ -503,56 +503,56 @@ mod tests {

#[test]
fn it_returns_the_correct_shard_group() {
let group = SubstateAddress::zero().to_shard_group(NumPreshards::Four, 2);
let group = SubstateAddress::zero().to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(1));

let group = plus_one(address_at(0, 4)).to_shard_group(NumPreshards::Four, 2);
let group = plus_one(address_at(0, 4)).to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(1));

let group = address_at(1, 4).to_shard_group(NumPreshards::Four, 2);
let group = address_at(1, 4).to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(1));

let group = address_at(2, 4).to_shard_group(NumPreshards::Four, 2);
let group = address_at(2, 4).to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(2)..=Shard::from(3));

let group = address_at(3, 4).to_shard_group(NumPreshards::Four, 2);
let group = address_at(3, 4).to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(2)..=Shard::from(3));

let group = SubstateAddress::max().to_shard_group(NumPreshards::Four, 2);
let group = SubstateAddress::max().to_shard_group(NumPreshards::P4, 2);
assert_eq!(group.as_range(), Shard::from(2)..=Shard::from(3));

let group = minus_one(address_at(1, 64)).to_shard_group(NumPreshards::SixtyFour, 16);
let group = minus_one(address_at(1, 64)).to_shard_group(NumPreshards::P64, 16);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(3));
let group = address_at(4, 64).to_shard_group(NumPreshards::SixtyFour, 16);
let group = address_at(4, 64).to_shard_group(NumPreshards::P64, 16);
assert_eq!(group.as_range(), Shard::from(4)..=Shard::from(7));

let group = address_at(8, 64).to_shard_group(NumPreshards::SixtyFour, 2);
let group = address_at(8, 64).to_shard_group(NumPreshards::P64, 2);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(31));
let group = address_at(5, 8).to_shard_group(NumPreshards::SixtyFour, 2);
let group = address_at(5, 8).to_shard_group(NumPreshards::P64, 2);
assert_eq!(group.as_range(), Shard::from(32)..=Shard::from(63));

// On boundary
let group = address_at(0, 8).to_shard_group(NumPreshards::SixtyFour, 2);
let group = address_at(0, 8).to_shard_group(NumPreshards::P64, 2);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(31));
let group = address_at(4, 8).to_shard_group(NumPreshards::SixtyFour, 2);
let group = address_at(4, 8).to_shard_group(NumPreshards::P64, 2);
assert_eq!(group.as_range(), Shard::from(32)..=Shard::from(63));

let group = address_at(8, 8).to_shard_group(NumPreshards::SixtyFour, 2);
let group = address_at(8, 8).to_shard_group(NumPreshards::P64, 2);
assert_eq!(group.as_range(), Shard::from(32)..=Shard::from(63));

let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::SixtyFour, 32);
let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::P64, 32);
assert_eq!(group.as_range(), Shard::from(2)..=Shard::from(3));

let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::SixtyFour, 32);
let group = plus_one(address_at(3, 64)).to_shard_group(NumPreshards::P64, 32);
assert_eq!(group.as_range(), Shard::from(2)..=Shard::from(3));

let group = address_at(16, 64).to_shard_group(NumPreshards::SixtyFour, 32);
let group = address_at(16, 64).to_shard_group(NumPreshards::P64, 32);
assert_eq!(group.as_range(), Shard::from(16)..=Shard::from(17));

let group = minus_one(address_at(1, 4)).to_shard_group(NumPreshards::SixtyFour, 64);
let group = minus_one(address_at(1, 4)).to_shard_group(NumPreshards::P64, 64);
assert_eq!(group.as_range(), Shard::from(16)..=Shard::from(16));

let group = address_at(66, 256).to_shard_group(NumPreshards::SixtyFour, 16);
let group = address_at(66, 256).to_shard_group(NumPreshards::P64, 16);
assert_eq!(group.as_range(), Shard::from(16)..=Shard::from(19));
}

Expand Down Expand Up @@ -588,66 +588,66 @@ mod tests {
fn it_returns_the_correct_shard_group_for_odd_num_committees() {
// All shard groups except the last have 3 shards each

let group = address_at(0, 64).to_shard_group(NumPreshards::SixtyFour, 3);
let group = address_at(0, 64).to_shard_group(NumPreshards::P64, 3);
// First shard group gets an extra shard to cover the remainder
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(21));
assert_eq!(group.len(), 22);
let group = address_at(31, 64).to_shard_group(NumPreshards::SixtyFour, 3);
let group = address_at(31, 64).to_shard_group(NumPreshards::P64, 3);
assert_eq!(group.as_range(), Shard::from(22)..=Shard::from(42));
assert_eq!(group.len(), 21);
let group = address_at(50, 64).to_shard_group(NumPreshards::SixtyFour, 3);
let group = address_at(50, 64).to_shard_group(NumPreshards::P64, 3);
assert_eq!(group.as_range(), Shard::from(43)..=Shard::from(63));
assert_eq!(group.len(), 21);

let group = address_at(3, 64).to_shard_group(NumPreshards::SixtyFour, 7);
let group = address_at(3, 64).to_shard_group(NumPreshards::P64, 7);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(9));
assert_eq!(group.len(), 10);
let group = address_at(11, 64).to_shard_group(NumPreshards::SixtyFour, 7);
let group = address_at(11, 64).to_shard_group(NumPreshards::P64, 7);
assert_eq!(group.as_range(), Shard::from(10)..=Shard::from(18));
assert_eq!(group.len(), 9);
let group = address_at(22, 64).to_shard_group(NumPreshards::SixtyFour, 7);
let group = address_at(22, 64).to_shard_group(NumPreshards::P64, 7);
assert_eq!(group.as_range(), Shard::from(19)..=Shard::from(27));
assert_eq!(group.len(), 9);
let group = address_at(60, 64).to_shard_group(NumPreshards::SixtyFour, 7);
let group = address_at(60, 64).to_shard_group(NumPreshards::P64, 7);
assert_eq!(group.as_range(), Shard::from(55)..=Shard::from(63));
assert_eq!(group.len(), 9);
let group = address_at(64, 64).to_shard_group(NumPreshards::SixtyFour, 7);
let group = address_at(64, 64).to_shard_group(NumPreshards::P64, 7);
assert_eq!(group.as_range(), Shard::from(55)..=Shard::from(63));
assert_eq!(group.len(), 9);
let group = SubstateAddress::zero().to_shard_group(NumPreshards::Eight, 3);
let group = SubstateAddress::zero().to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(2));

let group = address_at(1, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(1, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(2));

let group = address_at(1, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(1, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(0)..=Shard::from(2));

let group = address_at(3, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(3, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(5));

let group = address_at(4, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(4, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(5));

let group = address_at(5, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(5, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(3)..=Shard::from(5));
//
let group = address_at(6, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(6, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(6)..=Shard::from(7));

let group = address_at(7, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(7, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(6)..=Shard::from(7));
let group = address_at(8, 8).to_shard_group(NumPreshards::Eight, 3);
let group = address_at(8, 8).to_shard_group(NumPreshards::P8, 3);
assert_eq!(group.as_range(), Shard::from(6)..=Shard::from(7));

// Committee = 5
let group = address_at(4, 8).to_shard_group(NumPreshards::Eight, 5);
let group = address_at(4, 8).to_shard_group(NumPreshards::P8, 5);
assert_eq!(group.as_range(), Shard::from(4)..=Shard::from(5));

let group = address_at(7, 8).to_shard_group(NumPreshards::Eight, 5);
let group = address_at(7, 8).to_shard_group(NumPreshards::P8, 5);
assert_eq!(group.as_range(), Shard::from(7)..=Shard::from(7));

let group = address_at(8, 8).to_shard_group(NumPreshards::Eight, 5);
let group = address_at(8, 8).to_shard_group(NumPreshards::P8, 5);
assert_eq!(group.as_range(), Shard::from(7)..=Shard::from(7));
}
}
Expand All @@ -657,17 +657,17 @@ mod tests {

#[test]
fn it_works() {
let range = ShardGroup::new(0, 9).to_substate_address_range(NumPreshards::Sixteen);
let range = ShardGroup::new(0, 9).to_substate_address_range(NumPreshards::P16);
assert_range(range, SubstateAddress::zero()..address_at(10, 16));

let range = ShardGroup::new(1, 15).to_substate_address_range(NumPreshards::Sixteen);
let range = ShardGroup::new(1, 15).to_substate_address_range(NumPreshards::P16);
// Last shard always includes SubstateAddress::max
assert_range(range, address_at(1, 16)..=address_at(16, 16));

let range = ShardGroup::new(1, 8).to_substate_address_range(NumPreshards::Sixteen);
let range = ShardGroup::new(1, 8).to_substate_address_range(NumPreshards::P16);
assert_range(range, address_at(1, 16)..address_at(9, 16));

let range = ShardGroup::new(8, 15).to_substate_address_range(NumPreshards::Sixteen);
let range = ShardGroup::new(8, 15).to_substate_address_range(NumPreshards::P16);
assert_range(range, address_at(8, 16)..=address_at(16, 16));
}
}
Expand Down
Loading

0 comments on commit 6c1dbc6

Please sign in to comment.