Skip to content

Commit

Permalink
feat!: add new header field (#6686)
Browse files Browse the repository at this point in the history
Description
---
Add new header field, block output mmr which only tracks outputs in
current block

Motivation and Context
---
This is needed in order to validate that all outputs included in the
block is correctly captured in the pow of the header. The current
strategy is to only use the output smt, which commits to all outputs on
the blockchain. This is required for horizon sync and prune mode to
verify the blockchain as we sync.
But someone else wants to verify the block outputs, they have to have
the entire outputs set aka entire blockchain in order to verify that the
outputs are indeed locked behind pow.
This addition allows any person to take only the block and know that the
outputs contained in that block, is correctly locked behind the pow of
the header.

The output field is constructed as two merkle mountain ranges. 
the primary one used in the header field is constructed as
[coinbase_1, coinbase_2, .., coinbase_x,
non_coinbase_ouputs_merkle_root]
with the non_coinbase_ouputs_merkle_root constructed as
[output_1, output_2, .., output_x]

This double mountain range approach allows you to prove the output are
in the block header, as well as create a proof for a specific output.
But separating them allows p2pool to only need to store a single hash +
list of all outputs in order to verify all coin bases are correctly
included in the header. Rather than all outputs or before this pr, the
entire block.

Breaking change
---
Need to resync database
  • Loading branch information
SWvheerden authored Nov 14, 2024
1 parent a2aa20e commit 69a5872
Show file tree
Hide file tree
Showing 17 changed files with 159 additions and 32 deletions.
5 changes: 3 additions & 2 deletions applications/minotari_app_grpc/proto/block.proto
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,10 @@ message BlockHeader {
bytes prev_hash = 4;
// Timestamp at which the block was built.
uint64 timestamp = 5;
// This is the UTXO merkle root of the outputs
// This is calculated as Hash (txo MMR root || roaring bitmap hash of UTXO indices)
// This is the UTXO merkle root of the outputs in the blockchain
bytes output_mr = 6;
// This is the merkle root of all outputs in this block
bytes block_output_mr = 7;
// This is the MMR root of the kernels
bytes kernel_mr = 8;
// This is the Merkle root of the inputs in this block
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ impl From<BlockHeader> for grpc::BlockHeader {
timestamp: h.timestamp.as_u64(),
input_mr: h.input_mr.to_vec(),
output_mr: h.output_mr.to_vec(),
block_output_mr: h.block_output_mr.to_vec(),
output_mmr_size: h.output_smt_size,
kernel_mr: h.kernel_mr.to_vec(),
kernel_mmr_size: h.kernel_mmr_size,
Expand Down Expand Up @@ -76,6 +77,7 @@ impl TryFrom<grpc::BlockHeader> for BlockHeader {
timestamp: EpochTime::from(header.timestamp),
input_mr: FixedHash::try_from(header.input_mr).map_err(|err| err.to_string())?,
output_mr: FixedHash::try_from(header.output_mr).map_err(|err| err.to_string())?,
block_output_mr: FixedHash::try_from(header.block_output_mr).unwrap_or_default(),
output_smt_size: header.output_mmr_size,
kernel_mr: FixedHash::try_from(header.kernel_mr).map_err(|err| err.to_string())?,
kernel_mmr_size: header.kernel_mmr_size,
Expand Down
14 changes: 10 additions & 4 deletions base_layer/core/src/base_node/sync/header_sync/validator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -245,13 +245,18 @@ mod test {
test_helpers::blockchain::{create_new_blockchain, TempDatabase},
};

fn setup() -> (BlockHeaderSyncValidator<TempDatabase>, AsyncBlockchainDb<TempDatabase>) {
fn setup() -> (
BlockHeaderSyncValidator<TempDatabase>,
AsyncBlockchainDb<TempDatabase>,
ConsensusManager,
) {
let rules = ConsensusManager::builder(Network::LocalNet).build().unwrap();
let randomx_factory = RandomXFactory::default();
let db = create_new_blockchain();
(
BlockHeaderSyncValidator::new(db.clone().into(), rules, randomx_factory),
BlockHeaderSyncValidator::new(db.clone().into(), rules.clone(), randomx_factory),
db.into(),
rules,
)
}

Expand All @@ -262,10 +267,11 @@ mod test {
AsyncBlockchainDb<TempDatabase>,
ChainHeader,
) {
let (validator, db) = setup();
let (validator, db, cm) = setup();
let mut tip = db.fetch_tip_header().await.unwrap();
for _ in 0..n {
let mut header = BlockHeader::from_previous(tip.header());
header.version = cm.consensus_constants(header.height).blockchain_version();
// Needed to have unique keys for the blockchain db mmr count indexes (MDB_KEY_EXIST error)
header.kernel_mmr_size += 1;
header.output_smt_size += 1;
Expand Down Expand Up @@ -301,7 +307,7 @@ mod test {

#[tokio::test]
async fn it_errors_if_hash_does_not_exist() {
let (mut validator, _) = setup();
let (mut validator, _, _cm) = setup();
let start_hash = vec![0; 32];
let err = validator
.initialize_state(&start_hash.clone().try_into().unwrap())
Expand Down
19 changes: 13 additions & 6 deletions base_layer/core/src/blocks/block_header.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,10 @@ pub struct BlockHeader {
pub timestamp: EpochTime,
/// This is the Merkle root of the inputs in this block
pub input_mr: FixedHash,
/// This is the UTXO merkle root of the outputs
/// This is calculated as Hash (txo MMR root || roaring bitmap hash of UTXO indices)
/// This is the UTXO merkle root of the outputs on the blockchain
pub output_mr: FixedHash,
/// This is the block_output_mr
pub block_output_mr: FixedHash,
/// The size (number of leaves) of the output and range proof MMRs at the time of this header
pub output_smt_size: u64,
/// This is the MMR root of the kernels
Expand Down Expand Up @@ -130,6 +131,7 @@ impl BlockHeader {
prev_hash: FixedHash::zero(),
timestamp: EpochTime::now(),
output_mr: FixedHash::zero(),
block_output_mr: FixedHash::zero(),
output_smt_size: 0,
kernel_mr: FixedHash::zero(),
kernel_mmr_size: 0,
Expand Down Expand Up @@ -164,6 +166,7 @@ impl BlockHeader {
timestamp: EpochTime::now(),
output_mr: FixedHash::zero(),
output_smt_size: prev.output_smt_size,
block_output_mr: FixedHash::zero(),
kernel_mr: FixedHash::zero(),
kernel_mmr_size: prev.kernel_mmr_size,
input_mr: FixedHash::zero(),
Expand Down Expand Up @@ -222,7 +225,7 @@ impl BlockHeader {
/// Provides a mining hash of the header, used for the mining.
/// This differs from the normal hash by not hashing the nonce and kernel pow.
pub fn mining_hash(&self) -> FixedHash {
DomainSeparatedConsensusHasher::<BlocksHashDomain, Blake2b<U32>>::new("block_header")
let incomplete = DomainSeparatedConsensusHasher::<BlocksHashDomain, Blake2b<U32>>::new("block_header")
.chain(&self.version)
.chain(&self.height)
.chain(&self.prev_hash)
Expand All @@ -235,9 +238,12 @@ impl BlockHeader {
.chain(&self.total_kernel_offset)
.chain(&self.total_script_offset)
.chain(&self.validator_node_mr)
.chain(&self.validator_node_size)
.finalize()
.into()
.chain(&self.validator_node_size);

match self.version {
0 => incomplete.finalize().into(),
_ => incomplete.chain(&self.block_output_mr).finalize().into(),
}
}

pub fn merge_mining_hash(&self) -> FixedHash {
Expand Down Expand Up @@ -273,6 +279,7 @@ impl From<NewBlockHeaderTemplate> for BlockHeader {
prev_hash: header_template.prev_hash,
timestamp: EpochTime::now(),
output_mr: FixedHash::zero(),
block_output_mr: FixedHash::zero(),
output_smt_size: 0,
kernel_mr: FixedHash::zero(),
kernel_mmr_size: 0,
Expand Down
47 changes: 44 additions & 3 deletions base_layer/core/src/blocks/genesis_block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,8 @@ pub fn get_nextnet_genesis_block() -> ChainBlock {
// TODO: Fix this hack with the next nextnet reset!!
block.header.input_mr =
FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();
block.header.block_output_mr =
FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();

// Add pre-mine utxos - enable/disable as required
let add_pre_mine_utxos = false;
Expand Down Expand Up @@ -270,6 +272,8 @@ pub fn get_mainnet_genesis_block() -> ChainBlock {
FixedHash::from_hex("b7b38b76f5832b5b63691a8334dfa67d8c762b77b2b4aa4f648c4eb1dfb25c1e").unwrap();
block.header.output_mr =
FixedHash::from_hex("a77ecf05b20c426d3d400a63397be6c622843c66d5751ecbe3390c8a4885158e").unwrap();
block.header.block_output_mr =
FixedHash::from_hex("91e997520b0eee770914334692080f92d18db434d373561f8842c56d70c11b97").unwrap();
block.header.validator_node_mr =
FixedHash::from_hex("277da65c40b2cf99db86baedb903a3f0a38540f3a94d40c826eecac7e27d5dfc").unwrap();
}
Expand Down Expand Up @@ -371,6 +375,10 @@ pub fn get_esmeralda_genesis_block() -> ChainBlock {
// lets get the block
let mut block = get_esmeralda_genesis_block_raw();

// TODO: Fix this hack with the next esme reset!!
block.header.block_output_mr =
FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap();

// Add pre-mine utxos - enable/disable as required
let add_pre_mine_utxos = true;
if add_pre_mine_utxos {
Expand Down Expand Up @@ -482,7 +490,9 @@ fn get_raw_block(genesis_timestamp: &DateTime<FixedOffset>, not_before_proof: &P
height: 0,
prev_hash: FixedHash::zero(),
timestamp: timestamp.into(),
output_mr: FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000").unwrap(),
output_mr: FixedHash::zero(),
block_output_mr: FixedHash::from_hex("622720a6571c33d6bf6138d9e737d3468c77f1193640698ad459953d24ec0812")
.unwrap(),
output_smt_size: 0,
kernel_mr: FixedHash::from_hex("c14803066909d6d22abf0d2d2782e8936afc3f713f2af3a4ef5c42e8400c1303").unwrap(),
kernel_mmr_size: 0,
Expand Down Expand Up @@ -518,6 +528,7 @@ mod test {

use super::*;
use crate::{
block_output_mr_hash_from_pruned_mmr,
chain_storage::calculate_validator_node_mr,
consensus::ConsensusManager,
test_helpers::blockchain::create_new_blockchain_with_network,
Expand All @@ -527,8 +538,8 @@ mod test {
},
validation::{ChainBalanceValidator, FinalHorizonStateValidation},
KernelMmr,
PrunedOutputMmr,
};

#[test]
#[serial]
fn esmeralda_genesis_sanity_check() {
Expand Down Expand Up @@ -661,9 +672,15 @@ mod test {
kernel_mmr.push(k.hash().to_vec()).unwrap();
}
let mut output_smt = OutputSmt::new();

let mut block_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
let mut normal_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
let mut vn_nodes = Vec::new();
for o in block.block().body.outputs() {
if o.features.is_coinbase() {
block_output_mmr.push(o.hash().to_vec()).unwrap();
} else {
normal_output_mmr.push(o.hash().to_vec()).unwrap();
}
let smt_key = NodeKey::try_from(o.commitment.as_bytes()).unwrap();
let smt_node = ValueHash::try_from(o.smt_hash(block.header().height).as_slice()).unwrap();
output_smt.insert(smt_key, smt_node).unwrap();
Expand All @@ -681,6 +698,11 @@ mod test {
));
}
}

block_output_mmr
.push(normal_output_mmr.get_merkle_root().unwrap().to_vec())
.unwrap();

for i in block.block().body.inputs() {
let smt_key = NodeKey::try_from(i.commitment().unwrap().as_bytes()).unwrap();
output_smt.delete(&smt_key).unwrap();
Expand Down Expand Up @@ -718,6 +740,25 @@ mod test {
output_mr_hash_from_smt(&mut output_smt).unwrap().to_vec().to_hex(),
block.header().output_mr.to_vec().to_hex(),
);

// TODO: Fix this hack with the next nextnet/esme release reset!!
if network == Network::NextNet || network == Network::Esmeralda {
assert_eq!(
FixedHash::from_hex("0000000000000000000000000000000000000000000000000000000000000000")
.unwrap()
.to_vec()
.to_hex(),
block.header().block_output_mr.to_vec().to_hex(),
);
} else {
assert_eq!(
block_output_mr_hash_from_pruned_mmr(&block_output_mmr)
.unwrap()
.to_vec()
.to_hex(),
block.header().block_output_mr.to_vec().to_hex(),
);
}
if network == Network::NextNet {
// TODO: Fix this hack with the next nextnet reset!!
assert_eq!(
Expand Down
20 changes: 20 additions & 0 deletions base_layer/core/src/chain_storage/blockchain_database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ use tari_utilities::{epoch_time::EpochTime, hex::Hex, ByteArray};

use super::TemplateRegistrationEntry;
use crate::{
block_output_mr_hash_from_pruned_mmr,
blocks::{
Block,
BlockAccumulatedData,
Expand Down Expand Up @@ -105,6 +106,7 @@ use crate::{
OutputSmt,
PrunedInputMmr,
PrunedKernelMmr,
PrunedOutputMmr,
ValidatorNodeBMT,
};

Expand Down Expand Up @@ -912,6 +914,7 @@ where B: BlockchainBackend
block.header.kernel_mmr_size = roots.kernel_mmr_size;
block.header.input_mr = roots.input_mr;
block.header.output_mr = roots.output_mr;
block.header.block_output_mr = roots.block_output_mr;
block.header.output_smt_size = roots.output_smt_size;
block.header.validator_node_mr = roots.validator_node_mr;
block.header.validator_node_size = roots.validator_node_size;
Expand Down Expand Up @@ -1321,6 +1324,7 @@ pub struct MmrRoots {
pub kernel_mmr_size: u64,
pub input_mr: FixedHash,
pub output_mr: FixedHash,
pub block_output_mr: FixedHash,
pub output_smt_size: u64,
pub validator_node_mr: FixedHash,
pub validator_node_size: u64,
Expand All @@ -1333,6 +1337,7 @@ impl std::fmt::Display for MmrRoots {
writeln!(f, "Kernel MR : {}", self.kernel_mr)?;
writeln!(f, "Kernel MMR Size : {}", self.kernel_mmr_size)?;
writeln!(f, "Output MR : {}", self.output_mr)?;
writeln!(f, "Block Output MR : {}", self.block_output_mr)?;
writeln!(f, "Output SMT Size : {}", self.output_smt_size)?;
writeln!(f, "Validator MR : {}", self.validator_node_mr)?;
Ok(())
Expand Down Expand Up @@ -1372,13 +1377,20 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(

let mut kernel_mmr = PrunedKernelMmr::new(kernels);
let mut input_mmr = PrunedInputMmr::new(PrunedHashSet::default());
let mut block_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());
let mut normal_output_mmr = PrunedOutputMmr::new(PrunedHashSet::default());

for kernel in body.kernels() {
kernel_mmr.push(kernel.hash().to_vec())?;
}

let mut outputs_to_remove = Vec::new();
for output in body.outputs() {
if output.features.is_coinbase() {
block_output_mmr.push(output.hash().to_vec())?;
} else {
normal_output_mmr.push(output.hash().to_vec())?;
}
if !output.is_burned() {
let smt_key = NodeKey::try_from(output.commitment.as_bytes())?;
let smt_node = ValueHash::try_from(output.smt_hash(header.height).as_slice())?;
Expand All @@ -1393,6 +1405,7 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
}
}
}
block_output_mmr.push(normal_output_mmr.get_merkle_root()?.to_vec())?;

let mut outputs_to_add = Vec::new();
for input in body.inputs() {
Expand Down Expand Up @@ -1426,11 +1439,18 @@ pub fn calculate_mmr_roots<T: BlockchainBackend>(
(tip_header.validator_node_mr, 0)
};

let block_output_mr = if block.version() > 0 {
block_output_mr_hash_from_pruned_mmr(&block_output_mmr)?
} else {
FixedHash::zero()
};

let mmr_roots = MmrRoots {
kernel_mr: kernel_mr_hash_from_pruned_mmr(&kernel_mmr)?,
kernel_mmr_size: kernel_mmr.get_leaf_count()? as u64,
input_mr: input_mr_hash_from_pruned_mmr(&input_mmr)?,
output_mr: output_mr_hash_from_smt(output_smt)?,
block_output_mr,
output_smt_size: output_smt.size(),
validator_node_mr,
validator_node_size: validator_node_size as u64,
Expand Down
Loading

0 comments on commit 69a5872

Please sign in to comment.