Skip to content

Commit

Permalink
slightly simplify call to pre_validate_blocks_multiprocessing. batch …
Browse files Browse the repository at this point in the history
…size is always 4, check_filter is always True. Remove the member function wrapper around it in Blockchain, to prepare for being able to pass wrapped blockchains into it
  • Loading branch information
arvidn committed Sep 4, 2024
1 parent 1de9278 commit 76e5f77
Show file tree
Hide file tree
Showing 8 changed files with 83 additions and 89 deletions.
6 changes: 3 additions & 3 deletions chia/_tests/blockchain/blockchain_test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from chia.consensus.block_body_validation import ForkInfo
from chia.consensus.blockchain import AddBlockResult, Blockchain
from chia.consensus.multiprocess_validation import PreValidationResult
from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_blocks_multiprocessing
from chia.types.full_block import FullBlock
from chia.util.errors import Err
from chia.util.ints import uint32, uint64
Expand Down Expand Up @@ -63,8 +63,8 @@ async def _validate_and_add_block(
else:
# validate_signatures must be False in order to trigger add_block() to
# validate the signature.
pre_validation_results: List[PreValidationResult] = await blockchain.pre_validate_blocks_multiprocessing(
[block], {}, validate_signatures=False
pre_validation_results: List[PreValidationResult] = await pre_validate_blocks_multiprocessing(
blockchain.constants, blockchain, [block], blockchain.pool, {}, validate_signatures=False
)
assert pre_validation_results is not None
results = pre_validation_results[0]
Expand Down
56 changes: 33 additions & 23 deletions chia/_tests/blockchain/test_blockchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from chia.consensus.constants import ConsensusConstants
from chia.consensus.full_block_to_block_record import block_to_block_record
from chia.consensus.get_block_generator import get_block_generator
from chia.consensus.multiprocess_validation import PreValidationResult
from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_blocks_multiprocessing
from chia.consensus.pot_iterations import is_overflow_block
from chia.full_node.mempool_check_conditions import get_name_puzzle_conditions
from chia.simulator.block_tools import BlockTools, create_block_tools_async
Expand Down Expand Up @@ -1818,8 +1818,8 @@ async def test_pre_validation_fails_bad_blocks(self, empty_blockchain: Blockchai
block_bad = recursive_replace(
blocks[-1], "reward_chain_block.total_iters", blocks[-1].reward_chain_block.total_iters + 1
)
res = await empty_blockchain.pre_validate_blocks_multiprocessing(
[blocks[0], block_bad], {}, validate_signatures=True
res = await pre_validate_blocks_multiprocessing(
bt.constants, empty_blockchain, [blocks[0], block_bad], empty_blockchain.pool, {}, validate_signatures=True
)
assert res[0].error is None
assert res[1].error is not None
Expand All @@ -1837,8 +1837,8 @@ async def test_pre_validation(
end_i = min(i + n_at_a_time, len(blocks))
blocks_to_validate = blocks[i:end_i]
start_pv = time.time()
res = await empty_blockchain.pre_validate_blocks_multiprocessing(
blocks_to_validate, {}, validate_signatures=True
res = await pre_validate_blocks_multiprocessing(
bt.constants, empty_blockchain, blocks_to_validate, empty_blockchain.pool, {}, validate_signatures=True
)
end_pv = time.time()
times_pv.append(end_pv - start_pv)
Expand Down Expand Up @@ -1936,8 +1936,13 @@ async def test_conditions(
time_per_block=10,
)

pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[blocks[-1]], {}, validate_signatures=False
pre_validation_results: List[PreValidationResult] = await pre_validate_blocks_multiprocessing(
bt.constants,
b,
[blocks[-1]],
b.pool,
{},
validate_signatures=False,
)
# Ignore errors from pre-validation, we are testing block_body_validation
repl_preval_results = replace(pre_validation_results[0], error=None, required_iters=uint64(1))
Expand Down Expand Up @@ -2050,9 +2055,8 @@ async def test_timelock_conditions(
transaction_data=tx,
time_per_block=10,
)

pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[blocks[-1]], {}, validate_signatures=True
pre_validation_results = await pre_validate_blocks_multiprocessing(
bt.constants, b, [blocks[-1]], b.pool, {}, validate_signatures=True
)
assert pre_validation_results is not None
assert (await b.add_block(blocks[-1], pre_validation_results[0], None))[0] == expected
Expand Down Expand Up @@ -2123,8 +2127,8 @@ async def test_aggsig_garbage(
time_per_block=10,
)

pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[blocks[-1]], {}, validate_signatures=False
pre_validation_results = await pre_validate_blocks_multiprocessing(
bt.constants, b, [blocks[-1]], b.pool, {}, validate_signatures=False
)
# Ignore errors from pre-validation, we are testing block_body_validation
repl_preval_results = replace(pre_validation_results[0], error=None, required_iters=uint64(1))
Expand Down Expand Up @@ -2240,8 +2244,8 @@ async def test_ephemeral_timelock(
time_per_block=10,
)

pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[blocks[-1]], {}, validate_signatures=True
pre_validation_results = await pre_validate_blocks_multiprocessing(
bt.constants, b, [blocks[-1]], b.pool, {}, validate_signatures=True
)
assert pre_validation_results is not None
assert (await b.add_block(blocks[-1], pre_validation_results[0], None))[0] == expected
Expand Down Expand Up @@ -2587,8 +2591,8 @@ async def test_cost_exceeds_max(
]
assert err in [Err.BLOCK_COST_EXCEEDS_MAX]

results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[blocks[-1]], {}, validate_signatures=False
results = await pre_validate_blocks_multiprocessing(
bt.constants, b, [blocks[-1]], b.pool, {}, validate_signatures=False
)
assert results is not None
assert Err(results[0].error) == Err.BLOCK_COST_EXCEEDS_MAX
Expand Down Expand Up @@ -3151,7 +3155,9 @@ async def test_invalid_agg_sig(self, empty_blockchain: Blockchain, bt: BlockTool
await _validate_and_add_block(b, last_block, expected_error=Err.BAD_AGGREGATE_SIGNATURE, use_bls_cache=True)

# Bad signature also fails in prevalidation
preval_results = await b.pre_validate_blocks_multiprocessing([last_block], {}, validate_signatures=True)
preval_results = await pre_validate_blocks_multiprocessing(
bt.constants, b, [last_block], b.pool, {}, validate_signatures=True
)
assert preval_results is not None
assert preval_results[0].error == Err.BAD_AGGREGATE_SIGNATURE.value

Expand Down Expand Up @@ -3257,8 +3263,8 @@ async def test_long_reorg(
blocks = default_10000_blocks[:num_blocks_chain_1]

print(f"pre-validating {len(blocks)} blocks")
pre_validation_results: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
blocks, {}, validate_signatures=False
pre_validation_results = await pre_validate_blocks_multiprocessing(
b.constants, b, blocks, b.pool, {}, validate_signatures=False
)

for i, block in enumerate(blocks):
Expand Down Expand Up @@ -3805,12 +3811,14 @@ async def test_reorg_flip_flop(empty_blockchain: Blockchain, bt: BlockTools) ->
block1, block2 = b1, b2
counter += 1

preval: List[PreValidationResult] = await b.pre_validate_blocks_multiprocessing(
[block1], {}, validate_signatures=False
preval = await pre_validate_blocks_multiprocessing(
bt.constants, b, [block1], b.pool, {}, validate_signatures=False
)
_, err, _ = await b.add_block(block1, preval[0], None)
assert err is None
preval = await b.pre_validate_blocks_multiprocessing([block2], {}, validate_signatures=False)
preval = await pre_validate_blocks_multiprocessing(
bt.constants, b, [block2], b.pool, {}, validate_signatures=False
)
_, err, _ = await b.add_block(block2, preval[0], None)
assert err is None

Expand All @@ -3834,7 +3842,9 @@ async def test_get_tx_peak(default_400_blocks: List[FullBlock], empty_blockchain
bc = empty_blockchain
test_blocks = default_400_blocks[:100]

res = await bc.pre_validate_blocks_multiprocessing(test_blocks, {}, validate_signatures=False)
res = await pre_validate_blocks_multiprocessing(
bc.constants, bc, test_blocks, bc.pool, {}, validate_signatures=False
)

last_tx_block_record = None
for b, prevalidation_res in zip(test_blocks, res):
Expand Down
27 changes: 13 additions & 14 deletions chia/_tests/core/full_node/test_full_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from chia._tests.util.setup_nodes import SimulatorsAndWalletsServices
from chia._tests.util.time_out_assert import time_out_assert, time_out_assert_custom_interval, time_out_messages
from chia.consensus.block_body_validation import ForkInfo
from chia.consensus.multiprocess_validation import pre_validate_blocks_multiprocessing
from chia.consensus.pot_iterations import is_overflow_block
from chia.full_node.full_node import WalletUpdate
from chia.full_node.full_node_api import FullNodeAPI
Expand Down Expand Up @@ -436,25 +437,23 @@ async def check_transaction_confirmed(transaction) -> bool:
for reorg_block in reog_blocks[:r]:
await _validate_and_add_block_no_error(blockchain, reorg_block)
for i in range(1, height):
for batch_size in range(1, height, 3):
results = await blockchain.pre_validate_blocks_multiprocessing(
all_blocks[:i], {}, batch_size, validate_signatures=False
)
assert results is not None
for result in results:
assert result.error is None
results = await pre_validate_blocks_multiprocessing(
bt.constants, blockchain, all_blocks[:i], blockchain.pool, {}, validate_signatures=False
)
assert results is not None
for result in results:
assert result.error is None

for r in range(0, len(all_blocks), 3):
for block in all_blocks[:r]:
await _validate_and_add_block_no_error(blockchain, block)
for i in range(1, height):
for batch_size in range(1, height, 3):
results = await blockchain.pre_validate_blocks_multiprocessing(
all_blocks[:i], {}, batch_size, validate_signatures=False
)
assert results is not None
for result in results:
assert result.error is None
results = await pre_validate_blocks_multiprocessing(
bt.constants, blockchain, all_blocks[:i], blockchain.pool, {}, validate_signatures=False
)
assert results is not None
for result in results:
assert result.error is None


class TestFullNodeProtocol:
Expand Down
8 changes: 5 additions & 3 deletions chia/_tests/farmer_harvester/test_third_party_harvesters.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from chia._tests.util.time_out_assert import time_out_assert
from chia.consensus.blockchain import AddBlockResult
from chia.consensus.multiprocess_validation import PreValidationResult
from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_blocks_multiprocessing
from chia.farmer.farmer import Farmer, calculate_harvester_fee_quality
from chia.farmer.farmer_api import FarmerAPI
from chia.full_node.full_node import FullNode
Expand Down Expand Up @@ -422,8 +422,10 @@ async def add_test_blocks_into_full_node(blocks: List[FullBlock], full_node: Ful
# Inject full node with a pre-existing block to skip initial genesis sub-slot
# so that we have blocks generated that have our farmer reward address, instead
# of the GENESIS_PRE_FARM_FARMER_PUZZLE_HASH.
pre_validation_results: List[PreValidationResult] = await full_node.blockchain.pre_validate_blocks_multiprocessing(
blocks, {}, validate_signatures=True
constants = full_node.blockchain.constants
pool = full_node.blockchain.pool
pre_validation_results: List[PreValidationResult] = await pre_validate_blocks_multiprocessing(
constants, full_node.blockchain, blocks, pool, {}, validate_signatures=True
)
assert pre_validation_results is not None and len(pre_validation_results) == len(blocks)
for i in range(len(blocks)):
Expand Down
27 changes: 1 addition & 26 deletions chia/consensus/blockchain.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,7 @@
from chia.consensus.find_fork_point import lookup_fork_chain
from chia.consensus.full_block_to_block_record import block_to_block_record
from chia.consensus.get_block_generator import get_block_generator
from chia.consensus.multiprocess_validation import (
PreValidationResult,
_run_generator,
pre_validate_blocks_multiprocessing,
)
from chia.consensus.multiprocess_validation import PreValidationResult, _run_generator
from chia.full_node.block_height_map import BlockHeightMap
from chia.full_node.block_store import BlockStore
from chia.full_node.coin_store import CoinStore
Expand Down Expand Up @@ -800,27 +796,6 @@ async def validate_unfinished_block(

return PreValidationResult(None, required_iters, cost_result, False, uint32(0))

async def pre_validate_blocks_multiprocessing(
self,
blocks: List[FullBlock],
npc_results: Dict[uint32, NPCResult], # A cache of the result of running CLVM, optional (you can use {})
batch_size: int = 4,
wp_summaries: Optional[List[SubEpochSummary]] = None,
*,
validate_signatures: bool,
) -> List[PreValidationResult]:
return await pre_validate_blocks_multiprocessing(
self.constants,
self,
blocks,
self.pool,
True,
npc_results,
batch_size,
wp_summaries,
validate_signatures=validate_signatures,
)

async def run_generator(self, unfinished_block: bytes, generator: BlockGenerator, height: uint32) -> NPCResult:
task = asyncio.get_running_loop().run_in_executor(
self.pool,
Expand Down
9 changes: 3 additions & 6 deletions chia/consensus/multiprocess_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ def batch_pre_validate_blocks(
full_blocks_pickled: List[bytes],
prev_transaction_generators: List[Optional[List[bytes]]],
npc_results: Dict[uint32, bytes],
check_filter: bool,
expected_difficulty: List[uint64],
expected_sub_slot_iters: List[uint64],
validate_signatures: bool,
Expand Down Expand Up @@ -108,7 +107,7 @@ def batch_pre_validate_blocks(
constants,
BlockCache(blocks),
header_block,
check_filter,
True, # check_filter
expected_difficulty[i],
expected_sub_slot_iters[i],
)
Expand Down Expand Up @@ -158,9 +157,7 @@ async def pre_validate_blocks_multiprocessing(
block_records: BlocksProtocol,
blocks: Sequence[FullBlock],
pool: Executor,
check_filter: bool,
npc_results: Dict[uint32, NPCResult],
batch_size: int,
wp_summaries: Optional[List[SubEpochSummary]] = None,
*,
validate_signatures: bool = True,
Expand All @@ -171,7 +168,6 @@ async def pre_validate_blocks_multiprocessing(
if any validation issue occurs, returns False.
Args:
check_filter:
constants:
pool:
constants:
Expand Down Expand Up @@ -286,6 +282,8 @@ async def pre_validate_blocks_multiprocessing(
futures = []
# Pool of workers to validate blocks concurrently
recent_blocks_bytes = {bytes(k): bytes(v) for k, v in recent_blocks.items()} # convert to bytes

batch_size = 4
for i in range(0, len(blocks), batch_size):
end_i = min(i + batch_size, len(blocks))
blocks_to_validate = blocks[i:end_i]
Expand Down Expand Up @@ -329,7 +327,6 @@ async def pre_validate_blocks_multiprocessing(
b_pickled,
previous_generators,
npc_results_pickled,
check_filter,
[diff_ssis[j][0] for j in range(i, end_i)],
[diff_ssis[j][1] for j in range(i, end_i)],
validate_signatures,
Expand Down
23 changes: 18 additions & 5 deletions chia/full_node/full_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty
from chia.consensus.get_block_generator import get_block_generator
from chia.consensus.make_sub_epoch_summary import next_sub_epoch_summary
from chia.consensus.multiprocess_validation import PreValidationResult
from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_blocks_multiprocessing
from chia.consensus.pot_iterations import calculate_sp_iters
from chia.full_node.block_store import BlockStore
from chia.full_node.coin_store import CoinStore
Expand Down Expand Up @@ -1305,8 +1305,15 @@ async def add_block_batch(
# Validates signatures in multiprocessing since they take a while, and we don't have cached transactions
# for these blocks (unlike during normal operation where we validate one at a time)
pre_validate_start = time.monotonic()
pre_validation_results: List[PreValidationResult] = await self.blockchain.pre_validate_blocks_multiprocessing(
blocks_to_validate, {}, wp_summaries=wp_summaries, validate_signatures=True
pool = self.blockchain.pool
pre_validation_results: List[PreValidationResult] = await pre_validate_blocks_multiprocessing(
self.constants,
self.blockchain,
blocks_to_validate,
pool,
{},
wp_summaries=wp_summaries,
validate_signatures=True,
)
pre_validate_end = time.monotonic()
pre_validate_time = pre_validate_end - pre_validate_start
Expand Down Expand Up @@ -1752,8 +1759,14 @@ async def add_block(

# Don't validate signatures because we want to validate them in the main thread later, since we have a
# cache available
pre_validation_results = await self.blockchain.pre_validate_blocks_multiprocessing(
[block], npc_results, validate_signatures=False
pool = self.blockchain.pool
pre_validation_results = await pre_validate_blocks_multiprocessing(
self.constants,
self.blockchain,
[block],
pool,
npc_results,
validate_signatures=False,
)
added: Optional[AddBlockResult] = None
pre_validation_time = time.monotonic() - validation_start
Expand Down
Loading

0 comments on commit 76e5f77

Please sign in to comment.