Some ETH Protocol changes to prepare the room for eth/66

newsfragments/1672.internal.rst

@@ -0,0 +1,2 @@
+Improve test coverage of Ethereum Wire Protocol making sure that certain tests ran across
+all supported protocols from ETH/63 to ETH/65.

p2p/exchange/abc.py

@@ -38,9 +38,8 @@ class NormalizerAPI(ABC, Generic[TResponseCommand, TResult]):
     # a thread to ensure it doesn't block the main loop.
     is_normalization_slow: bool
 
-    @staticmethod
     @abstractmethod
-    def normalize_result(message: TResponseCommand) -> TResult:
+    def normalize_result(self, message: TResponseCommand) -> TResult:
         """
         Convert underlying peer message to final result
         """

p2p/exchange/normalizers.py

@@ -1,6 +1,34 @@
+from typing import Type, Callable
+
 from .abc import NormalizerAPI
 from .typing import TResponseCommand, TResult
 
 
 class BaseNormalizer(NormalizerAPI[TResponseCommand, TResult]):
     is_normalization_slow = False
+
+
+def _pick_payload(cmd: TResponseCommand) -> TResult:
+    return cmd.payload
+
+
+class DefaultNormalizer(NormalizerAPI[TResponseCommand, TResult]):
+    """
+    A normalizer that directly delegates to the payload of the response command.
+    """
+
+    def __init__(self,
+                 command_type: Type[TResponseCommand],
+                 payload_type: Type[TResult],
+                 is_normalization_slow: bool = False,
+                 normalize_fn: Callable[[TResponseCommand], TResult] = _pick_payload):
+        self.is_normalization_slow = is_normalization_slow
+        self.normalize_fn = normalize_fn
+
+        # The constructor allows mypy to defer typehints automatically from something like:
+        # normalizer = DefaultNormalizer(BlockHeadersV66, Tuple[BlockHeaderAPI, ...])
+        # instead of the slightly longer form:
+        # normalizer: DefaultNormalizer[BlockHeadersV66, Tuple[BlockHeaderAPI, ...]] = DefaultNormalizer() # noqa: E501
+
+    def normalize_result(self, cmd: TResponseCommand) -> TResult:
+        return self.normalize_fn(cmd)

tests/core/json-rpc/test_ipc.py

@@ -36,7 +36,7 @@
     SyncProgress
 )
 from trinity.tools.event_bus import mock_request_response
-from trinity.tools.factories import ETHPeerPairFactory
+from trinity.tools.factories import LatestETHPeerPairFactory
 
 from trinity._utils.version import construct_trinity_client_identifier
 
@@ -657,7 +657,7 @@ async def test_admin_peers(
         event_bus,
         ipc_server):
 
-    async with ETHPeerPairFactory() as (alice, bob):
+    async with LatestETHPeerPairFactory() as (alice, bob):
         peer_pool = MockPeerPoolWithConnectedPeers([alice, bob], event_bus=event_bus)
 
         async with run_peer_pool_event_server(event_bus, peer_pool):

tests/core/p2p-proto/test_eth_api.py

@@ -17,8 +17,8 @@
 from trinity.exceptions import WrongForkIDFailure
 from trinity.protocol.eth.api import ETHV65API, ETHV63API, ETHV64API
 from trinity.protocol.eth.commands import (
-    GetBlockHeaders,
-    GetNodeData,
+    GetBlockHeadersV65,
+    GetNodeDataV65,
     NewBlock,
     Status,
     StatusV63,
@@ -40,9 +40,8 @@
 from trinity.tools.factories import (
     BlockHashFactory,
     ChainContextFactory,
-    ETHPeerPairFactory,
-    ETHV63PeerPairFactory,
-    ETHV64PeerPairFactory,
+    LatestETHPeerPairFactory,
+    ALL_PEER_PAIR_FACTORIES,
 )
 
 
@@ -85,7 +84,9 @@ def alice_chain_on_fork(bob_chain):
     return chain
 
 
-@pytest.fixture(params=(ETHV63PeerPairFactory, ETHV64PeerPairFactory, ETHPeerPairFactory))
+@pytest.fixture(
+    params=ALL_PEER_PAIR_FACTORIES
+)
 async def alice_and_bob(alice_chain, bob_chain, request):
     pair_factory = request.param(
         alice_client_version='alice',
@@ -231,11 +232,11 @@ async def test_eth_api_send_get_node_data(alice, bob):
     async def _handle_cmd(connection, cmd):
         command_fut.set_result(cmd)
 
-    bob.connection.add_command_handler(GetNodeData, _handle_cmd)
+    bob.connection.add_command_handler(GetNodeDataV65, _handle_cmd)
     alice.eth_api.send_get_node_data(payload)
 
     result = await asyncio.wait_for(command_fut, timeout=1)
-    assert isinstance(result, GetNodeData)
+    assert isinstance(result, GetNodeDataV65)
     assert_type_equality(payload, result.payload)
 
 
@@ -248,7 +249,7 @@ async def test_eth_api_send_get_block_headers(alice, bob):
     async def _handle_cmd(connection, cmd):
         command_fut.set_result(cmd)
 
-    bob.connection.add_command_handler(GetBlockHeaders, _handle_cmd)
+    bob.connection.add_command_handler(GetBlockHeadersV65, _handle_cmd)
     alice.eth_api.send_get_block_headers(
         block_number_or_hash=payload.block_number_or_hash,
         max_headers=payload.block_number_or_hash,
@@ -257,7 +258,7 @@ async def _handle_cmd(connection, cmd):
     )
 
     result = await asyncio.wait_for(command_fut, timeout=1)
-    assert isinstance(result, GetBlockHeaders)
+    assert isinstance(result, GetBlockHeadersV65)
     assert_type_equality(payload, result.payload)
 
 
@@ -269,7 +270,7 @@ async def test_handshake_with_incompatible_fork_id(alice_chain, bob_chain):
         mine_block()
     )
 
-    pair_factory = ETHPeerPairFactory(
+    pair_factory = LatestETHPeerPairFactory(
         alice_peer_context=ChainContextFactory(
             headerdb=AsyncHeaderDB(alice_chain.headerdb.db),
             vm_configuration=((1, PetersburgVM), (2, MuirGlacierVM))

tests/core/p2p-proto/test_eth_proto.py

@@ -4,16 +4,16 @@
 
 from trinity._utils.assertions import assert_type_equality
 from trinity.protocol.eth.commands import (
-    BlockBodies,
-    BlockHeaders,
-    GetBlockBodies,
-    GetBlockHeaders,
-    GetNodeData,
-    GetReceipts,
+    BlockBodiesV65,
+    BlockHeadersV65,
+    GetBlockBodiesV65,
+    GetBlockHeadersV65,
+    GetNodeDataV65,
+    GetReceiptsV65,
     NewBlock,
     NewBlockHashes,
-    NodeData,
-    Receipts,
+    NodeDataV65,
+    ReceiptsV65,
     Status,
     Transactions,
     StatusV63,
@@ -44,16 +44,19 @@
         (Status, StatusPayloadFactory()),
         (NewBlockHashes, tuple(NewBlockHashFactory.create_batch(2))),
         (Transactions, tuple(BaseTransactionFieldsFactory.create_batch(2))),
-        (GetBlockHeaders, BlockHeadersQueryFactory()),
-        (GetBlockHeaders, BlockHeadersQueryFactory(block_number_or_hash=BlockHashFactory())),
-        (BlockHeaders, tuple(BlockHeaderFactory.create_batch(2))),
-        (GetBlockBodies, tuple(BlockHashFactory.create_batch(2))),
-        (BlockBodies, tuple(BlockBodyFactory.create_batch(2))),
+        (GetBlockHeadersV65, BlockHeadersQueryFactory()),
+        (GetBlockHeadersV65, BlockHeadersQueryFactory(block_number_or_hash=BlockHashFactory())),
+        (BlockHeadersV65, tuple(BlockHeaderFactory.create_batch(2))),
+        (GetBlockBodiesV65, tuple(BlockHashFactory.create_batch(2))),
+        (BlockBodiesV65, tuple(BlockBodyFactory.create_batch(2))),
         (NewBlock, NewBlockPayloadFactory()),
-        (GetNodeData, tuple(BlockHashFactory.create_batch(2))),
-        (NodeData, (secrets.token_bytes(10), secrets.token_bytes(100))),
-        (GetReceipts, tuple(BlockHashFactory.create_batch(2))),
-        (Receipts, (tuple(ReceiptFactory.create_batch(2)), tuple(ReceiptFactory.create_batch(3)))),
+        (GetNodeDataV65, tuple(BlockHashFactory.create_batch(2))),
+        (NodeDataV65, (secrets.token_bytes(10), secrets.token_bytes(100))),
+        (GetReceiptsV65, tuple(BlockHashFactory.create_batch(2))),
+        (
+            ReceiptsV65,
+            (tuple(ReceiptFactory.create_batch(2)), tuple(ReceiptFactory.create_batch(3)))
+        ),
     ),
 )
 @pytest.mark.parametrize(

tests/core/p2p-proto/test_peer.py

@@ -16,7 +16,7 @@
 )
 
 from trinity.tools.factories import (
-    ETHPeerPairFactory,
+    LatestETHPeerPairFactory,
     LESV1PeerPairFactory,
     LESV2PeerPairFactory,
 )
@@ -59,7 +59,7 @@ async def test_ETH_v63_peers():
 
 @pytest.mark.asyncio
 async def test_ETH_peers():
-    async with ETHPeerPairFactory() as (alice, bob):
+    async with LatestETHPeerPairFactory() as (alice, bob):
         assert isinstance(alice, ETHPeer)
         assert isinstance(bob, ETHPeer)
 
@@ -69,9 +69,9 @@ async def test_ETH_peers():
 
 @pytest.mark.asyncio
 async def test_peer_pool_iter(event_loop):
-    factory_a = ETHPeerPairFactory()
-    factory_b = ETHPeerPairFactory()
-    factory_c = ETHPeerPairFactory()
+    factory_a = LatestETHPeerPairFactory()
+    factory_b = LatestETHPeerPairFactory()
+    factory_c = LatestETHPeerPairFactory()
     async with factory_a as (peer1, _), factory_b as (peer2, _), factory_c as (peer3, _):
         pool = MockPeerPoolWithConnectedPeers([peer1, peer2, peer3])
         peers = list([peer async for peer in pool])
@@ -100,7 +100,7 @@ async def validate_remote_dao_fork_block():
         nonlocal dao_fork_validator_called
         dao_fork_validator_called = True
 
-    async with ETHPeerPairFactory() as (alice, _):
+    async with LatestETHPeerPairFactory() as (alice, _):
         boot_manager = alice.get_boot_manager()
         monkeypatch.setattr(
             boot_manager, 'validate_remote_dao_fork_block', validate_remote_dao_fork_block)

tests/core/p2p-proto/test_peer_block_body_validator_api.py

@@ -19,7 +19,7 @@
 
 from trinity.rlp.block_body import BlockBody
 
-from trinity.tools.factories import ETHPeerPairFactory
+from trinity.tools.factories import LatestETHPeerPairFactory
 
 
 def mk_uncle(block_number):
@@ -75,7 +75,7 @@ def mk_headers(*counts):
 
 @pytest.fixture
 async def eth_peer_and_remote():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         yield peer, remote
 
 

tests/core/p2p-proto/test_peer_block_header_validator_api.py

@@ -10,7 +10,7 @@
 from trinity.protocol.les.commands import GetBlockHeaders
 
 from trinity.tools.factories import (
-    ETHPeerPairFactory,
+    LatestETHPeerPairFactory,
     LESV1PeerPairFactory,
     LESV2PeerPairFactory,
 )
@@ -47,7 +47,7 @@ def mk_header_chain(length):
 
 @pytest.fixture
 async def eth_peer_and_remote():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         yield peer, remote
 
 

tests/core/p2p-proto/test_peer_node_data_validator_api.py

@@ -8,12 +8,12 @@
     keccak,
 )
 
-from trinity.tools.factories import ETHPeerPairFactory
+from trinity.tools.factories import LatestETHPeerPairFactory
 
 
 @pytest.fixture
 async def eth_peer_and_remote():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         yield peer, remote
 
 

tests/core/p2p-proto/test_peer_receipts_validator_api.py

@@ -10,12 +10,12 @@
 from eth.rlp.headers import BlockHeader
 from eth.rlp.receipts import Receipt
 
-from trinity.tools.factories import ETHPeerPairFactory
+from trinity.tools.factories import LatestETHPeerPairFactory
 
 
 @pytest.fixture
 async def eth_peer_and_remote():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         yield peer, remote
 
 

tests/core/p2p-proto/test_requests.py

@@ -16,11 +16,12 @@
     ETHProxyPeerPool,
     ETHPeerPoolEventServer,
 )
+from trinity.protocol.eth.proto import ETHProtocolV65
 from trinity.protocol.eth.servers import ETHRequestServer
 
 from trinity.tools.factories import (
     ChainContextFactory,
-    ETHPeerPairFactory,
+    ALL_PEER_PAIR_FACTORIES,
 )
 
 from tests.core.integration_test_helpers import (
@@ -31,9 +32,18 @@
 )
 
 
-@pytest.fixture
-async def client_and_server(chaindb_fresh, chaindb_20):
-    peer_pair = ETHPeerPairFactory(
+def get_highest_eth_protocol_version(client):
+    return max(
+        protocol.as_capability()[1] for protocol in client.connection.get_protocols()
+        if protocol.as_capability()[0] == 'eth'
+    )
+
+
+@pytest.fixture(
+    params=ALL_PEER_PAIR_FACTORIES
+)
+async def client_and_server(chaindb_fresh, chaindb_20, request):
+    peer_pair = request.param(
         alice_peer_context=ChainContextFactory(headerdb__db=chaindb_fresh.db),
         bob_peer_context=ChainContextFactory(headerdb__db=chaindb_20.db),
     )
@@ -109,6 +119,9 @@ async def test_get_pooled_transactions_request(request,
     client_event_bus = other_event_bus
     client_peer, server_peer = client_and_server
 
+    if get_highest_eth_protocol_version(client_peer) < ETHProtocolV65.version:
+        pytest.skip("Test not applicable below eth/65")
+
     client_peer_pool = MockPeerPoolWithConnectedPeers([client_peer], event_bus=client_event_bus)
     server_peer_pool = MockPeerPoolWithConnectedPeers([server_peer], event_bus=server_event_bus)
 

tests/core/p2p-proto/test_stats.py

@@ -9,7 +9,7 @@
 
 from trinity.protocol.les.commands import GetBlockHeaders
 
-from trinity.tools.factories import ETHPeerPairFactory, LESV2PeerPairFactory
+from trinity.tools.factories import LatestETHPeerPairFactory, LESV2PeerPairFactory
 
 
 class RequestIDMonitor(PeerSubscriber):
@@ -43,15 +43,15 @@ def mk_header_chain(length):
 
 @pytest.mark.asyncio
 async def test_eth_get_headers_empty_stats():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         stats = peer.eth_api.get_extra_stats()
         assert all('None' in line for line in stats)
         assert any('BlockHeader' in line for line in stats)
 
 
 @pytest.mark.asyncio
 async def test_eth_get_headers_stats():
-    async with ETHPeerPairFactory() as (peer, remote):
+    async with LatestETHPeerPairFactory() as (peer, remote):
         async def send_headers():
             remote.eth_api.send_block_headers(mk_header_chain(1))