From d3f018fde839fa7ee3fb5eeb14a9eb7e34870636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 28 Jun 2021 18:40:56 +0300 Subject: [PATCH 01/60] eth: drop eth/65, the last non-reqid protocol version --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 +--------- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 ++-- eth/protocols/eth/handler.go | 47 ++--- eth/protocols/eth/handler_test.go | 134 +++++--------- eth/protocols/eth/handlers.go | 107 ----------- eth/protocols/eth/handshake_test.go | 1 - eth/protocols/eth/peer.go | 208 +++++++--------------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 +---------- eth/sync_test.go | 1 - les/client_handler.go | 2 +- 15 files changed, 179 insertions(+), 609 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6e3217151a52..6d14404e6624 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.AllEthTests() { + for _, test := range suite.Eth66Tests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index e8a4a76ca21f..f0fab1b7a958 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH65 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) + if p.version < eth.ETH66 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 794160993bb0..17cd3630c98e 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,10 +522,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } - func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -549,9 +545,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } -func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } - func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -634,10 +627,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } -func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } -func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } - func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -667,10 +656,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } -func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } -func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } - func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -702,10 +687,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } -func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } -func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } - func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -736,16 +717,6 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync65Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) -} -func TestBoundedHeavyForkedSync65Fast(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) -} -func TestBoundedHeavyForkedSync65Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) -} - func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -800,10 +771,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } -func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } -func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } - func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -833,10 +800,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } -func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } -func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } - func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -863,10 +826,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } -func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } -func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } - func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -881,8 +840,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) + //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -891,7 +850,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{65, 66} { + for _, version := range []int{66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -901,10 +860,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } -func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } -func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } - func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -955,10 +910,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } -func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } -func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } - func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -987,10 +938,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } -func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } -func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } - func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -1024,7 +971,6 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1115,16 +1061,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack65Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FullSync) -} -func TestHighTDStarvationAttack65Fast(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FastSync) -} -func TestHighTDStarvationAttack65Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, LightSync) -} - func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1149,7 +1085,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1202,10 +1137,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } -func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } -func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } - func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1286,10 +1217,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } -func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } -func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } - func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1362,10 +1289,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } -func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } -func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } - func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1435,10 +1358,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } -func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } -func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } - func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1512,10 +1431,6 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } -func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } -func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } - func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1673,12 +1588,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } -func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } -func TestCheckpointEnforcement65Light(t *testing.T) { - testCheckpointEnforcement(t, eth.ETH65, LightSync) -} - func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 066a366315a7..863294832971 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index aff4871afa42..06a8088bf07c 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,7 +117,6 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop - txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -140,7 +139,6 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, - txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -408,9 +406,8 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(2) + h.wg.Add(1) go h.chainSync.loop() - go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 038de469908e..039091244fec 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,7 +80,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +235,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +292,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -306,7 +303,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -380,7 +377,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -521,8 +517,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -543,30 +539,39 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } - // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { - t.Fatalf("challenge mismatch: %v", err) + msg, err := p2pRemote.ReadMsg() + if err != nil { + t.Fatalf("failed to read checkpoint challenge: %v", err) + } + request := new(eth.GetBlockHeadersPacket66) + if err := msg.Decode(request); err != nil { + t.Fatalf("failed to decode checkpoint challenge: %v", err) + } + query := request.GetBlockHeadersPacket + if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { + t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", + query.Origin.Number, query.Amount, query.Skip, query.Reverse, + response.Number.Uint64(), 1, 0, false) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } - // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -619,8 +624,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -671,7 +676,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 6bbaa2f555f1..8289300141f0 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,39 +171,21 @@ type Decoder interface { Time() time.Time } -var eth65 = map[uint64]msgHandler{ - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - // eth66 messages with request-id - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -219,10 +201,11 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth65 - if peer.Version() >= ETH66 { - handlers = eth66 - } + var handlers = eth66 + //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added + // handlers = eth67 + //} + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 473be3f9b77e..809f17e36cf5 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,7 +110,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -254,44 +253,30 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } } } @@ -299,7 +284,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -369,28 +353,20 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockBodiesMsg, hashes) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) } } } // Tests that the state trie nodes can be retrieved based on hashes. -func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -449,14 +425,10 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - if protocol <= ETH65 { - p2p.Send(peer.app, GetNodeDataMsg, hashes) - } else { - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) - } + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -464,18 +436,14 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var data [][]byte - if protocol <= ETH65 { - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - } else { - var res NodeDataPacket66 - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - data = res.NodeDataPacket + var ( + data [][]byte + res NodeDataPacket66 + ) + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) } + data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -506,7 +474,6 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -566,21 +533,14 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetReceiptsMsg, hashes) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) - } - } else { - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) - } + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index d7d993a23dee..e54838cbc56e 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,17 +27,6 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// handleGetBlockHeaders handles Block header query, collect the requested headers and reply -func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // Decode the complex header query - var query GetBlockHeadersPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockHeadersQuery(backend, &query, peer) - return peer.SendBlockHeaders(response) -} - // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -135,16 +124,6 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } -func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block body retrieval message - var query GetBlockBodiesPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockBodiesQuery(backend, query, peer) - return peer.SendBlockBodiesRLP(response) -} - func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -174,16 +153,6 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } -func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { - // Decode the trie node data retrieval message - var query GetNodeDataPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetNodeDataQuery(backend, query, peer) - return peer.SendNodeData(response) -} - func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -223,16 +192,6 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } -func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetReceiptsQuery(backend, query, peer) - return peer.SendReceiptsRLP(response) -} - func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -312,15 +271,6 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } -func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // A batch of headers arrived to one of our previous requests - res := new(BlockHeadersPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -332,15 +282,6 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } -func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // A batch of block bodies arrived to one of our previous requests - res := new(BlockBodiesPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -352,15 +293,6 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } -func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { - // A batch of node state data arrived to one of our previous requests - res := new(NodeDataPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -372,15 +304,6 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } -func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { - // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -409,16 +332,6 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } -func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Decode the pooled transactions retrieval message - var query GetPooledTransactionsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - hashes, txs := answerGetPooledTransactions(backend, query, peer) - return peer.SendPooledTransactionsRLP(hashes, txs) -} - func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -477,26 +390,6 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } -func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("%w: transaction %d is nil", errDecode, i) - } - peer.markTransaction(tx.Hash()) - } - return backend.Handle(peer, &txs) -} - func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 3bebda2dcc9b..05d473e05795 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index e619c183ba2f..98273ccfc9dc 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,9 +108,8 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - if version >= ETH65 { - go peer.announceTransactions() - } + go peer.announceTransactions() + return peer } @@ -252,22 +251,6 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } -// SendPooledTransactionsRLP sends requested transactions to the peer and adds the -// hashes in its transaction hash set for future reference. -// -// Note, the method assumes the hashes are correct and correspond to the list of -// transactions being sent. -func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding -} - // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -346,11 +329,6 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *Peer) SendBlockHeaders(headers []*types.Header) error { - return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) -} - // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -359,12 +337,6 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from -// an already RLP encoded format. -func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { - return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding -} - // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -374,12 +346,6 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } -// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the -// hashes requested. -func (p *Peer) SendNodeData(data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) -} - // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -388,12 +354,6 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the -// ones requested from an already RLP encoded format. -func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding -} - // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -406,138 +366,102 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + }, + }) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) -} - -// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side -// of the RequestHeadersByNumber operation. -func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - req := &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) - } - return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) - } - return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index de1b0ed1ee7f..3c3da30fa503 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH65 = 65 ETH66 = 66 ) @@ -40,31 +39,28 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65} +var ProtocolVersions = []uint{ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} +var protocolLengths = map[uint]uint64{ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - // Protocol messages in eth/64 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 - - // Protocol messages overloaded in eth/65 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -128,7 +124,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket represents a block header query over eth/66 +// GetBlockHeadersPacket66 represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index ab114b59f3e1..27941158f3df 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,7 +18,6 @@ package eth import ( "math/big" - "math/rand" "sync/atomic" "time" @@ -28,23 +27,13 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing - - // This is the target size for the packs of transactions sent by txsyncLoop64. - // A pack can get larger than this if a single transactions exceeds this size. - txsyncPackSize = 100 * 1024 ) -type txsync struct { - p *eth.Peer - txs []*types.Transaction -} - // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -64,94 +53,11 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - if p.Version() >= eth.ETH65 { - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() - } - p.AsyncSendPooledTransactionHashes(hashes) - return - } - // Out of luck, peer is running legacy protocols, drop the txs over - select { - case h.txsyncCh <- &txsync{p: p, txs: txs}: - case <-h.quitSync: - } -} - -// txsyncLoop64 takes care of the initial transaction sync for each new -// connection. When a new peer appears, we relay all currently pending -// transactions. In order to minimise egress bandwidth usage, we send -// the transactions in small packs to one peer at a time. -func (h *handler) txsyncLoop64() { - defer h.wg.Done() - - var ( - pending = make(map[enode.ID]*txsync) - sending = false // whether a send is active - pack = new(txsync) // the pack that is being sent - done = make(chan error, 1) // result of the send - ) - - // send starts a sending a pack of transactions from the sync. - send := func(s *txsync) { - if s.p.Version() >= eth.ETH65 { - panic("initial transaction syncer running on eth/65+") - } - // Fill pack with transactions up to the target size. - size := common.StorageSize(0) - pack.p = s.p - pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.Peer.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() - } - // pick chooses the next pending sync. - pick := func() *txsync { - if len(pending) == 0 { - return nil - } - n := rand.Intn(len(pending)) + 1 - for _, s := range pending { - if n--; n == 0 { - return s - } - } - return nil - } - - for { - select { - case s := <-h.txsyncCh: - pending[s.p.Peer.ID()] = s - if !sending { - send(s) - } - case err := <-done: - sending = false - // Stop tracking peers that cause send failures. - if err != nil { - pack.p.Log().Debug("Transaction send failed", "err", err) - delete(pending, pack.p.Peer.ID()) - } - // Schedule the next send. - if s := pick(); s != nil { - send(s) - } - case <-h.quitSync: - return - } + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() } + p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index a0c6f8602327..e96b9ee81f69 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. -func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index e95996c51f6f..b903b11065c8 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From a1f16bc74c7efb593db2982c92222d1e4a201c25 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 22 Jul 2021 16:45:22 +0200 Subject: [PATCH 02/60] params: begin v1.10.7 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 8d3804a0cd73..61fb82dda719 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From bbfa6488ac9e882d30c58b1b380cc894c8ef8a58 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 27 Jul 2021 05:27:28 +0200 Subject: [PATCH 03/60] Use `hexutil.Uint` for `blockCount` parameter in `feeHistory` method (#23239) * internal/ethapi/api: use hexutil.uint for blockCount parameter instead of int for feeHistory * return hex value for oldestBlock instead of number * return uint64 from oracle.resolveBlockRange * eth/gasprice: fixed test Co-authored-by: Zsolt Felfoldi --- eth/api_backend.go | 2 +- eth/gasprice/feehistory.go | 37 +++++++++++++++++---------------- eth/gasprice/feehistory_test.go | 4 ++-- internal/ethapi/api.go | 8 +++---- internal/ethapi/backend.go | 2 +- les/api_backend.go | 2 +- 6 files changed, 28 insertions(+), 27 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index f22462c7c88a..49de70e21069 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -287,7 +287,7 @@ func (b *EthAPIBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) return b.gpo.SuggestTipCap(ctx) } -func (b *EthAPIBackend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (firstBlock rpc.BlockNumber, reward [][]*big.Int, baseFee []*big.Int, gasUsedRatio []float64, err error) { +func (b *EthAPIBackend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (firstBlock *big.Int, reward [][]*big.Int, baseFee []*big.Int, gasUsedRatio []float64, err error) { return b.gpo.FeeHistory(ctx, blockCount, lastBlock, rewardPercentiles) } diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index a14dd594be04..2c4486c6f4c2 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -24,6 +24,7 @@ import ( "sort" "sync/atomic" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -48,7 +49,7 @@ const ( // blockFees represents a single block for processing type blockFees struct { // set by the caller - blockNumber rpc.BlockNumber + blockNumber uint64 header *types.Header block *types.Block // only set if reward percentiles are requested receipts types.Receipts @@ -133,7 +134,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, rpc.BlockNumber, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( headBlock rpc.BlockNumber pendingBlock *types.Block @@ -181,7 +182,7 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block if rpc.BlockNumber(blocks) > lastBlock+1 { blocks = int(lastBlock + 1) } - return pendingBlock, pendingReceipts, lastBlock, blocks, nil + return pendingBlock, pendingReceipts, uint64(lastBlock), blocks, nil } // FeeHistory returns data relevant for fee estimation based on the specified range of blocks. @@ -197,9 +198,9 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block // - gasUsedRatio: gasUsed/gasLimit in the given block // Note: baseFee includes the next block after the newest of the returned range, because this // value can be derived from the newest block. -func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (rpc.BlockNumber, [][]*big.Int, []*big.Int, []float64, error) { +func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { if blocks < 1 { - return 0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks + return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks } if blocks > maxFeeHistory { log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory) @@ -207,10 +208,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc. } for i, p := range rewardPercentiles { if p < 0 || p > 100 { - return 0, nil, nil, nil, fmt.Errorf("%w: %f", errInvalidPercentile, p) + return common.Big0, nil, nil, nil, fmt.Errorf("%w: %f", errInvalidPercentile, p) } if i > 0 && p < rewardPercentiles[i-1] { - return 0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) + return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } // Only process blocks if reward percentiles were requested @@ -223,36 +224,36 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc. pendingReceipts []*types.Receipt err error ) - pendingBlock, pendingReceipts, lastBlock, blocks, err = oracle.resolveBlockRange(ctx, lastBlock, blocks, maxHistory) + pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory) if err != nil || blocks == 0 { - return 0, nil, nil, nil, err + return common.Big0, nil, nil, nil, err } - oldestBlock := lastBlock + 1 - rpc.BlockNumber(blocks) + oldestBlock := lastBlock + 1 - uint64(blocks) var ( - next = int64(oldestBlock) + next = oldestBlock results = make(chan *blockFees, blocks) ) for i := 0; i < maxBlockFetchers && i < blocks; i++ { go func() { for { // Retrieve the next block number to fetch with this goroutine - blockNumber := rpc.BlockNumber(atomic.AddInt64(&next, 1) - 1) + blockNumber := atomic.AddUint64(&next, 1) - 1 if blockNumber > lastBlock { return } fees := &blockFees{blockNumber: blockNumber} - if pendingBlock != nil && blockNumber >= rpc.BlockNumber(pendingBlock.NumberU64()) { + if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() { fees.block, fees.receipts = pendingBlock, pendingReceipts } else { if len(rewardPercentiles) != 0 { - fees.block, fees.err = oracle.backend.BlockByNumber(ctx, blockNumber) + fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) if fees.block != nil && fees.err == nil { fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) } } else { - fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, blockNumber) + fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) } } if fees.block != nil { @@ -275,7 +276,7 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc. for ; blocks > 0; blocks-- { fees := <-results if fees.err != nil { - return 0, nil, nil, nil, fees.err + return common.Big0, nil, nil, nil, fees.err } i := int(fees.blockNumber - oldestBlock) if fees.header != nil { @@ -288,7 +289,7 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc. } } if firstMissing == 0 { - return 0, nil, nil, nil, nil + return common.Big0, nil, nil, nil, nil } if len(rewardPercentiles) != 0 { reward = reward[:firstMissing] @@ -296,5 +297,5 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, lastBlock rpc. reward = nil } baseFee, gasUsedRatio = baseFee[:firstMissing+1], gasUsedRatio[:firstMissing] - return oldestBlock, reward, baseFee, gasUsedRatio, nil + return new(big.Int).SetUint64(oldestBlock), reward, baseFee, gasUsedRatio, nil } diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index 57cfb260c410..16c74b7db46a 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -32,7 +32,7 @@ func TestFeeHistory(t *testing.T) { count int last rpc.BlockNumber percent []float64 - expFirst rpc.BlockNumber + expFirst uint64 expCount int expErr error }{ @@ -70,7 +70,7 @@ func TestFeeHistory(t *testing.T) { expBaseFee++ } - if first != c.expFirst { + if first.Uint64() != c.expFirst { t.Fatalf("Test case %d: first block mismatch, want %d, got %d", i, c.expFirst, first) } if len(reward) != expReward { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b65f98836c31..59555ea46307 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -81,19 +81,19 @@ func (s *PublicEthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*hexutil. } type feeHistoryResult struct { - OldestBlock rpc.BlockNumber `json:"oldestBlock"` + OldestBlock *hexutil.Big `json:"oldestBlock"` Reward [][]*hexutil.Big `json:"reward,omitempty"` BaseFee []*hexutil.Big `json:"baseFeePerGas,omitempty"` GasUsedRatio []float64 `json:"gasUsedRatio"` } -func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { - oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, blockCount, lastBlock, rewardPercentiles) +func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount hexutil.Uint, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { + oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { return nil, err } results := &feeHistoryResult{ - OldestBlock: oldest, + OldestBlock: (*hexutil.Big)(oldest), GasUsedRatio: gasUsed, } if reward != nil { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index fe55ec59c82a..995454582193 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -42,7 +42,7 @@ type Backend interface { // General Ethereum API Downloader() *downloader.Downloader SuggestGasTipCap(ctx context.Context) (*big.Int, error) - FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (rpc.BlockNumber, [][]*big.Int, []*big.Int, []float64, error) + FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) ChainDb() ethdb.Database AccountManager() *accounts.Manager ExtRPCEnabled() bool diff --git a/les/api_backend.go b/les/api_backend.go index 326b540b6f85..9c80270da0a1 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -269,7 +269,7 @@ func (b *LesApiBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) return b.gpo.SuggestTipCap(ctx) } -func (b *LesApiBackend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (firstBlock rpc.BlockNumber, reward [][]*big.Int, baseFee []*big.Int, gasUsedRatio []float64, err error) { +func (b *LesApiBackend) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (firstBlock *big.Int, reward [][]*big.Int, baseFee []*big.Int, gasUsedRatio []float64, err error) { return b.gpo.FeeHistory(ctx, blockCount, lastBlock, rewardPercentiles) } From 5c89ec9b989084dfc67dcaf81736067ebb8f078c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 27 Jul 2021 16:19:48 +0200 Subject: [PATCH 04/60] cmd/geth: update vulnerability testdata (#23252) --- cmd/geth/testdata/vcheck/vulnerabilities.json | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/cmd/geth/testdata/vcheck/vulnerabilities.json b/cmd/geth/testdata/vcheck/vulnerabilities.json index 36509f95a98a..92de0c9ccc70 100644 --- a/cmd/geth/testdata/vcheck/vulnerabilities.json +++ b/cmd/geth/testdata/vcheck/vulnerabilities.json @@ -52,13 +52,16 @@ "check": "Geth\\/v1\\.9\\.(7|8|9|10|11|12|13|14|15|16).*$" }, { - "name": "GethCrash", + "name": "Geth DoS via MULMOD", "uid": "GETH-2020-04", "summary": "A denial-of-service issue can be used to crash Geth nodes during block processing", - "description": "Full details to be disclosed at a later date", + "description": "Affected versions suffer from a vulnerability which can be exploited through the `MULMOD` operation, by specifying a modulo of `0`: `mulmod(a,b,0)`, causing a `panic` in the underlying library. \nThe crash was in the `uint256` library, where a buffer [underflowed](https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L442).\n\n\tif `d == 0`, `dLen` remains `0`\n\nand https://github.com/holiman/uint256/blob/4ce82e695c10ddad57215bdbeafb68b8c5df2c30/uint256.go#L451 will try to access index `[-1]`.\n\nThe `uint256` library was first merged in this [commit](https://github.com/ethereum/go-ethereum/commit/cf6674539c589f80031f3371a71c6a80addbe454), on 2020-06-08. \nExploiting this vulnerabilty would cause all vulnerable nodes to drop off the network. \n\nThe issue was brought to our attention through a [bug report](https://github.com/ethereum/go-ethereum/issues/21367), showing a `panic` occurring on sync from genesis on the Ropsten network.\n \nIt was estimated that the least obvious way to fix this would be to merge the fix into `uint256`, make a new release of that library and then update the geth-dependency.\n", "links": [ "https://blog.ethereum.org/2020/11/12/geth_security_release/", - "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m" + "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-jm5c-rv3w-w83m", + "https://github.com/holiman/uint256/releases/tag/v1.1.1", + "https://github.com/holiman/uint256/pull/80", + "https://github.com/ethereum/go-ethereum/pull/21368" ], "introduced": "v1.9.16", "fixed": "v1.9.18", @@ -66,5 +69,51 @@ "severity": "Critical", "CVE": "CVE-2020-26242", "check": "Geth\\/v1\\.9.(16|17).*$" + }, + { + "name": "LES Server DoS via GetProofsV2", + "uid": "GETH-2020-05", + "summary": "A DoS vulnerability can make a LES server crash.", + "description": "A DoS vulnerability can make a LES server crash via malicious GetProofsV2 request from a connected LES client.\n\nThe vulnerability was patched in #21896.\n\nThis vulnerability only concern users explicitly running geth as a light server", + "links": [ + "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-r33q-22hv-j29q", + "https://github.com/ethereum/go-ethereum/pull/21896" + ], + "introduced": "v1.8.0", + "fixed": "v1.9.25", + "published": "2020-12-10", + "severity": "Medium", + "CVE": "CVE-2020-26264", + "check": "(Geth\\/v1\\.8\\.*)|(Geth\\/v1\\.9\\.\\d-.*)|(Geth\\/v1\\.9\\.1\\d-.*)|(Geth\\/v1\\.9\\.(20|21|22|23|24)-.*)$" + }, + { + "name": "SELFDESTRUCT-recreate consensus flaw", + "uid": "GETH-2020-06", + "introduced": "v1.9.4", + "fixed": "v1.9.20", + "summary": "A consensus-vulnerability in Geth could cause a chain split, where vulnerable versions refuse to accept the canonical chain.", + "description": "A flaw was repoted at 2020-08-11 by John Youngseok Yang (Software Platform Lab), where a particular sequence of transactions could cause a consensus failure.\n\n- Tx 1:\n - `sender` invokes `caller`.\n - `caller` invokes `0xaa`. `0xaa` has 3 wei, does a self-destruct-to-self\n - `caller` does a `1 wei` -call to `0xaa`, who thereby has 1 wei (the code in `0xaa` still executed, since the tx is still ongoing, but doesn't redo the selfdestruct, it takes a different path if callvalue is non-zero)\n\n-Tx 2:\n - `sender` does a 5-wei call to 0xaa. No exec (since no code). \n\nIn geth, the result would be that `0xaa` had `6 wei`, whereas OE reported (correctly) `5` wei. Furthermore, in geth, if the second tx was not executed, the `0xaa` would be destructed, resulting in `0 wei`. Thus obviously wrong. \n\nIt was determined that the root cause was this [commit](https://github.com/ethereum/go-ethereum/commit/223b950944f494a5b4e0957fd9f92c48b09037ad) from [this PR](https://github.com/ethereum/go-ethereum/pull/19953). The semantics of `createObject` was subtly changd, into returning a non-nil object (with `deleted=true`) where it previously did not if the account had been destructed. This return value caused the new object to inherit the old `balance`.\n", + "links": [ + "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-xw37-57qp-9mm4" + ], + "published": "2020-12-10", + "severity": "High", + "CVE": "CVE-2020-26265", + "check": "(Geth\\/v1\\.9\\.(4|5|6|7|8|9)-.*)|(Geth\\/v1\\.9\\.1\\d-.*)$" + }, + { + "name": "Not ready for London upgrade", + "uid": "GETH-2021-01", + "summary": "The client is not ready for the 'London' technical upgrade, and will deviate from the canonical chain when the London upgrade occurs (at block '12965000' around August 4, 2021.", + "description": "At (or around) August 4, Ethereum will undergo a technical upgrade called 'London'. Clients not upgraded will fail to progress on the canonical chain.", + "links": [ + "https://github.com/ethereum/eth1.0-specs/blob/master/network-upgrades/mainnet-upgrades/london.md", + "https://notes.ethereum.org/@timbeiko/ropsten-postmortem" + ], + "introduced": "v1.10.1", + "fixed": "v1.10.6", + "published": "2020-12-10", + "severity": "High", + "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$" } ] From b20bc5c0cae6901209610c5f53f01401b6f7974e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 27 Jul 2021 16:22:21 +0200 Subject: [PATCH 05/60] accounts/abi/bind: parse ABI only once, create metadata struct (#22583) --- accounts/abi/bind/base.go | 25 +++++++++++++++++++++++++ accounts/abi/bind/template.go | 34 ++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index 274f6e4d946f..354632b257a5 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "math/big" + "strings" + "sync" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" @@ -76,6 +78,29 @@ type WatchOpts struct { Context context.Context // Network context to support cancellation and timeouts (nil = no timeout) } +// MetaData collects all metadata for a bound contract. +type MetaData struct { + mu sync.Mutex + Sigs map[string]string + Bin string + ABI string + ab *abi.ABI +} + +func (m *MetaData) GetAbi() (*abi.ABI, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.ab != nil { + return m.ab, nil + } + if parsed, err := abi.JSON(strings.NewReader(m.ABI)); err != nil { + return nil, err + } else { + m.ab = &parsed + } + return m.ab, nil +} + // BoundContract is the base wrapper object that reflects a contract on the // Ethereum network. It contains a collection of methods that are used by the // higher level contract bindings to operate. diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index e9bdd3d4149b..492bad8c577f 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -90,6 +90,7 @@ package {{.Package}} import ( "math/big" "strings" + "errors" ethereum "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" @@ -101,6 +102,7 @@ import ( // Reference imports to suppress errors if they are not otherwise used. var ( + _ = errors.New _ = big.NewInt _ = strings.NewReader _ = ethereum.NotFound @@ -120,32 +122,48 @@ var ( {{end}} {{range $contract := .Contracts}} + // {{.Type}}MetaData contains all meta data concerning the {{.Type}} contract. + var {{.Type}}MetaData = &bind.MetaData{ + ABI: "{{.InputABI}}", + {{if $contract.FuncSigs -}} + Sigs: map[string]string{ + {{range $strsig, $binsig := .FuncSigs}}"{{$binsig}}": "{{$strsig}}", + {{end}} + }, + {{end -}} + {{if .InputBin -}} + Bin: "0x{{.InputBin}}", + {{end}} + } // {{.Type}}ABI is the input ABI used to generate the binding from. - const {{.Type}}ABI = "{{.InputABI}}" + // Deprecated: Use {{.Type}}MetaData.ABI instead. + var {{.Type}}ABI = {{.Type}}MetaData.ABI {{if $contract.FuncSigs}} + // Deprecated: Use {{.Type}}MetaData.Sigs instead. // {{.Type}}FuncSigs maps the 4-byte function signature to its string representation. - var {{.Type}}FuncSigs = map[string]string{ - {{range $strsig, $binsig := .FuncSigs}}"{{$binsig}}": "{{$strsig}}", - {{end}} - } + var {{.Type}}FuncSigs = {{.Type}}MetaData.Sigs {{end}} {{if .InputBin}} // {{.Type}}Bin is the compiled bytecode used for deploying new contracts. - var {{.Type}}Bin = "0x{{.InputBin}}" + // Deprecated: Use {{.Type}}MetaData.Bin instead. + var {{.Type}}Bin = {{.Type}}MetaData.Bin // Deploy{{.Type}} deploys a new Ethereum contract, binding an instance of {{.Type}} to it. func Deploy{{.Type}}(auth *bind.TransactOpts, backend bind.ContractBackend {{range .Constructor.Inputs}}, {{.Name}} {{bindtype .Type $structs}}{{end}}) (common.Address, *types.Transaction, *{{.Type}}, error) { - parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI)) + parsed, err := {{.Type}}MetaData.GetAbi() if err != nil { return common.Address{}, nil, nil, err } + if parsed == nil { + return common.Address{}, nil, nil, errors.New("GetABI returned nil") + } {{range $pattern, $name := .Libraries}} {{decapitalise $name}}Addr, _, _, _ := Deploy{{capitalise $name}}(auth, backend) {{$contract.Type}}Bin = strings.Replace({{$contract.Type}}Bin, "__${{$pattern}}$__", {{decapitalise $name}}Addr.String()[2:], -1) {{end}} - address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex({{.Type}}Bin), backend {{range .Constructor.Inputs}}, {{.Name}}{{end}}) + address, tx, contract, err := bind.DeployContract(auth, *parsed, common.FromHex({{.Type}}Bin), backend {{range .Constructor.Inputs}}, {{.Name}}{{end}}) if err != nil { return common.Address{}, nil, nil, err } From 3aea432b35f0344d9f7c1f1843a4eca496229ffc Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 27 Jul 2021 16:24:27 +0200 Subject: [PATCH 06/60] accounts/abi/bind: set Context in TransactOpts (#23188) --- accounts/abi/bind/auth.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index b8065e8488ed..a4307a952910 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -17,6 +17,7 @@ package bind import ( + "context" "crypto/ecdsa" "errors" "io" @@ -74,6 +75,7 @@ func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account } return tx.WithSignature(signer, signature) }, + Context: context.Background(), }, nil } @@ -97,6 +99,7 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts { } return tx.WithSignature(signer, signature) }, + Context: context.Background(), } } @@ -133,6 +136,7 @@ func NewKeyStoreTransactorWithChainID(keystore *keystore.KeyStore, account accou } return tx.WithSignature(signer, signature) }, + Context: context.Background(), }, nil } @@ -156,6 +160,7 @@ func NewKeyedTransactorWithChainID(key *ecdsa.PrivateKey, chainID *big.Int) (*Tr } return tx.WithSignature(signer, signature) }, + Context: context.Background(), }, nil } @@ -170,5 +175,6 @@ func NewClefTransactor(clef *external.ExternalSigner, account accounts.Account) } return clef.SignTx(account, transaction, nil) // Clef enforces its own chain id }, + Context: context.Background(), } } From 2faf796d2a502ef6d3c02681a649bd3f41999ccc Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 28 Jul 2021 14:21:35 +0200 Subject: [PATCH 07/60] internal/ethapi: fix panic in accesslist creation (#23225) * internal/ethapi: revert + fix properly in al tracer * internal/ethapi: use toMessage instead of creating new message * internal/ethapi: remove ineffassign * core: fix invalid unmarshalling, fix test Co-authored-by: Martin Holst Swende --- core/vm/access_list_tracer.go | 2 +- ethclient/gethclient/gethclient_test.go | 2 +- internal/ethapi/api.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/vm/access_list_tracer.go b/core/vm/access_list_tracer.go index b5bc961c8437..cc5461d1c310 100644 --- a/core/vm/access_list_tracer.go +++ b/core/vm/access_list_tracer.go @@ -96,7 +96,7 @@ func (al accessList) equal(other accessList) bool { func (al accessList) accessList() types.AccessList { acl := make(types.AccessList, 0, len(al)) for addr, slots := range al { - tuple := types.AccessTuple{Address: addr} + tuple := types.AccessTuple{Address: addr, StorageKeys: []common.Hash{}} for slot := range slots { tuple.StorageKeys = append(tuple.StorageKeys, slot) } diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 3c408c22593e..c0b9d0c2d099 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -138,7 +138,7 @@ func testAccessList(t *testing.T, client *rpc.Client) { From: testAddr, To: &common.Address{}, Gas: 21000, - GasPrice: big.NewInt(1), + GasPrice: big.NewInt(765625000), Value: big.NewInt(1), } al, gas, vmErr, err := ec.CreateAccessList(context.Background(), msg) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 59555ea46307..826934a9c1bd 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1441,7 +1441,12 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH } // Copy the original db so we don't modify it statedb := db.Copy() - msg := types.NewMessage(args.from(), args.To, uint64(*args.Nonce), args.Value.ToInt(), uint64(*args.Gas), args.GasPrice.ToInt(), big.NewInt(0), big.NewInt(0), args.data(), accessList, false) + // Set the accesslist to the last al + args.AccessList = &accessList + msg, err := args.ToMessage(b.RPCGasCap(), header.BaseFee) + if err != nil { + return nil, 0, nil, err + } // Apply the transaction with the access list tracer tracer := vm.NewAccessListTracer(accessList, args.from(), to, precompiles) From 0730acc5a01e683176b31ef0fcdf0d965a47d14a Mon Sep 17 00:00:00 2001 From: "@edgararout" Date: Wed, 28 Jul 2021 08:24:41 -0400 Subject: [PATCH 08/60] consensus/ethash: less allocation during mining (#23199) --- consensus/ethash/sealer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 1830e672b1ff..6fa60ef6a8bb 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -140,8 +140,9 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s ) // Start generating random nonces until we abort or find a good one var ( - attempts = int64(0) - nonce = seed + attempts = int64(0) + nonce = seed + powBuffer = new(big.Int) ) logger := ethash.config.Log.New("miner", id) logger.Trace("Started ethash search for new nonces", "seed", seed) @@ -163,7 +164,7 @@ search: } // Compute the PoW value of this nonce digest, result := hashimotoFull(dataset.dataset, hash, nonce) - if new(big.Int).SetBytes(result).Cmp(target) <= 0 { + if powBuffer.SetBytes(result).Cmp(target) <= 0 { // Correct nonce found, create a new header with it header = types.CopyHeader(header) header.Nonce = types.EncodeNonce(nonce) From 56e9001a1a8ddecc478943170b00207ef46109b9 Mon Sep 17 00:00:00 2001 From: ligi Date: Wed, 28 Jul 2021 18:14:46 +0200 Subject: [PATCH 09/60] README: fix default sync mode (#23282) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb55f565cd42..c7adb135d6ec 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ $ geth console ``` This command will: - * Start `geth` in fast sync mode (default, can be changed with the `--syncmode` flag), + * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), causing it to download more data in exchange for avoiding processing the entire history of the Ethereum network, which is very CPU intensive. * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), From 523866c2ccbfd75ee88b309167cdba7311ba9695 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Thu, 29 Jul 2021 10:17:40 +0200 Subject: [PATCH 10/60] all: change blacklist terms --- cmd/puppeth/module_ethstats.go | 2 +- cmd/puppeth/wizard_ethstats.go | 10 +++++----- consensus/ethash/consensus.go | 2 +- core/blockchain.go | 4 ++-- core/blockchain_test.go | 4 ++-- core/error.go | 4 ++-- core/headerchain.go | 4 ++-- light/lightchain_test.go | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index 58ecb83951e0..abed4db5fc50 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -158,7 +158,7 @@ func checkEthstats(client *sshClient, network string) (*ethstatsInfos, error) { if port != 80 && port != 443 { config += fmt.Sprintf(":%d", port) } - // Retrieve the IP blacklist + // Retrieve the IP banned list banned := strings.Split(infos.envvars["BANNED"], ",") // Run a sanity check to see if the port is reachable diff --git a/cmd/puppeth/wizard_ethstats.go b/cmd/puppeth/wizard_ethstats.go index 58ff3efbe986..95cab9da4633 100644 --- a/cmd/puppeth/wizard_ethstats.go +++ b/cmd/puppeth/wizard_ethstats.go @@ -63,20 +63,20 @@ func (w *wizard) deployEthstats() { fmt.Printf("What should be the secret password for the API? (default = %s)\n", infos.secret) infos.secret = w.readDefaultString(infos.secret) } - // Gather any blacklists to ban from reporting + // Gather any banned lists to ban from reporting if existed { fmt.Println() - fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned) + fmt.Printf("Keep existing IP %v in the banned list (y/n)? (default = yes)\n", infos.banned) if !w.readDefaultYesNo(true) { // The user might want to clear the entire list, although generally probably not fmt.Println() - fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n") + fmt.Printf("Clear out the banned list and start over (y/n)? (default = no)\n") if w.readDefaultYesNo(false) { infos.banned = nil } // Offer the user to explicitly add/remove certain IP addresses fmt.Println() - fmt.Println("Which additional IP addresses should be blacklisted?") + fmt.Println("Which additional IP addresses should be in the banned list?") for { if ip := w.readIPAddress(); ip != "" { infos.banned = append(infos.banned, ip) @@ -85,7 +85,7 @@ func (w *wizard) deployEthstats() { break } fmt.Println() - fmt.Println("Which IP addresses should not be blacklisted?") + fmt.Println("Which IP addresses should not be in the banned list?") for { if ip := w.readIPAddress(); ip != "" { for i, addr := range infos.banned { diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 9b9657e19009..5743387f6c7f 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -215,7 +215,7 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo ancestors[parent] = ancestorHeader // If the ancestor doesn't have any uncles, we don't have to iterate them if ancestorHeader.UncleHash != types.EmptyUncleHash { - // Need to add those uncles to the blacklist too + // Need to add those uncles to the banned list too ancestor := chain.GetBlock(parent, number) if ancestor == nil { break diff --git a/core/blockchain.go b/core/blockchain.go index 18e126657c49..48a23da79bd8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1785,8 +1785,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er } // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { - bc.reportBlock(block, nil, ErrBlacklistedHash) - return it.index, ErrBlacklistedHash + bc.reportBlock(block, nil, ErrBannedHash) + return it.index, ErrBannedHash } // If the block is known (in the middle of the chain), it's a special case for // Clique blocks where they can share state among each other, so importing an diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 7fb4c6f69523..4e5df633b7e3 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -473,8 +473,8 @@ func testBadHashes(t *testing.T, full bool) { _, err = blockchain.InsertHeaderChain(headers, 1) } - if !errors.Is(err, ErrBlacklistedHash) { - t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash) + if !errors.Is(err, ErrBannedHash) { + t.Errorf("error mismatch: have: %v, want: %v", err, ErrBannedHash) } } diff --git a/core/error.go b/core/error.go index a5a1230099a7..bcbce09ba431 100644 --- a/core/error.go +++ b/core/error.go @@ -26,8 +26,8 @@ var ( // ErrKnownBlock is returned when a block to import is already known locally. ErrKnownBlock = errors.New("block already known") - // ErrBlacklistedHash is returned if a block to import is on the blacklist. - ErrBlacklistedHash = errors.New("blacklisted hash") + // ErrBannedHash is returned if a block to import is on the banned list. + ErrBannedHash = errors.New("banned hash") // ErrNoGenesis is returned when there is no Genesis Block. ErrNoGenesis = errors.New("genesis not found in chain") diff --git a/core/headerchain.go b/core/headerchain.go index 07307c710cae..7ef7dd43f704 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -315,11 +315,11 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) } // If the header is a banned one, straight out abort if BadHashes[chain[i].ParentHash] { - return i - 1, ErrBlacklistedHash + return i - 1, ErrBannedHash } // If it's the last header in the cunk, we need to check it too if i == len(chain)-1 && BadHashes[chain[i].Hash()] { - return i, ErrBlacklistedHash + return i, ErrBannedHash } } diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 2aed08d74edf..af36ebd96a23 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -322,8 +322,8 @@ func TestBadHeaderHashes(t *testing.T) { var err error headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) core.BadHashes[headers[2].Hash()] = true - if _, err = bc.InsertHeaderChain(headers, 1); !errors.Is(err, core.ErrBlacklistedHash) { - t.Errorf("error mismatch: have: %v, want %v", err, core.ErrBlacklistedHash) + if _, err = bc.InsertHeaderChain(headers, 1); !errors.Is(err, core.ErrBannedHash) { + t.Errorf("error mismatch: have: %v, want %v", err, core.ErrBannedHash) } } From 5c13012b5681218aec6cf6039349adb7308d58e0 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Jul 2021 14:00:06 +0200 Subject: [PATCH 11/60] accounts/external, internal/ethapi: fixes for London tx signing (#23274) Ticket #23273 found a flaw where we were unable to sign legacy-transactions using the external signer, even if we're still on non-london network. That's fixed in this PR. Additionally, I found that even when supplying all parameters, it was impossible to sign a london-transaction on an unsynched node. It's a pretty common usecase that someone wants to sign a transaction using an unsynced 'vanilla' node, providing all necessary data. Our setDefaults, however, insisted on checking the current block against the config. This PR therefore adds a case, so that if both MaxPriorityFeePerGas and MaxFeePerGas are provided, we accept them as given. OBS This PR fixes a regression -- on current master, we are unable to sign a london-transaction unless the node is synched, which may break scenarios where geth (or clef) is used as a cold wallet. Fixes #23273 --- accounts/external/backend.go | 9 ++-- internal/ethapi/transaction_args.go | 67 ++++++++++++++++------------- 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 4f6ca399673c..f6715a84eb21 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -211,11 +211,14 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio To: to, From: common.NewMixedcaseAddress(account.Address), } - if tx.GasFeeCap() != nil { + switch tx.Type() { + case types.LegacyTxType, types.AccessListTxType: + args.GasPrice = (*hexutil.Big)(tx.GasPrice()) + case types.DynamicFeeTxType: args.MaxFeePerGas = (*hexutil.Big)(tx.GasFeeCap()) args.MaxPriorityFeePerGas = (*hexutil.Big)(tx.GasTipCap()) - } else { - args.GasPrice = (*hexutil.Big)(tx.GasPrice()) + default: + return nil, fmt.Errorf("Unsupported tx type %d", tx.Type()) } // We should request the default chain id that we're operating with // (the chain we're executing on) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 1fbaaeacbf06..9076e49b2a66 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -80,40 +80,45 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { } // After london, default to 1559 unless gasPrice is set head := b.CurrentHeader() - if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { - if args.MaxPriorityFeePerGas == nil { - tip, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err + // If user specifies both maxPriorityfee and maxFee, then we do not + // need to consult the chain for defaults. It's definitely a London tx. + if args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil { + // In this clause, user left some fields unspecified. + if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) } - args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) - } - if args.MaxFeePerGas == nil { - gasFeeCap := new(big.Int).Add( - (*big.Int)(args.MaxPriorityFeePerGas), - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), - ) - args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) - } - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } - } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { - return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") - } - if args.GasPrice == nil { - price, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err + if args.MaxFeePerGas == nil { + gasFeeCap := new(big.Int).Add( + (*big.Int)(args.MaxPriorityFeePerGas), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) + } + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") } - if b.ChainConfig().IsLondon(head.Number) { - // The legacy tx gas price suggestion should not add 2x base fee - // because all fees are consumed, so it would result in a spiral - // upwards. - price.Add(price, head.BaseFee) + if args.GasPrice == nil { + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + if b.ChainConfig().IsLondon(head.Number) { + // The legacy tx gas price suggestion should not add 2x base fee + // because all fees are consumed, so it would result in a spiral + // upwards. + price.Add(price, head.BaseFee) + } + args.GasPrice = (*hexutil.Big)(price) } - args.GasPrice = (*hexutil.Big)(price) } } if args.Value == nil { From fb8ea5993f466bd398d33a42a8273c9fc2bf1fb6 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 29 Jul 2021 14:05:22 +0200 Subject: [PATCH 12/60] tests: update tests/testdata to v9.0.4 (london) (#23279) --- tests/block_test_util.go | 18 +++++++++--------- tests/gen_btheader.go | 10 +++++----- tests/state_test.go | 8 ++++++++ tests/state_test_util.go | 21 ++++++++++++++++++--- tests/testdata | 2 +- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index c9c0afb1ba25..bcf861e09b9c 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -85,17 +85,17 @@ type btHeader struct { GasLimit uint64 GasUsed uint64 Timestamp uint64 - BaseFee *big.Int + BaseFeePerGas *big.Int } type btHeaderMarshaling struct { - ExtraData hexutil.Bytes - Number *math.HexOrDecimal256 - Difficulty *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - GasUsed math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + ExtraData hexutil.Bytes + Number *math.HexOrDecimal256 + Difficulty *math.HexOrDecimal256 + GasLimit math.HexOrDecimal64 + GasUsed math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + BaseFeePerGas *math.HexOrDecimal256 } func (t *BlockTest) Run(snapshotter bool) error { @@ -170,7 +170,7 @@ func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { Mixhash: t.json.Genesis.MixHash, Coinbase: t.json.Genesis.Coinbase, Alloc: t.json.Pre, - BaseFee: t.json.Genesis.BaseFee, + BaseFee: t.json.Genesis.BaseFeePerGas, } } diff --git a/tests/gen_btheader.go b/tests/gen_btheader.go index 07df1ecb3657..4387f8db41c8 100644 --- a/tests/gen_btheader.go +++ b/tests/gen_btheader.go @@ -33,7 +33,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { GasLimit math.HexOrDecimal64 GasUsed math.HexOrDecimal64 Timestamp math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + BaseFeePerGas *math.HexOrDecimal256 } var enc btHeader enc.Bloom = b.Bloom @@ -52,7 +52,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { enc.GasLimit = math.HexOrDecimal64(b.GasLimit) enc.GasUsed = math.HexOrDecimal64(b.GasUsed) enc.Timestamp = math.HexOrDecimal64(b.Timestamp) - enc.BaseFee = (*math.HexOrDecimal256)(b.BaseFee) + enc.BaseFeePerGas = (*math.HexOrDecimal256)(b.BaseFeePerGas) return json.Marshal(&enc) } @@ -75,7 +75,7 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { GasLimit *math.HexOrDecimal64 GasUsed *math.HexOrDecimal64 Timestamp *math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + BaseFeePerGas *math.HexOrDecimal256 } var dec btHeader if err := json.Unmarshal(input, &dec); err != nil { @@ -129,8 +129,8 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { if dec.Timestamp != nil { b.Timestamp = uint64(*dec.Timestamp) } - if dec.BaseFee != nil { - b.BaseFee = (*big.Int)(dec.BaseFee) + if dec.BaseFeePerGas != nil { + b.BaseFeePerGas = (*big.Int)(dec.BaseFeePerGas) } return nil } diff --git a/tests/state_test.go b/tests/state_test.go index 2ed98b650987..c2ca0e8d6948 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -68,6 +68,10 @@ func TestState(t *testing.T) { t.Run(key+"/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { _, _, err := test.Run(subtest, vmconfig, false) + if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { + // Ignore expected errors (TODO MariusVanDerWijden check error string) + return nil + } return st.checkFailure(t, err) }) }) @@ -79,6 +83,10 @@ func TestState(t *testing.T) { return err } } + if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { + // Ignore expected errors (TODO MariusVanDerWijden check error string) + return nil + } return st.checkFailure(t, err) }) }) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 97fd3fb6a89e..4f57fee8909b 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -65,9 +65,11 @@ type stJSON struct { } type stPostState struct { - Root common.UnprefixedHash `json:"hash"` - Logs common.UnprefixedHash `json:"logs"` - Indexes struct { + Root common.UnprefixedHash `json:"hash"` + Logs common.UnprefixedHash `json:"logs"` + TxBytes hexutil.Bytes `json:"txbytes"` + ExpectException string `json:"expectException"` + Indexes struct { Data int `json:"data"` Gas int `json:"gas"` Value int `json:"value"` @@ -198,6 +200,19 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return nil, nil, common.Hash{}, err } + // Try to recover tx with current signer + if len(post.TxBytes) != 0 { + var ttx types.Transaction + err := ttx.UnmarshalBinary(post.TxBytes) + if err != nil { + return nil, nil, common.Hash{}, err + } + + if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { + return nil, nil, common.Hash{}, err + } + } + // Prepare the EVM. txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) diff --git a/tests/testdata b/tests/testdata index fa0ab110f3f4..5d534e37b80e 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit fa0ab110f3f45d1f6786f978ea596a18ecbe8275 +Subproject commit 5d534e37b80e9310e8c7751f805ca481a451123e From 64a5e125c5b61b6182fbbec27dcc78a33fe6ac67 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 29 Jul 2021 21:20:15 +0800 Subject: [PATCH 13/60] go.mod: upgrade to goupnp v1.0.2 (#23197) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 046578016f09..58cc2b4b6d06 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.0 - github.com/huin/goupnp v1.0.1-0.20210626160114-33cdcbb30dda + github.com/huin/goupnp v1.0.2 github.com/influxdata/influxdb v1.8.3 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e diff --git a/go.sum b/go.sum index 8fde7ce0f7a0..dbb08ce3e41b 100644 --- a/go.sum +++ b/go.sum @@ -213,8 +213,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.1-0.20210626160114-33cdcbb30dda h1:Vofqyy/Ysqit++X33unU0Gr08b6P35hKm3juytDrBVI= -github.com/huin/goupnp v1.0.1-0.20210626160114-33cdcbb30dda/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.2 h1:RfGLP+h3mvisuWEyybxNq5Eft3NWhHLPeUN72kpKZoI= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= From b157bae2c9815fa08be19c7533c079b720bd19a1 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Thu, 29 Jul 2021 15:20:45 +0200 Subject: [PATCH 14/60] go.mod: bump golang.org/x/text to v0.3.6 (#23291) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 58cc2b4b6d06..4621bc9ab42e 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 - golang.org/x/text v0.3.4 + golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 diff --git a/go.sum b/go.sum index dbb08ce3e41b..4c494e170bdd 100644 --- a/go.sum +++ b/go.sum @@ -492,6 +492,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= From 8f11d279d241cfdf2571d4fa772dc38efc6175d0 Mon Sep 17 00:00:00 2001 From: Evolution404 <35091674+Evolution404@users.noreply.github.com> Date: Thu, 29 Jul 2021 22:03:50 +0800 Subject: [PATCH 15/60] p2p/simulations: fix unlikely crash in probabilistic connect (#23200) When the nodeCount is less than 10, it will panic with the out of bound error. How about we just skip this round, when rand1 and rand2 are equal? --- p2p/simulations/mocker.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index 8ce777a0103f..fd25e2c918dd 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -123,20 +123,12 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { randWait := time.Duration(rand.Intn(5000)+1000) * time.Millisecond rand1 := rand.Intn(nodeCount - 1) rand2 := rand.Intn(nodeCount - 1) - if rand1 < rand2 { + if rand1 <= rand2 { lowid = rand1 highid = rand2 } else if rand1 > rand2 { highid = rand1 lowid = rand2 - } else { - if rand1 == 0 { - rand2 = 9 - } else if rand1 == 9 { - rand1 = 0 - } - lowid = rand1 - highid = rand2 } var steps = highid - lowid wg.Add(steps) From 295bc35ecf9b29f72db8d71f61e2d65b4d4bb24c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Jul 2021 16:06:44 +0200 Subject: [PATCH 16/60] signer/core: move API JSON types to separate package (#23275) This PR moves (some) account types into a standalone package, to avoid depending on signer/core from accounts/external. --- accounts/external/backend.go | 4 ++-- cmd/abidump/main.go | 4 ++-- cmd/clef/main.go | 10 ++++---- signer/core/api.go | 37 +++++++++++++++-------------- signer/core/api_test.go | 5 ++-- signer/core/{ => apitypes}/types.go | 6 ++--- signer/core/auditlog.go | 3 ++- signer/core/gnosis_safe.go | 5 ++-- signer/core/signed_data.go | 3 ++- signer/fourbyte/validation.go | 8 +++---- signer/fourbyte/validation_test.go | 6 ++--- signer/rules/rules_test.go | 7 +++--- 12 files changed, 52 insertions(+), 46 deletions(-) rename signer/core/{ => apitypes}/types.go (96%) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index f6715a84eb21..b2de9e932af9 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) type ExternalBackend struct { @@ -203,7 +203,7 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio t := common.NewMixedcaseAddress(*tx.To()) to = &t } - args := &core.SendTxArgs{ + args := &apitypes.SendTxArgs{ Data: &data, Nonce: hexutil.Uint64(tx.Nonce()), Value: hexutil.Big(*tx.Value()), diff --git a/cmd/abidump/main.go b/cmd/abidump/main.go index 35cbcbb0ed2d..4f942749dfdf 100644 --- a/cmd/abidump/main.go +++ b/cmd/abidump/main.go @@ -23,7 +23,7 @@ import ( "os" "strings" - "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/ethereum/go-ethereum/signer/fourbyte" ) @@ -41,7 +41,7 @@ func parse(data []byte) { if err != nil { die(err) } - messages := core.ValidationMessages{} + messages := apitypes.ValidationMessages{} db.ValidateCallData(nil, data, &messages) for _, m := range messages.Messages { fmt.Printf("%v: %v\n", m.Typ, m.Message) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 84e1dda99edf..cd2320518778 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -50,10 +50,10 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/ethereum/go-ethereum/signer/fourbyte" "github.com/ethereum/go-ethereum/signer/rules" "github.com/ethereum/go-ethereum/signer/storage" - "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "gopkg.in/urfave/cli.v1" @@ -923,7 +923,7 @@ func testExternalUI(api *core.SignerAPI) { time.Sleep(delay) data := hexutil.Bytes([]byte{}) to := common.NewMixedcaseAddress(a) - tx := core.SendTxArgs{ + tx := apitypes.SendTxArgs{ Data: &data, Nonce: 0x1, Value: hexutil.Big(*big.NewInt(6)), @@ -1055,11 +1055,11 @@ func GenDoc(ctx *cli.Context) { data := hexutil.Bytes([]byte{0x01, 0x02, 0x03, 0x04}) add("SignTxRequest", desc, &core.SignTxRequest{ Meta: meta, - Callinfo: []core.ValidationInfo{ + Callinfo: []apitypes.ValidationInfo{ {Typ: "Warning", Message: "Something looks odd, show this message as a warning"}, {Typ: "Info", Message: "User should see this as well"}, }, - Transaction: core.SendTxArgs{ + Transaction: apitypes.SendTxArgs{ Data: &data, Nonce: 0x1, Value: hexutil.Big(*big.NewInt(6)), @@ -1075,7 +1075,7 @@ func GenDoc(ctx *cli.Context) { add("SignTxResponse - approve", "Response to request to sign a transaction. This response needs to contain the `transaction`"+ ", because the UI is free to make modifications to the transaction.", &core.SignTxResponse{Approved: true, - Transaction: core.SendTxArgs{ + Transaction: apitypes.SendTxArgs{ Data: &data, Nonce: 0x4, Value: hexutil.Big(*big.NewInt(6)), diff --git a/signer/core/api.go b/signer/core/api.go index 03daeeb6e7c7..fb68018a6bcb 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/ethereum/go-ethereum/signer/storage" ) @@ -52,7 +53,7 @@ type ExternalAPI interface { // New request to create a new account New(ctx context.Context) (common.Address, error) // SignTransaction request to sign the specified transaction - SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) + SignTransaction(ctx context.Context, args apitypes.SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) // SignData - request to sign the given data (plus prefix) SignData(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (hexutil.Bytes, error) // SignTypedData - request to sign the given structured data (plus prefix) @@ -104,7 +105,7 @@ type Validator interface { // ValidateTransaction does a number of checks on the supplied transaction, and // returns either a list of warnings, or an error (indicating that the transaction // should be immediately rejected). - ValidateTransaction(selector *string, tx *SendTxArgs) (*ValidationMessages, error) + ValidateTransaction(selector *string, tx *apitypes.SendTxArgs) (*apitypes.ValidationMessages, error) } // SignerAPI defines the actual implementation of ExternalAPI @@ -220,24 +221,24 @@ func (m Metadata) String() string { type ( // SignTxRequest contains info about a Transaction to sign SignTxRequest struct { - Transaction SendTxArgs `json:"transaction"` - Callinfo []ValidationInfo `json:"call_info"` - Meta Metadata `json:"meta"` + Transaction apitypes.SendTxArgs `json:"transaction"` + Callinfo []apitypes.ValidationInfo `json:"call_info"` + Meta Metadata `json:"meta"` } // SignTxResponse result from SignTxRequest SignTxResponse struct { //The UI may make changes to the TX - Transaction SendTxArgs `json:"transaction"` - Approved bool `json:"approved"` + Transaction apitypes.SendTxArgs `json:"transaction"` + Approved bool `json:"approved"` } SignDataRequest struct { - ContentType string `json:"content_type"` - Address common.MixedcaseAddress `json:"address"` - Rawdata []byte `json:"raw_data"` - Messages []*NameValueType `json:"messages"` - Callinfo []ValidationInfo `json:"call_info"` - Hash hexutil.Bytes `json:"hash"` - Meta Metadata `json:"meta"` + ContentType string `json:"content_type"` + Address common.MixedcaseAddress `json:"address"` + Rawdata []byte `json:"raw_data"` + Messages []*NameValueType `json:"messages"` + Callinfo []apitypes.ValidationInfo `json:"call_info"` + Hash hexutil.Bytes `json:"hash"` + Meta Metadata `json:"meta"` } SignDataResponse struct { Approved bool `json:"approved"` @@ -537,7 +538,7 @@ func (api *SignerAPI) lookupOrQueryPassword(address common.Address, title, promp } // SignTransaction signs the given Transaction and returns it both as json and rlp-encoded form -func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) { +func (api *SignerAPI) SignTransaction(ctx context.Context, args apitypes.SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) { var ( err error result SignTxResponse @@ -548,7 +549,7 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth } // If we are in 'rejectMode', then reject rather than show the user warnings if api.rejectMode { - if err := msgs.getWarnings(); err != nil { + if err := msgs.GetWarnings(); err != nil { return nil, err } } @@ -585,7 +586,7 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth return nil, err } // Convert fields into a real transaction - var unsignedTx = result.Transaction.toTransaction() + var unsignedTx = result.Transaction.ToTransaction() // Get the password for the transaction pw, err := api.lookupOrQueryPassword(acc.Address, "Account password", fmt.Sprintf("Please enter the password for account %s", acc.Address.String())) @@ -621,7 +622,7 @@ func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common } // If we are in 'rejectMode', then reject rather than show the user warnings if api.rejectMode { - if err := msgs.getWarnings(); err != nil { + if err := msgs.GetWarnings(); err != nil { return nil, err } } diff --git a/signer/core/api_test.go b/signer/core/api_test.go index ff4d281d4540..36f12f71a52d 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/ethereum/go-ethereum/signer/fourbyte" "github.com/ethereum/go-ethereum/signer/storage" ) @@ -223,14 +224,14 @@ func TestNewAcc(t *testing.T) { } } -func mkTestTx(from common.MixedcaseAddress) core.SendTxArgs { +func mkTestTx(from common.MixedcaseAddress) apitypes.SendTxArgs { to := common.NewMixedcaseAddress(common.HexToAddress("0x1337")) gas := hexutil.Uint64(21000) gasPrice := (hexutil.Big)(*big.NewInt(2000000000)) value := (hexutil.Big)(*big.NewInt(1e18)) nonce := (hexutil.Uint64)(0) data := hexutil.Bytes(common.Hex2Bytes("01020304050607080a")) - tx := core.SendTxArgs{ + tx := apitypes.SendTxArgs{ From: from, To: &to, Gas: gas, diff --git a/signer/core/types.go b/signer/core/apitypes/types.go similarity index 96% rename from signer/core/types.go rename to signer/core/apitypes/types.go index 85ea6170d635..591d253abb1a 100644 --- a/signer/core/types.go +++ b/signer/core/apitypes/types.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package core +package apitypes import ( "encoding/json" @@ -52,7 +52,7 @@ func (vs *ValidationMessages) Info(msg string) { } /// getWarnings returns an error with all messages of type WARN of above, or nil if no warnings were present -func (v *ValidationMessages) getWarnings() error { +func (v *ValidationMessages) GetWarnings() error { var messages []string for _, msg := range v.Messages { if msg.Typ == WARN || msg.Typ == CRIT { @@ -97,7 +97,7 @@ func (args SendTxArgs) String() string { return err.Error() } -func (args *SendTxArgs) toTransaction() *types.Transaction { +func (args *SendTxArgs) ToTransaction() *types.Transaction { txArgs := ethapi.TransactionArgs{ Gas: &args.Gas, GasPrice: args.GasPrice, diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index bda88a8b2e55..84877ee712cf 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) type AuditLogger struct { @@ -43,7 +44,7 @@ func (l *AuditLogger) New(ctx context.Context) (common.Address, error) { return l.api.New(ctx) } -func (l *AuditLogger) SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) { +func (l *AuditLogger) SignTransaction(ctx context.Context, args apitypes.SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) { sel := "" if methodSelector != nil { sel = *methodSelector diff --git a/signer/core/gnosis_safe.go b/signer/core/gnosis_safe.go index 050425f0afc0..bdf7f837a0f8 100644 --- a/signer/core/gnosis_safe.go +++ b/signer/core/gnosis_safe.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) // GnosisSafeTx is a type to parse the safe-tx returned by the relayer, @@ -76,9 +77,9 @@ func (tx *GnosisSafeTx) ToTypedData() TypedData { // ArgsForValidation returns a SendTxArgs struct, which can be used for the // common validations, e.g. look up 4byte destinations -func (tx *GnosisSafeTx) ArgsForValidation() *SendTxArgs { +func (tx *GnosisSafeTx) ArgsForValidation() *apitypes.SendTxArgs { gp := hexutil.Big(tx.GasPrice) - args := &SendTxArgs{ + args := &apitypes.SendTxArgs{ From: tx.Safe, To: &tx.To, Gas: hexutil.Uint64(tx.SafeTxGas.Uint64()), diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 3bff1e1f200d..6c2fd5f8fefc 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) type SigFormat struct { @@ -323,7 +324,7 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd // signTypedData is identical to the capitalized version, except that it also returns the hash (preimage) // - the signature preimage (hash) func (api *SignerAPI) signTypedData(ctx context.Context, addr common.MixedcaseAddress, - typedData TypedData, validationMessages *ValidationMessages) (hexutil.Bytes, hexutil.Bytes, error) { + typedData TypedData, validationMessages *apitypes.ValidationMessages) (hexutil.Bytes, hexutil.Bytes, error) { domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) if err != nil { return nil, nil, err diff --git a/signer/fourbyte/validation.go b/signer/fourbyte/validation.go index f311c89e5240..58111e8e00c8 100644 --- a/signer/fourbyte/validation.go +++ b/signer/fourbyte/validation.go @@ -23,14 +23,14 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) // ValidateTransaction does a number of checks on the supplied transaction, and // returns either a list of warnings, or an error (indicating that the transaction // should be immediately rejected). -func (db *Database) ValidateTransaction(selector *string, tx *core.SendTxArgs) (*core.ValidationMessages, error) { - messages := new(core.ValidationMessages) +func (db *Database) ValidateTransaction(selector *string, tx *apitypes.SendTxArgs) (*apitypes.ValidationMessages, error) { + messages := new(apitypes.ValidationMessages) // Prevent accidental erroneous usage of both 'input' and 'data' (show stopper) if tx.Data != nil && tx.Input != nil && !bytes.Equal(*tx.Data, *tx.Input) { @@ -90,7 +90,7 @@ func (db *Database) ValidateTransaction(selector *string, tx *core.SendTxArgs) ( // ValidateCallData checks if the ABI call-data + method selector (if given) can // be parsed and seems to match. -func (db *Database) ValidateCallData(selector *string, data []byte, messages *core.ValidationMessages) { +func (db *Database) ValidateCallData(selector *string, data []byte, messages *apitypes.ValidationMessages) { // If the data is empty, we have a plain value transfer, nothing more to do if len(data) == 0 { return diff --git a/signer/fourbyte/validation_test.go b/signer/fourbyte/validation_test.go index b088cf30978b..c3085696f432 100644 --- a/signer/fourbyte/validation_test.go +++ b/signer/fourbyte/validation_test.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" ) func mixAddr(a string) (*common.MixedcaseAddress, error) { @@ -36,7 +36,7 @@ func toHexUint(h string) hexutil.Uint64 { b := big.NewInt(0).SetBytes(common.FromHex(h)) return hexutil.Uint64(b.Uint64()) } -func dummyTxArgs(t txtestcase) *core.SendTxArgs { +func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { to, _ := mixAddr(t.to) from, _ := mixAddr(t.from) n := toHexUint(t.n) @@ -55,7 +55,7 @@ func dummyTxArgs(t txtestcase) *core.SendTxArgs { input = &a } - return &core.SendTxArgs{ + return &apitypes.SendTxArgs{ From: *from, To: to, Value: value, diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index ec11e37121d5..d506ef2db09f 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/core/apitypes" "github.com/ethereum/go-ethereum/signer/storage" ) @@ -180,7 +181,7 @@ func TestSignTxRequest(t *testing.T) { } t.Logf("to %v", to.Address().String()) resp, err := r.ApproveTx(&core.SignTxRequest{ - Transaction: core.SendTxArgs{ + Transaction: apitypes.SendTxArgs{ From: *from, To: to}, Callinfo: nil, @@ -432,7 +433,7 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { gasPrice := hexutil.Big(*big.NewInt(2000000)) return &core.SignTxRequest{ - Transaction: core.SendTxArgs{ + Transaction: apitypes.SendTxArgs{ From: *from, To: to, Value: value, @@ -440,7 +441,7 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { GasPrice: &gasPrice, Gas: gas, }, - Callinfo: []core.ValidationInfo{ + Callinfo: []apitypes.ValidationInfo{ {Typ: "Warning", Message: "All your base are bellong to us"}, }, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, From 3526f690478482a02a152988f4d31074c176b136 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Thu, 29 Jul 2021 17:36:15 +0200 Subject: [PATCH 17/60] all: remove term "whitelist" in comments and log messages (#23294) --- build/ci.go | 2 +- core/tx_pool.go | 4 ++-- rpc/websocket.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/ci.go b/build/ci.go index 63dd606321a8..ca87913eb00f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -450,7 +450,7 @@ func maybeSkipArchive(env build.Environment) { os.Exit(0) } if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping archive creation because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) + log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) os.Exit(0) } } diff --git a/core/tx_pool.go b/core/tx_pool.go index c5b6047486e9..f6228df2fad4 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -635,8 +635,8 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { // pending or queued one, it overwrites the previous transaction if its price is higher. // // If a newly added transaction is marked as local, its sending account will be -// whitelisted, preventing any associated transaction from being dropped out of the pool -// due to pricing constraints. +// be added to the allowlist, preventing any associated transaction from being dropped +// out of the pool due to pricing constraints. func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err error) { // If the transaction is already known, discard it hash := tx.Hash() diff --git a/rpc/websocket.go b/rpc/websocket.go index ab55ae69c100..afeb4c2081b8 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -96,7 +96,7 @@ func wsHandshakeValidator(allowedOrigins []string) func(*http.Request) bool { if _, ok := req.Header["Origin"]; !ok { return true } - // Verify origin against whitelist. + // Verify origin against allow list. origin := strings.ToLower(req.Header.Get("Origin")) if allowAllOrigins || originIsAllowed(origins, origin) { return true From 860184d54285738322ca697b1666040183e05e76 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Thu, 29 Jul 2021 17:50:18 +0200 Subject: [PATCH 18/60] p2p: remove term "whitelist" (#23295) Co-authored-by: Felix Lange --- p2p/dial.go | 6 +++--- p2p/discover/common.go | 2 +- p2p/discover/v4_udp.go | 2 +- p2p/server.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index d36d6655019a..83ced3cb3280 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -77,7 +77,7 @@ var ( errAlreadyDialing = errors.New("already dialing") errAlreadyConnected = errors.New("already connected") errRecentlyDialed = errors.New("recently dialed") - errNotWhitelisted = errors.New("not contained in netrestrict whitelist") + errNetRestrict = errors.New("not contained in netrestrict list") errNoPort = errors.New("node does not provide TCP port") ) @@ -133,7 +133,7 @@ type dialConfig struct { self enode.ID // our own ID maxDialPeers int // maximum number of dialed peers maxActiveDials int // maximum number of active dials - netRestrict *netutil.Netlist // IP whitelist, disabled if nil + netRestrict *netutil.Netlist // IP netrestrict list, disabled if nil resolver nodeResolver dialer NodeDialer log log.Logger @@ -402,7 +402,7 @@ func (d *dialScheduler) checkDial(n *enode.Node) error { return errAlreadyConnected } if d.netRestrict != nil && !d.netRestrict.Contains(n.IP()) { - return errNotWhitelisted + return errNetRestrict } if d.history.contains(string(n.ID().Bytes())) { return errRecentlyDialed diff --git a/p2p/discover/common.go b/p2p/discover/common.go index 3708bfb72c4b..e389821fda8b 100644 --- a/p2p/discover/common.go +++ b/p2p/discover/common.go @@ -41,7 +41,7 @@ type Config struct { PrivateKey *ecdsa.PrivateKey // These settings are optional: - NetRestrict *netutil.Netlist // network whitelist + NetRestrict *netutil.Netlist // list of allowed IP networks Bootnodes []*enode.Node // list of bootstrap nodes Unhandled chan<- ReadPacket // unhandled packets are sent on this channel Log log.Logger // if set, log messages go here diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 2b3eb48391b2..334716aebed0 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -583,7 +583,7 @@ func (t *UDPv4) nodeFromRPC(sender *net.UDPAddr, rn v4wire.Node) (*node, error) return nil, err } if t.netrestrict != nil && !t.netrestrict.Contains(rn.IP) { - return nil, errors.New("not contained in netrestrict whitelist") + return nil, errors.New("not contained in netrestrict list") } key, err := v4wire.DecodePubkey(crypto.S256(), rn.ID) if err != nil { diff --git a/p2p/server.go b/p2p/server.go index 04fdecaec1a4..bcfc1bd10bd7 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -353,7 +353,7 @@ func (srv *Server) RemovePeer(node *enode.Node) { } } -// AddTrustedPeer adds the given node to a reserved whitelist which allows the +// AddTrustedPeer adds the given node to a reserved trusted list which allows the // node to always connect, even if the slot are full. func (srv *Server) AddTrustedPeer(node *enode.Node) { select { @@ -903,7 +903,7 @@ func (srv *Server) checkInboundConn(remoteIP net.IP) error { } // Reject connections that do not match NetRestrict. if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not whitelisted in NetRestrict") + return fmt.Errorf("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() From 85afdeef37e17bffd9d098e86312c569995705e0 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Thu, 29 Jul 2021 20:23:37 +0200 Subject: [PATCH 19/60] tests: remove whitelist feature (#23297) --- tests/init_test.go | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/tests/init_test.go b/tests/init_test.go index 1638f863e1c1..f1a4ca2cf087 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -89,11 +89,10 @@ func findLine(data []byte, offset int64) (line int) { // testMatcher controls skipping and chain config assignment to tests. type testMatcher struct { - configpat []testConfig - failpat []testFailure - skiploadpat []*regexp.Regexp - slowpat []*regexp.Regexp - whitelistpat *regexp.Regexp + configpat []testConfig + failpat []testFailure + skiploadpat []*regexp.Regexp + slowpat []*regexp.Regexp } type testConfig struct { @@ -124,10 +123,6 @@ func (tm *testMatcher) fails(pattern string, reason string) { tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason}) } -func (tm *testMatcher) whitelist(pattern string) { - tm.whitelistpat = regexp.MustCompile(pattern) -} - // config defines chain config for tests matching the pattern. func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) { tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg}) @@ -217,11 +212,6 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte if r, _ := tm.findSkip(name); r != "" { t.Skip(r) } - if tm.whitelistpat != nil { - if !tm.whitelistpat.MatchString(name) { - t.Skip("Skipped by whitelist") - } - } t.Parallel() // Load the file as map[string]. @@ -275,14 +265,3 @@ func runTestFunc(runTest interface{}, t *testing.T, name string, m reflect.Value m.MapIndex(reflect.ValueOf(key)), }) } - -func TestMatcherWhitelist(t *testing.T) { - t.Parallel() - tm := new(testMatcher) - tm.whitelist("invalid*") - tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) { - if name[:len("invalidRLPTest.json")] != "invalidRLPTest.json" { - t.Fatalf("invalid test found: %s != invalidRLPTest.json", name) - } - }) -} From 95bbd46eabc5d95d9fb2108ec232dd62df2f44ab Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Mon, 2 Aug 2021 15:43:01 +0200 Subject: [PATCH 20/60] node, cmd/clef: remove term "whitelist" (#23296) * node: remove term "whitelist" * include cmd/clef --- cmd/clef/main.go | 2 +- node/rpcstack.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index cd2320518778..61d2811f6569 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -657,7 +657,7 @@ func signer(c *cli.Context) error { cors := utils.SplitAndTrim(c.GlobalString(utils.HTTPCORSDomainFlag.Name)) srv := rpc.NewServer() - err := node.RegisterApisFromWhitelist(rpcAPI, []string{"account"}, srv, false) + err := node.RegisterApis(rpcAPI, []string{"account"}, srv, false) if err != nil { utils.Fatalf("Could not register API: %w", err) } diff --git a/node/rpcstack.go b/node/rpcstack.go index 19490411b237..2c55a070b229 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -280,7 +280,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApisFromWhitelist(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv, false); err != nil { return err } h.httpConfig = config @@ -312,7 +312,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApisFromWhitelist(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv, false); err != nil { return err } h.wsConfig = config @@ -515,20 +515,20 @@ func (is *ipcServer) stop() error { return err } -// RegisterApisFromWhitelist checks the given modules' availability, generates a whitelist based on the allowed modules, +// RegisterApis checks the given modules' availability, generates an allowlist based on the allowed modules, // and then registers all of the APIs exposed by the services. -func RegisterApisFromWhitelist(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll bool) error { +func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll bool) error { if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 { log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available) } - // Generate the whitelist based on the allowed modules - whitelist := make(map[string]bool) + // Generate the allow list based on the allowed modules + allowList := make(map[string]bool) for _, module := range modules { - whitelist[module] = true + allowList[module] = true } // Register all the APIs exposed by the services for _, api := range apis { - if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) { + if exposeAll || allowList[api.Namespace] || (len(allowList) == 0 && api.Public) { if err := srv.RegisterName(api.Namespace, api.Service); err != nil { return err } From 82c508539939e025ad9f31aa784f6b4307afcbbb Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Tue, 3 Aug 2021 02:06:58 -0400 Subject: [PATCH 21/60] cre/state: fix outdated statedb Prepare comment (#23320) --- core/state/statedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index db74513057a2..61f0f5b9f6eb 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -878,7 +878,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { return s.trie.Hash() } -// Prepare sets the current transaction hash and index and block hash which is +// Prepare sets the current transaction hash and index which are // used when the EVM emits new state logs. func (s *StateDB) Prepare(thash common.Hash, ti int) { s.thash = thash From 4cd6a1458e37c78ec172cf62b9168e3b92c65ca7 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 3 Aug 2021 11:21:39 +0200 Subject: [PATCH 22/60] cmd/devp2p: fix ping/pong race in discv4 tests (#23306) This PR modifies the post-PING-send expectations to both be laxer and stricter: it doesn't care what order the packets arrive, but also verifies that exactly one PING and one PONG is returned. --- cmd/devp2p/internal/v4test/discv4tests.go | 57 ++++++++++++++++------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/cmd/devp2p/internal/v4test/discv4tests.go b/cmd/devp2p/internal/v4test/discv4tests.go index 1b5e5304ed9f..04ad67637a66 100644 --- a/cmd/devp2p/internal/v4test/discv4tests.go +++ b/cmd/devp2p/internal/v4test/discv4tests.go @@ -79,14 +79,44 @@ func BasicPing(t *utesting.T) { To: te.remoteEndpoint(), Expiration: futureExpiration(), }) - - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if err := te.checkPingPong(pingHash); err != nil { t.Fatal(err) } } -// checkPong verifies that reply is a valid PONG matching the given ping hash. +// checkPingPong verifies that the remote side sends both a PONG with the +// correct hash, and a PING. +// The two packets do not have to be in any particular order. +func (te *testenv) checkPingPong(pingHash []byte) error { + var ( + pings int + pongs int + ) + for i := 0; i < 2; i++ { + reply, _, err := te.read(te.l1) + if err != nil { + return err + } + switch reply.Kind() { + case v4wire.PongPacket: + if err := te.checkPong(reply, pingHash); err != nil { + return err + } + pongs++ + case v4wire.PingPacket: + pings++ + default: + return fmt.Errorf("expected PING or PONG, got %v %v", reply.Name(), reply) + } + } + if pongs == 1 && pings == 1 { + return nil + } + return fmt.Errorf("expected 1 PING (got %d) and 1 PONG (got %d)", pings, pongs) +} + +// checkPong verifies that reply is a valid PONG matching the given ping hash, +// and a PING. The two packets do not have to be in any particular order. func (te *testenv) checkPong(reply v4wire.Packet, pingHash []byte) error { if reply == nil { return fmt.Errorf("expected PONG reply, got nil") @@ -119,9 +149,7 @@ func PingWrongTo(t *utesting.T) { To: wrongEndpoint, Expiration: futureExpiration(), }) - - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if err := te.checkPingPong(pingHash); err != nil { t.Fatal(err) } } @@ -139,8 +167,7 @@ func PingWrongFrom(t *utesting.T) { Expiration: futureExpiration(), }) - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if err := te.checkPingPong(pingHash); err != nil { t.Fatal(err) } } @@ -161,8 +188,7 @@ func PingExtraData(t *utesting.T) { JunkData2: []byte{9, 8, 7, 6, 5, 4, 3, 2, 1}, }) - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if err := te.checkPingPong(pingHash); err != nil { t.Fatal(err) } } @@ -183,8 +209,7 @@ func PingExtraDataWrongFrom(t *utesting.T) { JunkData2: []byte{9, 8, 7, 6, 5, 4, 3, 2, 1}, } pingHash := te.send(te.l1, &req) - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if err := te.checkPingPong(pingHash); err != nil { t.Fatal(err) } } @@ -240,9 +265,9 @@ func BondThenPingWithWrongFrom(t *utesting.T) { To: te.remoteEndpoint(), Expiration: futureExpiration(), }) - - reply, _, _ := te.read(te.l1) - if err := te.checkPong(reply, pingHash); err != nil { + if reply, _, err := te.read(te.l1); err != nil { + t.Fatal(err) + } else if err := te.checkPong(reply, pingHash); err != nil { t.Fatal(err) } } From c38fab912ba817e5f502f4190ab315738e42949b Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 3 Aug 2021 17:29:47 +0200 Subject: [PATCH 23/60] core: get header from block cache (#23299) --- core/blockchain.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index 48a23da79bd8..e14291aa54c3 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2417,12 +2417,22 @@ func (bc *BlockChain) GetTdByHash(hash common.Hash) *big.Int { // GetHeader retrieves a block header from the database by hash and number, // caching it if found. func (bc *BlockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + // Blockchain might have cached the whole block, only if not go to headerchain + if block, ok := bc.blockCache.Get(hash); ok { + return block.(*types.Block).Header() + } + return bc.hc.GetHeader(hash, number) } // GetHeaderByHash retrieves a block header from the database by hash, caching it if // found. func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { + // Blockchain might have cached the whole block, only if not go to headerchain + if block, ok := bc.blockCache.Get(hash); ok { + return block.(*types.Block).Header() + } + return bc.hc.GetHeaderByHash(hash) } From f311488d2cfc25292d9ee8b1ef40a2bdd10a6d94 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 3 Aug 2021 23:32:13 +0800 Subject: [PATCH 24/60] internal/ethapi: fix trace log marshalling (#23292) --- internal/ethapi/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 826934a9c1bd..30f6dd06ab67 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1120,7 +1120,7 @@ type StructLogRes struct { Gas uint64 `json:"gas"` GasCost uint64 `json:"gasCost"` Depth int `json:"depth"` - Error error `json:"error,omitempty"` + Error string `json:"error,omitempty"` Stack *[]string `json:"stack,omitempty"` Memory *[]string `json:"memory,omitempty"` Storage *map[string]string `json:"storage,omitempty"` @@ -1136,7 +1136,7 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { Gas: trace.Gas, GasCost: trace.GasCost, Depth: trace.Depth, - Error: trace.Err, + Error: trace.ErrorString(), } if trace.Stack != nil { stack := make([]string, len(trace.Stack)) From 28ba686cbfa6a46a9b4f03b620460867d502f34e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 3 Aug 2021 17:35:25 +0200 Subject: [PATCH 25/60] core/state: add trie prefetcher tests (#23216) * core/state: add trie prefetcher tests * core/state: add missing license --- core/state/trie_prefetcher_test.go | 110 +++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 core/state/trie_prefetcher_test.go diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go new file mode 100644 index 000000000000..35dc7a2c0da4 --- /dev/null +++ b/core/state/trie_prefetcher_test.go @@ -0,0 +1,110 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" +) + +func filledStateDB() *StateDB { + state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil) + + // Create an account and check if the retrieved balance is correct + addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") + skey := common.HexToHash("aaa") + sval := common.HexToHash("bbb") + + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie + for i := 0; i < 100; i++ { + sk := common.BigToHash(big.NewInt(int64(i))) + state.SetState(addr, sk, sk) // Change the storage trie + } + return state +} + +func TestCopyAndClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + time.Sleep(1 * time.Second) + a := prefetcher.trie(db.originalRoot) + prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + b := prefetcher.trie(db.originalRoot) + cpy := prefetcher.copy() + cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + c := cpy.trie(db.originalRoot) + prefetcher.close() + cpy2 := cpy.copy() + cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + d := cpy2.trie(db.originalRoot) + cpy.close() + cpy2.close() + if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { + t.Fatalf("Invalid trie, hashes should be equal: %v %v %v %v", a.Hash(), b.Hash(), c.Hash(), d.Hash()) + } +} + +func TestUseAfterClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + a := prefetcher.trie(db.originalRoot) + prefetcher.close() + b := prefetcher.trie(db.originalRoot) + if a == nil { + t.Fatal("Prefetching before close should not return nil") + } + if b != nil { + t.Fatal("Trie after close should return nil") + } +} + +func TestCopyClose(t *testing.T) { + db := filledStateDB() + prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") + skey := common.HexToHash("aaa") + prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + cpy := prefetcher.copy() + a := prefetcher.trie(db.originalRoot) + b := cpy.trie(db.originalRoot) + prefetcher.close() + c := prefetcher.trie(db.originalRoot) + d := cpy.trie(db.originalRoot) + if a == nil { + t.Fatal("Prefetching before close should not return nil") + } + if b == nil { + t.Fatal("Copy trie should return nil") + } + if c != nil { + t.Fatal("Trie after close should return nil") + } + if d == nil { + t.Fatal("Copy trie should not return nil") + } +} From d3e3a460ec947c9e1e963d1a35f887d95f23f99d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 4 Aug 2021 01:10:37 -0700 Subject: [PATCH 26/60] core/rawdb: fix logs to print block number, not address (#23328) --- core/rawdb/accessors_indexes.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index d6dab6808d3a..25a44354bfd3 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -106,7 +106,7 @@ func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, com } body := ReadBody(db, blockHash, *blockNumber) if body == nil { - log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash) + log.Error("Transaction referenced missing", "number", *blockNumber, "hash", blockHash) return nil, common.Hash{}, 0, 0 } for txIndex, tx := range body.Transactions { @@ -114,7 +114,7 @@ func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, com return tx, blockHash, *blockNumber, uint64(txIndex) } } - log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + log.Error("Transaction not found", "number", *blockNumber, "hash", blockHash, "txhash", hash) return nil, common.Hash{}, 0, 0 } @@ -137,7 +137,7 @@ func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) return receipt, blockHash, *blockNumber, uint64(receiptIndex) } } - log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + log.Error("Receipt not found", "number", *blockNumber, "hash", blockHash, "txhash", hash) return nil, common.Hash{}, 0, 0 } From 0658712f6599cf8d84309ea8b74f8c0fcd83687d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 7 Aug 2021 19:38:18 +0200 Subject: [PATCH 27/60] core: check if sender is EOA (#23303) This adds a check to verify that a sender-account does not have code, which means that the codehash is either `emptyCodeHash` _OR_ not present. The latter occurs IFF the sender did not previously exist, a situation which can only occur with zero cost gasprices. --- core/error.go | 3 +++ core/state_processor_test.go | 42 +++++++++++++++++++++++++++++++++++- core/state_transition.go | 8 +++++++ eth/tracers/api_test.go | 10 ++++----- tests/init_test.go | 37 ++++++++++++++++++++++++------- tests/state_test.go | 7 ++++-- 6 files changed, 91 insertions(+), 16 deletions(-) diff --git a/core/error.go b/core/error.go index bcbce09ba431..33cf874d15e4 100644 --- a/core/error.go +++ b/core/error.go @@ -87,4 +87,7 @@ var ( // ErrFeeCapTooLow is returned if the transaction fee cap is less than the // the base fee of the block. ErrFeeCapTooLow = errors.New("max fee per gas less than block base fee") + + // ErrSenderNoEOA is returned if the sender of a transaction is a contract. + ErrSenderNoEOA = errors.New("sender not an eoa") ) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index ce6d2bcdde8b..13a9eb810df6 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -195,7 +195,7 @@ func TestStateProcessorErrors(t *testing.T) { } } - // One final error is ErrTxTypeNotSupported. For this, we need an older chain + // ErrTxTypeNotSupported, For this, we need an older chain { var ( db = rawdb.NewMemoryDatabase() @@ -244,6 +244,46 @@ func TestStateProcessorErrors(t *testing.T) { } } } + + // ErrSenderNoEOA, for this we need the sender to have contract code + { + var ( + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + Config: config, + Alloc: GenesisAlloc{ + common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ + Balance: big.NewInt(1000000000000000000), // 1 ether + Nonce: 0, + Code: common.FromHex("0xB0B0FACE"), + }, + }, + } + genesis = gspec.MustCommit(db) + blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ) + defer blockchain.Stop() + for i, tt := range []struct { + txs []*types.Transaction + want string + }{ + { // ErrSenderNoEOA + txs: []*types.Transaction{ + mkDynamicTx(0, common.Address{}, params.TxGas-1000, big.NewInt(0), big.NewInt(0)), + }, + want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: sender not an eoa: address 0x71562b71999873DB5b286dF957af199Ec94617F7, codehash: 0x9280914443471259d4570a8661015ae4a5b80186dbc619658fb494bebc3da3d1", + }, + } { + block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + _, err := blockchain.InsertChain(types.Blocks{block}) + if err == nil { + t.Fatal("block imported without errors") + } + if have, want := err.Error(), tt.want; have != want { + t.Errorf("test %d:\nhave \"%v\"\nwant \"%v\"\n", i, have, want) + } + } + } } // GenerateBadBlock constructs a "block" which contains the transactions. The transactions are not expected to be diff --git a/core/state_transition.go b/core/state_transition.go index b5b0ae990a41..68f3c74a3121 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -25,9 +25,12 @@ import ( cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" ) +var emptyCodeHash = crypto.Keccak256Hash(nil) + /* The State Transitioning Model @@ -220,6 +223,11 @@ func (st *StateTransition) preCheck() error { st.msg.From().Hex(), msgNonce, stNonce) } } + // Make sure the sender is an EOA + if codeHash := st.state.GetCodeHash(st.msg.From()); codeHash != emptyCodeHash && codeHash != (common.Hash{}) { + return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, + st.msg.From().Hex(), codeHash) + } // Make sure that transaction gasFeeCap is greater than the baseFee (post london) if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { // Skip the checks if gas fields are zero and baseFee was explicitly disabled (eth_call) diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 4a4e5bbc33d2..9afd59d596bc 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -417,24 +417,24 @@ func TestOverriddenTraceCall(t *testing.T) { }, }, } - for _, testspec := range testSuite { + for i, testspec := range testSuite { result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) if testspec.expectErr != nil { if err == nil { - t.Errorf("Expect error %v, get nothing", testspec.expectErr) + t.Errorf("test %d: want error %v, have nothing", i, testspec.expectErr) continue } if !errors.Is(err, testspec.expectErr) { - t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + t.Errorf("test %d: error mismatch, want %v, have %v", i, testspec.expectErr, err) } } else { if err != nil { - t.Errorf("Expect no error, get %v", err) + t.Errorf("test %d: want no error, have %v", i, err) continue } ret := new(callTrace) if err := json.Unmarshal(result.(json.RawMessage), ret); err != nil { - t.Fatalf("failed to unmarshal trace result: %v", err) + t.Fatalf("test %d: failed to unmarshal trace result: %v", i, err) } if !jsonEqual(ret, testspec.expect) { // uncomment this for easier debugging diff --git a/tests/init_test.go b/tests/init_test.go index f1a4ca2cf087..969cf7139915 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -34,10 +34,10 @@ import ( ) var ( - baseDir = filepath.Join(".", "testdata") - blockTestDir = filepath.Join(baseDir, "BlockchainTests") - stateTestDir = filepath.Join(baseDir, "GeneralStateTests") - legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") + baseDir = filepath.Join(".", "testdata") + blockTestDir = filepath.Join(baseDir, "BlockchainTests") + stateTestDir = filepath.Join(baseDir, "GeneralStateTests") + //legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") transactionTestDir = filepath.Join(baseDir, "TransactionTests") vmTestDir = filepath.Join(baseDir, "VMTests") rlpTestDir = filepath.Join(baseDir, "RLPTests") @@ -89,10 +89,11 @@ func findLine(data []byte, offset int64) (line int) { // testMatcher controls skipping and chain config assignment to tests. type testMatcher struct { - configpat []testConfig - failpat []testFailure - skiploadpat []*regexp.Regexp - slowpat []*regexp.Regexp + configpat []testConfig + failpat []testFailure + skiploadpat []*regexp.Regexp + slowpat []*regexp.Regexp + runonlylistpat *regexp.Regexp } type testConfig struct { @@ -123,6 +124,10 @@ func (tm *testMatcher) fails(pattern string, reason string) { tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason}) } +func (tm *testMatcher) runonly(pattern string) { + tm.runonlylistpat = regexp.MustCompile(pattern) +} + // config defines chain config for tests matching the pattern. func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) { tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg}) @@ -212,6 +217,11 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte if r, _ := tm.findSkip(name); r != "" { t.Skip(r) } + if tm.runonlylistpat != nil { + if !tm.runonlylistpat.MatchString(name) { + t.Skip("Skipped by runonly") + } + } t.Parallel() // Load the file as map[string]. @@ -265,3 +275,14 @@ func runTestFunc(runTest interface{}, t *testing.T, name string, m reflect.Value m.MapIndex(reflect.ValueOf(key)), }) } + +func TestMatcherRunonlylist(t *testing.T) { + t.Parallel() + tm := new(testMatcher) + tm.runonly("invalid*") + tm.walk(t, rlpTestDir, func(t *testing.T, name string, test *RLPTest) { + if name[:len("invalidRLPTest.json")] != "invalidRLPTest.json" { + t.Fatalf("invalid test found: %s != invalidRLPTest.json", name) + } + }) +} diff --git a/tests/state_test.go b/tests/state_test.go index c2ca0e8d6948..242e9712a312 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -45,7 +45,8 @@ func TestState(t *testing.T) { // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) - + // Un-skip this when https://github.com/ethereum/tests/issues/908 is closed + st.skipLoad(`^stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy`) // Broken tests: // Expected failures: //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") @@ -58,7 +59,9 @@ func TestState(t *testing.T) { // For Istanbul, older tests were moved into LegacyTests for _, dir := range []string{ stateTestDir, - legacyStateTestDir, + // legacy state tests are disabled, due to them not being + // regenerated for the no-sender-eoa change. + //legacyStateTestDir, } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { From 8a24b563312a0ab0a808770e464c5598ab7e35ea Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 7 Aug 2021 23:04:34 +0200 Subject: [PATCH 28/60] cmd/evm: implement input txs via rlp in t8n tool (#23138) In many cases, it's desireable to use already-signed transactions as input to the state transition, instead of having the evm sign them internally (for example to use malformed or not-yet-valid transactions). This PR adds support + docs for that feature. --- cmd/evm/README.md | 117 ++++++++++++++++++++----- cmd/evm/internal/t8ntool/flags.go | 6 +- cmd/evm/internal/t8ntool/transition.go | 43 ++++++++- cmd/evm/testdata/13/alloc.json | 23 +++++ cmd/evm/testdata/13/env.json | 12 +++ cmd/evm/testdata/13/readme.md | 4 + cmd/evm/testdata/13/txs.json | 34 +++++++ cmd/evm/transition-test.sh | 37 ++++++-- 8 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 cmd/evm/testdata/13/alloc.json create mode 100644 cmd/evm/testdata/13/env.json create mode 100644 cmd/evm/testdata/13/readme.md create mode 100644 cmd/evm/testdata/13/txs.json diff --git a/cmd/evm/README.md b/cmd/evm/README.md index cdff41f904c1..d5257069f271 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -4,15 +4,15 @@ The `evm t8n` tool is a stateless state transition utility. It is a utility which can 1. Take a prestate, including - - Accounts, - - Block context information, - - Previous blockshashes (*optional) +- Accounts, +- Block context information, +- Previous blockshashes (*optional) 2. Apply a set of transactions, 3. Apply a mining-reward (*optional), 4. And generate a post-state, including - - State root, transaction root, receipt root, - - Information about rejected transactions, - - Optionally: a full or partial post-state dump +- State root, transaction root, receipt root, +- Information about rejected transactions, +- Optionally: a full or partial post-state dump ## Specification @@ -37,6 +37,8 @@ Command line params that has to be supported are --output.result result Determines where to put the result (stateroot, txroot etc) of the post-state. `stdout` - into the stdout output `stderr` - into the stderr output + --output.body value If set, the RLP of the transactions (block body) will be written to this file. + --input.txs stdin stdin or file name of where to find the transactions to apply. If the file prefix is '.rlp', then the data is interpreted as an RLP list of signed transactions.The '.rlp' format is identical to the output.body format. (default: "txs.json") --state.fork value Name of ruleset to use. --state.chainid value ChainID to use (default: 1) --state.reward value Mining reward. Set to -1 to disable (default: 0) @@ -110,7 +112,10 @@ Two resulting files: } ], "rejected": [ - 1 + { + "index": 1, + "error": "nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" + } ] } ``` @@ -156,7 +161,10 @@ Output: } ], "rejected": [ - 1 + { + "index": 1, + "error": "nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" + } ] } } @@ -168,9 +176,9 @@ Mining rewards and ommer rewards might need to be added. This is how those are a - `block_reward` is the block mining reward for the miner (`0xaa`), of a block at height `N`. - For each ommer (mined by `0xbb`), with blocknumber `N-delta` - - (where `delta` is the difference between the current block and the ommer) - - The account `0xbb` (ommer miner) is awarded `(8-delta)/ 8 * block_reward` - - The account `0xaa` (block miner) is awarded `block_reward / 32` + - (where `delta` is the difference between the current block and the ommer) + - The account `0xbb` (ommer miner) is awarded `(8-delta)/ 8 * block_reward` + - The account `0xaa` (block miner) is awarded `block_reward / 32` To make `state_t8n` apply these, the following inputs are required: @@ -220,7 +228,7 @@ Output: ### Future EIPS It is also possible to experiment with future eips that are not yet defined in a hard fork. -Example, putting EIP-1344 into Frontier: +Example, putting EIP-1344 into Frontier: ``` ./evm t8n --state.fork=Frontier+1344 --input.pre=./testdata/1/pre.json --input.txs=./testdata/1/txs.json --input.env=/testdata/1/env.json ``` @@ -229,41 +237,102 @@ Example, putting EIP-1344 into Frontier: The `BLOCKHASH` opcode requires blockhashes to be provided by the caller, inside the `env`. If a required blockhash is not provided, the exit code should be `4`: -Example where blockhashes are provided: +Example where blockhashes are provided: ``` -./evm t8n --input.alloc=./testdata/3/alloc.json --input.txs=./testdata/3/txs.json --input.env=./testdata/3/env.json --trace +./evm --verbosity=1 t8n --input.alloc=./testdata/3/alloc.json --input.txs=./testdata/3/txs.json --input.env=./testdata/3/env.json --trace +INFO [07-27|11:53:40.960] Trie dumping started root=b7341d..857ea1 +INFO [07-27|11:53:40.960] Trie dumping complete accounts=3 elapsed="103.298µs" +INFO [07-27|11:53:40.960] Wrote file file=alloc.json +INFO [07-27|11:53:40.960] Wrote file file=result.json + ``` + ``` cat trace-0-0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81.jsonl | grep BLOCKHASH -C2 ``` ``` -{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"PUSH1","error":""} -{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"BLOCKHASH","error":""} -{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnStack":[],"returnData":"0x","depth":1,"refund":0,"opName":"STOP","error":""} -{"output":"","gasUsed":"0x17","time":142709} +{"pc":0,"op":96,"gas":"0x5f58ef8","gasCost":"0x3","memory":"0x","memSize":0,"stack":[],"returnData":"0x","depth":1,"refund":0,"opName":"PUSH1","error":""} +{"pc":2,"op":64,"gas":"0x5f58ef5","gasCost":"0x14","memory":"0x","memSize":0,"stack":["0x1"],"returnData":"0x","depth":1,"refund":0,"opName":"BLOCKHASH","error":""} +{"pc":3,"op":0,"gas":"0x5f58ee1","gasCost":"0x0","memory":"0x","memSize":0,"stack":["0xdac58aa524e50956d0c0bae7f3f8bb9d35381365d07804dd5b48a5a297c06af4"],"returnData":"0x","depth":1,"refund":0,"opName":"STOP","error":""} +{"output":"","gasUsed":"0x17","time":156276} ``` In this example, the caller has not provided the required blockhash: ``` ./evm t8n --input.alloc=./testdata/4/alloc.json --input.txs=./testdata/4/txs.json --input.env=./testdata/4/env.json --trace -``` -``` ERROR(4): getHash(3) invoked, blockhash for that block not provided ``` Error code: 4 + ### Chaining Another thing that can be done, is to chain invocations: ``` ./evm t8n --input.alloc=./testdata/1/alloc.json --input.txs=./testdata/1/txs.json --input.env=./testdata/1/env.json --output.alloc=stdout | ./evm t8n --input.alloc=stdin --input.env=./testdata/1/env.json --input.txs=./testdata/1/txs.json -INFO [01-21|22:41:22.963] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" -INFO [01-21|22:41:22.966] rejected tx index=0 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" -INFO [01-21|22:41:22.967] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [07-27|11:53:41.049] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [07-27|11:53:41.050] Trie dumping started root=84208a..ae4e13 +INFO [07-27|11:53:41.050] Trie dumping complete accounts=3 elapsed="59.412µs" +INFO [07-27|11:53:41.050] Wrote file file=result.json +INFO [07-27|11:53:41.051] rejected tx index=0 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [07-27|11:53:41.051] rejected tx index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" +INFO [07-27|11:53:41.052] Trie dumping started root=84208a..ae4e13 +INFO [07-27|11:53:41.052] Trie dumping complete accounts=3 elapsed="45.734µs" +INFO [07-27|11:53:41.052] Wrote file file=alloc.json +INFO [07-27|11:53:41.052] Wrote file file=result.json ``` -What happened here, is that we first applied two identical transactions, so the second one was rejected. +What happened here, is that we first applied two identical transactions, so the second one was rejected. Then, taking the poststate alloc as the input for the next state, we tried again to include the same two transactions: this time, both failed due to too low nonce. In order to meaningfully chain invocations, one would need to provide meaningful new `env`, otherwise the actual blocknumber (exposed to the EVM) would not increase. + +### Transactions in RLP form + +It is possible to provide already-signed transactions as input to, using an `input.txs` which ends with the `rlp` suffix. +The input format for RLP-form transactions is _identical_ to the _output_ format for block bodies. Therefore, it's fully possible +to use the evm to go from `json` input to `rlp` input. + +The following command takes **json** the transactions in `./testdata/13/txs.json` and signs them. After execution, they are output to `signed_txs.rlp`.: +``` +./evm t8n --state.fork=London --input.alloc=./testdata/13/alloc.json --input.txs=./testdata/13/txs.json --input.env=./testdata/13/env.json --output.result=alloc_jsontx.json --output.body=signed_txs.rlp +INFO [07-27|11:53:41.124] Trie dumping started root=e4b924..6aef61 +INFO [07-27|11:53:41.124] Trie dumping complete accounts=3 elapsed="94.284µs" +INFO [07-27|11:53:41.125] Wrote file file=alloc.json +INFO [07-27|11:53:41.125] Wrote file file=alloc_jsontx.json +INFO [07-27|11:53:41.125] Wrote file file=signed_txs.rlp + +``` + +The `output.body` is the rlp-list of transactions, encoded in hex and placed in a string a'la `json` encoding rules: +``` +cat signed_txs.rlp +"0xf8d2b86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9" +``` + +We can use `rlpdump` to check what the contents are: +``` +rlpdump -hex $(cat signed_txs.rlp | jq -r ) +[ + 02f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904, + 02f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9, +] +``` +Now, we can now use those (or any other already signed transactions), as input, like so: +``` +./evm t8n --state.fork=London --input.alloc=./testdata/13/alloc.json --input.txs=./signed_txs.rlp --input.env=./testdata/13/env.json --output.result=alloc_rlptx.json +INFO [07-27|11:53:41.253] Trie dumping started root=e4b924..6aef61 +INFO [07-27|11:53:41.253] Trie dumping complete accounts=3 elapsed="128.445µs" +INFO [07-27|11:53:41.253] Wrote file file=alloc.json +INFO [07-27|11:53:41.255] Wrote file file=alloc_rlptx.json + +``` + +You might have noticed that the results from these two invocations were stored in two separate files. +And we can now finally check that they match. +``` +cat alloc_jsontx.json | jq .stateRoot && cat alloc_rlptx.json | jq .stateRoot +"0xe4b924a6adb5959fccf769d5b7bb2f6359e26d1e76a2443c5a91a36d826aef61" +"0xe4b924a6adb5959fccf769d5b7bb2f6359e26d1e76a2443c5a91a36d826aef61" +``` diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index a599462cc61d..626f974a0b0d 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -79,8 +79,10 @@ var ( Value: "env.json", } InputTxsFlag = cli.StringFlag{ - Name: "input.txs", - Usage: "`stdin` or file name of where to find the transactions to apply.", + Name: "input.txs", + Usage: "`stdin` or file name of where to find the transactions to apply. " + + "If the file prefix is '.rlp', then the data is interpreted as an RLP list of signed transactions." + + "The '.rlp' format is identical to the output.body format.", Value: "txs.json", } RewardFlag = cli.Int64Flag{ diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index bab6e63faa6a..8334aa01d4f9 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -25,6 +25,7 @@ import ( "math/big" "os" "path" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -72,6 +73,7 @@ type input struct { Alloc core.GenesisAlloc `json:"alloc,omitempty"` Env *stEnv `json:"env,omitempty"` Txs []*txWithKey `json:"txs,omitempty"` + TxRlp string `json:"txsRlp,omitempty"` } func Main(ctx *cli.Context) error { @@ -199,11 +201,44 @@ func Main(ctx *cli.Context) error { } defer inFile.Close() decoder := json.NewDecoder(inFile) - if err := decoder.Decode(&txsWithKeys); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + if strings.HasSuffix(txStr, ".rlp") { + var body hexutil.Bytes + if err := decoder.Decode(&body); err != nil { + return err + } + var txs types.Transactions + if err := rlp.DecodeBytes(body, &txs); err != nil { + return err + } + for _, tx := range txs { + txsWithKeys = append(txsWithKeys, &txWithKey{ + key: nil, + tx: tx, + }) + } + } else { + if err := decoder.Decode(&txsWithKeys); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + } } } else { - txsWithKeys = inputData.Txs + if len(inputData.TxRlp) > 0 { + // Decode the body of already signed transactions + body := common.FromHex(inputData.TxRlp) + var txs types.Transactions + if err := rlp.DecodeBytes(body, &txs); err != nil { + return err + } + for _, tx := range txs { + txsWithKeys = append(txsWithKeys, &txWithKey{ + key: nil, + tx: tx, + }) + } + } else { + // JSON encoded transactions + txsWithKeys = inputData.Txs + } } // We may have to sign the transactions. signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number))) @@ -365,6 +400,7 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } os.Stdout.Write(b) + os.Stdout.Write([]byte("\n")) } if len(stdErrObject) > 0 { b, err := json.MarshalIndent(stdErrObject, "", " ") @@ -372,6 +408,7 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } os.Stderr.Write(b) + os.Stderr.Write([]byte("\n")) } return nil } diff --git a/cmd/evm/testdata/13/alloc.json b/cmd/evm/testdata/13/alloc.json new file mode 100644 index 000000000000..6e98e7513c45 --- /dev/null +++ b/cmd/evm/testdata/13/alloc.json @@ -0,0 +1,23 @@ +{ + "0x1111111111111111111111111111111111111111" : { + "balance" : "0x010000000000", + "code" : "0xfe", + "nonce" : "0x01", + "storage" : { + } + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x010000000000", + "code" : "0x", + "nonce" : "0x01", + "storage" : { + } + }, + "0xd02d72e067e77158444ef2020ff2d325f929b363" : { + "balance" : "0x01000000000000", + "code" : "0x", + "nonce" : "0x01", + "storage" : { + } + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/13/env.json b/cmd/evm/testdata/13/env.json new file mode 100644 index 000000000000..3a82d46a774b --- /dev/null +++ b/cmd/evm/testdata/13/env.json @@ -0,0 +1,12 @@ +{ + "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty" : "0x020000", + "currentNumber" : "0x01", + "currentTimestamp" : "0x079e", + "previousHash" : "0xcb23ee65a163121f640673b41788ee94633941405f95009999b502eedfbbfd4f", + "currentGasLimit" : "0x40000000", + "currentBaseFee" : "0x036b", + "blockHashes" : { + "0" : "0xcb23ee65a163121f640673b41788ee94633941405f95009999b502eedfbbfd4f" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/13/readme.md b/cmd/evm/testdata/13/readme.md new file mode 100644 index 000000000000..64f52fc9a910 --- /dev/null +++ b/cmd/evm/testdata/13/readme.md @@ -0,0 +1,4 @@ +## Input transactions in RLP form + +This testdata folder is used to examplify how transaction input can be provided in rlp form. +Please see the README in `evm` folder for how this is performed. \ No newline at end of file diff --git a/cmd/evm/testdata/13/txs.json b/cmd/evm/testdata/13/txs.json new file mode 100644 index 000000000000..c45ef1e13d1f --- /dev/null +++ b/cmd/evm/testdata/13/txs.json @@ -0,0 +1,34 @@ +[ + { + "input" : "0x", + "gas" : "0x84d0", + "nonce" : "0x1", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x0", + "s" : "0x0", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "maxFeePerGas" : "0xfa0", + "maxPriorityFeePerGas" : "0x0", + "accessList" : [] + }, + { + "input" : "0x", + "gas" : "0x84d0", + "nonce" : "0x2", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "v" : "0x0", + "r" : "0x0", + "s" : "0x0", + "secretKey" : "0x41f6e321b31e72173f8ff2e292359e1862f24fba42fe6f97efaf641980eff298", + "chainId" : "0x1", + "type" : "0x2", + "maxFeePerGas" : "0xfa0", + "maxPriorityFeePerGas" : "0x0", + "accessList" : [] + } +] \ No newline at end of file diff --git a/cmd/evm/transition-test.sh b/cmd/evm/transition-test.sh index 34c92498559a..250238d1694a 100644 --- a/cmd/evm/transition-test.sh +++ b/cmd/evm/transition-test.sh @@ -11,6 +11,8 @@ function showjson(){ function demo(){ echo "$ticks" echo "$1" + $1 + echo "" echo "$ticks" echo "" } @@ -152,9 +154,7 @@ echo "" echo "The \`BLOCKHASH\` opcode requires blockhashes to be provided by the caller, inside the \`env\`." echo "If a required blockhash is not provided, the exit code should be \`4\`:" echo "Example where blockhashes are provided: " -cmd="./evm t8n --input.alloc=./testdata/3/alloc.json --input.txs=./testdata/3/txs.json --input.env=./testdata/3/env.json --trace" -tick && echo $cmd && tick -$cmd 2>&1 >/dev/null +demo "./evm --verbosity=1 t8n --input.alloc=./testdata/3/alloc.json --input.txs=./testdata/3/txs.json --input.env=./testdata/3/env.json --trace" cmd="cat trace-0-0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81.jsonl | grep BLOCKHASH -C2" tick && echo $cmd && tick echo "$ticks" @@ -164,13 +164,11 @@ echo "" echo "In this example, the caller has not provided the required blockhash:" cmd="./evm t8n --input.alloc=./testdata/4/alloc.json --input.txs=./testdata/4/txs.json --input.env=./testdata/4/env.json --trace" -tick && echo $cmd && tick -tick -$cmd +tick && echo $cmd && $cmd errc=$? tick echo "Error code: $errc" - +echo "" echo "### Chaining" echo "" @@ -189,3 +187,28 @@ echo "" echo "In order to meaningfully chain invocations, one would need to provide meaningful new \`env\`, otherwise the" echo "actual blocknumber (exposed to the EVM) would not increase." echo "" + +echo "### Transactions in RLP form" +echo "" +echo "It is possible to provide already-signed transactions as input to, using an \`input.txs\` which ends with the \`rlp\` suffix." +echo "The input format for RLP-form transactions is _identical_ to the _output_ format for block bodies. Therefore, it's fully possible" +echo "to use the evm to go from \`json\` input to \`rlp\` input." +echo "" +echo "The following command takes **json** the transactions in \`./testdata/13/txs.json\` and signs them. After execution, they are output to \`signed_txs.rlp\`.:" +demo "./evm t8n --state.fork=London --input.alloc=./testdata/13/alloc.json --input.txs=./testdata/13/txs.json --input.env=./testdata/13/env.json --output.result=alloc_jsontx.json --output.body=signed_txs.rlp" +echo "The \`output.body\` is the rlp-list of transactions, encoded in hex and placed in a string a'la \`json\` encoding rules:" +demo "cat signed_txs.rlp" +echo "We can use \`rlpdump\` to check what the contents are: " +echo "$ticks" +echo "rlpdump -hex \$(cat signed_txs.rlp | jq -r )" +rlpdump -hex $(cat signed_txs.rlp | jq -r ) +echo "$ticks" +echo "Now, we can now use those (or any other already signed transactions), as input, like so: " +demo "./evm t8n --state.fork=London --input.alloc=./testdata/13/alloc.json --input.txs=./signed_txs.rlp --input.env=./testdata/13/env.json --output.result=alloc_rlptx.json" + +echo "You might have noticed that the results from these two invocations were stored in two separate files. " +echo "And we can now finally check that they match." +echo "$ticks" +echo "cat alloc_jsontx.json | jq .stateRoot && cat alloc_rlptx.json | jq .stateRoot" +cat alloc_jsontx.json | jq .stateRoot && cat alloc_rlptx.json | jq .stateRoot +echo "$ticks" From 9e59474e46f383cd2eace981d498dd7176ea1319 Mon Sep 17 00:00:00 2001 From: Shihao Xia Date: Sun, 8 Aug 2021 09:44:42 -0400 Subject: [PATCH 29/60] core/rawdb: close database in test to avoid goroutine leak (#23287) * add db close to avoid goroutine leak * core/rawdb: move close to defer Co-authored-by: Martin Holst Swende --- core/rawdb/accessors_chain_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index ea9dc436cfdd..f20e8b1fffb5 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -444,6 +444,7 @@ func TestAncientStorage(t *testing.T) { if err != nil { t.Fatalf("failed to create database with ancient backend") } + defer db.Close() // Create a test block block := types.NewBlockWithHeader(&types.Header{ Number: big.NewInt(0), From d60cfd2604a4615078696da20d474828be65ab70 Mon Sep 17 00:00:00 2001 From: shawn <36943337+lxex@users.noreply.github.com> Date: Mon, 9 Aug 2021 22:34:20 +0800 Subject: [PATCH 30/60] core: fix london-check to avoid duplication (#23333) Co-authored-by: lxex --- core/state_transition.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 68f3c74a3121..4c4187df152c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -287,6 +287,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { sender := vm.AccountRef(msg.From()) homestead := st.evm.ChainConfig().IsHomestead(st.evm.Context.BlockNumber) istanbul := st.evm.ChainConfig().IsIstanbul(st.evm.Context.BlockNumber) + london := st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) contractCreation := msg.To() == nil // Check clauses 4-5, subtract intrinsic gas if everything is correct @@ -319,7 +320,8 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1) ret, st.gas, vmerr = st.evm.Call(sender, st.to(), st.data, st.gas, st.value) } - if !st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + + if !london { // Before EIP-3529: refunds were capped to gasUsed / 2 st.refundGas(params.RefundQuotient) } else { @@ -327,7 +329,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { st.refundGas(params.RefundQuotientEIP3529) } effectiveTip := st.gasPrice - if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + if london { effectiveTip = cmath.BigMin(st.gasTipCap, new(big.Int).Sub(st.gasFeeCap, st.evm.Context.BaseFee)) } st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip)) From 97bd6cd2167d45d557a29f12133c42faccd61efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 10 Aug 2021 08:53:40 +0200 Subject: [PATCH 31/60] internal/ethapi: accept both hex and decimal for blockCount (#23363) --- internal/ethapi/api.go | 2 +- rpc/types.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 30f6dd06ab67..06a0dcf8eb0f 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -87,7 +87,7 @@ type feeHistoryResult struct { GasUsedRatio []float64 `json:"gasUsedRatio"` } -func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount hexutil.Uint, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { +func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { return nil, err diff --git a/rpc/types.go b/rpc/types.go index d1b878c7858f..ad068defa886 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math" + "strconv" "strings" "github.com/ethereum/go-ethereum/common" @@ -191,3 +192,24 @@ func BlockNumberOrHashWithHash(hash common.Hash, canonical bool) BlockNumberOrHa RequireCanonical: canonical, } } + +// DecimalOrHex unmarshals a non-negative decimal or hex parameter into a uint64. +type DecimalOrHex uint64 + +// UnmarshalJSON implements json.Unmarshaler. +func (dh *DecimalOrHex) UnmarshalJSON(data []byte) error { + input := strings.TrimSpace(string(data)) + if len(input) >= 2 && input[0] == '"' && input[len(input)-1] == '"' { + input = input[1 : len(input)-1] + } + + value, err := strconv.ParseUint(input, 10, 64) + if err != nil { + value, err = hexutil.DecodeUint64(input) + } + if err != nil { + return err + } + *dh = DecimalOrHex(value) + return nil +} From 3b38a83274b5fab8afaa5fde78d6007787f77ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 10 Aug 2021 10:38:49 +0300 Subject: [PATCH 32/60] core/state/pruner: fix state bloom sync permission in Windows --- core/state/pruner/bloom.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 4aeeb176e888..1cd03cf5363b 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -90,7 +90,7 @@ func (bloom *stateBloom) Commit(filename, tempname string) error { return err } // Ensure the file is synced to disk - f, err := os.Open(tempname) + f, err := os.OpenFile(tempname, os.O_RDWR, 0666) if err != nil { return err } From 6d175460dfb4ceddd1de41a90b520159902c6bc1 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 10 Aug 2021 16:28:33 +0800 Subject: [PATCH 33/60] cmd, core, eth, miner: deprecate miner.gastarget flag (#23213) --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 11 +++------- cmd/utils/flags_legacy.go | 16 +++++++++++--- core/block_validator.go | 41 +++--------------------------------- core/block_validator_test.go | 13 ++++++------ core/chain_makers.go | 5 ++--- eth/ethconfig/config.go | 1 - miner/stress/1559/main.go | 1 - miner/stress/clique/main.go | 1 - miner/stress/ethash/main.go | 1 - miner/worker.go | 8 +++---- miner/worker_test.go | 1 - 13 files changed, 32 insertions(+), 71 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 1a27a3255adb..25b35ca93c4b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -118,7 +118,7 @@ var ( utils.MiningEnabledFlag, utils.MinerThreadsFlag, utils.MinerNotifyFlag, - utils.MinerGasTargetFlag, + utils.LegacyMinerGasTargetFlag, utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, utils.MinerEtherbaseFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index dea0b7c08a09..3c401eae8ab6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -182,7 +182,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MinerNotifyFlag, utils.MinerNotifyFullFlag, utils.MinerGasPriceFlag, - utils.MinerGasTargetFlag, utils.MinerGasLimitFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, @@ -226,6 +225,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.LegacyRPCCORSDomainFlag, utils.LegacyRPCVirtualHostsFlag, utils.LegacyRPCApiFlag, + utils.LegacyMinerGasTargetFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7ed5907dba85..7fa3d8b86a98 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -444,11 +444,6 @@ var ( Name: "miner.notify.full", Usage: "Notify with pending block headers instead of work packages", } - MinerGasTargetFlag = cli.Uint64Flag{ - Name: "miner.gastarget", - Usage: "Target gas floor for mined blocks", - Value: ethconfig.Defaults.Miner.GasFloor, - } MinerGasLimitFlag = cli.Uint64Flag{ Name: "miner.gaslimit", Usage: "Target gas ceiling for mined blocks", @@ -1386,9 +1381,6 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } - if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { - cfg.GasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name) - } if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { cfg.GasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) } @@ -1401,6 +1393,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { cfg.Noverify = ctx.GlobalBool(MinerNoVerfiyFlag.Name) } + if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { + log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") + } } func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index fb5fde657695..df597e3657d6 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -33,7 +34,9 @@ var ShowDeprecated = cli.Command{ Description: "Show flags that have been deprecated and will soon be removed", } -var DeprecatedFlags = []cli.Flag{} +var DeprecatedFlags = []cli.Flag{ + LegacyMinerGasTargetFlag, +} var ( // (Deprecated May 2020, shown in aliased flags section) @@ -66,6 +69,12 @@ var ( Usage: "API's offered over the HTTP-RPC interface (deprecated and will be removed June 2021, use --http.api)", Value: "", } + // (Deprecated July 2021, shown in aliased flags section) + LegacyMinerGasTargetFlag = cli.Uint64Flag{ + Name: "miner.gastarget", + Usage: "Target gas floor for mined blocks (deprecated)", + Value: ethconfig.Defaults.Miner.GasFloor, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. @@ -74,7 +83,8 @@ func showDeprecated(*cli.Context) { fmt.Println("The following flags are deprecated and will be removed in the future!") fmt.Println("--------------------------------------------------------------------") fmt.Println() - // TODO remove when there are newly deprecated flags - fmt.Println("no deprecated flags to show at this time") + for _, flag := range DeprecatedFlags { + fmt.Println(flag.String()) + } fmt.Println() } diff --git a/core/block_validator.go b/core/block_validator.go index d317d82ed48d..3763be0be08d 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -103,44 +103,9 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD } // CalcGasLimit computes the gas limit of the next block after parent. It aims -// to keep the baseline gas above the provided floor, and increase it towards the -// ceil if the blocks are full. If the ceil is exceeded, it will always decrease -// the gas allowance. -func CalcGasLimit(parentGasUsed, parentGasLimit, gasFloor, gasCeil uint64) uint64 { - // contrib = (parentGasUsed * 3 / 2) / 1024 - contrib := (parentGasUsed + parentGasUsed/2) / params.GasLimitBoundDivisor - - // decay = parentGasLimit / 1024 -1 - decay := parentGasLimit/params.GasLimitBoundDivisor - 1 - - /* - strategy: gasLimit of block-to-mine is set based on parent's - gasUsed value. if parentGasUsed > parentGasLimit * (2/3) then we - increase it, otherwise lower it (or leave it unchanged if it's right - at that usage) the amount increased/decreased depends on how far away - from parentGasLimit * (2/3) parentGasUsed is. - */ - limit := parentGasLimit - decay + contrib - if limit < params.MinGasLimit { - limit = params.MinGasLimit - } - // If we're outside our allowed gas range, we try to hone towards them - if limit < gasFloor { - limit = parentGasLimit + decay - if limit > gasFloor { - limit = gasFloor - } - } else if limit > gasCeil { - limit = parentGasLimit - decay - if limit < gasCeil { - limit = gasCeil - } - } - return limit -} - -// CalcGasLimit1559 calculates the next block gas limit under 1559 rules. -func CalcGasLimit1559(parentGasLimit, desiredLimit uint64) uint64 { +// to keep the baseline gas close to the provided target, and increase it towards +// the target if the baseline gas is lower. +func CalcGasLimit(parentGasLimit, desiredLimit uint64) uint64 { delta := parentGasLimit/params.GasLimitBoundDivisor - 1 limit := parentGasLimit if desiredLimit < params.MinGasLimit { diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 3b4de337891c..86f9835a0150 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -198,8 +198,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { } } -func TestCalcGasLimit1559(t *testing.T) { - +func TestCalcGasLimit(t *testing.T) { for i, tc := range []struct { pGasLimit uint64 max uint64 @@ -209,23 +208,23 @@ func TestCalcGasLimit1559(t *testing.T) { {40000000, 40039061, 39960939}, } { // Increase - if have, want := CalcGasLimit1559(tc.pGasLimit, 2*tc.pGasLimit), tc.max; have != want { + if have, want := CalcGasLimit(tc.pGasLimit, 2*tc.pGasLimit), tc.max; have != want { t.Errorf("test %d: have %d want <%d", i, have, want) } // Decrease - if have, want := CalcGasLimit1559(tc.pGasLimit, 0), tc.min; have != want { + if have, want := CalcGasLimit(tc.pGasLimit, 0), tc.min; have != want { t.Errorf("test %d: have %d want >%d", i, have, want) } // Small decrease - if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit-1), tc.pGasLimit-1; have != want { + if have, want := CalcGasLimit(tc.pGasLimit, tc.pGasLimit-1), tc.pGasLimit-1; have != want { t.Errorf("test %d: have %d want %d", i, have, want) } // Small increase - if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit+1), tc.pGasLimit+1; have != want { + if have, want := CalcGasLimit(tc.pGasLimit, tc.pGasLimit+1), tc.pGasLimit+1; have != want { t.Errorf("test %d: have %d want %d", i, have, want) } // No change - if have, want := CalcGasLimit1559(tc.pGasLimit, tc.pGasLimit), tc.pGasLimit; have != want { + if have, want := CalcGasLimit(tc.pGasLimit, tc.pGasLimit), tc.pGasLimit; have != want { t.Errorf("test %d: have %d want %d", i, have, want) } } diff --git a/core/chain_makers.go b/core/chain_makers.go index 929a2aa3a43a..b113c0d1be9d 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -273,11 +273,10 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S } if chain.Config().IsLondon(header.Number) { header.BaseFee = misc.CalcBaseFee(chain.Config(), parent.Header()) - parentGasLimit := parent.GasLimit() if !chain.Config().IsLondon(parent.Number()) { - parentGasLimit = parent.GasLimit() * params.ElasticityMultiplier + parentGasLimit := parent.GasLimit() * params.ElasticityMultiplier + header.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit) } - header.GasLimit = CalcGasLimit1559(parentGasLimit, parentGasLimit) } return header } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 0913b69d7ffd..23ccf2484310 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -83,7 +83,6 @@ var Defaults = Config{ TrieTimeout: 60 * time.Minute, SnapshotCache: 102, Miner: miner.Config{ - GasFloor: 8000000, GasCeil: 8000000, GasPrice: big.NewInt(params.GWei), Recommit: 3 * time.Second, diff --git a/miner/stress/1559/main.go b/miner/stress/1559/main.go index 7d697e8ce466..90f210b2726d 100644 --- a/miner/stress/1559/main.go +++ b/miner/stress/1559/main.go @@ -242,7 +242,6 @@ func makeMiner(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { GPO: ethconfig.Defaults.GPO, Ethash: ethconfig.Defaults.Ethash, Miner: miner.Config{ - GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, GasPrice: big.NewInt(1), Recommit: time.Second, diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go index dea1ab7453d3..2aad40bb51ce 100644 --- a/miner/stress/clique/main.go +++ b/miner/stress/clique/main.go @@ -193,7 +193,6 @@ func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { TxPool: core.DefaultTxPoolConfig, GPO: ethconfig.Defaults.GPO, Miner: miner.Config{ - GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, GasPrice: big.NewInt(1), Recommit: time.Second, diff --git a/miner/stress/ethash/main.go b/miner/stress/ethash/main.go index 0f27c5e74cc1..7958e9ab8ef7 100644 --- a/miner/stress/ethash/main.go +++ b/miner/stress/ethash/main.go @@ -171,7 +171,6 @@ func makeMiner(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { GPO: ethconfig.Defaults.GPO, Ethash: ethconfig.Defaults.Ethash, Miner: miner.Config{ - GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, GasPrice: big.NewInt(1), Recommit: time.Second, diff --git a/miner/worker.go b/miner/worker.go index accf3dac9096..8bdb1eff7ca0 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -897,19 +897,17 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) header := &types.Header{ ParentHash: parent.Hash(), Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent.GasUsed(), parent.GasLimit(), w.config.GasFloor, w.config.GasCeil), + GasLimit: core.CalcGasLimit(parent.GasLimit(), w.config.GasCeil), Extra: w.extra, Time: uint64(timestamp), } // Set baseFee and GasLimit if we are on an EIP-1559 chain if w.chainConfig.IsLondon(header.Number) { header.BaseFee = misc.CalcBaseFee(w.chainConfig, parent.Header()) - parentGasLimit := parent.GasLimit() if !w.chainConfig.IsLondon(parent.Number()) { - // Bump by 2x - parentGasLimit = parent.GasLimit() * params.ElasticityMultiplier + parentGasLimit := parent.GasLimit() * params.ElasticityMultiplier + header.GasLimit = core.CalcGasLimit(parentGasLimit, w.config.GasCeil) } - header.GasLimit = core.CalcGasLimit1559(parentGasLimit, w.config.GasCeil) } // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) if w.isRunning() { diff --git a/miner/worker_test.go b/miner/worker_test.go index 2bb6c9407bbe..9c4dc0f3788e 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -67,7 +67,6 @@ var ( testConfig = &Config{ Recommit: time.Second, - GasFloor: params.GenesisGasLimit, GasCeil: params.GenesisGasLimit, } ) From e4b687cf462870538743b3218906940ae590e7fd Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 10 Aug 2021 04:40:54 -0400 Subject: [PATCH 34/60] mobile: remove deprecated code (#23357) --- mobile/types.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/mobile/types.go b/mobile/types.go index de6457e7e17c..d1427ac11d38 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -292,19 +292,6 @@ func (tx *Transaction) GetNonce() int64 { return int64(tx.tx.Nonce()) } func (tx *Transaction) GetHash() *Hash { return &Hash{tx.tx.Hash()} } func (tx *Transaction) GetCost() *BigInt { return &BigInt{tx.tx.Cost()} } -// Deprecated: GetSigHash cannot know which signer to use. -func (tx *Transaction) GetSigHash() *Hash { return &Hash{types.HomesteadSigner{}.Hash(tx.tx)} } - -// Deprecated: use EthereumClient.TransactionSender -func (tx *Transaction) GetFrom(chainID *BigInt) (address *Address, _ error) { - var signer types.Signer = types.HomesteadSigner{} - if chainID != nil { - signer = types.NewEIP155Signer(chainID.bigint) - } - from, err := types.Sender(signer, tx.tx) - return &Address{from}, err -} - func (tx *Transaction) GetTo() *Address { if to := tx.tx.To(); to != nil { return &Address{*to} From 57d9e0ac759d3ff6bda8bd83016c83af450cbe42 Mon Sep 17 00:00:00 2001 From: Ziyuan Zhong Date: Tue, 10 Aug 2021 17:04:29 +0800 Subject: [PATCH 35/60] core/state/snapshot: fix typo in comment (#23219) --- core/state/snapshot/snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index cb8ec7a7072c..e799dab9738a 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -469,7 +469,7 @@ func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer { if flattened.memory < aggregatorMemoryLimit { // Accumulator layer is smaller than the limit, so we can abort, unless // there's a snapshot being generated currently. In that case, the trie - // will move fron underneath the generator so we **must** merge all the + // will move from underneath the generator so we **must** merge all the // partial data down into the snapshot and restart the generation. if flattened.parent.(*diskLayer).genAbort == nil { return nil From 66948316f7e94f04edbd25906fd09a648a489129 Mon Sep 17 00:00:00 2001 From: Tyler Chambers <2775339+tylerchambers@users.noreply.github.com> Date: Tue, 10 Aug 2021 05:16:53 -0400 Subject: [PATCH 36/60] core/state/snapshot: clarify comment about snapshot repair (#23305) Co-authored-by: Tyler Chambers Co-authored-by: Felix Lange --- core/state/snapshot/snapshot.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index e799dab9738a..828e58d042b2 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -169,11 +169,17 @@ type Tree struct { // store (with a number of memory layers from a journal), ensuring that the head // of the snapshot matches the expected one. // -// If the snapshot is missing or the disk layer is broken, the entire is deleted -// and will be reconstructed from scratch based on the tries in the key-value -// store, on a background thread. If the memory layers from the journal is not -// continuous with disk layer or the journal is missing, all diffs will be discarded -// iff it's in "recovery" mode, otherwise rebuild is mandatory. +// If the snapshot is missing or the disk layer is broken, the snapshot will be +// reconstructed using both the existing data and the state trie. +// The repair happens on a background thread. +// +// If the memory layers in the journal do not match the disk layer (e.g. there is +// a gap) or the journal is missing, there are two repair cases: +// +// - if the 'recovery' parameter is true, all memory diff-layers will be discarded. +// This case happens when the snapshot is 'ahead' of the state trie. +// - otherwise, the entire snapshot is considered invalid and will be recreated on +// a background thread. func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ From 39fe7eca6b5ce78be8e4fdb88aa13d1ec41090d2 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 10 Aug 2021 07:39:06 -0600 Subject: [PATCH 37/60] internal/ethapi: return maxFeePerGas for gasPrice for EIP-1559 txs (#23345) --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 06a0dcf8eb0f..f7c99512f279 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1323,7 +1323,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber price := math.BigMin(new(big.Int).Add(tx.GasTipCap(), baseFee), tx.GasFeeCap()) result.GasPrice = (*hexutil.Big)(price) } else { - result.GasPrice = nil + result.GasPrice = (*hexutil.Big)(tx.GasFeeCap()) } } return result From a879c42bd34e91a4e08c0247138c96957eca2b3f Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 10 Aug 2021 07:56:34 -0600 Subject: [PATCH 38/60] internal/ethapi, accounts/abi/bind: cap highest gas limit by account balance for 1559 fee parameters (#23309) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * internal/ethapi/api: cap highest gas limit by account balance for 1559 fee parameters * accounts/abi/bind: port gas limit cap for 1559 parameters to simulated backend * accounts/abi/bind: add test for 1559 gas estimates for the simulated backend * internal/ethapi/api: fix comment * accounts/abi/bind/backends, internal/ethapi: unify naming style Co-authored-by: Péter Szilágyi --- accounts/abi/bind/backends/simulated.go | 17 ++++++++++++--- accounts/abi/bind/backends/simulated_test.go | 23 ++++++++++++++++++++ internal/ethapi/api.go | 19 ++++++++++++---- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 06cd0908e746..cb4eea3a0dc0 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -488,8 +488,19 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs } else { hi = b.pendingBlock.GasLimit() } + // Normalize the max fee per gas the call is willing to spend. + var feeCap *big.Int + if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { + return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } else if call.GasPrice != nil { + feeCap = call.GasPrice + } else if call.GasFeeCap != nil { + feeCap = call.GasFeeCap + } else { + feeCap = common.Big0 + } // Recap the highest gas allowance with account's balance. - if call.GasPrice != nil && call.GasPrice.BitLen() != 0 { + if feeCap.BitLen() != 0 { balance := b.pendingState.GetBalance(call.From) // from can't be nil available := new(big.Int).Set(balance) if call.Value != nil { @@ -498,14 +509,14 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs } available.Sub(available, call.Value) } - allowance := new(big.Int).Div(available, call.GasPrice) + allowance := new(big.Int).Div(available, feeCap) if allowance.IsUint64() && hi > allowance.Uint64() { transfer := call.Value if transfer == nil { transfer = new(big.Int) } log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, - "sent", transfer, "gasprice", call.GasPrice, "fundable", allowance) + "sent", transfer, "feecap", feeCap, "fundable", allowance) hi = allowance.Uint64() } } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 0257e3e8702c..613267810744 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -580,6 +580,26 @@ func TestEstimateGasWithPrice(t *testing.T) { Value: big.NewInt(100000000000), Data: nil, }, 21000, errors.New("gas required exceeds allowance (10999)")}, // 10999=(2.2ether-1000wei)/(2e14) + + {"EstimateEIP1559WithHighFees", ethereum.CallMsg{ + From: addr, + To: &addr, + Gas: 0, + GasFeeCap: big.NewInt(1e14), // maxgascost = 2.1ether + GasTipCap: big.NewInt(1), + Value: big.NewInt(1e17), // the remaining balance for fee is 2.1ether + Data: nil, + }, params.TxGas, nil}, + + {"EstimateEIP1559WithSuperHighFees", ethereum.CallMsg{ + From: addr, + To: &addr, + Gas: 0, + GasFeeCap: big.NewInt(1e14), // maxgascost = 2.1ether + GasTipCap: big.NewInt(1), + Value: big.NewInt(1e17 + 1), // the remaining balance for fee is 2.1ether + Data: nil, + }, params.TxGas, errors.New("gas required exceeds allowance (20999)")}, // 20999=(2.2ether-0.1ether-1wei)/(1e14) } for i, c := range cases { got, err := sim.EstimateGas(context.Background(), c.message) @@ -592,6 +612,9 @@ func TestEstimateGasWithPrice(t *testing.T) { } continue } + if c.expectError == nil && err != nil { + t.Fatalf("test %d: didn't expect error, got %v", i, err) + } if got != c.expect { t.Fatalf("test %d: gas estimation mismatch, want %d, got %d", i, c.expect, got) } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f7c99512f279..9a82824ada14 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -477,7 +477,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio if args.Nonce == nil { return nil, fmt.Errorf("nonce not specified") } - // Before actually sign the transaction, ensure the transaction fee is reasonable. + // Before actually signing the transaction, ensure the transaction fee is reasonable. tx := args.toTransaction() if err := checkTxFee(tx.GasPrice(), tx.Gas(), s.b.RPCTxFeeCap()); err != nil { return nil, err @@ -1008,8 +1008,19 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } hi = block.GasLimit() } + // Normalize the max fee per gas the call is willing to spend. + var feeCap *big.Int + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } else if args.GasPrice != nil { + feeCap = args.GasPrice.ToInt() + } else if args.MaxFeePerGas != nil { + feeCap = args.MaxFeePerGas.ToInt() + } else { + feeCap = common.Big0 + } // Recap the highest gas limit with account's available balance. - if args.GasPrice != nil && args.GasPrice.ToInt().BitLen() != 0 { + if feeCap.BitLen() != 0 { state, _, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if err != nil { return 0, err @@ -1022,7 +1033,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } available.Sub(available, args.Value.ToInt()) } - allowance := new(big.Int).Div(available, args.GasPrice.ToInt()) + allowance := new(big.Int).Div(available, feeCap) // If the allowance is larger than maximum uint64, skip checking if allowance.IsUint64() && hi > allowance.Uint64() { @@ -1031,7 +1042,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr transfer = new(hexutil.Big) } log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, - "sent", transfer.ToInt(), "gasprice", args.GasPrice.ToInt(), "fundable", allowance) + "sent", transfer.ToInt(), "maxFeePerGas", feeCap, "fundable", allowance) hi = allowance.Uint64() } } From 9d6480c3cd7e7380fd9fb90935d108bca12d33e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 10 Aug 2021 16:58:38 +0300 Subject: [PATCH 39/60] core/state/snapshot: gofmt --- core/state/snapshot/snapshot.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 828e58d042b2..8889fa9fec12 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -172,10 +172,10 @@ type Tree struct { // If the snapshot is missing or the disk layer is broken, the snapshot will be // reconstructed using both the existing data and the state trie. // The repair happens on a background thread. -// +// // If the memory layers in the journal do not match the disk layer (e.g. there is // a gap) or the journal is missing, there are two repair cases: -// +// // - if the 'recovery' parameter is true, all memory diff-layers will be discarded. // This case happens when the snapshot is 'ahead' of the state trie. // - otherwise, the entire snapshot is considered invalid and will be recreated on From 9097d0a325e3309387aa90e01cbe843941d60fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 10 Aug 2021 17:13:06 +0300 Subject: [PATCH 40/60] travis: transition from docker auto builds to manual pushes --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94d2dda73604..a7db711ccc92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,7 +41,7 @@ jobs: before_install: - export DOCKER_CLI_EXPERIMENTAL=enabled script: - - go run build/ci.go docker -image -manifest amd64,arm64 -upload karalabe/geth-docker-test + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go - stage: build if: type = push @@ -58,7 +58,7 @@ jobs: before_install: - export DOCKER_CLI_EXPERIMENTAL=enabled script: - - go run build/ci.go docker -image -manifest amd64,arm64 -upload karalabe/geth-docker-test + - go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go # This builder does the Ubuntu PPA upload - stage: build From c27bd3481e5acb2cd88226eb3d6edf58aa1c5096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 10 Aug 2021 17:49:52 +0300 Subject: [PATCH 41/60] build: fix docker tag to include `v` prefix in version string --- build/ci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci.go b/build/ci.go index ca87913eb00f..d7d2ce72e06c 100644 --- a/build/ci.go +++ b/build/ci.go @@ -492,7 +492,7 @@ func doDocker(cmdline []string) { case env.Branch == "master": tags = []string{"latest"} case strings.HasPrefix(env.Tag, "v1."): - tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), params.Version} + tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} } // If architecture specific image builds are requested, build and push them if *image { From 278ec7176ab1ec57cc0ab56ba3cbdbc3b2635bf6 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 12 Aug 2021 08:14:21 +0200 Subject: [PATCH 42/60] internal/ethapi: add back missing check for maxfee < maxPriorityFee --- internal/ethapi/transaction_args.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 9076e49b2a66..8c12743dcda4 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -120,6 +120,11 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { args.GasPrice = (*hexutil.Big)(price) } } + } else { + // Both maxPriorityfee and maxFee set by caller. Sanity-check their internal relation + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } } if args.Value == nil { args.Value = new(hexutil.Big) From f4ad49387025cad7d278a05c5afaee6089b573a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 12 Aug 2021 10:11:39 +0300 Subject: [PATCH 43/60] params: update CHTs for the 1.10.7 release --- params/config.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/params/config.go b/params/config.go index aa155c50db89..591c36de2a37 100644 --- a/params/config.go +++ b/params/config.go @@ -75,10 +75,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 389, - SectionHead: common.HexToHash("0x8f96e510cf64abf34095c5aa3937acdf5316de5540945b9688f4a2e083cddc73"), - CHTRoot: common.HexToHash("0xa2362493848d6dbc50dcbbf74c017ea808b8938bfb129217d507bd276950d7ac"), - BloomRoot: common.HexToHash("0x72fc78a841bde7e08e1fb7c187b622c49dc8271db12db748ff5d0f27bdb41413"), + SectionIndex: 395, + SectionHead: common.HexToHash("0xbfca95b8c1de014e252288e9c32029825fadbff58285f5b54556525e480dbb5b"), + CHTRoot: common.HexToHash("0x2ccf3dbb58eb6375e037fdd981ca5778359e4b8fa0270c2878b14361e64161e7"), + BloomRoot: common.HexToHash("0x2d46ec65a6941a2dc1e682f8f81f3d24192021f492fdf6ef0fdd51acb0f4ba0f"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -116,10 +116,10 @@ var ( // RopstenTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. RopstenTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 322, - SectionHead: common.HexToHash("0xe3f2fb70acd752bbcac06b67688db8430815c788a31213011ed51b966108a5f4"), - CHTRoot: common.HexToHash("0xb2993a6bc28b23b84159cb477c38c0ec5607434faae6b3657ad44cbcf116f288"), - BloomRoot: common.HexToHash("0x871841e5c2ada9dab2011a550d38e9fe0a30047cfc81f1ffc7ebc09f4f230732"), + SectionIndex: 329, + SectionHead: common.HexToHash("0xe66f7038333a01fb95dc9ea03e5a2bdaf4b833cdcb9e393b9127e013bd64d39b"), + CHTRoot: common.HexToHash("0x1b0c883338ac0d032122800c155a2e73105fbfebfaa50436893282bc2d9feec5"), + BloomRoot: common.HexToHash("0x3cc98c88d283bf002378246f22c653007655cbcea6ed89f98d739f73bd341a01"), } // RopstenCheckpointOracle contains a set of configs for the Ropsten test network oracle. @@ -160,10 +160,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 270, - SectionHead: common.HexToHash("0x03ef8982c93bbf18c859bc1b20ae05b439f04cf1ff592656e941d2c3fcff5d68"), - CHTRoot: common.HexToHash("0x9eb80685e8ece479e105b170439779bc0f89997ab7f4dee425f85c4234e8a6b5"), - BloomRoot: common.HexToHash("0xc3673721c5697efe5fe4cb825d178f4a335dbfeda6a197fb75c9256a767379dc"), + SectionIndex: 276, + SectionHead: common.HexToHash("0xea89a4b04e3da9bd688e316f8de669396b6d4a38a19d2cd96a00b70d58b836aa"), + CHTRoot: common.HexToHash("0xd6889d0bf6673c0d2c1cf6e9098a6fe5b30888a115b6112796aa8ee8efc4a723"), + BloomRoot: common.HexToHash("0x6009a9256b34b8bde3a3f094afb647ba5d73237546017b9025d64ac1ff54c47c"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -202,10 +202,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 154, - SectionHead: common.HexToHash("0xf4cb74cc0e3683589f4992902184241fb892d7c3859d0044c16ec864605ff80d"), - CHTRoot: common.HexToHash("0xead95f9f2504b2c7c6d82c51d30e50b40631c3ea2f590cddcc9721cfc0ae79de"), - BloomRoot: common.HexToHash("0xc6dd6cfe88ac9c4a6d19c9a8651944fa9d941a2340a8f5ddaf673d4d39779d81"), + SectionIndex: 160, + SectionHead: common.HexToHash("0xb5a666c790dc35a5613d04ebba8ba47a850b45a15d9b95ad7745c35ae034b5a5"), + CHTRoot: common.HexToHash("0x6b4e00df52bdc38fa6c26c8ef595c2ad6184963ea36ab08ee744af460aa735e1"), + BloomRoot: common.HexToHash("0x8fa88f5e50190cb25243aeee262a1a9e4434a06f8d455885dcc1b5fc48c33836"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From 12f0ff40b1bfb484cafc3be6e0040262e96743bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 12 Aug 2021 10:14:03 +0300 Subject: [PATCH 44/60] params: release Geth v1.10.7 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 61fb82dda719..2d3155d35e11 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From a580f7d6c54812ef47df94c6ffc974c9dbc48245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 12 Aug 2021 10:15:49 +0300 Subject: [PATCH 45/60] params: begin v1.10.8 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 2d3155d35e11..3c50a9e8d017 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5f98020a21d64fa22516c04aaf3bc4ed0629e7cc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 13 Aug 2021 10:51:01 +0200 Subject: [PATCH 46/60] core/rawdb: implement sequential reads in freezer_table (#23117) * core/rawdb: implement sequential reads in freezer_table * core/rawdb, ethdb: add sequential reader to db interface * core/rawdb: lint nitpicks * core/rawdb: fix some nitpicks * core/rawdb: fix flaw with deferred reads not being performed * core/rawdb: better documentation --- core/rawdb/database.go | 5 + core/rawdb/freezer.go | 12 ++ core/rawdb/freezer_table.go | 214 +++++++++++++++++++++++-------- core/rawdb/freezer_table_test.go | 117 ++++++++++++++++- core/rawdb/table.go | 6 + ethdb/database.go | 7 + 6 files changed, 309 insertions(+), 52 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c8bfdbace14e..90619169a09b 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -89,6 +89,11 @@ func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { return nil, errNotSupported } +// ReadAncients returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) ReadAncients(kind string, start, max, maxByteSize uint64) ([][]byte, error) { + return nil, errNotSupported +} + // Ancients returns an error as we don't have a backing chain freezer. func (db *nofreezedb) Ancients() (uint64, error) { return 0, errNotSupported diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ff8919b59e92..253de9f7cfc3 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -180,6 +180,18 @@ func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) { return nil, errUnknownTable } +// ReadAncients retrieves multiple items in sequence, starting from the index 'start'. +// It will return +// - at most 'max' items, +// - at least 1 item (even if exceeding the maxByteSize), but will otherwise +// return as many items as fit into maxByteSize. +func (f *freezer) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) { + if table := f.tables[kind]; table != nil { + return table.RetrieveItems(start, count, maxBytes) + } + return nil, errUnknownTable +} + // Ancients returns the length of the frozen items. func (f *freezer) Ancients() (uint64, error) { return atomic.LoadUint64(&f.frozen), nil diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index d7bfe18e020f..9d052f7cd8d3 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -70,6 +70,19 @@ func (i *indexEntry) marshallBinary() []byte { return b } +// bounds returns the start- and end- offsets, and the file number of where to +// read there data item marked by the two index entries. The two entries are +// assumed to be sequential. +func (start *indexEntry) bounds(end *indexEntry) (startOffset, endOffset, fileId uint32) { + if start.filenum != end.filenum { + // If a piece of data 'crosses' a data-file, + // it's actually in one piece on the second data-file. + // We return a zero-indexEntry for the second file as start + return 0, end.offset, end.filenum + } + return start.offset, end.offset, end.filenum +} + // freezerTable represents a single chained data table within the freezer (e.g. blocks). // It consists of a data file (snappy encoded arbitrary data blobs) and an indexEntry // file (uncompressed 64 bit indices into the data file). @@ -546,84 +559,183 @@ func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool return false, nil } -// getBounds returns the indexes for the item -// returns start, end, filenumber and error -func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) { - buffer := make([]byte, indexEntrySize) - var startIdx, endIdx indexEntry - // Read second index - if _, err := t.index.ReadAt(buffer, int64((item+1)*indexEntrySize)); err != nil { - return 0, 0, 0, err - } - endIdx.unmarshalBinary(buffer) - // Read first index (unless it's the very first item) - if item != 0 { - if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { - return 0, 0, 0, err - } - startIdx.unmarshalBinary(buffer) - } else { +// getIndices returns the index entries for the given from-item, covering 'count' items. +// N.B: The actual number of returned indices for N items will always be N+1 (unless an +// error is returned). +// OBS: This method assumes that the caller has already verified (and/or trimmed) the range +// so that the items are within bounds. If this method is used to read out of bounds, +// it will return error. +func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { + // Apply the table-offset + from = from - uint64(t.itemOffset) + // For reading N items, we need N+1 indices. + buffer := make([]byte, (count+1)*indexEntrySize) + if _, err := t.index.ReadAt(buffer, int64(from*indexEntrySize)); err != nil { + return nil, err + } + var ( + indices []*indexEntry + offset int + ) + for i := from; i <= from+count; i++ { + index := new(indexEntry) + index.unmarshalBinary(buffer[offset:]) + offset += indexEntrySize + indices = append(indices, index) + } + if from == 0 { // Special case if we're reading the first item in the freezer. We assume that // the first item always start from zero(regarding the deletion, we // only support deletion by files, so that the assumption is held). // This means we can use the first item metadata to carry information about // the 'global' offset, for the deletion-case - return 0, endIdx.offset, endIdx.filenum, nil + indices[0].offset = 0 + indices[0].filenum = indices[1].filenum } - if startIdx.filenum != endIdx.filenum { - // If a piece of data 'crosses' a data-file, - // it's actually in one piece on the second data-file. - // We return a zero-indexEntry for the second file as start - return 0, endIdx.offset, endIdx.filenum, nil - } - return startIdx.offset, endIdx.offset, endIdx.filenum, nil + return indices, nil } // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { - blob, err := t.retrieve(item) + items, err := t.RetrieveItems(item, 1, 0) if err != nil { return nil, err } - if t.noCompression { - return blob, nil + return items[0], nil +} + +// RetrieveItems returns multiple items in sequence, starting from the index 'start'. +// It will return at most 'max' items, but will abort earlier to respect the +// 'maxBytes' argument. However, if the 'maxBytes' is smaller than the size of one +// item, it _will_ return one element and possibly overflow the maxBytes. +func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, error) { + // First we read the 'raw' data, which might be compressed. + diskData, sizes, err := t.retrieveItems(start, count, maxBytes) + if err != nil { + return nil, err } - return snappy.Decode(nil, blob) + var ( + output = make([][]byte, 0, count) + offset int // offset for reading + outputSize int // size of uncompressed data + ) + // Now slice up the data and decompress. + for i, diskSize := range sizes { + item := diskData[offset : offset+diskSize] + offset += diskSize + decompressedSize := diskSize + if !t.noCompression { + decompressedSize, _ = snappy.DecodedLen(item) + } + if i > 0 && uint64(outputSize+decompressedSize) > maxBytes { + break + } + if !t.noCompression { + data, err := snappy.Decode(nil, item) + if err != nil { + return nil, err + } + output = append(output, data) + } else { + output = append(output, item) + } + outputSize += decompressedSize + } + return output, nil } -// retrieve looks up the data offset of an item with the given number and retrieves -// the raw binary blob from the data file. OBS! This method does not decode -// compressed data. -func (t *freezerTable) retrieve(item uint64) ([]byte, error) { +// retrieveItems reads up to 'count' items from the table. It reads at least +// one item, but otherwise avoids reading more than maxBytes bytes. +// It returns the (potentially compressed) data, and the sizes. +func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) { t.lock.RLock() defer t.lock.RUnlock() // Ensure the table and the item is accessible if t.index == nil || t.head == nil { - return nil, errClosed + return nil, nil, errClosed } - if atomic.LoadUint64(&t.items) <= item { - return nil, errOutOfBounds + itemCount := atomic.LoadUint64(&t.items) // max number + // Ensure the start is written, not deleted from the tail, and that the + // caller actually wants something + if itemCount <= start || uint64(t.itemOffset) > start || count == 0 { + return nil, nil, errOutOfBounds } - // Ensure the item was not deleted from the tail either - if uint64(t.itemOffset) > item { - return nil, errOutOfBounds + if start+count > itemCount { + count = itemCount - start } - startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset)) - if err != nil { - return nil, err + var ( + output = make([]byte, maxBytes) // Buffer to read data into + outputSize int // Used size of that buffer + ) + // readData is a helper method to read a single data item from disk. + readData := func(fileId, start uint32, length int) error { + // In case a small limit is used, and the elements are large, may need to + // realloc the read-buffer when reading the first (and only) item. + if len(output) < length { + output = make([]byte, length) + } + dataFile, exist := t.files[fileId] + if !exist { + return fmt.Errorf("missing data file %d", fileId) + } + if _, err := dataFile.ReadAt(output[outputSize:outputSize+length], int64(start)); err != nil { + return err + } + outputSize += length + return nil } - dataFile, exist := t.files[filenum] - if !exist { - return nil, fmt.Errorf("missing data file %d", filenum) + // Read all the indexes in one go + indices, err := t.getIndices(start, count) + if err != nil { + return nil, nil, err } - // Retrieve the data itself, decompress and return - blob := make([]byte, endOffset-startOffset) - if _, err := dataFile.ReadAt(blob, int64(startOffset)); err != nil { - return nil, err + var ( + sizes []int // The sizes for each element + totalSize = 0 // The total size of all data read so far + readStart = indices[0].offset // Where, in the file, to start reading + unreadSize = 0 // The size of the as-yet-unread data + ) + + for i, firstIndex := range indices[:len(indices)-1] { + secondIndex := indices[i+1] + // Determine the size of the item. + offset1, offset2, _ := firstIndex.bounds(secondIndex) + size := int(offset2 - offset1) + // Crossing a file boundary? + if secondIndex.filenum != firstIndex.filenum { + // If we have unread data in the first file, we need to do that read now. + if unreadSize > 0 { + if err := readData(firstIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + unreadSize = 0 + } + readStart = 0 + } + if i > 0 && uint64(totalSize+size) > maxBytes { + // About to break out due to byte limit being exceeded. We don't + // read this last item, but we need to do the deferred reads now. + if unreadSize > 0 { + if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + } + break + } + // Defer the read for later + unreadSize += size + totalSize += size + sizes = append(sizes, size) + if i == len(indices)-2 || uint64(totalSize) > maxBytes { + // Last item, need to do the read now + if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + break + } } - t.readMeter.Mark(int64(len(blob) + 2*indexEntrySize)) - return blob, nil + return output[:outputSize], sizes, nil } // has returns an indicator whether the specified number data diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0df28f236d2d..e8a8b5c46309 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -74,7 +74,7 @@ func TestFreezerBasics(t *testing.T) { exp := getChunk(15, y) got, err := f.Retrieve(uint64(y)) if err != nil { - t.Fatal(err) + t.Fatalf("reading item %d: %v", y, err) } if !bytes.Equal(got, exp) { t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) @@ -692,3 +692,118 @@ func TestAppendTruncateParallel(t *testing.T) { } } } + +// TestSequentialRead does some basic tests on the RetrieveItems. +func TestSequentialRead(t *testing.T) { + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("batchread-%d", rand.Uint64()) + { // Fill table + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + if err != nil { + t.Fatal(err) + } + // Write 15 bytes 30 times + for x := 0; x < 30; x++ { + data := getChunk(15, x) + f.Append(uint64(x), data) + } + f.DumpIndex(0, 30) + f.Close() + } + { // Open it, iterate, verify iteration + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, 10000, 100000) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), 30; have != want { + t.Fatalf("want %d items, have %d ", want, have) + } + for i, have := range items { + want := getChunk(15, i) + if !bytes.Equal(want, have) { + t.Fatalf("data corruption: have\n%x\n, want \n%x\n", have, want) + } + } + f.Close() + } + { // Open it, iterate, verify byte limit. The byte limit is less than item + // size, so each lookup should only return one item + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, 10000, 10) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), 1; have != want { + t.Fatalf("want %d items, have %d ", want, have) + } + for i, have := range items { + want := getChunk(15, i) + if !bytes.Equal(want, have) { + t.Fatalf("data corruption: have\n%x\n, want \n%x\n", have, want) + } + } + f.Close() + } +} + +// TestSequentialReadByteLimit does some more advanced tests on batch reads. +// These tests check that when the byte limit hits, we correctly abort in time, +// but also properly do all the deferred reads for the previous data, regardless +// of whether the data crosses a file boundary or not. +func TestSequentialReadByteLimit(t *testing.T) { + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("batchread-2-%d", rand.Uint64()) + { // Fill table + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + if err != nil { + t.Fatal(err) + } + // Write 10 bytes 30 times, + // Splitting it at every 100 bytes (10 items) + for x := 0; x < 30; x++ { + data := getChunk(10, x) + f.Append(uint64(x), data) + } + f.Close() + } + for i, tc := range []struct { + items uint64 + limit uint64 + want int + }{ + {9, 89, 8}, + {10, 99, 9}, + {11, 109, 10}, + {100, 89, 8}, + {100, 99, 9}, + {100, 109, 10}, + } { + { + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, tc.items, tc.limit) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), tc.want; have != want { + t.Fatalf("test %d: want %d items, have %d ", i, want, have) + } + for ii, have := range items { + want := getChunk(10, ii) + if !bytes.Equal(want, have) { + t.Fatalf("test %d: data corruption item %d: have\n%x\n, want \n%x\n", i, ii, have, want) + } + } + f.Close() + } + } +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index d5ef60ae50ef..586451c0644d 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -62,6 +62,12 @@ func (t *table) Ancient(kind string, number uint64) ([]byte, error) { return t.db.Ancient(kind, number) } +// ReadAncients is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) { + return t.db.ReadAncients(kind, start, count, maxBytes) +} + // Ancients is a noop passthrough that just forwards the request to the underlying // database. func (t *table) Ancients() (uint64, error) { diff --git a/ethdb/database.go b/ethdb/database.go index 0dc14624b98e..bdc09d5e9877 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -76,6 +76,13 @@ type AncientReader interface { // Ancient retrieves an ancient binary blob from the append-only immutable files. Ancient(kind string, number uint64) ([]byte, error) + // ReadAncients retrieves multiple items in sequence, starting from the index 'start'. + // It will return + // - at most 'count' items, + // - at least 1 item (even if exceeding the maxBytes), but will otherwise + // return as many items as fit into maxBytes. + ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) + // Ancients returns the ancient item numbers in the ancient store. Ancients() (uint64, error) From 0a68558e7e025afebf67b81bf48ecb8b0fa7c06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 13 Aug 2021 15:39:51 +0300 Subject: [PATCH 47/60] accounts/external: handle 0 chainid as not-set for the Clef API (#23394) * accounts/external: handle 0 chainid as not-set for the Clef API * accounts/external: document SignTx Co-authored-by: Felix Lange --- accounts/external/backend.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index b2de9e932af9..e3f754eafcc4 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -196,6 +196,10 @@ type signTransactionResult struct { Tx *types.Transaction `json:"tx"` } +// SignTx sends the transaction to the external signer. +// If chainID is nil, or tx.ChainID is zero, the chain ID will be assigned +// by the external signer. For non-legacy transactions, the chain ID of the +// transaction overrides the chainID parameter. func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { data := hexutil.Bytes(tx.Data()) var to *common.MixedcaseAddress @@ -218,17 +222,17 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio args.MaxFeePerGas = (*hexutil.Big)(tx.GasFeeCap()) args.MaxPriorityFeePerGas = (*hexutil.Big)(tx.GasTipCap()) default: - return nil, fmt.Errorf("Unsupported tx type %d", tx.Type()) + return nil, fmt.Errorf("unsupported tx type %d", tx.Type()) } // We should request the default chain id that we're operating with // (the chain we're executing on) - if chainID != nil { + if chainID != nil && chainID.Sign() != 0 { args.ChainID = (*hexutil.Big)(chainID) } if tx.Type() != types.LegacyTxType { // However, if the user asked for a particular chain id, then we should // use that instead. - if tx.ChainId() != nil { + if tx.ChainId().Sign() != 0 { args.ChainID = (*hexutil.Big)(tx.ChainId()) } accessList := tx.AccessList() From fb4007bb2208a5b76f76287c03001ef906261691 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 17 Aug 2021 17:30:21 +0200 Subject: [PATCH 48/60] tests: update, enable legacy tests, remove vm tests (#23350) * tests: update, enable legacy tests, remove vm tests * tests: minor fixes --- tests/gen_vmexec.go | 90 ---------------------- tests/init_test.go | 9 +-- tests/state_test.go | 7 +- tests/state_test_util.go | 4 + tests/testdata | 2 +- tests/vm_test.go | 39 ---------- tests/vm_test_util.go | 161 --------------------------------------- 7 files changed, 11 insertions(+), 301 deletions(-) delete mode 100644 tests/gen_vmexec.go delete mode 100644 tests/vm_test.go delete mode 100644 tests/vm_test_util.go diff --git a/tests/gen_vmexec.go b/tests/gen_vmexec.go deleted file mode 100644 index 2fe155152d0c..000000000000 --- a/tests/gen_vmexec.go +++ /dev/null @@ -1,90 +0,0 @@ -// Code generated by github.com/fjl/gencodec. DO NOT EDIT. - -package tests - -import ( - "encoding/json" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/math" -) - -var _ = (*vmExecMarshaling)(nil) - -// MarshalJSON marshals as JSON. -func (v vmExec) MarshalJSON() ([]byte, error) { - type vmExec struct { - Address common.UnprefixedAddress `json:"address" gencodec:"required"` - Caller common.UnprefixedAddress `json:"caller" gencodec:"required"` - Origin common.UnprefixedAddress `json:"origin" gencodec:"required"` - Code hexutil.Bytes `json:"code" gencodec:"required"` - Data hexutil.Bytes `json:"data" gencodec:"required"` - Value *math.HexOrDecimal256 `json:"value" gencodec:"required"` - GasLimit math.HexOrDecimal64 `json:"gas" gencodec:"required"` - GasPrice *math.HexOrDecimal256 `json:"gasPrice" gencodec:"required"` - } - var enc vmExec - enc.Address = common.UnprefixedAddress(v.Address) - enc.Caller = common.UnprefixedAddress(v.Caller) - enc.Origin = common.UnprefixedAddress(v.Origin) - enc.Code = v.Code - enc.Data = v.Data - enc.Value = (*math.HexOrDecimal256)(v.Value) - enc.GasLimit = math.HexOrDecimal64(v.GasLimit) - enc.GasPrice = (*math.HexOrDecimal256)(v.GasPrice) - return json.Marshal(&enc) -} - -// UnmarshalJSON unmarshals from JSON. -func (v *vmExec) UnmarshalJSON(input []byte) error { - type vmExec struct { - Address *common.UnprefixedAddress `json:"address" gencodec:"required"` - Caller *common.UnprefixedAddress `json:"caller" gencodec:"required"` - Origin *common.UnprefixedAddress `json:"origin" gencodec:"required"` - Code *hexutil.Bytes `json:"code" gencodec:"required"` - Data *hexutil.Bytes `json:"data" gencodec:"required"` - Value *math.HexOrDecimal256 `json:"value" gencodec:"required"` - GasLimit *math.HexOrDecimal64 `json:"gas" gencodec:"required"` - GasPrice *math.HexOrDecimal256 `json:"gasPrice" gencodec:"required"` - } - var dec vmExec - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - if dec.Address == nil { - return errors.New("missing required field 'address' for vmExec") - } - v.Address = common.Address(*dec.Address) - if dec.Caller == nil { - return errors.New("missing required field 'caller' for vmExec") - } - v.Caller = common.Address(*dec.Caller) - if dec.Origin == nil { - return errors.New("missing required field 'origin' for vmExec") - } - v.Origin = common.Address(*dec.Origin) - if dec.Code == nil { - return errors.New("missing required field 'code' for vmExec") - } - v.Code = *dec.Code - if dec.Data == nil { - return errors.New("missing required field 'data' for vmExec") - } - v.Data = *dec.Data - if dec.Value == nil { - return errors.New("missing required field 'value' for vmExec") - } - v.Value = (*big.Int)(dec.Value) - if dec.GasLimit == nil { - return errors.New("missing required field 'gas' for vmExec") - } - v.GasLimit = uint64(*dec.GasLimit) - if dec.GasPrice == nil { - return errors.New("missing required field 'gasPrice' for vmExec") - } - v.GasPrice = (*big.Int)(dec.GasPrice) - return nil -} diff --git a/tests/init_test.go b/tests/init_test.go index 969cf7139915..312ad8869a37 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -34,12 +34,11 @@ import ( ) var ( - baseDir = filepath.Join(".", "testdata") - blockTestDir = filepath.Join(baseDir, "BlockchainTests") - stateTestDir = filepath.Join(baseDir, "GeneralStateTests") - //legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") + baseDir = filepath.Join(".", "testdata") + blockTestDir = filepath.Join(baseDir, "BlockchainTests") + stateTestDir = filepath.Join(baseDir, "GeneralStateTests") + legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") transactionTestDir = filepath.Join(baseDir, "TransactionTests") - vmTestDir = filepath.Join(baseDir, "VMTests") rlpTestDir = filepath.Join(baseDir, "RLPTests") difficultyTestDir = filepath.Join(baseDir, "BasicTests") ) diff --git a/tests/state_test.go b/tests/state_test.go index 242e9712a312..c2ca0e8d6948 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -45,8 +45,7 @@ func TestState(t *testing.T) { // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) - // Un-skip this when https://github.com/ethereum/tests/issues/908 is closed - st.skipLoad(`^stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy`) + // Broken tests: // Expected failures: //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") @@ -59,9 +58,7 @@ func TestState(t *testing.T) { // For Istanbul, older tests were moved into LegacyTests for _, dir := range []string{ stateTestDir, - // legacy state tests are disabled, due to them not being - // regenerated for the no-sender-eoa change. - //legacyStateTestDir, + legacyStateTestDir, } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 4f57fee8909b..8b8a6a4fdacf 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -358,3 +358,7 @@ func rlpHash(x interface{}) (h common.Hash) { hw.Sum(h[:0]) return h } + +func vmTestBlockHash(n uint64) common.Hash { + return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) +} diff --git a/tests/testdata b/tests/testdata index 5d534e37b80e..092a8834dc44 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 5d534e37b80e9310e8c7751f805ca481a451123e +Subproject commit 092a8834dc445e683103689d6f0e75a5d380a190 diff --git a/tests/vm_test.go b/tests/vm_test.go deleted file mode 100644 index 2150df9e23a5..000000000000 --- a/tests/vm_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package tests - -import ( - "testing" - - "github.com/ethereum/go-ethereum/core/vm" -) - -func TestVM(t *testing.T) { - t.Parallel() - vmt := new(testMatcher) - vmt.slow("^vmPerformance") - vmt.fails("^vmSystemOperationsTest.json/createNameRegistrator$", "fails without parallel execution") - - vmt.walk(t, vmTestDir, func(t *testing.T, name string, test *VMTest) { - withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, test.Run(vmconfig, false)) - }) - withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, test.Run(vmconfig, true)) - }) - }) -} diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go deleted file mode 100644 index 418cc6716864..000000000000 --- a/tests/vm_test_util.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package tests - -import ( - "bytes" - "encoding/json" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" -) - -// VMTest checks EVM execution without block or transaction context. -// See https://github.com/ethereum/tests/wiki/VM-Tests for the test format specification. -type VMTest struct { - json vmJSON -} - -func (t *VMTest) UnmarshalJSON(data []byte) error { - return json.Unmarshal(data, &t.json) -} - -type vmJSON struct { - Env stEnv `json:"env"` - Exec vmExec `json:"exec"` - Logs common.UnprefixedHash `json:"logs"` - GasRemaining *math.HexOrDecimal64 `json:"gas"` - Out hexutil.Bytes `json:"out"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"post"` - PostStateRoot common.Hash `json:"postStateRoot"` -} - -//go:generate gencodec -type vmExec -field-override vmExecMarshaling -out gen_vmexec.go - -type vmExec struct { - Address common.Address `json:"address" gencodec:"required"` - Caller common.Address `json:"caller" gencodec:"required"` - Origin common.Address `json:"origin" gencodec:"required"` - Code []byte `json:"code" gencodec:"required"` - Data []byte `json:"data" gencodec:"required"` - Value *big.Int `json:"value" gencodec:"required"` - GasLimit uint64 `json:"gas" gencodec:"required"` - GasPrice *big.Int `json:"gasPrice" gencodec:"required"` -} - -type vmExecMarshaling struct { - Address common.UnprefixedAddress - Caller common.UnprefixedAddress - Origin common.UnprefixedAddress - Code hexutil.Bytes - Data hexutil.Bytes - Value *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - GasPrice *math.HexOrDecimal256 -} - -func (t *VMTest) Run(vmconfig vm.Config, snapshotter bool) error { - snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) - if snapshotter { - preRoot := statedb.IntermediateRoot(false) - defer func() { - if _, err := snaps.Journal(preRoot); err != nil { - panic(err) - } - }() - } - ret, gasRemaining, err := t.exec(statedb, vmconfig) - - if t.json.GasRemaining == nil { - if err == nil { - return fmt.Errorf("gas unspecified (indicating an error), but VM returned no error") - } - if gasRemaining > 0 { - return fmt.Errorf("gas unspecified (indicating an error), but VM returned gas remaining > 0") - } - return nil - } - // Test declares gas, expecting outputs to match. - if !bytes.Equal(ret, t.json.Out) { - return fmt.Errorf("return data mismatch: got %x, want %x", ret, t.json.Out) - } - if gasRemaining != uint64(*t.json.GasRemaining) { - return fmt.Errorf("remaining gas %v, want %v", gasRemaining, *t.json.GasRemaining) - } - for addr, account := range t.json.Post { - for k, wantV := range account.Storage { - if haveV := statedb.GetState(addr, k); haveV != wantV { - return fmt.Errorf("wrong storage value at %x:\n got %x\n want %x", k, haveV, wantV) - } - } - } - // if root := statedb.IntermediateRoot(false); root != t.json.PostStateRoot { - // return fmt.Errorf("post state root mismatch, got %x, want %x", root, t.json.PostStateRoot) - // } - if logs := rlpHash(statedb.Logs()); logs != common.Hash(t.json.Logs) { - return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, t.json.Logs) - } - return nil -} - -func (t *VMTest) exec(statedb *state.StateDB, vmconfig vm.Config) ([]byte, uint64, error) { - evm := t.newEVM(statedb, vmconfig) - e := t.json.Exec - return evm.Call(vm.AccountRef(e.Caller), e.Address, e.Data, e.GasLimit, e.Value) -} - -func (t *VMTest) newEVM(statedb *state.StateDB, vmconfig vm.Config) *vm.EVM { - initialCall := true - canTransfer := func(db vm.StateDB, address common.Address, amount *big.Int) bool { - if initialCall { - initialCall = false - return true - } - return core.CanTransfer(db, address, amount) - } - transfer := func(db vm.StateDB, sender, recipient common.Address, amount *big.Int) {} - txContext := vm.TxContext{ - Origin: t.json.Exec.Origin, - GasPrice: t.json.Exec.GasPrice, - } - context := vm.BlockContext{ - CanTransfer: canTransfer, - Transfer: transfer, - GetHash: vmTestBlockHash, - Coinbase: t.json.Env.Coinbase, - BlockNumber: new(big.Int).SetUint64(t.json.Env.Number), - Time: new(big.Int).SetUint64(t.json.Env.Timestamp), - GasLimit: t.json.Env.GasLimit, - Difficulty: t.json.Env.Difficulty, - } - vmconfig.NoRecursion = true - return vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vmconfig) -} - -func vmTestBlockHash(n uint64) common.Hash { - return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) -} From 690248576704d9dc312d5746400e035c6dccaea6 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 17 Aug 2021 18:40:14 +0200 Subject: [PATCH 49/60] cmd, metrics: add support for influxdb-v2 (cherry-picking from italoacasas' changes), leave existing support for v1 to maintain backwards-compatibility. (#23194) This PR adds flag to enable InfluxDB v2 (--metrics.influxdbv2), flags for v2-specific features (--metrics.influxdb.token, --metrics.influxdb.bucket), also carries over addition of support for specifying organization (--metrics.influxdb.organization), but still retains backwards compatibility with InfluxDB v1. --- cmd/geth/chaincmd.go | 4 + cmd/geth/config.go | 12 ++ cmd/geth/main.go | 4 + cmd/utils/flags.go | 64 +++++++++- go.mod | 13 +- go.sum | 64 ++++++++-- metrics/config.go | 11 ++ metrics/influxdb/influxdbv2.go | 223 +++++++++++++++++++++++++++++++++ 8 files changed, 377 insertions(+), 18 deletions(-) create mode 100644 metrics/influxdb/influxdbv2.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9c5d4f9b2c9c..071a3cd65b23 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -92,11 +92,15 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsHTTPFlag, utils.MetricsPortFlag, utils.MetricsEnableInfluxDBFlag, + utils.MetricsEnableInfluxDBV2Flag, utils.MetricsInfluxDBEndpointFlag, utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBTokenFlag, + utils.MetricsInfluxDBBucketFlag, + utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, }, Category: "BLOCKCHAIN COMMANDS", diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 117be96ae971..604f907b7fb1 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -233,6 +233,18 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) } + if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { + cfg.Metrics.EnableInfluxDBV2 = ctx.GlobalBool(utils.MetricsEnableInfluxDBV2Flag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBTokenFlag.Name) { + cfg.Metrics.InfluxDBToken = ctx.GlobalString(utils.MetricsInfluxDBTokenFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBBucketFlag.Name) { + cfg.Metrics.InfluxDBBucket = ctx.GlobalString(utils.MetricsInfluxDBBucketFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { + cfg.Metrics.InfluxDBOrganization = ctx.GlobalString(utils.MetricsInfluxDBOrganizationFlag.Name) + } } func deprecated(field string) bool { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 25b35ca93c4b..642405c50d9f 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -195,6 +195,10 @@ var ( utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsEnableInfluxDBV2Flag, + utils.MetricsInfluxDBTokenFlag, + utils.MetricsInfluxDBBucketFlag, + utils.MetricsInfluxDBOrganizationFlag, } ) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7fa3d8b86a98..453a553eb267 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -756,6 +756,29 @@ var ( Value: metrics.DefaultConfig.InfluxDBTags, } + MetricsEnableInfluxDBV2Flag = cli.BoolFlag{ + Name: "metrics.influxdbv2", + Usage: "Enable metrics export/push to an external InfluxDB v2 database", + } + + MetricsInfluxDBTokenFlag = cli.StringFlag{ + Name: "metrics.influxdb.token", + Usage: "Token to authorize access to the database (v2 only)", + Value: metrics.DefaultConfig.InfluxDBToken, + } + + MetricsInfluxDBBucketFlag = cli.StringFlag{ + Name: "metrics.influxdb.bucket", + Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", + Value: metrics.DefaultConfig.InfluxDBBucket, + } + + MetricsInfluxDBOrganizationFlag = cli.StringFlag{ + Name: "metrics.influxdb.organization", + Usage: "InfluxDB organization name (v2 only)", + Value: metrics.DefaultConfig.InfluxDBOrganization, + } + CatalystFlag = cli.BoolFlag{ Name: "catalyst", Usage: "Catalyst mode (eth2 integration testing)", @@ -1739,11 +1762,36 @@ func SetupMetrics(ctx *cli.Context) { log.Info("Enabling metrics collection") var ( - enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) - endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) + enableExportV2 = ctx.GlobalBool(MetricsEnableInfluxDBV2Flag.Name) + ) + + if enableExport || enableExportV2 { + CheckExclusive(ctx, MetricsEnableInfluxDBFlag, MetricsEnableInfluxDBV2Flag) + + v1FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBUsernameFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBPasswordFlag.Name) + + v2FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBTokenFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBOrganizationFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBBucketFlag.Name) + + if enableExport && v2FlagIsSet { + Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + } else if enableExportV2 && v1FlagIsSet { + Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + } + } + + var ( + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + + token = ctx.GlobalString(MetricsInfluxDBTokenFlag.Name) + bucket = ctx.GlobalString(MetricsInfluxDBBucketFlag.Name) + organization = ctx.GlobalString(MetricsInfluxDBOrganizationFlag.Name) ) if enableExport { @@ -1752,6 +1800,12 @@ func SetupMetrics(ctx *cli.Context) { log.Info("Enabling metrics export to InfluxDB") go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + } else if enableExportV2 { + tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + + log.Info("Enabling metrics export to InfluxDB (v2)") + + go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) } if ctx.GlobalIsSet(MetricsHTTPFlag.Name) { diff --git a/go.mod b/go.mod index 4621bc9ab42e..7e8407f53302 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 @@ -39,13 +40,15 @@ require ( github.com/holiman/uint256 v1.2.0 github.com/huin/goupnp v1.0.2 github.com/influxdata/influxdb v1.8.3 + github.com/influxdata/influxdb-client-go/v2 v2.4.0 + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.0 - github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/mattn/go-colorable v0.1.8 + github.com/mattn/go-isatty v0.0.12 github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 @@ -60,12 +63,14 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 + golang.org/x/sys v0.0.0-20210423082822-04245dca01da golang.org/x/text v0.3.6 - golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 + gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 4c494e170bdd..b4d0b725c5df 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -111,6 +112,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= @@ -136,8 +140,12 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= @@ -147,6 +155,8 @@ github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80n github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -178,6 +188,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -200,6 +211,7 @@ github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= @@ -221,8 +233,13 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= @@ -263,18 +280,27 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -360,6 +386,7 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= @@ -372,6 +399,9 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -389,6 +419,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -436,10 +468,13 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -461,6 +496,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -468,36 +504,45 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -593,8 +638,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/metrics/config.go b/metrics/config.go index d05d6642653c..2eb09fb48a33 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -28,6 +28,11 @@ type Config struct { InfluxDBUsername string `toml:",omitempty"` InfluxDBPassword string `toml:",omitempty"` InfluxDBTags string `toml:",omitempty"` + + EnableInfluxDBV2 bool `toml:",omitempty"` + InfluxDBToken string `toml:",omitempty"` + InfluxDBBucket string `toml:",omitempty"` + InfluxDBOrganization string `toml:",omitempty"` } // DefaultConfig is the default config for metrics used in go-ethereum. @@ -42,4 +47,10 @@ var DefaultConfig = Config{ InfluxDBUsername: "test", InfluxDBPassword: "test", InfluxDBTags: "host=localhost", + + // influxdbv2-specific flags + EnableInfluxDBV2: false, + InfluxDBToken: "test", + InfluxDBBucket: "geth", + InfluxDBOrganization: "geth", } diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go new file mode 100644 index 000000000000..00901f52c9f4 --- /dev/null +++ b/metrics/influxdb/influxdbv2.go @@ -0,0 +1,223 @@ +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package influxdb + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" +) + +type v2Reporter struct { + reg metrics.Registry + interval time.Duration + + endpoint string + token string + bucket string + organization string + namespace string + tags map[string]string + + client influxdb2.Client + write api.WriteAPI + + cache map[string]int64 +} + +// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, token string, bucket string, organization string, namespace string, tags map[string]string) { + rep := &v2Reporter{ + reg: r, + interval: d, + endpoint: endpoint, + token: token, + bucket: bucket, + organization: organization, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + + rep.client = influxdb2.NewClient(rep.endpoint, rep.token) + defer rep.client.Close() + + // async write client + rep.write = rep.client.WriteAPI(rep.organization, rep.bucket) + errorsCh := rep.write.Errors() + + // have to handle write errors in a separate goroutine like this b/c the channel is unbuffered and will block writes if not read + go func() { + for err := range errorsCh { + log.Warn("write error", "err", err.Error()) + } + }() + rep.run() +} + +func (r *v2Reporter) run() { + intervalTicker := time.Tick(r.interval) + pingTicker := time.Tick(time.Second * 5) + + for { + select { + case <-intervalTicker: + r.send() + case <-pingTicker: + _, err := r.client.Health(context.Background()) + if err != nil { + log.Warn("Got error from influxdb client health check", "err", err.Error()) + } + } + } + +} + +func (r *v2Reporter) send() { + r.reg.Each(func(name string, i interface{}) { + now := time.Now() + namespace := r.namespace + + switch metric := i.(type) { + + case metrics.Counter: + v := metric.Count() + l := r.cache[name] + + measurement := fmt.Sprintf("%s%s.count", namespace, name) + fields := map[string]interface{}{ + "value": v - l, + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + r.cache[name] = v + + case metrics.Gauge: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.gauge", namespace, name) + fields := map[string]interface{}{ + "value": ms.Value(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.GaugeFloat64: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.gauge", namespace, name) + fields := map[string]interface{}{ + "value": ms.Value(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.Histogram: + ms := metric.Snapshot() + + if ms.Count() > 0 { + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + measurement := fmt.Sprintf("%s%s.histogram", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + } + + case metrics.Meter: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.meter", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "mean": ms.RateMean(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.Timer: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + + measurement := fmt.Sprintf("%s%s.timer", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "meanrate": ms.RateMean(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.ResettingTimer: + t := metric.Snapshot() + + if len(t.Values()) > 0 { + ps := t.Percentiles([]float64{50, 95, 99}) + val := t.Values() + + measurement := fmt.Sprintf("%s%s.span", namespace, name) + fields := map[string]interface{}{ + "count": len(val), + "max": val[len(val)-1], + "mean": t.Mean(), + "min": val[0], + "p50": ps[0], + "p95": ps[1], + "p99": ps[2], + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + } + } + }) + + // Force all unwritten data to be sent + r.write.Flush() +} From 85b9bdd6413797f6720c7d9ef46e5fdeb47b0cb6 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 17 Aug 2021 12:43:25 -0400 Subject: [PATCH 50/60] cmd, core: remove calaveras testnet (#23366) Removes references to the short-lived Calaveras testnet --- cmd/geth/chaincmd.go | 1 - cmd/geth/consolecmd.go | 2 -- cmd/geth/dbcmd.go | 8 -------- cmd/geth/main.go | 4 ---- cmd/geth/usage.go | 1 - cmd/utils/flags.go | 20 +------------------- core/genesis.go | 14 -------------- core/genesis_test.go | 4 ---- params/bootnodes.go | 6 ------ params/config.go | 30 ++++-------------------------- 10 files changed, 5 insertions(+), 85 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 071a3cd65b23..436e558b5e9a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -68,7 +68,6 @@ It expects the genesis file as argument.`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 776412df6fc5..0e156fde9a50 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -134,8 +134,6 @@ func remoteConsole(ctx *cli.Context) error { path = filepath.Join(path, "rinkeby") } else if ctx.GlobalBool(utils.GoerliFlag.Name) { path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.CalaverasFlag.Name) { - path = filepath.Join(path, "calaveras") } } endpoint = fmt.Sprintf("%s/geth.ipc", path) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index bb3cf03909db..54ada408902b 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -75,7 +75,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, @@ -91,7 +90,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, } dbCompactCmd = cli.Command{ @@ -105,7 +103,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, @@ -125,7 +122,6 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -141,7 +137,6 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, @@ -158,7 +153,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, @@ -175,7 +169,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -191,7 +184,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command displays information about the freezer index.", } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 642405c50d9f..da62b8d4d4d2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -138,7 +138,6 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -278,9 +277,6 @@ func prepare(ctx *cli.Context) { case ctx.GlobalIsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") - case ctx.GlobalIsSet(utils.CalaverasFlag.Name): - log.Info("Starting Geth on Calaveras testnet...") - case ctx.GlobalIsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 3c401eae8ab6..708edcc79325 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -44,7 +44,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, - utils.CalaverasFlag, utils.RopstenFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 453a553eb267..df97c8cf02b5 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -151,10 +151,6 @@ var ( Name: "goerli", Usage: "Görli network: pre-configured proof-of-authority test network", } - CalaverasFlag = cli.BoolFlag{ - Name: "calaveras", - Usage: "Calaveras network: pre-configured proof-of-authority shortlived test network.", - } RinkebyFlag = cli.BoolFlag{ Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", @@ -801,9 +797,6 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(CalaverasFlag.Name) { - return filepath.Join(path, "calaveras") - } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -856,8 +849,6 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.RinkebyBootnodes case ctx.GlobalBool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(CalaverasFlag.Name): - urls = params.CalaverasBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -1301,8 +1292,6 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(CalaverasFlag.Name) && cfg.DataDir == node.DefaultDataDir(): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "calaveras") } } @@ -1488,7 +1477,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, CalaverasFlag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { @@ -1641,11 +1630,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(CalaverasFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = 123 // https://gist.github.com/holiman/c5697b041b3dc18c50a5cdd382cbdd16 - } - cfg.Genesis = core.DefaultCalaverasGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1866,8 +1850,6 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(CalaverasFlag.Name): - genesis = core.DefaultCalaverasGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/core/genesis.go b/core/genesis.go index dc50ff475ec7..c1f226c34a39 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -248,8 +248,6 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return params.RinkebyChainConfig case ghash == params.GoerliGenesisHash: return params.GoerliChainConfig - case ghash == params.CalaverasGenesisHash: - return params.CalaverasChainConfig default: return params.AllEthashProtocolChanges } @@ -399,18 +397,6 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -func DefaultCalaverasGenesisBlock() *Genesis { - // Full genesis: https://gist.github.com/holiman/c6ed9269dce28304ad176314caa75e97 - return &Genesis{ - Config: params.CalaverasChainConfig, - Timestamp: 0x60b3877f, - ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000005211cea3870c7ba7c6c44b185e62eecdb864cd8c560228ce57d31efbf64c200b2c200aacec78cf17a7148e784fe95a7a750335f8b9572ee28d72e7650000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), - GasLimit: 0x47b760, - Difficulty: big.NewInt(1), - Alloc: decodePrealloc(calaverasAllocData), - } -} - // DeveloperGenesisBlock returns the 'geth --dev' genesis block. func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one diff --git a/core/genesis_test.go b/core/genesis_test.go index 487c30ed6f35..52c4384abe4b 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -185,10 +185,6 @@ func TestGenesisHashes(t *testing.T) { genesis: DefaultRinkebyGenesisBlock(), hash: params.RinkebyGenesisHash, }, - { - genesis: DefaultCalaverasGenesisBlock(), - hash: params.CalaverasGenesisHash, - }, } for i, c := range cases { b := c.genesis.MustCommit(rawdb.NewMemoryDatabase()) diff --git a/params/bootnodes.go b/params/bootnodes.go index 1610ea81926d..bc291449e285 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -67,12 +67,6 @@ var GoerliBootnodes = []string{ "enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303", } -// CalaverasBootnodes are the enode URLs of the P2P bootstrap nodes running on the -// Calaveras ephemeral test network. -var CalaverasBootnodes = []string{ - "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", -} - var V5Bootnodes = []string{ // Teku team's bootnode "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", diff --git a/params/config.go b/params/config.go index 591c36de2a37..16c801825776 100644 --- a/params/config.go +++ b/params/config.go @@ -27,11 +27,10 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - CalaverasGenesisHash = common.HexToHash("0xeb9233d066c275efcdfed8037f4fc082770176aefdbcb7691c71da412a5670f2") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of @@ -221,27 +220,6 @@ var ( Threshold: 2, } - CalaverasChainConfig = &ChainConfig{ - ChainID: big.NewInt(123), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(500), - Clique: &CliqueConfig{ - Period: 30, - Epoch: 30000, - }, - } - // AllEthashProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Ethash consensus. // From 16ecdd583984162923256e563320b8b381da7f46 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 17 Aug 2021 12:49:19 -0400 Subject: [PATCH 51/60] cmd/utils: add --nousb to the list of deprecated flags (#23388) Adds --nousb as a deprecated flag when someone runs the geth show-deprecated-flags command. --- cmd/utils/flags.go | 4 ---- cmd/utils/flags_legacy.go | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index df97c8cf02b5..8a53f525f6bf 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -125,10 +125,6 @@ var ( Name: "keystore", Usage: "Directory for the keystore (default = inside the datadir)", } - NoUSBFlag = cli.BoolFlag{ - Name: "nousb", - Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", - } USBFlag = cli.BoolFlag{ Name: "usb", Usage: "Enable monitoring and management of USB hardware wallets", diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index df597e3657d6..627f77475319 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -36,10 +36,15 @@ var ShowDeprecated = cli.Command{ var DeprecatedFlags = []cli.Flag{ LegacyMinerGasTargetFlag, + NoUSBFlag, } var ( // (Deprecated May 2020, shown in aliased flags section) + NoUSBFlag = cli.BoolFlag{ + Name: "nousb", + Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + } LegacyRPCEnabledFlag = cli.BoolFlag{ Name: "rpc", Usage: "Enable the HTTP-RPC server (deprecated and will be removed June 2021, use --http)", From 57feabea663496109e59df669238398239438fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=BD=B3?= Date: Wed, 18 Aug 2021 00:55:18 +0800 Subject: [PATCH 52/60] eth, internal/ethapi: make RPC block miner field show block sealer correctly (#23312) Makes the RPC block return the POA sealer for clique blocks on the 'miner' field (was previously zeroes) --- eth/api.go | 2 +- internal/ethapi/api.go | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eth/api.go b/eth/api.go index 8b96d1f316d7..0f57128d755b 100644 --- a/eth/api.go +++ b/eth/api.go @@ -342,7 +342,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, } else { blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { + if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.engine); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} } results = append(results, &BadBlockArgs{ diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9a82824ada14..1af98e1071da 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" @@ -1175,7 +1176,8 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { } // RPCMarshalHeader converts the given header to the RPC output . -func RPCMarshalHeader(head *types.Header) map[string]interface{} { +func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]interface{} { + miner, _ := engine.Author(head) result := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), "hash": head.Hash(), @@ -1185,7 +1187,7 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { "sha3Uncles": head.UncleHash, "logsBloom": head.Bloom, "stateRoot": head.Root, - "miner": head.Coinbase, + "miner": miner, "difficulty": (*hexutil.Big)(head.Difficulty), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(head.Size()), @@ -1206,8 +1208,8 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields := RPCMarshalHeader(block.Header()) +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consensus.Engine) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header(), engine) fields["size"] = hexutil.Uint64(block.Size()) if inclTx { @@ -1242,7 +1244,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { - fields := RPCMarshalHeader(header) + fields := RPCMarshalHeader(header, s.b.Engine()) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } @@ -1250,7 +1252,7 @@ func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *type // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx) + fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.Engine()) if err != nil { return nil, err } From 5566e5d152c490e5e533f3bc7735dddd57a428eb Mon Sep 17 00:00:00 2001 From: chuwt Date: Wed, 18 Aug 2021 18:03:41 +0800 Subject: [PATCH 53/60] eth/downloader: fix typo in comment (#23413) --- core/rawdb/freezer.go | 2 +- eth/downloader/downloader.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 253de9f7cfc3..4262a615a7b6 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -414,7 +414,7 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { } batch.Reset() - // Wipe out side chains also and track dangling side chians + // Wipe out side chains also and track dangling side chains var dangling []common.Hash for number := first; number < f.frozen; number++ { // Always keep the genesis block in active database diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f0fab1b7a958..a6bf87acbd22 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -465,7 +465,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } if mode == FastSync && pivot == nil { // If no pivot block was returned, the head is below the min full block - // threshold (i.e. new chian). In that case we won't really fast sync + // threshold (i.e. new chain). In that case we won't really fast sync // anyway, but still need a valid pivot block to avoid some code hitting // nil panics on an access. pivot = d.blockchain.CurrentBlock().Header() @@ -681,7 +681,7 @@ func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *ty return head, nil, nil } // At this point we have 2 headers in total and the first is the - // validated head of the chian. Check the pivot number and return, + // validated head of the chain. Check the pivot number and return, pivot := headers[1] if pivot.Number.Uint64() != head.Number.Uint64()-uint64(fsMinFullBlocks) { return nil, nil, fmt.Errorf("%w: remote pivot %d != requested %d", errInvalidChain, pivot.Number, head.Number.Uint64()-uint64(fsMinFullBlocks)) From c368f728c19e7fd7a9613513edda68ffcb503af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 20 Aug 2021 15:14:21 +0300 Subject: [PATCH 54/60] Revert "eth: drop eth/65, the last non-reqid protocol version" (#23426) --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 +++++++++- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 ++-- eth/protocols/eth/handler.go | 47 +++-- eth/protocols/eth/handler_test.go | 134 +++++++++----- eth/protocols/eth/handlers.go | 107 +++++++++++ eth/protocols/eth/handshake_test.go | 1 + eth/protocols/eth/peer.go | 208 +++++++++++++++------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 ++++++++++- eth/sync_test.go | 1 + les/client_handler.go | 2 +- 15 files changed, 609 insertions(+), 179 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6d14404e6624..6e3217151a52 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.AllEthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a6bf87acbd22..1767506a33e6 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH66 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) + if p.version < eth.ETH65 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 17cd3630c98e..794160993bb0 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,6 +522,10 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } +func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } +func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } +func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } + func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -545,6 +549,9 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. +func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } +func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } + func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -627,6 +634,10 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. +func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } +func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } +func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } + func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -656,6 +667,10 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. +func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } +func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } +func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } + func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -687,6 +702,10 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. +func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } +func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } +func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } + func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -717,6 +736,16 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. +func TestBoundedHeavyForkedSync65Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) +} +func TestBoundedHeavyForkedSync65Fast(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) +} +func TestBoundedHeavyForkedSync65Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) +} + func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -771,6 +800,10 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. +func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } +func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } +func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } + func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -800,6 +833,10 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). +func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } +func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } +func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } + func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -826,6 +863,10 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. +func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } +func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } +func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } + func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -840,8 +881,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type + tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) - //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -850,7 +891,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{66} { + for _, version := range []int{65, 66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -860,6 +901,10 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. +func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } +func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } +func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } + func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -910,6 +955,10 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. +func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } +func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } +func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } + func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -938,6 +987,10 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. +func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } +func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } +func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } + func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -971,6 +1024,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. +func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1061,6 +1115,16 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. +func TestHighTDStarvationAttack65Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FullSync) +} +func TestHighTDStarvationAttack65Fast(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FastSync) +} +func TestHighTDStarvationAttack65Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, LightSync) +} + func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1085,6 +1149,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. +func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1137,6 +1202,10 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. +func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } +func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } +func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } + func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1217,6 +1286,10 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). +func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } +func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } +func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } + func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1289,6 +1362,10 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. +func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } +func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } +func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } + func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1358,6 +1435,10 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. +func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } +func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } +func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } + func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1431,6 +1512,10 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. +func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } +func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } +func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } + func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1588,6 +1673,12 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. +func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } +func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } +func TestCheckpointEnforcement65Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH65, LightSync) +} + func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 863294832971..066a366315a7 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index 06a8088bf07c..aff4871afa42 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,6 +117,7 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop + txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -139,6 +140,7 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, + txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -406,8 +408,9 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(1) + h.wg.Add(2) go h.chainSync.loop() + go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 039091244fec..038de469908e 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,6 +80,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. +func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -235,6 +236,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -292,6 +294,7 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -303,7 +306,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -377,6 +380,7 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. +func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -517,8 +521,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -539,39 +543,30 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } + // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - msg, err := p2pRemote.ReadMsg() - if err != nil { - t.Fatalf("failed to read checkpoint challenge: %v", err) - } - request := new(eth.GetBlockHeadersPacket66) - if err := msg.Decode(request); err != nil { - t.Fatalf("failed to decode checkpoint challenge: %v", err) - } - query := request.GetBlockHeadersPacket - if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { - t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", - query.Origin.Number, query.Amount, query.Skip, query.Reverse, - response.Number.Uint64(), 1, 0, false) + if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { + t.Fatalf("challenge mismatch: %v", err) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } + // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -624,8 +619,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -676,6 +671,7 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. +func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 8289300141f0..6bbaa2f555f1 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,21 +171,39 @@ type Decoder interface { Time() time.Time } +var eth65 = map[uint64]msgHandler{ + GetBlockHeadersMsg: handleGetBlockHeaders, + BlockHeadersMsg: handleBlockHeaders, + GetBlockBodiesMsg: handleGetBlockBodies, + BlockBodiesMsg: handleBlockBodies, + GetNodeDataMsg: handleGetNodeData, + NodeDataMsg: handleNodeData, + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetPooledTransactionsMsg: handleGetPooledTransactions, + PooledTransactionsMsg: handlePooledTransactions, +} + var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + // eth66 messages with request-id + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -201,11 +219,10 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth66 - //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added - // handlers = eth67 - //} - + var handlers = eth65 + if peer.Version() >= ETH66 { + handlers = eth66 + } // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 809f17e36cf5..473be3f9b77e 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,6 +110,7 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. +func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -253,30 +254,44 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } } } @@ -284,6 +299,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. +func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -353,20 +369,28 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockBodiesMsg, hashes) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } } } } // Tests that the state trie nodes can be retrieved based on hashes. +func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -425,10 +449,14 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) + if protocol <= ETH65 { + p2p.Send(peer.app, GetNodeDataMsg, hashes) + } else { + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) + } msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -436,14 +464,18 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var ( - data [][]byte - res NodeDataPacket66 - ) - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) + var data [][]byte + if protocol <= ETH65 { + if err := msg.Decode(&data); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + } else { + var res NodeDataPacket66 + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + data = res.NodeDataPacket } - data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -474,6 +506,7 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. +func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -533,14 +566,21 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetReceiptsMsg, hashes) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { + t.Errorf("receipts mismatch: %v", err) + } + } else { + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) + } } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index e54838cbc56e..d7d993a23dee 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,6 +27,17 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// handleGetBlockHeaders handles Block header query, collect the requested headers and reply +func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // Decode the complex header query + var query GetBlockHeadersPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockHeadersQuery(backend, &query, peer) + return peer.SendBlockHeaders(response) +} + // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -124,6 +135,16 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } +func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block body retrieval message + var query GetBlockBodiesPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockBodiesQuery(backend, query, peer) + return peer.SendBlockBodiesRLP(response) +} + func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -153,6 +174,16 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } +func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { + // Decode the trie node data retrieval message + var query GetNodeDataPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetNodeDataQuery(backend, query, peer) + return peer.SendNodeData(response) +} + func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -192,6 +223,16 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } +func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block receipts retrieval message + var query GetReceiptsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetReceiptsQuery(backend, query, peer) + return peer.SendReceiptsRLP(response) +} + func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -271,6 +312,15 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } +func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // A batch of headers arrived to one of our previous requests + res := new(BlockHeadersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -282,6 +332,15 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } +func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // A batch of block bodies arrived to one of our previous requests + res := new(BlockBodiesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -293,6 +352,15 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } +func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { + // A batch of node state data arrived to one of our previous requests + res := new(NodeDataPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -304,6 +372,15 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } +func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { + // A batch of receipts arrived to one of our previous requests + res := new(ReceiptsPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -332,6 +409,16 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } +func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Decode the pooled transactions retrieval message + var query GetPooledTransactionsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + hashes, txs := answerGetPooledTransactions(backend, query, peer) + return peer.SendPooledTransactionsRLP(hashes, txs) +} + func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -390,6 +477,26 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } +func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs PooledTransactionsPacket + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + return backend.Handle(peer, &txs) +} + func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 05d473e05795..3bebda2dcc9b 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,6 +27,7 @@ import ( ) // Tests that handshake failures are detected and reported correctly. +func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98273ccfc9dc..e619c183ba2f 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,8 +108,9 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - go peer.announceTransactions() - + if version >= ETH65 { + go peer.announceTransactions() + } return peer } @@ -251,6 +252,22 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } +// SendPooledTransactionsRLP sends requested transactions to the peer and adds the +// hashes in its transaction hash set for future reference. +// +// Note, the method assumes the hashes are correct and correspond to the list of +// transactions being sent. +func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding +} + // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -329,6 +346,11 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } +// SendBlockHeaders sends a batch of block headers to the remote peer. +func (p *Peer) SendBlockHeaders(headers []*types.Header) error { + return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) +} + // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -337,6 +359,12 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } +// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// an already RLP encoded format. +func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { + return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding +} + // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -346,6 +374,12 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } +// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the +// hashes requested. +func (p *Peer) SendNodeData(data [][]byte) error { + return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) +} + // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -354,6 +388,12 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } +// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the +// ones requested from an already RLP encoded format. +func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { + return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding +} + // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -366,102 +406,138 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - }, - }) + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + } + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - }, - }) + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - }, - }) + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) +} + +// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side +// of the RequestHeadersByNumber operation. +func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + req := &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) + } + return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) + } + return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) + } + return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - id := rand.Uint64() - - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) + if p.Version() >= ETH66 { + id := rand.Uint64() + + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) + } + return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 3c3da30fa503..de1b0ed1ee7f 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,6 +30,7 @@ import ( // Constants to match up protocol versions and messages const ( + ETH65 = 65 ETH66 = 66 ) @@ -39,28 +40,31 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66} +var ProtocolVersions = []uint{ETH66, ETH65} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17} +var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 + // Protocol messages in eth/64 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 + + // Protocol messages overloaded in eth/65 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -124,7 +128,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket66 represents a block header query over eth/66 +// GetBlockHeadersPacket represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index 27941158f3df..ab114b59f3e1 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,6 +18,7 @@ package eth import ( "math/big" + "math/rand" "sync/atomic" "time" @@ -27,13 +28,23 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing + + // This is the target size for the packs of transactions sent by txsyncLoop64. + // A pack can get larger than this if a single transactions exceeds this size. + txsyncPackSize = 100 * 1024 ) +type txsync struct { + p *eth.Peer + txs []*types.Transaction +} + // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -53,11 +64,94 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() + if p.Version() >= eth.ETH65 { + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() + } + p.AsyncSendPooledTransactionHashes(hashes) + return + } + // Out of luck, peer is running legacy protocols, drop the txs over + select { + case h.txsyncCh <- &txsync{p: p, txs: txs}: + case <-h.quitSync: + } +} + +// txsyncLoop64 takes care of the initial transaction sync for each new +// connection. When a new peer appears, we relay all currently pending +// transactions. In order to minimise egress bandwidth usage, we send +// the transactions in small packs to one peer at a time. +func (h *handler) txsyncLoop64() { + defer h.wg.Done() + + var ( + pending = make(map[enode.ID]*txsync) + sending = false // whether a send is active + pack = new(txsync) // the pack that is being sent + done = make(chan error, 1) // result of the send + ) + + // send starts a sending a pack of transactions from the sync. + send := func(s *txsync) { + if s.p.Version() >= eth.ETH65 { + panic("initial transaction syncer running on eth/65+") + } + // Fill pack with transactions up to the target size. + size := common.StorageSize(0) + pack.p = s.p + pack.txs = pack.txs[:0] + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.Peer.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendTransactions(pack.txs) }() + } + // pick chooses the next pending sync. + pick := func() *txsync { + if len(pending) == 0 { + return nil + } + n := rand.Intn(len(pending)) + 1 + for _, s := range pending { + if n--; n == 0 { + return s + } + } + return nil + } + + for { + select { + case s := <-h.txsyncCh: + pending[s.p.Peer.ID()] = s + if !sending { + send(s) + } + case err := <-done: + sending = false + // Stop tracking peers that cause send failures. + if err != nil { + pack.p.Log().Debug("Transaction send failed", "err", err) + delete(pending, pack.p.Peer.ID()) + } + // Schedule the next send. + if s := pick(); s != nil { + send(s) + } + case <-h.quitSync: + return + } } - p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index e96b9ee81f69..a0c6f8602327 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,6 +28,7 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. +func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index b903b11065c8..e95996c51f6f 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From d02c60536799698888d21f093f7c379acdad3147 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 23 Aug 2021 11:49:39 +0200 Subject: [PATCH 55/60] core: only check sendernoeoa in non fake mode (#23424) --- accounts/abi/bind/backends/simulated.go | 2 +- core/state_transition.go | 17 +++++++++-------- core/types/transaction.go | 10 +++++----- internal/ethapi/transaction_args.go | 4 ++-- les/odr_test.go | 4 ++-- light/odr_test.go | 2 +- tests/state_test_util.go | 2 +- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index cb4eea3a0dc0..e410522ac8bf 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -795,7 +795,7 @@ type callMsg struct { func (m callMsg) From() common.Address { return m.CallMsg.From } func (m callMsg) Nonce() uint64 { return 0 } -func (m callMsg) CheckNonce() bool { return false } +func (m callMsg) IsFake() bool { return true } func (m callMsg) To() *common.Address { return m.CallMsg.To } func (m callMsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } func (m callMsg) GasFeeCap() *big.Int { return m.CallMsg.GasFeeCap } diff --git a/core/state_transition.go b/core/state_transition.go index 4c4187df152c..6a09f6adc441 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -74,7 +74,7 @@ type Message interface { Value() *big.Int Nonce() uint64 - CheckNonce() bool + IsFake() bool Data() []byte AccessList() types.AccessList } @@ -212,8 +212,9 @@ func (st *StateTransition) buyGas() error { } func (st *StateTransition) preCheck() error { - // Make sure this transaction's nonce is correct. - if st.msg.CheckNonce() { + // Only check transactions that are not fake + if !st.msg.IsFake() { + // Make sure this transaction's nonce is correct. stNonce := st.state.GetNonce(st.msg.From()) if msgNonce := st.msg.Nonce(); stNonce < msgNonce { return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooHigh, @@ -222,11 +223,11 @@ func (st *StateTransition) preCheck() error { return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooLow, st.msg.From().Hex(), msgNonce, stNonce) } - } - // Make sure the sender is an EOA - if codeHash := st.state.GetCodeHash(st.msg.From()); codeHash != emptyCodeHash && codeHash != (common.Hash{}) { - return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, - st.msg.From().Hex(), codeHash) + // Make sure the sender is an EOA + if codeHash := st.state.GetCodeHash(st.msg.From()); codeHash != emptyCodeHash && codeHash != (common.Hash{}) { + return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, + st.msg.From().Hex(), codeHash) + } } // Make sure that transaction gasFeeCap is greater than the baseFee (post london) if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { diff --git a/core/types/transaction.go b/core/types/transaction.go index a556f4b57fba..e21cf2bda826 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -579,10 +579,10 @@ type Message struct { gasTipCap *big.Int data []byte accessList AccessList - checkNonce bool + isFake bool } -func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice, gasFeeCap, gasTipCap *big.Int, data []byte, accessList AccessList, checkNonce bool) Message { +func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice, gasFeeCap, gasTipCap *big.Int, data []byte, accessList AccessList, isFake bool) Message { return Message{ from: from, to: to, @@ -594,7 +594,7 @@ func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *b gasTipCap: gasTipCap, data: data, accessList: accessList, - checkNonce: checkNonce, + isFake: isFake, } } @@ -610,7 +610,7 @@ func (tx *Transaction) AsMessage(s Signer, baseFee *big.Int) (Message, error) { amount: tx.Value(), data: tx.Data(), accessList: tx.AccessList(), - checkNonce: true, + isFake: false, } // If baseFee provided, set gasPrice to effectiveGasPrice. if baseFee != nil { @@ -631,4 +631,4 @@ func (m Message) Gas() uint64 { return m.gasLimit } func (m Message) Nonce() uint64 { return m.nonce } func (m Message) Data() []byte { return m.data } func (m Message) AccessList() AccessList { return m.accessList } -func (m Message) CheckNonce() bool { return m.checkNonce } +func (m Message) IsFake() bool { return m.isFake } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 8c12743dcda4..52811b2a9a21 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -171,7 +171,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { return nil } -// ToMessage converts th transaction arguments to the Message type used by the +// ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Message, error) { @@ -238,7 +238,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t if args.AccessList != nil { accessList = *args.AccessList } - msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, false) + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, true) return msg, nil } diff --git a/les/odr_test.go b/les/odr_test.go index 91419ec80bba..ea88495d198d 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -135,7 +135,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai from := statedb.GetOrNewStateObject(bankAddr) from.SetBalance(math.MaxBig256) - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} context := core.NewEVMBlockContext(header, bc, nil) txContext := core.NewEVMTxContext(msg) @@ -150,7 +150,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai header := lc.GetHeaderByHash(bhash) state := light.NewState(ctx, header, lc.Odr()) state.SetBalance(bankAddr, math.MaxBig256) - msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} context := core.NewEVMBlockContext(header, lc, nil) txContext := core.NewEVMTxContext(msg) vmenv := vm.NewEVM(context, txContext, state, config, vm.Config{NoBaseFee: true}) diff --git a/light/odr_test.go b/light/odr_test.go index b31e54d28a90..fdf657a82ec5 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -194,7 +194,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain // Perform read-only call. st.SetBalance(testBankAddress, math.MaxBig256) - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, chain, nil) vmenv := vm.NewEVM(context, txContext, st, config, vm.Config{NoBaseFee: true}) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 8b8a6a4fdacf..f7fb08bfbc8d 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -348,7 +348,7 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (core.Messa } msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, gasPrice, - tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, data, accessList, true) + tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, data, accessList, false) return msg, nil } From bb1f7ebf203f40dae714a3b8445918cfcfc9a7db Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 23 Aug 2021 12:20:56 +0200 Subject: [PATCH 56/60] signer/core/apitypes: remove dependency on internal/ethapi (#23362) --- signer/core/apitypes/types.go | 69 +++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 591d253abb1a..625959219f66 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -19,12 +19,12 @@ package apitypes import ( "encoding/json" "fmt" + "math/big" "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/internal/ethapi" ) type ValidationInfo struct { @@ -97,23 +97,60 @@ func (args SendTxArgs) String() string { return err.Error() } +// ToTransaction converts the arguments to a transaction. func (args *SendTxArgs) ToTransaction() *types.Transaction { - txArgs := ethapi.TransactionArgs{ - Gas: &args.Gas, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: &args.Value, - Nonce: &args.Nonce, - Data: args.Data, - Input: args.Input, - AccessList: args.AccessList, - ChainID: args.ChainID, - } // Add the To-field, if specified + var to *common.Address if args.To != nil { - to := args.To.Address() - txArgs.To = &to + dstAddr := args.To.Address() + to = &dstAddr + } + + var input []byte + if args.Input != nil { + input = *args.Input + } else if args.Data != nil { + input = *args.Data + } + + var data types.TxData + switch { + case args.MaxFeePerGas != nil: + al := types.AccessList{} + if args.AccessList != nil { + al = *args.AccessList + } + data = &types.DynamicFeeTx{ + To: to, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasFeeCap: (*big.Int)(args.MaxFeePerGas), + GasTipCap: (*big.Int)(args.MaxPriorityFeePerGas), + Value: (*big.Int)(&args.Value), + Data: input, + AccessList: al, + } + case args.AccessList != nil: + data = &types.AccessListTx{ + To: to, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + AccessList: *args.AccessList, + } + default: + data = &types.LegacyTx{ + To: to, + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + } } - return txArgs.ToTransaction() + return types.NewTx(data) } From dfeb2f7e8001aef1005a8d5e1605bae1de0b4f12 Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 23 Aug 2021 19:45:16 +0800 Subject: [PATCH 57/60] go.mod: upgrade golang.org/x/sys for go1.17 support (#23406) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7e8407f53302..ee9ffa0454e2 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210423082822-04245dca01da + golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index b4d0b725c5df..d6e9639d2ef6 100644 --- a/go.sum +++ b/go.sum @@ -525,8 +525,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From f38abc55f135587ce3cf60c9d574ec2a4ebb8197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 23 Aug 2021 23:50:24 +0200 Subject: [PATCH 58/60] eth/gasprice: feeHistory improvements (#23422) * eth/gasprice: cache feeHistory results * eth/gasprice: changed feeHistory block count limitation * eth/gasprice: do not use embedded struct in blockFees * eth/gasprice: fee processing logic cleanup * eth/gasprice: purge feeHistory cache at chain reorgs --- eth/ethconfig/config.go | 4 +- eth/gasprice/feehistory.go | 105 +++++++++++++++++--------------- eth/gasprice/feehistory_test.go | 22 +++---- eth/gasprice/gasprice.go | 20 ++++++ eth/gasprice/gasprice_test.go | 5 ++ 5 files changed, 95 insertions(+), 61 deletions(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 23ccf2484310..89cdb75597e0 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -43,8 +43,8 @@ import ( var FullNodeGPO = gasprice.Config{ Blocks: 20, Percentile: 60, - MaxHeaderHistory: 0, - MaxBlockHistory: 0, + MaxHeaderHistory: 1024, + MaxBlockHistory: 1024, MaxPrice: gasprice.DefaultMaxPrice, IgnorePrice: gasprice.DefaultIgnorePrice, } diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 2c4486c6f4c2..970dfd4467a5 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -18,8 +18,10 @@ package gasprice import ( "context" + "encoding/binary" "errors" "fmt" + "math" "math/big" "sort" "sync/atomic" @@ -37,10 +39,6 @@ var ( ) const ( - // maxFeeHistory is the maximum number of blocks that can be retrieved for a - // fee history request. - maxFeeHistory = 1024 - // maxBlockFetchers is the max number of goroutines to spin up to pull blocks // for the fee history calculation (mostly relevant for LES). maxBlockFetchers = 4 @@ -54,10 +52,15 @@ type blockFees struct { block *types.Block // only set if reward percentiles are requested receipts types.Receipts // filled by processBlock + results processedFees + err error +} + +// processedFees contains the results of a processed block and is also used for caching +type processedFees struct { reward []*big.Int baseFee, nextBaseFee *big.Int gasUsedRatio float64 - err error } // txGasAndReward is sorted in ascending order based on reward @@ -82,15 +85,15 @@ func (s sortGasAndReward) Less(i, j int) bool { // fills in the rest of the fields. func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { chainconfig := oracle.backend.ChainConfig() - if bf.baseFee = bf.header.BaseFee; bf.baseFee == nil { - bf.baseFee = new(big.Int) + if bf.results.baseFee = bf.header.BaseFee; bf.results.baseFee == nil { + bf.results.baseFee = new(big.Int) } if chainconfig.IsLondon(big.NewInt(int64(bf.blockNumber + 1))) { - bf.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) + bf.results.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) } else { - bf.nextBaseFee = new(big.Int) + bf.results.nextBaseFee = new(big.Int) } - bf.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) + bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if len(percentiles) == 0 { // rewards were not requested, return null return @@ -100,11 +103,11 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { return } - bf.reward = make([]*big.Int, len(percentiles)) + bf.results.reward = make([]*big.Int, len(percentiles)) if len(bf.block.Transactions()) == 0 { // return an all zero row if there are no transactions to gather data from - for i := range bf.reward { - bf.reward[i] = new(big.Int) + for i := range bf.results.reward { + bf.results.reward[i] = new(big.Int) } return } @@ -125,7 +128,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { txIndex++ sumGasUsed += sorter[txIndex].gasUsed } - bf.reward[i] = sorter[txIndex].reward + bf.results.reward[i] = sorter[txIndex].reward } } @@ -134,7 +137,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( headBlock rpc.BlockNumber pendingBlock *types.Block @@ -167,17 +170,6 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block } else if pendingBlock == nil && lastBlock > headBlock { return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) } - if maxHistory != 0 { - // limit retrieval to the given number of latest blocks - if tooOldCount := int64(headBlock) - int64(maxHistory) - int64(lastBlock) + int64(blocks); tooOldCount > 0 { - // tooOldCount is the number of requested blocks that are too old to be served - if int64(blocks) > tooOldCount { - blocks -= int(tooOldCount) - } else { - return nil, nil, 0, 0, nil - } - } - } // ensure not trying to retrieve before genesis if rpc.BlockNumber(blocks) > lastBlock+1 { blocks = int(lastBlock + 1) @@ -202,6 +194,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast if blocks < 1 { return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks } + maxFeeHistory := oracle.maxHeaderHistory + if len(rewardPercentiles) != 0 { + maxFeeHistory = oracle.maxBlockHistory + } if blocks > maxFeeHistory { log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory) blocks = maxFeeHistory @@ -214,17 +210,12 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } - // Only process blocks if reward percentiles were requested - maxHistory := oracle.maxHeaderHistory - if len(rewardPercentiles) != 0 { - maxHistory = oracle.maxBlockHistory - } var ( pendingBlock *types.Block pendingReceipts []*types.Receipt err error ) - pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory) + pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks) if err != nil || blocks == 0 { return common.Big0, nil, nil, nil, err } @@ -234,6 +225,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast next = oldestBlock results = make(chan *blockFees, blocks) ) + percentileKey := make([]byte, 8*len(rewardPercentiles)) + for i, p := range rewardPercentiles { + binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p)) + } for i := 0; i < maxBlockFetchers && i < blocks; i++ { go func() { for { @@ -246,24 +241,38 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast fees := &blockFees{blockNumber: blockNumber} if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() { fees.block, fees.receipts = pendingBlock, pendingReceipts + fees.header = fees.block.Header() + oracle.processBlock(fees, rewardPercentiles) + results <- fees } else { - if len(rewardPercentiles) != 0 { - fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) - if fees.block != nil && fees.err == nil { - fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) - } + cacheKey := struct { + number uint64 + percentiles string + }{blockNumber, string(percentileKey)} + + if p, ok := oracle.historyCache.Get(cacheKey); ok { + fees.results = p.(processedFees) + results <- fees } else { - fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + if len(rewardPercentiles) != 0 { + fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) + if fees.block != nil && fees.err == nil { + fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) + fees.header = fees.block.Header() + } + } else { + fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + } + if fees.header != nil && fees.err == nil { + oracle.processBlock(fees, rewardPercentiles) + if fees.err == nil { + oracle.historyCache.Add(cacheKey, fees.results) + } + } + // send to results even if empty to guarantee that blocks items are sent in total + results <- fees } } - if fees.block != nil { - fees.header = fees.block.Header() - } - if fees.header != nil { - oracle.processBlock(fees, rewardPercentiles) - } - // send to results even if empty to guarantee that blocks items are sent in total - results <- fees } }() } @@ -279,8 +288,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fees.err } i := int(fees.blockNumber - oldestBlock) - if fees.header != nil { - reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.reward, fees.baseFee, fees.nextBaseFee, fees.gasUsedRatio + if fees.results.baseFee != nil { + reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.results.reward, fees.results.baseFee, fees.results.nextBaseFee, fees.results.gasUsedRatio } else { // getting no block and no error means we are requesting into the future (might happen because of a reorg) if i < firstMissing { diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index 16c74b7db46a..c259eb0acf76 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -36,20 +36,20 @@ func TestFeeHistory(t *testing.T) { expCount int expErr error }{ - {false, 0, 0, 10, 30, nil, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{0, 10}, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, - {false, 0, 0, 1000000000, 30, nil, 0, 31, nil}, - {false, 0, 0, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, - {false, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, - {true, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {false, 1000, 1000, 10, 30, nil, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{0, 10}, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, + {false, 1000, 1000, 1000000000, 30, nil, 0, 31, nil}, + {false, 1000, 1000, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, + {false, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {true, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, {false, 20, 2, 100, rpc.LatestBlockNumber, nil, 13, 20, nil}, {false, 20, 2, 100, rpc.LatestBlockNumber, []float64{0, 10}, 31, 2, nil}, {false, 20, 2, 100, 32, []float64{0, 10}, 31, 2, nil}, - {false, 0, 0, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, - {false, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, + {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 407eeaa2899c..8feb5ef24ba4 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -23,10 +23,13 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -53,6 +56,7 @@ type OracleBackend interface { GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) PendingBlockAndReceipts() (*types.Block, types.Receipts) ChainConfig() *params.ChainConfig + SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } // Oracle recommends gas prices based on the content of recent @@ -68,6 +72,7 @@ type Oracle struct { checkBlocks, percentile int maxHeaderHistory, maxBlockHistory int + historyCache *lru.Cache } // NewOracle returns a new gasprice oracle which can recommend suitable @@ -99,6 +104,20 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { } else if ignorePrice.Int64() > 0 { log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice) } + + cache, _ := lru.New(2048) + headEvent := make(chan core.ChainHeadEvent, 1) + backend.SubscribeChainHeadEvent(headEvent) + go func() { + var lastHead common.Hash + for ev := range headEvent { + if ev.Block.ParentHash() != lastHead { + cache.Purge() + } + lastHead = ev.Block.Hash() + } + }() + return &Oracle{ backend: backend, lastPrice: params.Default, @@ -108,6 +127,7 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { percentile: percent, maxHeaderHistory: params.MaxHeaderHistory, maxBlockHistory: params.MaxBlockHistory, + historyCache: cache, } } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index dea8fea95ae3..feecfddec730 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -90,6 +91,10 @@ func (b *testBackend) ChainConfig() *params.ChainConfig { return b.chain.Config() } +func (b *testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} + func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend { var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") From 1d995731923ca899964371ddb213d40b7e773818 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 09:00:10 +0200 Subject: [PATCH 59/60] core/vm: faster code analysis (#23381) * core/vm: more detailed benchmark for jumpdest analysis * core/vm: make jd analysis benchmark alloc free * core/vm: improve jumpdest analysis * core/vm: improve worst-case * core/vm: further improvements in analysis * core/vm: improve jumpdest analysis >PUSH15 * core/vm: make jd analysis ref by value * core/vm: fix misspell * core/vm: improve set8 and set16 a bit * core/vm: reduce amount of code * core/vm: optimize byte copying --- core/vm/analysis.go | 94 +++++++++++++++++++++++++++++++++------- core/vm/analysis_test.go | 24 +++++++++- core/vm/instructions.go | 4 ++ core/vm/interpreter.go | 2 +- 4 files changed, 105 insertions(+), 19 deletions(-) diff --git a/core/vm/analysis.go b/core/vm/analysis.go index 0ccf47b97903..449cded2a896 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -16,17 +16,49 @@ package vm +const ( + set2BitsMask = uint16(0b1100_0000_0000_0000) + set3BitsMask = uint16(0b1110_0000_0000_0000) + set4BitsMask = uint16(0b1111_0000_0000_0000) + set5BitsMask = uint16(0b1111_1000_0000_0000) + set6BitsMask = uint16(0b1111_1100_0000_0000) + set7BitsMask = uint16(0b1111_1110_0000_0000) +) + // bitvec is a bit vector which maps bytes in a program. // An unset bit means the byte is an opcode, a set bit means // it's data (i.e. argument of PUSHxx). type bitvec []byte -func (bits *bitvec) set(pos uint64) { - (*bits)[pos/8] |= 0x80 >> (pos % 8) +var lookup = [8]byte{ + 0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1, +} + +func (bits bitvec) set1(pos uint64) { + bits[pos/8] |= lookup[pos%8] +} + +func (bits bitvec) setN(flag uint16, pos uint64) { + a := flag >> (pos % 8) + bits[pos/8] |= byte(a >> 8) + if b := byte(a); b != 0 { + // If the bit-setting affects the neighbouring byte, we can assign - no need to OR it, + // since it's the first write to that byte + bits[pos/8+1] = b + } +} + +func (bits bitvec) set8(pos uint64) { + a := byte(0xFF >> (pos % 8)) + bits[pos/8] |= a + bits[pos/8+1] = ^a } -func (bits *bitvec) set8(pos uint64) { - (*bits)[pos/8] |= 0xFF >> (pos % 8) - (*bits)[pos/8+1] |= ^(0xFF >> (pos % 8)) + +func (bits bitvec) set16(pos uint64) { + a := byte(0xFF >> (pos % 8)) + bits[pos/8] |= a + bits[pos/8+1] = 0xFF + bits[pos/8+2] = ^a } // codeSegment checks if the position is in a code segment. @@ -40,22 +72,52 @@ func codeBitmap(code []byte) bitvec { // ends with a PUSH32, the algorithm will push zeroes onto the // bitvector outside the bounds of the actual code. bits := make(bitvec, len(code)/8+1+4) + return codeBitmapInternal(code, bits) +} + +// codeBitmapInternal is the internal implementation of codeBitmap. +// It exists for the purpose of being able to run benchmark tests +// without dynamic allocations affecting the results. +func codeBitmapInternal(code, bits bitvec) bitvec { for pc := uint64(0); pc < uint64(len(code)); { op := OpCode(code[pc]) - - if op >= PUSH1 && op <= PUSH32 { - numbits := op - PUSH1 + 1 - pc++ + pc++ + if op < PUSH1 || op > PUSH32 { + continue + } + numbits := op - PUSH1 + 1 + if numbits >= 8 { + for ; numbits >= 16; numbits -= 16 { + bits.set16(pc) + pc += 16 + } for ; numbits >= 8; numbits -= 8 { - bits.set8(pc) // 8 + bits.set8(pc) pc += 8 } - for ; numbits > 0; numbits-- { - bits.set(pc) - pc++ - } - } else { - pc++ + } + switch numbits { + case 1: + bits.set1(pc) + pc += 1 + case 2: + bits.setN(set2BitsMask, pc) + pc += 2 + case 3: + bits.setN(set3BitsMask, pc) + pc += 3 + case 4: + bits.setN(set4BitsMask, pc) + pc += 4 + case 5: + bits.setN(set5BitsMask, pc) + pc += 5 + case 6: + bits.setN(set6BitsMask, pc) + pc += 6 + case 7: + bits.setN(set7BitsMask, pc) + pc += 7 } } return bits diff --git a/core/vm/analysis_test.go b/core/vm/analysis_test.go index fd2d744d87f4..585bb3097f44 100644 --- a/core/vm/analysis_test.go +++ b/core/vm/analysis_test.go @@ -47,10 +47,10 @@ func TestJumpDestAnalysis(t *testing.T) { {[]byte{byte(PUSH32)}, 0xFF, 1}, {[]byte{byte(PUSH32)}, 0xFF, 2}, } - for _, test := range tests { + for i, test := range tests { ret := codeBitmap(test.code) if ret[test.which] != test.exp { - t.Fatalf("expected %x, got %02x", test.exp, ret[test.which]) + t.Fatalf("test %d: expected %x, got %02x", i, test.exp, ret[test.which]) } } } @@ -73,3 +73,23 @@ func BenchmarkJumpdestHashing_1200k(bench *testing.B) { } bench.StopTimer() } + +func BenchmarkJumpdestOpAnalysis(bench *testing.B) { + var op OpCode + bencher := func(b *testing.B) { + code := make([]byte, 32*b.N) + for i := range code { + code[i] = byte(op) + } + bits := make(bitvec, len(code)/8+1+4) + b.ResetTimer() + codeBitmapInternal(code, bits) + } + for op = PUSH1; op <= PUSH32; op++ { + bench.Run(op.String(), bencher) + } + op = JUMPDEST + bench.Run(op.String(), bencher) + op = STOP + bench.Run(op.String(), bencher) +} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index f7ef2f900e20..6c8c6e6e6fa1 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -669,6 +669,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -703,6 +704,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -730,6 +732,7 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -757,6 +760,7 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 9cf0c4e2c1e0..9fb83799c98c 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -262,7 +262,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // if the operation clears the return data (e.g. it has returning data) // set the last return to the result of the operation. if operation.returns { - in.returnData = common.CopyBytes(res) + in.returnData = res } switch { From 26675454bf93bf904be7a43cce6b3f550115ff90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 10:04:30 +0300 Subject: [PATCH 60/60] params: release Geth v1.10.8 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3c50a9e8d017..8d222ac04708 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string.