From ff41a3503981036c088fa4c110f1e1c7e3ae8ab6 Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Thu, 21 Mar 2024 17:57:34 -0700 Subject: [PATCH 01/44] Add updateRoot method to fetchers --- packages/client/src/sync/fetcher/accountfetcher.ts | 5 +++++ packages/client/src/sync/fetcher/storagefetcher.ts | 4 ++++ packages/client/src/sync/fetcher/trienodefetcher.ts | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 79c9c48adc..ca0e0b1ba4 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -580,6 +580,11 @@ export class AccountFetcher extends Fetcher updateStateRoot(stateRoot: Uint8Array) { this.root = stateRoot + + this.storageFetcher.updateStateRoot(stateRoot) + + // TODO trieNodeFetcher needs to be constantly healing while other fetchers work + this.trieNodeFetcher.updateStateRoot(stateRoot) } nextTasks(): void { diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index ed173499b4..fa4807a1dc 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -612,6 +612,10 @@ export class StorageFetcher extends Fetcher return tasks } + updateStateRoot(stateRoot: Uint8Array) { + this.root = stateRoot + } + nextTasks(): void { try { if (this.in.length === 0) { From 6c00e9c60ff94873abe80f8ac85a64aea3a418f8 Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Thu, 21 Mar 2024 17:58:01 -0700 Subject: [PATCH 02/44] Update the fetcher roots in syncWithPeer if fetcher already exists --- packages/client/src/sync/snapsync.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 0ab6c362ad..ad1887a41d 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -239,15 +239,22 @@ export class SnapSynchronizer extends Synchronizer { : 'previous fetcher errored=' + this.fetcher.syncErrored?.message }` ) - this.fetcher = new AccountFetcher({ - config: this.config, - pool: this.pool, - stateManager: this.execution.vm.stateManager as DefaultStateManager, - root: stateRoot, - // This needs to be determined from the current state of the MPT dump - first: BigInt(0), - fetcherDoneFlags: this.fetcherDoneFlags, - }) + + if (this.fetcher === undefined) { + this.fetcher = new AccountFetcher({ + config: this.config, + pool: this.pool, + stateManager: this.execution.vm.stateManager as DefaultStateManager, + root: stateRoot, + // This needs to be determined from the current state of the MPT dump + first: BigInt(0), + fetcherDoneFlags: this.fetcherDoneFlags, + }) + } else { + // update root + this.config.logger.info(`UPDATING ROOTS`) + this.fetcher?.updateStateRoot(stateRoot) + } } else { return false } From dd21e7309aa228f1f8decd18d40ca378006c9017 Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Thu, 4 Apr 2024 16:46:05 -0700 Subject: [PATCH 03/44] Update fetcher roots on FCUs --- packages/client/src/service/fullethereumservice.ts | 6 +++++- packages/client/src/sync/snapsync.ts | 7 ++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index c1521bdc73..e2eadab116 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -243,7 +243,11 @@ export class FullEthereumService extends Service { * vm execution */ async buildHeadState(): Promise { - if (this.building) return + // If during building FCU identifies a new root/height to sync to, evalueate new height with sync + // strategy and initiate state healing and fetcher root updates for snap sync + if (this.building) { + this.snapsync?.sync() + } this.building = true try { diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index ad1887a41d..f65e83b750 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -220,10 +220,7 @@ export class SnapSynchronizer extends Synchronizer { this.config.logger.info(`New sync target height=${height} hash=${bytesToHex(latest.hash())}`) } - if ( - (this.fetcher === null || this.fetcher.syncErrored !== undefined) && - this.config.syncTargetHeight <= latest.number + this.config.snapAvailabilityDepth - ) { + if (this.config.syncTargetHeight <= latest.number + this.config.snapAvailabilityDepth) { if ((this.fetcherDoneFlags.snapTargetHeight ?? BIGINT_0) < latest.number) { this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot @@ -240,7 +237,7 @@ export class SnapSynchronizer extends Synchronizer { }` ) - if (this.fetcher === undefined) { + if (this.fetcher === null || this.fetcher.syncErrored !== undefined) { this.fetcher = new AccountFetcher({ config: this.config, pool: this.pool, From 19adeea3465d62be94dedc795e5ed0c7d60c00bc Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Thu, 4 Apr 2024 23:54:29 -0700 Subject: [PATCH 04/44] Call syncWithPeer directly from buildHeadState for updating root --- packages/client/src/service/fullethereumservice.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index e2eadab116..eeeea114aa 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -246,7 +246,8 @@ export class FullEthereumService extends Service { // If during building FCU identifies a new root/height to sync to, evalueate new height with sync // strategy and initiate state healing and fetcher root updates for snap sync if (this.building) { - this.snapsync?.sync() + const peer = await this.snapsync?.best() + void this.snapsync?.syncWithPeer(peer) } this.building = true From 1e1b1cc8d717904d5dcc69c38fb8ac90230f1ae7 Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Tue, 9 Apr 2024 12:57:04 -0700 Subject: [PATCH 05/44] Keep track of requested pathStrings to remove duplicate requests in TrieNodeFetcher --- .../src/sync/fetcher/trienodefetcher.ts | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/client/src/sync/fetcher/trienodefetcher.ts b/packages/client/src/sync/fetcher/trienodefetcher.ts index da2510c927..1a1da1afad 100644 --- a/packages/client/src/sync/fetcher/trienodefetcher.ts +++ b/packages/client/src/sync/fetcher/trienodefetcher.ts @@ -60,6 +60,7 @@ type FetchedNodeData = { } type NodeRequestData = { + requested: boolean nodeHash: string nodeParentHash: string parentAccountHash?: string // for leaf account nodes that contain a storage component @@ -116,6 +117,7 @@ export class TrieNodeFetcher extends Fetcher // will always start with root node as first set of node requests this.pathToNodeRequestData.setElement('', { + requested: false, nodeHash: bytesToHex(this.root), nodeParentHash: '', // root node does not have a parent } as NodeRequestData) @@ -256,6 +258,7 @@ export class TrieNodeFetcher extends Fetcher storagePath, ].join('/') this.pathToNodeRequestData.setElement(syncPath, { + requested: false, nodeHash: bytesToHex(storageRoot), nodeParentHash: nodeHash, parentAccountHash: nodeHash, @@ -292,6 +295,7 @@ export class TrieNodeFetcher extends Fetcher pathString ) as NodeRequestData this.pathToNodeRequestData.setElement(childNode.path, { + requested: false, nodeHash: bytesToHex(childNode.nodeHash as Uint8Array), nodeParentHash: nodeHash, // TODO root node does not have a parent, so handle that in the leaf callback when checking if dependencies are met recursively parentAccountHash, @@ -404,17 +408,25 @@ export class TrieNodeFetcher extends Fetcher if (this.pathToNodeRequestData.size() > 0) { let { pathStrings } = this.getSortedPathStrings() // TODO pass in number of paths to return while (tasks.length < maxTasks && pathStrings.length > 0) { - const requestedPathStrings = pathStrings.slice(0, max) + const pendingPathStrings = pathStrings.slice(0, max) pathStrings = pathStrings.slice(max + 1) - for (const pathString of requestedPathStrings) { - const nodeHash = this.pathToNodeRequestData.getElementByKey(pathString)?.nodeHash // TODO return node set too from sorted function and avoid lookups here - if (nodeHash === undefined) throw Error('Path should exist') - this.requestedNodeToPath.set(nodeHash as unknown as string, pathString) + const neededPathStrings = [] + for (const pathString of pendingPathStrings) { + const requestData = this.pathToNodeRequestData.getElementByKey(pathString) // TODO return node set too from sorted function and avoid lookups here + if (requestData === undefined) throw Error('Path should exist') + if (requestData.requested === false) { + this.requestedNodeToPath.set(requestData.nodeHash as unknown as string, pathString) + this.pathToNodeRequestData.setElement(pathString, { + ...requestData, + requested: true, + }) + neededPathStrings.push(pathString) + } } this.debug('At start of mergeAndFormatPaths') - const paths = mergeAndFormatKeyPaths(requestedPathStrings) as unknown as Uint8Array[][] + const paths = mergeAndFormatKeyPaths(neededPathStrings) as unknown as Uint8Array[][] tasks.push({ - pathStrings: requestedPathStrings, + pathStrings: neededPathStrings, paths, }) this.debug(`Created new tasks num=${tasks.length}`) From 8077f4dd43168aa36cc4738ba05b992dc6e0c5bd Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Tue, 9 Apr 2024 18:35:17 -0700 Subject: [PATCH 06/44] Fix test --- packages/client/test/sync/fetcher/trienodefetcher.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/client/test/sync/fetcher/trienodefetcher.spec.ts b/packages/client/test/sync/fetcher/trienodefetcher.spec.ts index d4b1202801..f4999f6f41 100644 --- a/packages/client/test/sync/fetcher/trienodefetcher.spec.ts +++ b/packages/client/test/sync/fetcher/trienodefetcher.spec.ts @@ -212,6 +212,7 @@ describe('[TrieNodeFetcher]', async () => { fetcher.pathToNodeRequestData = new OrderedMap() fetcher.pathToNodeRequestData.setElement('', { + requested: false, nodeHash: bytesToHex(new Uint8Array(0)), nodeParentHash: '', }) From 6a2b266545f854b88b2f7019ff4d09df0ebd23bd Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Thu, 11 Apr 2024 12:47:07 +0200 Subject: [PATCH 07/44] Client -> Peer: add lastEthStatusUpdate property to Peer, set initially on protocol binding, small local refactor --- packages/client/src/net/peer/peer.ts | 28 ++++++++++++++-------------- packages/client/src/net/peerpool.ts | 4 +++- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index 8fb2bffa5e..70660d711d 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -49,6 +49,8 @@ export abstract class Peer extends EventEmitter { public snap?: BoundSnapProtocol public les?: BoundLesProtocol + public lastEthStatusUpdate?: number + /* If the peer is in the PeerPool. If true, messages are handled immediately. @@ -108,28 +110,26 @@ export abstract class Peer extends EventEmitter { if (protocol.name === 'eth') { bound = new BoundEthProtocol(boundOpts) + + await bound!.handshake(sender) + this.lastEthStatusUpdate = Date.now() + + this.eth = bound } else if (protocol.name === 'les') { bound = new BoundLesProtocol(boundOpts) - } else if (protocol.name === 'snap') { - bound = new BoundSnapProtocol(boundOpts) - } else { - throw new Error(`addProtocol: ${protocol.name} protocol not supported`) - } - // Handshake only when snap, else - if (protocol.name !== 'snap') { await bound!.handshake(sender) - } else { - if (sender.status === undefined) throw Error('Snap can only be bound on handshaked peer') - } - if (protocol.name === 'eth') { - this.eth = bound + this.les = bound } else if (protocol.name === 'snap') { + bound = new BoundSnapProtocol(boundOpts) + if (sender.status === undefined) throw Error('Snap can only be bound on handshaked peer') + this.snap = bound - } else if (protocol.name === 'les') { - this.les = bound + } else { + throw new Error(`addProtocol: ${protocol.name} protocol not supported`) } + this.boundProtocols.push(bound) } diff --git a/packages/client/src/net/peerpool.ts b/packages/client/src/net/peerpool.ts index fdc5af70d1..f792da1bb5 100644 --- a/packages/client/src/net/peerpool.ts +++ b/packages/client/src/net/peerpool.ts @@ -146,7 +146,9 @@ export class PeerPool { * @param filterFn filter function to apply before finding idle peers */ idle(filterFn = (_peer: Peer) => true): Peer | undefined { - const idle = this.peers.filter((p) => p.idle && filterFn(p)) + const idle = this.peers.filter((p) => { + return p.idle && filterFn(p) ? true : false + }) if (idle.length > 0) { const index = Math.floor(Math.random() * idle.length) return idle[index] From 4dd9bf86abca02bf7d96c894ba39d12e0e5a969e Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Thu, 11 Apr 2024 13:39:42 +0200 Subject: [PATCH 08/44] Redo handshake in client peer pool periodically, add force option to sendStatus for devp2p protocols --- packages/client/src/net/peer/peer.ts | 7 ++-- packages/client/src/net/peer/rlpxpeer.ts | 1 + packages/client/src/net/peerpool.ts | 35 ++++++++++++++++++- .../client/src/net/protocol/rlpxsender.ts | 2 +- packages/devp2p/src/protocol/eth.ts | 4 +-- packages/devp2p/src/protocol/les.ts | 4 +-- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index 70660d711d..15b3011a11 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'events' import { BoundEthProtocol, BoundLesProtocol, BoundSnapProtocol } from '../protocol' import type { Config } from '../../config' -import type { BoundProtocol, Protocol, Sender } from '../protocol' +import type { BoundProtocol, Protocol, RlpxSender, Sender } from '../protocol' import type { Server } from '../server' export interface PeerOptions { @@ -46,11 +46,12 @@ export abstract class Peer extends EventEmitter { // TODO check if this should be moved into RlpxPeer public eth?: BoundEthProtocol + public lastEthStatusUpdate?: number + public ethSender?: RlpxSender + public snap?: BoundSnapProtocol public les?: BoundLesProtocol - public lastEthStatusUpdate?: number - /* If the peer is in the PeerPool. If true, messages are handled immediately. diff --git a/packages/client/src/net/peer/rlpxpeer.ts b/packages/client/src/net/peer/rlpxpeer.ts index 39b3469c5f..0bf06d209a 100644 --- a/packages/client/src/net/peer/rlpxpeer.ts +++ b/packages/client/src/net/peer/rlpxpeer.ts @@ -168,6 +168,7 @@ export class RlpxPeer extends Peer { // handshake, and can just use the eth handshake if (protocol && name !== 'snap') { const sender = new RlpxSender(rlpxProtocol as Devp2pETH | Devp2pLES | Devp2pSNAP) + this.ethSender = sender return this.addProtocol(sender, protocol).then(() => { if (name === 'eth') { const snapRlpxProtocol = rlpxPeer diff --git a/packages/client/src/net/peerpool.ts b/packages/client/src/net/peerpool.ts index f792da1bb5..f23712073a 100644 --- a/packages/client/src/net/peerpool.ts +++ b/packages/client/src/net/peerpool.ts @@ -1,9 +1,12 @@ import { Hardfork } from '@ethereumjs/common' +import { bytesToUnprefixedHex } from '@ethereumjs/util' import { Event } from '../types' +import { type Peer, RlpxPeer } from './peer' +import { RlpxSender } from './protocol' + import type { Config } from '../config' -import type { Peer } from './peer' export interface PeerPoolOptions { /* Config */ @@ -31,7 +34,13 @@ export class PeerPool { */ private DEFAULT_STATUS_CHECK_INTERVAL = 20000 + /** + * Default status check interval (in ms) + */ + private DEFAULT_PEER_ETH_STATUS_CHECK_INTERVAL = 5000 + private _statusCheckInterval: NodeJS.Timeout | undefined /* global NodeJS */ + private _peerEthStatusCheckInterval: NodeJS.Timeout | undefined private _reconnectTimeout: NodeJS.Timeout | undefined /** @@ -87,6 +96,12 @@ export class PeerPool { this.DEFAULT_STATUS_CHECK_INTERVAL ) + this._peerEthStatusCheckInterval = setInterval( + // eslint-disable-next-line @typescript-eslint/await-thenable + await this._peerEthStatusCheck.bind(this), + this.DEFAULT_PEER_ETH_STATUS_CHECK_INTERVAL + ) + this.running = true return true } @@ -99,6 +114,7 @@ export class PeerPool { await this.close() } clearInterval(this._statusCheckInterval as NodeJS.Timeout) + clearInterval(this._peerEthStatusCheckInterval as NodeJS.Timeout) clearTimeout(this._reconnectTimeout as NodeJS.Timeout) this.running = false return true @@ -254,4 +270,21 @@ export class PeerPool { this.noPeerPeriods = 0 } } + + /** + * Periodically check pooled peers for the last ETH status exchange + * and trigger new status msg exchanges if too old + */ + async _peerEthStatusCheck() { + for (const p of this.peers) { + if (!p.idle && p.eth !== undefined && p.ethSender !== undefined && p instanceof RlpxPeer) { + p.idle = true + console.log('here!') + console.log(bytesToUnprefixedHex(p.eth.status.bestHash)) + await p.eth.handshake(p.ethSender!) + console.log(bytesToUnprefixedHex(p.eth.status.bestHash)) + p.idle = false + } + } + } } diff --git a/packages/client/src/net/protocol/rlpxsender.ts b/packages/client/src/net/protocol/rlpxsender.ts index f7e248d61c..3c371da70f 100644 --- a/packages/client/src/net/protocol/rlpxsender.ts +++ b/packages/client/src/net/protocol/rlpxsender.ts @@ -33,7 +33,7 @@ export class RlpxSender extends Sender { */ sendStatus(status: any) { try { - this.sender.sendStatus(status) + this.sender.sendStatus(status, true) } catch (err: any) { this.emit('error', err) } diff --git a/packages/devp2p/src/protocol/eth.ts b/packages/devp2p/src/protocol/eth.ts index 5772808ad2..33eb08a6a4 100644 --- a/packages/devp2p/src/protocol/eth.ts +++ b/packages/devp2p/src/protocol/eth.ts @@ -271,8 +271,8 @@ export class ETH extends Protocol { return sStr } - sendStatus(status: ETH.StatusOpts) { - if (this._status !== null) return + sendStatus(status: ETH.StatusOpts, force = false) { + if (this._status !== null && !force) return this._status = [ intToBytes(this._version), bigIntToBytes(this._peer.common.chainId()), diff --git a/packages/devp2p/src/protocol/les.ts b/packages/devp2p/src/protocol/les.ts index bb38eff6c6..18f746acf7 100644 --- a/packages/devp2p/src/protocol/les.ts +++ b/packages/devp2p/src/protocol/les.ts @@ -182,8 +182,8 @@ export class LES extends Protocol { return sStr } - sendStatus(status: LES.Status) { - if (this._status !== null) return + sendStatus(status: LES.Status, force = false) { + if (this._status !== null && !force) return if (status.announceType === undefined) { status['announceType'] = intToBytes(DEFAULT_ANNOUNCE_TYPE) From e268f9ac4e7d3982ed1c97129ffab3fa300cf897 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 11:25:29 +0200 Subject: [PATCH 09/44] Some basic refinement of peer pool periodic request logic (refactor preparation) --- packages/client/src/net/peer/peer.ts | 3 +-- packages/client/src/net/peer/rlpxpeer.ts | 1 - packages/client/src/net/peerpool.ts | 17 ++++++++--------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index 15b3011a11..07e2e5bb72 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'events' import { BoundEthProtocol, BoundLesProtocol, BoundSnapProtocol } from '../protocol' import type { Config } from '../../config' -import type { BoundProtocol, Protocol, RlpxSender, Sender } from '../protocol' +import type { BoundProtocol, Protocol, Sender } from '../protocol' import type { Server } from '../server' export interface PeerOptions { @@ -47,7 +47,6 @@ export abstract class Peer extends EventEmitter { // TODO check if this should be moved into RlpxPeer public eth?: BoundEthProtocol public lastEthStatusUpdate?: number - public ethSender?: RlpxSender public snap?: BoundSnapProtocol public les?: BoundLesProtocol diff --git a/packages/client/src/net/peer/rlpxpeer.ts b/packages/client/src/net/peer/rlpxpeer.ts index 0bf06d209a..39b3469c5f 100644 --- a/packages/client/src/net/peer/rlpxpeer.ts +++ b/packages/client/src/net/peer/rlpxpeer.ts @@ -168,7 +168,6 @@ export class RlpxPeer extends Peer { // handshake, and can just use the eth handshake if (protocol && name !== 'snap') { const sender = new RlpxSender(rlpxProtocol as Devp2pETH | Devp2pLES | Devp2pSNAP) - this.ethSender = sender return this.addProtocol(sender, protocol).then(() => { if (name === 'eth') { const snapRlpxProtocol = rlpxPeer diff --git a/packages/client/src/net/peerpool.ts b/packages/client/src/net/peerpool.ts index f23712073a..bf052e7db3 100644 --- a/packages/client/src/net/peerpool.ts +++ b/packages/client/src/net/peerpool.ts @@ -4,7 +4,6 @@ import { bytesToUnprefixedHex } from '@ethereumjs/util' import { Event } from '../types' import { type Peer, RlpxPeer } from './peer' -import { RlpxSender } from './protocol' import type { Config } from '../config' @@ -98,7 +97,7 @@ export class PeerPool { this._peerEthStatusCheckInterval = setInterval( // eslint-disable-next-line @typescript-eslint/await-thenable - await this._peerEthStatusCheck.bind(this), + await this._peerBestBlockUpdate.bind(this), this.DEFAULT_PEER_ETH_STATUS_CHECK_INTERVAL ) @@ -272,16 +271,16 @@ export class PeerPool { } /** - * Periodically check pooled peers for the last ETH status exchange - * and trigger new status msg exchanges if too old + * Periodically check for for the latest best block known by a + * peer to allow for a more accurate best peer choice */ - async _peerEthStatusCheck() { + async _peerBestBlockUpdate() { for (const p of this.peers) { - if (!p.idle && p.eth !== undefined && p.ethSender !== undefined && p instanceof RlpxPeer) { + if (!p.idle && p.eth !== undefined && p instanceof RlpxPeer) { p.idle = true - console.log('here!') - console.log(bytesToUnprefixedHex(p.eth.status.bestHash)) - await p.eth.handshake(p.ethSender!) + console.log(this.config.syncTargetHeight) + //const block = await p.eth.getBlockHeaders() + //console.log(block) console.log(bytesToUnprefixedHex(p.eth.status.bestHash)) p.idle = false } From b0532774e6a7d7ed6ae41315cff21a4a9241a2bb Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 11:29:57 +0200 Subject: [PATCH 10/44] Client: consolidate latest() method in base Synchronizer class --- packages/client/src/sync/beaconsync.ts | 13 +------------ packages/client/src/sync/fullsync.ts | 11 ----------- packages/client/src/sync/lightsync.ts | 11 ----------- packages/client/src/sync/snapsync.ts | 14 -------------- packages/client/src/sync/sync.ts | 12 ++++++++++++ 5 files changed, 13 insertions(+), 48 deletions(-) diff --git a/packages/client/src/sync/beaconsync.ts b/packages/client/src/sync/beaconsync.ts index 6732c50d19..15c4869d09 100644 --- a/packages/client/src/sync/beaconsync.ts +++ b/packages/client/src/sync/beaconsync.ts @@ -116,7 +116,7 @@ export class BeaconSynchronizer extends Synchronizer { if (peers.length < this.config.minPeers && !this.forceSync) return for (const peer of peers) { const latest = await this.latest(peer) - if (latest) { + if (latest !== undefined) { const { number } = latest if (!best || best[1] < number) { best = [peer, number] @@ -126,17 +126,6 @@ export class BeaconSynchronizer extends Synchronizer { return best ? best[0] : undefined } - /** - * Get latest header of peer - */ - async latest(peer: Peer) { - const result = await peer.eth?.getBlockHeaders({ - block: peer.eth!.status.bestHash, - max: 1, - }) - return result ? result[1][0] : undefined - } - /** * Start synchronizer. * If passed a block, will initialize sync starting from the block. diff --git a/packages/client/src/sync/fullsync.ts b/packages/client/src/sync/fullsync.ts index a82e631709..2f3427a7d2 100644 --- a/packages/client/src/sync/fullsync.ts +++ b/packages/client/src/sync/fullsync.ts @@ -138,17 +138,6 @@ export class FullSynchronizer extends Synchronizer { return best } - /** - * Get latest header of peer - */ - async latest(peer: Peer) { - const result = await peer.eth?.getBlockHeaders({ - block: peer.eth!.status.bestHash, - max: 1, - }) - return result ? result[1][0] : undefined - } - /** * Checks if tx pool should be started */ diff --git a/packages/client/src/sync/lightsync.ts b/packages/client/src/sync/lightsync.ts index 411d433110..985cb19c93 100644 --- a/packages/client/src/sync/lightsync.ts +++ b/packages/client/src/sync/lightsync.ts @@ -84,17 +84,6 @@ export class LightSynchronizer extends Synchronizer { return best } - /** - * Get latest header of peer - */ - async latest(peer: Peer) { - const result = await peer.les?.getBlockHeaders({ - block: peer.les!.status.headHash, - max: 1, - }) - return result?.headers[0] - } - /** * Called from `sync()` to sync headers and state from peer starting from current height. * @param peer remote peer to sync with diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 0ab6c362ad..3d4e415f9a 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -99,20 +99,6 @@ export class SnapSynchronizer extends Synchronizer { return best ? best[0] : undefined } - /** - * Get latest header of peer - */ - async latest(peer: Peer) { - // TODO: refine the way to query latest to fetch for the peer - const blockHash = peer.eth!.status.bestHash - // const blockHash = this.skeleton?.headHash() ?? peer.eth!.status.bestHash - const result = await peer.eth?.getBlockHeaders({ - block: blockHash, - max: 1, - }) - return result ? result[1][0] : undefined - } - /** * Start synchronizer. */ diff --git a/packages/client/src/sync/sync.ts b/packages/client/src/sync/sync.ts index 3ee493986f..a7da72ebb0 100644 --- a/packages/client/src/sync/sync.ts +++ b/packages/client/src/sync/sync.ts @@ -10,6 +10,7 @@ import type { Config } from '../config' import type { Peer } from '../net/peer/peer' import type { PeerPool } from '../net/peerpool' import type { AccountFetcher, BlockFetcher, HeaderFetcher, ReverseBlockFetcher } from './fetcher' +import type { BlockHeader } from '@ethereumjs/block' export interface SynchronizerOptions { /* Config */ @@ -163,6 +164,17 @@ export abstract class Synchronizer { } } + /** + * Get latest header of peer + */ + async latest(peer: Peer): Promise { + const result = await peer.eth?.getBlockHeaders({ + block: peer.eth!.status.bestHash, + max: 1, + }) + return result ? result[1][0] : undefined + } + /** * Fetch all blocks from current height up to highest found amongst peers * @returns when sync is completed From 72a28d15786a027c52fde2b84431fe832041b671 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 11:54:47 +0200 Subject: [PATCH 11/44] Client: add preparatory updatedBestHeader to Peer, refactor latest() to move from Sync -> Peer --- packages/client/src/net/peer/peer.ts | 20 ++++++++++++++++--- .../client/src/net/protocol/boundprotocol.ts | 9 +++++++++ packages/client/src/rpc/modules/eth.ts | 2 +- packages/client/src/sync/beaconsync.ts | 4 ++-- packages/client/src/sync/fullsync.ts | 2 +- packages/client/src/sync/lightsync.ts | 2 +- packages/client/src/sync/snapsync.ts | 6 +++--- packages/client/src/sync/sync.ts | 12 ----------- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index 07e2e5bb72..d6d696ccfa 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -5,6 +5,7 @@ import { BoundEthProtocol, BoundLesProtocol, BoundSnapProtocol } from '../protoc import type { Config } from '../../config' import type { BoundProtocol, Protocol, Sender } from '../protocol' import type { Server } from '../server' +import type { BlockHeader } from '@ethereumjs/block' export interface PeerOptions { /* Config */ @@ -46,8 +47,6 @@ export abstract class Peer extends EventEmitter { // TODO check if this should be moved into RlpxPeer public eth?: BoundEthProtocol - public lastEthStatusUpdate?: number - public snap?: BoundSnapProtocol public les?: BoundLesProtocol @@ -92,6 +91,22 @@ export abstract class Peer extends EventEmitter { abstract connect(): Promise + /** + * Get latest header of peer + */ + async latest(): Promise { + const result = await this.eth?.getBlockHeaders({ + block: this.eth!.status.bestHash, + max: 1, + }) + if (result !== undefined) { + const header = result[1][0] + this.eth!.updatedBestHeader = header + return header + } + return + } + /** * Handle unhandled messages along handshake */ @@ -112,7 +127,6 @@ export abstract class Peer extends EventEmitter { bound = new BoundEthProtocol(boundOpts) await bound!.handshake(sender) - this.lastEthStatusUpdate = Date.now() this.eth = bound } else if (protocol.name === 'les') { diff --git a/packages/client/src/net/protocol/boundprotocol.ts b/packages/client/src/net/protocol/boundprotocol.ts index 14e56c1072..74cb3bd465 100644 --- a/packages/client/src/net/protocol/boundprotocol.ts +++ b/packages/client/src/net/protocol/boundprotocol.ts @@ -43,6 +43,15 @@ export class BoundProtocol { private resolvers: Map private messageQueue: Message[] = [] + /** + * An eventual updated best head. + * + * If set this is by design known to be greater or equal the block hash from + * the initial `STATUS` exchange (`_status` property here) and `updatedBestHash` + * number/hash should take precedence. + */ + public updatedBestHeader?: BlockHeader + /** * Create bound protocol */ diff --git a/packages/client/src/rpc/modules/eth.ts b/packages/client/src/rpc/modules/eth.ts index d1477a97c6..e32f89fc3f 100644 --- a/packages/client/src/rpc/modules/eth.ts +++ b/packages/client/src/rpc/modules/eth.ts @@ -1118,7 +1118,7 @@ export class Eth { message: `no peer available for synchronization`, } } - const highestBlockHeader = await synchronizer.latest(bestPeer) + const highestBlockHeader = await bestPeer.latest() if (!highestBlockHeader) { throw { code: INTERNAL_ERROR, diff --git a/packages/client/src/sync/beaconsync.ts b/packages/client/src/sync/beaconsync.ts index 15c4869d09..30fe54b24d 100644 --- a/packages/client/src/sync/beaconsync.ts +++ b/packages/client/src/sync/beaconsync.ts @@ -115,7 +115,7 @@ export class BeaconSynchronizer extends Synchronizer { const peers = this.pool.peers.filter(this.syncable.bind(this)) if (peers.length < this.config.minPeers && !this.forceSync) return for (const peer of peers) { - const latest = await this.latest(peer) + const latest = await peer.latest() if (latest !== undefined) { const { number } = latest if (!best || best[1] < number) { @@ -210,7 +210,7 @@ export class BeaconSynchronizer extends Synchronizer { return false } - const latest = peer ? await this.latest(peer) : undefined + const latest = peer ? await peer.latest() : undefined if (!latest) return false const height = latest.number diff --git a/packages/client/src/sync/fullsync.ts b/packages/client/src/sync/fullsync.ts index 2f3427a7d2..470aa1569b 100644 --- a/packages/client/src/sync/fullsync.ts +++ b/packages/client/src/sync/fullsync.ts @@ -164,7 +164,7 @@ export class FullSynchronizer extends Synchronizer { * @returns a boolean if the setup was successful */ async syncWithPeer(peer?: Peer): Promise { - const latest = peer ? await this.latest(peer) : undefined + const latest = peer ? await peer.latest() : undefined if (!latest) return false const height = latest.number diff --git a/packages/client/src/sync/lightsync.ts b/packages/client/src/sync/lightsync.ts index 985cb19c93..2c33a19aa4 100644 --- a/packages/client/src/sync/lightsync.ts +++ b/packages/client/src/sync/lightsync.ts @@ -90,7 +90,7 @@ export class LightSynchronizer extends Synchronizer { * @returns a boolean if the setup was successful */ async syncWithPeer(peer?: Peer): Promise { - const latest = peer ? await this.latest(peer) : undefined + const latest = peer ? await peer.latest() : undefined if (!latest) return false const height = peer!.les!.status.headNum diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 3d4e415f9a..bd3657eade 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -84,8 +84,8 @@ export class SnapSynchronizer extends Synchronizer { const peers = this.pool.peers.filter(this.syncable.bind(this)) if (peers.length < this.config.minPeers && !this.forceSync) return for (const peer of peers) { - const latest = await this.latest(peer) - if (latest) { + const latest = await peer.latest() + if (latest !== undefined) { const { number } = latest if ( (!best && @@ -192,7 +192,7 @@ export class SnapSynchronizer extends Synchronizer { return false } - const latest = peer ? await this.latest(peer) : undefined + const latest = peer ? await peer.latest() : undefined if (!latest) { return false } diff --git a/packages/client/src/sync/sync.ts b/packages/client/src/sync/sync.ts index a7da72ebb0..3ee493986f 100644 --- a/packages/client/src/sync/sync.ts +++ b/packages/client/src/sync/sync.ts @@ -10,7 +10,6 @@ import type { Config } from '../config' import type { Peer } from '../net/peer/peer' import type { PeerPool } from '../net/peerpool' import type { AccountFetcher, BlockFetcher, HeaderFetcher, ReverseBlockFetcher } from './fetcher' -import type { BlockHeader } from '@ethereumjs/block' export interface SynchronizerOptions { /* Config */ @@ -164,17 +163,6 @@ export abstract class Synchronizer { } } - /** - * Get latest header of peer - */ - async latest(peer: Peer): Promise { - const result = await peer.eth?.getBlockHeaders({ - block: peer.eth!.status.bestHash, - max: 1, - }) - return result ? result[1][0] : undefined - } - /** * Fetch all blocks from current height up to highest found amongst peers * @returns when sync is completed From f29c3c6cd33ca6f8aedf870d82499ae3a6c7df20 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 15:50:14 +0200 Subject: [PATCH 12/44] Client: add potential best header num differentiated logic to Peer, latest() call on peers in peer pool --- packages/client/src/net/peer/peer.ts | 57 ++++++++++++++++++++++++---- packages/client/src/net/peerpool.ts | 23 +++++------ 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index d6d696ccfa..947b998944 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -1,5 +1,7 @@ +import { BIGINT_0 } from '@ethereumjs/util' import { EventEmitter } from 'events' +import { short } from '../../util' import { BoundEthProtocol, BoundLesProtocol, BoundSnapProtocol } from '../protocol' import type { Config } from '../../config' @@ -92,19 +94,60 @@ export abstract class Peer extends EventEmitter { abstract connect(): Promise /** - * Get latest header of peer + * Eventually updates and returns the latest header of peer */ async latest(): Promise { - const result = await this.eth?.getBlockHeaders({ - block: this.eth!.status.bestHash, + if (!this.eth) { + return + } + let block: bigint | Uint8Array + if (!this.eth!.updatedBestHeader) { + // If there is no updated best header stored yet, start with the status hash + block = this.eth!.status.bestHash + } else { + block = this.getPotentialBestHeaderNum() + } + const result = await this.eth!.getBlockHeaders({ + block, max: 1, }) if (result !== undefined) { - const header = result[1][0] - this.eth!.updatedBestHeader = header - return header + const latest = result[1][0] + this.eth!.updatedBestHeader = latest + if (latest !== undefined) { + const height = latest.number + if ( + this.config.syncTargetHeight === undefined || + this.config.syncTargetHeight === BIGINT_0 || + this.config.syncTargetHeight < latest.number + ) { + this.config.syncTargetHeight = height + this.config.logger.info(`New sync target height=${height} hash=${short(latest.hash())}`) + } + } + } + return this.eth!.updatedBestHeader + } + + /** + * Returns a potential best block header number for the peer + * (not necessarily verified by block request) derived from + * either the client-wide sync target height or the last best + * header timestamp "forward-calculated" by block/slot times (12s). + */ + getPotentialBestHeaderNum(): bigint { + let forwardCalculatedNum = BIGINT_0 + const bestSyncTargetNum = this.config.syncTargetHeight ?? BIGINT_0 + if (this.eth?.updatedBestHeader !== undefined) { + const bestHeaderNum = this.eth!.updatedBestHeader.number + const nowSec = Math.floor(Date.now() / 1000) + const diffSec = nowSec - Number(this.eth!.updatedBestHeader.timestamp) + const SLOT_TIME = 12 + const diffBlocks = BigInt(Math.floor(diffSec / SLOT_TIME)) + forwardCalculatedNum = bestHeaderNum + diffBlocks } - return + const best = forwardCalculatedNum > bestSyncTargetNum ? forwardCalculatedNum : bestSyncTargetNum + return best } /** diff --git a/packages/client/src/net/peerpool.ts b/packages/client/src/net/peerpool.ts index bf052e7db3..6897e63e6d 100644 --- a/packages/client/src/net/peerpool.ts +++ b/packages/client/src/net/peerpool.ts @@ -1,5 +1,4 @@ import { Hardfork } from '@ethereumjs/common' -import { bytesToUnprefixedHex } from '@ethereumjs/util' import { Event } from '../types' @@ -34,9 +33,9 @@ export class PeerPool { private DEFAULT_STATUS_CHECK_INTERVAL = 20000 /** - * Default status check interval (in ms) + * Default peer best header update interval (in ms) */ - private DEFAULT_PEER_ETH_STATUS_CHECK_INTERVAL = 5000 + private DEFAULT_PEER_BEST_HEADER_UPDATE_INTERVAL = 5000 private _statusCheckInterval: NodeJS.Timeout | undefined /* global NodeJS */ private _peerEthStatusCheckInterval: NodeJS.Timeout | undefined @@ -97,8 +96,8 @@ export class PeerPool { this._peerEthStatusCheckInterval = setInterval( // eslint-disable-next-line @typescript-eslint/await-thenable - await this._peerBestBlockUpdate.bind(this), - this.DEFAULT_PEER_ETH_STATUS_CHECK_INTERVAL + await this._peerBestHeaderUpdate.bind(this), + this.DEFAULT_PEER_BEST_HEADER_UPDATE_INTERVAL ) this.running = true @@ -271,18 +270,14 @@ export class PeerPool { } /** - * Periodically check for for the latest best block known by a - * peer to allow for a more accurate best peer choice + * Periodically update the latest best known header for peers */ - async _peerBestBlockUpdate() { + async _peerBestHeaderUpdate() { for (const p of this.peers) { - if (!p.idle && p.eth !== undefined && p instanceof RlpxPeer) { - p.idle = true - console.log(this.config.syncTargetHeight) - //const block = await p.eth.getBlockHeaders() - //console.log(block) - console.log(bytesToUnprefixedHex(p.eth.status.bestHash)) + if (p.idle && p.eth !== undefined && p instanceof RlpxPeer) { p.idle = false + await p.latest() + p.idle = true } } } From 6ea23b08d6a0cd24a28898aefdcd4d4d4918ddb4 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 16:02:21 +0200 Subject: [PATCH 13/44] Client: add Fetcher hack to call peer.latest() --- packages/client/src/sync/fetcher/accountfetcher.ts | 5 +++++ packages/client/src/sync/fetcher/blockfetcher.ts | 6 ++++++ packages/client/src/sync/fetcher/bytecodefetcher.ts | 5 +++++ packages/client/src/sync/fetcher/headerfetcher.ts | 7 +++++++ packages/client/src/sync/fetcher/storagefetcher.ts | 6 ++++++ packages/client/src/sync/fetcher/trienodefetcher.ts | 6 ++++++ 6 files changed, 35 insertions(+) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 79c9c48adc..58725681ee 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -372,6 +372,11 @@ export class AccountFetcher extends Fetcher job: Job ): Promise { const { peer } = job + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() const origin = this.getOrigin(job) const limit = this.getLimit(job) diff --git a/packages/client/src/sync/fetcher/blockfetcher.ts b/packages/client/src/sync/fetcher/blockfetcher.ts index 7aa496604b..79e89b7708 100644 --- a/packages/client/src/sync/fetcher/blockfetcher.ts +++ b/packages/client/src/sync/fetcher/blockfetcher.ts @@ -28,6 +28,12 @@ export class BlockFetcher extends BlockFetcherBase { */ async request(job: Job): Promise { const { task, peer, partialResult } = job + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() + let { first, count } = task if (partialResult) { first = !this.reverse diff --git a/packages/client/src/sync/fetcher/bytecodefetcher.ts b/packages/client/src/sync/fetcher/bytecodefetcher.ts index ea6bb6dd5c..7f76dfc390 100644 --- a/packages/client/src/sync/fetcher/bytecodefetcher.ts +++ b/packages/client/src/sync/fetcher/bytecodefetcher.ts @@ -86,6 +86,11 @@ export class ByteCodeFetcher extends Fetcher job: Job ): Promise { const { task, peer } = job + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() this.debug(`requested code hashes: ${Array.from(task.hashes).map((h) => bytesToHex(h))}`) diff --git a/packages/client/src/sync/fetcher/headerfetcher.ts b/packages/client/src/sync/fetcher/headerfetcher.ts index c2caf9cd5e..9e338f6333 100644 --- a/packages/client/src/sync/fetcher/headerfetcher.ts +++ b/packages/client/src/sync/fetcher/headerfetcher.ts @@ -37,6 +37,13 @@ export class HeaderFetcher extends BlockFetcherBase) { const { task, peer, partialResult } = job + + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() + if (this.flow.maxRequestCount(peer!, 'GetBlockHeaders') < this.config.maxPerRequest) { // we reached our request limit. try with a different peer. return diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index ed173499b4..beb5a60dc4 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -219,6 +219,12 @@ export class StorageFetcher extends Fetcher ): Promise { const { task, peer } = job + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() + const origin = this.getOrigin(job) const limit = this.getLimit(job) diff --git a/packages/client/src/sync/fetcher/trienodefetcher.ts b/packages/client/src/sync/fetcher/trienodefetcher.ts index 6b0a440264..bbe6c19b29 100644 --- a/packages/client/src/sync/fetcher/trienodefetcher.ts +++ b/packages/client/src/sync/fetcher/trienodefetcher.ts @@ -142,6 +142,12 @@ export class TrieNodeFetcher extends Fetcher job: Job ): Promise { const { task, peer } = job + // Currently this is the only safe place to call peer.latest() without interfering with the fetcher + // TODOs: + // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() + // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool + await peer?.latest() + const { paths, pathStrings } = task const rangeResult = await peer!.snap!.getTrieNodes({ From fabf53c995c65a77b5617ecb66b91b8499872778 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Mon, 15 Apr 2024 16:15:56 +0200 Subject: [PATCH 14/44] Various cleanups --- packages/client/src/net/peerpool.ts | 10 ++++------ packages/client/src/net/protocol/rlpxsender.ts | 2 +- packages/devp2p/src/protocol/eth.ts | 4 ++-- packages/devp2p/src/protocol/les.ts | 4 ++-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/client/src/net/peerpool.ts b/packages/client/src/net/peerpool.ts index 6897e63e6d..a504d02e42 100644 --- a/packages/client/src/net/peerpool.ts +++ b/packages/client/src/net/peerpool.ts @@ -38,7 +38,7 @@ export class PeerPool { private DEFAULT_PEER_BEST_HEADER_UPDATE_INTERVAL = 5000 private _statusCheckInterval: NodeJS.Timeout | undefined /* global NodeJS */ - private _peerEthStatusCheckInterval: NodeJS.Timeout | undefined + private _peerBestHeaderUpdateInterval: NodeJS.Timeout | undefined private _reconnectTimeout: NodeJS.Timeout | undefined /** @@ -94,7 +94,7 @@ export class PeerPool { this.DEFAULT_STATUS_CHECK_INTERVAL ) - this._peerEthStatusCheckInterval = setInterval( + this._peerBestHeaderUpdateInterval = setInterval( // eslint-disable-next-line @typescript-eslint/await-thenable await this._peerBestHeaderUpdate.bind(this), this.DEFAULT_PEER_BEST_HEADER_UPDATE_INTERVAL @@ -112,7 +112,7 @@ export class PeerPool { await this.close() } clearInterval(this._statusCheckInterval as NodeJS.Timeout) - clearInterval(this._peerEthStatusCheckInterval as NodeJS.Timeout) + clearInterval(this._peerBestHeaderUpdateInterval as NodeJS.Timeout) clearTimeout(this._reconnectTimeout as NodeJS.Timeout) this.running = false return true @@ -160,9 +160,7 @@ export class PeerPool { * @param filterFn filter function to apply before finding idle peers */ idle(filterFn = (_peer: Peer) => true): Peer | undefined { - const idle = this.peers.filter((p) => { - return p.idle && filterFn(p) ? true : false - }) + const idle = this.peers.filter((p) => p.idle && filterFn(p)) if (idle.length > 0) { const index = Math.floor(Math.random() * idle.length) return idle[index] diff --git a/packages/client/src/net/protocol/rlpxsender.ts b/packages/client/src/net/protocol/rlpxsender.ts index 3c371da70f..f7e248d61c 100644 --- a/packages/client/src/net/protocol/rlpxsender.ts +++ b/packages/client/src/net/protocol/rlpxsender.ts @@ -33,7 +33,7 @@ export class RlpxSender extends Sender { */ sendStatus(status: any) { try { - this.sender.sendStatus(status, true) + this.sender.sendStatus(status) } catch (err: any) { this.emit('error', err) } diff --git a/packages/devp2p/src/protocol/eth.ts b/packages/devp2p/src/protocol/eth.ts index 33eb08a6a4..5772808ad2 100644 --- a/packages/devp2p/src/protocol/eth.ts +++ b/packages/devp2p/src/protocol/eth.ts @@ -271,8 +271,8 @@ export class ETH extends Protocol { return sStr } - sendStatus(status: ETH.StatusOpts, force = false) { - if (this._status !== null && !force) return + sendStatus(status: ETH.StatusOpts) { + if (this._status !== null) return this._status = [ intToBytes(this._version), bigIntToBytes(this._peer.common.chainId()), diff --git a/packages/devp2p/src/protocol/les.ts b/packages/devp2p/src/protocol/les.ts index 18f746acf7..bb38eff6c6 100644 --- a/packages/devp2p/src/protocol/les.ts +++ b/packages/devp2p/src/protocol/les.ts @@ -182,8 +182,8 @@ export class LES extends Protocol { return sStr } - sendStatus(status: LES.Status, force = false) { - if (this._status !== null && !force) return + sendStatus(status: LES.Status) { + if (this._status !== null) return if (status.announceType === undefined) { status['announceType'] = intToBytes(DEFAULT_ANNOUNCE_TYPE) From ccf9366f9c5b420a92dfcab0611c335b03d5d7a2 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 10:55:15 +0200 Subject: [PATCH 15/44] Client: Fix lightsync.spec.ts tests --- packages/client/test/sync/lightsync.spec.ts | 109 +++++++++++++------- 1 file changed, 72 insertions(+), 37 deletions(-) diff --git a/packages/client/test/sync/lightsync.spec.ts b/packages/client/test/sync/lightsync.spec.ts index 2eb77c7c41..6bd6ecec87 100644 --- a/packages/client/test/sync/lightsync.spec.ts +++ b/packages/client/test/sync/lightsync.spec.ts @@ -4,6 +4,7 @@ import { assert, describe, it, vi } from 'vitest' import { Chain } from '../../src/blockchain' import { Config } from '../../src/config' +import { Peer } from '../../src/net/peer/peer' import { HeaderFetcher } from '../../src/sync/fetcher/headerfetcher' import { Event } from '../../src/types' @@ -20,6 +21,9 @@ HeaderFetcher.prototype.clear = td.func() HeaderFetcher.prototype.destroy = td.func() vi.mock('../../src/sync/fetcher/headerfetcher', () => td.object()) +Peer.prototype.latest = td.func() +vi.mock('../../src/net/peer/peer', () => td.object()) + const { LightSynchronizer } = await import('../../src/sync/lightsync') describe('[LightSynchronizer]', async () => { it('should initialize correctly', async () => { @@ -72,15 +76,24 @@ describe('[LightSynchronizer]', async () => { chain, }) sync.best = td.func() - sync.latest = td.func() - td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } } } as any) - td.when(sync.latest(td.matchers.anything())).thenResolve({ + //sync.latest = td.func() + td.when(sync.best()).thenResolve({ + les: { status: { headNum: BigInt(2) } }, + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) + /*td.when(sync.latest(td.matchers.anything())).thenResolve({ number: BigInt(2), hash: () => new Uint8Array(0), - }) + })*/ + td.when(HeaderFetcher.prototype.fetch(), { delay: 20, times: 2 }).thenResolve(true) ;(sync as any).chain = { headers: { height: BigInt(3) } } - assert.notOk(await sync.sync(), 'local height > remote height') + //assert.notOk(await sync.sync(), 'local height > remote height') ;(sync as any).chain = { headers: { height: BigInt(0) } } setTimeout(() => { config.events.emit(Event.SYNC_SYNCHRONIZED, BigInt(0)) @@ -94,42 +107,51 @@ describe('[LightSynchronizer]', async () => { await sync.stop() await sync.close() vi.unmock('../../src/sync/fetcher/headerfetcher') + vi.unmock('../../src/net/peer/peer') } }) }) describe('sync errors', async () => { - td.reset() - const config = new Config({ accountCache: 10000, storageCache: 1000 }) - const pool = new PeerPool() as any - const chain = await Chain.create({ config }) - const sync = new LightSynchronizer({ - config, - interval: 1, - pool, - chain, - }) - sync.best = td.func() - sync.latest = td.func() - td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } } } as any) - td.when(sync.latest(td.matchers.anything())).thenResolve({ + it('should produce correct errors', async () => { + td.reset() + const config = new Config({ accountCache: 10000, storageCache: 1000 }) + const pool = new PeerPool() as any + const chain = await Chain.create({ config }) + const sync = new LightSynchronizer({ + config, + interval: 1, + pool, + chain, + }) + sync.best = td.func() + //sync.latest = td.func() + td.when(sync.best()).thenResolve({ + les: { status: { headNum: BigInt(2) } }, + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) + /*td.when(sync.latest(td.matchers.anything())).thenResolve({ number: BigInt(2), hash: () => new Uint8Array(0), - }) - td.when(HeaderFetcher.prototype.fetch()).thenResolve(true) - td.when(HeaderFetcher.prototype.fetch()).thenDo(() => - config.events.emit(Event.SYNC_FETCHED_HEADERS, [] as BlockHeader[]) - ) - config.logger.on('data', async (data) => { - if ((data.message as string).includes('No headers fetched are applicable for import')) { - it('should generate correct warning', () => { + })*/ + td.when(HeaderFetcher.prototype.fetch()).thenResolve(true) + td.when(HeaderFetcher.prototype.fetch()).thenDo(() => + config.events.emit(Event.SYNC_FETCHED_HEADERS, [] as BlockHeader[]) + ) + config.logger.on('data', async (data) => { + if ((data.message as string).includes('No headers fetched are applicable for import')) { assert.ok(true, 'generated correct warning message when no headers received') - }) - config.logger.removeAllListeners() - await sync.stop() - await sync.close() - } + config.logger.removeAllListeners() + await sync.stop() + await sync.close() + } + }) + await sync.sync() }) - await sync.sync() }) describe('import headers', () => { it('should import header', async () => { @@ -138,6 +160,10 @@ describe('import headers', () => { HeaderFetcher.prototype.clear = td.func() HeaderFetcher.prototype.destroy = td.func() vi.mock('../../src/sync/fetcher/headerfetcher', () => td.object()) + + Peer.prototype.latest = td.func() + vi.mock('../../src/net/peer/peer', () => td.object()) + const { LightSynchronizer } = await import('../../src/sync/lightsync') const config = new Config({ accountCache: 10000, @@ -153,12 +179,20 @@ describe('import headers', () => { chain, }) sync.best = td.func() - sync.latest = td.func() - td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } } } as any) - td.when(sync.latest(td.matchers.anything())).thenResolve({ + //sync.latest = td.func() + td.when(sync.best()).thenResolve({ + les: { status: { headNum: BigInt(2) } }, + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) + /*td.when(sync.latest(td.matchers.anything())).thenResolve({ number: BigInt(2), hash: () => new Uint8Array(0), - }) + })*/ td.when(HeaderFetcher.prototype.fetch()).thenResolve(true) td.when(HeaderFetcher.prototype.fetch()).thenDo(() => config.events.emit(Event.SYNC_FETCHED_HEADERS, [BlockHeader.fromHeaderData({})]) @@ -169,6 +203,7 @@ describe('import headers', () => { config.logger.removeAllListeners() await sync.stop() await sync.close() + vi.unmock('../../src/net/peer/peer') vi.unmock('../../src/sync/fetcher/headerfetcher') } }) From 4cd01aae4f00ea5b339fe7770d9a642552c858ae Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 10:58:33 +0200 Subject: [PATCH 16/44] Some clean-ups --- packages/client/test/sync/lightsync.spec.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/client/test/sync/lightsync.spec.ts b/packages/client/test/sync/lightsync.spec.ts index 6bd6ecec87..a4798be5af 100644 --- a/packages/client/test/sync/lightsync.spec.ts +++ b/packages/client/test/sync/lightsync.spec.ts @@ -76,7 +76,6 @@ describe('[LightSynchronizer]', async () => { chain, }) sync.best = td.func() - //sync.latest = td.func() td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } }, latest: () => { @@ -86,14 +85,10 @@ describe('[LightSynchronizer]', async () => { } }, } as any) - /*td.when(sync.latest(td.matchers.anything())).thenResolve({ - number: BigInt(2), - hash: () => new Uint8Array(0), - })*/ td.when(HeaderFetcher.prototype.fetch(), { delay: 20, times: 2 }).thenResolve(true) ;(sync as any).chain = { headers: { height: BigInt(3) } } - //assert.notOk(await sync.sync(), 'local height > remote height') + assert.notOk(await sync.sync(), 'local height > remote height') ;(sync as any).chain = { headers: { height: BigInt(0) } } setTimeout(() => { config.events.emit(Event.SYNC_SYNCHRONIZED, BigInt(0)) @@ -124,7 +119,6 @@ describe('sync errors', async () => { chain, }) sync.best = td.func() - //sync.latest = td.func() td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } }, latest: () => { @@ -134,10 +128,6 @@ describe('sync errors', async () => { } }, } as any) - /*td.when(sync.latest(td.matchers.anything())).thenResolve({ - number: BigInt(2), - hash: () => new Uint8Array(0), - })*/ td.when(HeaderFetcher.prototype.fetch()).thenResolve(true) td.when(HeaderFetcher.prototype.fetch()).thenDo(() => config.events.emit(Event.SYNC_FETCHED_HEADERS, [] as BlockHeader[]) @@ -179,7 +169,6 @@ describe('import headers', () => { chain, }) sync.best = td.func() - //sync.latest = td.func() td.when(sync.best()).thenResolve({ les: { status: { headNum: BigInt(2) } }, latest: () => { @@ -189,10 +178,6 @@ describe('import headers', () => { } }, } as any) - /*td.when(sync.latest(td.matchers.anything())).thenResolve({ - number: BigInt(2), - hash: () => new Uint8Array(0), - })*/ td.when(HeaderFetcher.prototype.fetch()).thenResolve(true) td.when(HeaderFetcher.prototype.fetch()).thenDo(() => config.events.emit(Event.SYNC_FETCHED_HEADERS, [BlockHeader.fromHeaderData({})]) From 5777c42ce52acfd058ed42cb7be5855ae5d70bdf Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 11:11:58 +0200 Subject: [PATCH 17/44] Client: Fix beaconsync.spec.ts tests --- packages/client/test/sync/beaconsync.spec.ts | 60 ++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/packages/client/test/sync/beaconsync.spec.ts b/packages/client/test/sync/beaconsync.spec.ts index 5467035ff4..419a923066 100644 --- a/packages/client/test/sync/beaconsync.spec.ts +++ b/packages/client/test/sync/beaconsync.spec.ts @@ -61,10 +61,18 @@ describe('[BeaconSynchronizer]', async () => { const chain = await Chain.create({ config }) const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) const sync = new BeaconSynchronizer({ config, pool, chain, execution, skeleton }) - const peer = { eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash' } } } + const peer = { + eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash' } }, + latest: async () => { + return { + number: BigInt(5), + hash: () => new Uint8Array(0), + } + }, + } const headers = [{ number: BigInt(5) }] td.when(peer.eth.getBlockHeaders({ block: 'hash', max: 1 })).thenResolve([BigInt(1), headers]) - const latest = await sync.latest(peer as any) + const latest = await peer.latest() assert.ok(latest!.number === BigInt(5), 'got height') await sync.stop() await sync.close() @@ -78,8 +86,24 @@ describe('[BeaconSynchronizer]', async () => { const sync = new BeaconSynchronizer({ config, pool, chain, execution, skeleton }) ;(sync as any).running = true const peers = [ - { eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash1' }, inbound: false } }, - { eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash2' }, inbound: false } }, + { + eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash1' }, inbound: false }, + latest: () => { + return { + number: BigInt(5), + hash: () => new Uint8Array(0), + } + }, + }, + { + eth: { getBlockHeaders: td.func(), status: { bestHash: 'hash2' }, inbound: false }, + latest: () => { + return { + number: BigInt(10), + hash: () => new Uint8Array(0), + } + }, + }, ] td.when(peers[0].eth.getBlockHeaders({ block: 'hash1', max: 1 })).thenResolve([ BigInt(1), @@ -111,12 +135,14 @@ describe('[BeaconSynchronizer]', async () => { const sync = new BeaconSynchronizer({ config, pool, chain, execution, skeleton }) sync.best = td.func() - sync.latest = td.func() - td.when(sync.best()).thenResolve('peer') - td.when(sync.latest('peer' as any)).thenResolve({ - number: BigInt(2), - hash: () => new Uint8Array(0), - }) + td.when(sync.best()).thenResolve({ + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) td.when(ReverseBlockFetcher.prototype.fetch(), { delay: 100, times: 3 }).thenResolve(false) ;(skeleton as any).status.progress.subchains = [ { head: BigInt(10), tail: BigInt(6) }, @@ -162,12 +188,14 @@ describe('[BeaconSynchronizer]', async () => { await skeleton.open() const sync = new BeaconSynchronizer({ config, pool, chain, execution, skeleton }) sync.best = td.func() - sync.latest = td.func() - td.when(sync.best()).thenResolve('peer') - td.when(sync.latest('peer' as any)).thenResolve({ - number: BigInt(2), - hash: () => new Uint8Array(0), - }) + td.when(sync.best()).thenResolve({ + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) td.when(ReverseBlockFetcher.prototype.fetch(), { delay: 100, times: 1 }).thenResolve(false) ;(skeleton as any).status.progress.subchains = [{ head: BigInt(10), tail: BigInt(6) }] ;(sync as any).chain = { From 9ea28d8f2f8b65dc28579b312d535179fe4fb056 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 11:19:00 +0200 Subject: [PATCH 18/44] Client: Fix fullsync.spec.tst tests --- packages/client/test/sync/fullsync.spec.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/client/test/sync/fullsync.spec.ts b/packages/client/test/sync/fullsync.spec.ts index 3639b024bf..7214fce000 100644 --- a/packages/client/test/sync/fullsync.spec.ts +++ b/packages/client/test/sync/fullsync.spec.ts @@ -1,4 +1,5 @@ import { Block } from '@ethereumjs/block' +import * as td from 'testdouble' import { assert, describe, it, vi } from 'vitest' import { Chain } from '../../src/blockchain' @@ -74,8 +75,14 @@ describe('[FullSynchronizer]', async () => { }), status: { bestHash: 'hash' }, }, + latest: async () => { + return { + number: BigInt(5), + hash: () => new Uint8Array(0), + } + }, } - const latest = await sync.latest(peer as any) + const latest = await peer.latest() assert.equal(latest!.number, BigInt(5), 'got height') await sync.stop() await sync.close() @@ -128,14 +135,16 @@ describe('[FullSynchronizer]', async () => { txPool, execution, }) - sync.best = vi.fn().mockResolvedValue('peer') - sync.latest = vi.fn((input) => { - if (input === ('peer' as any)) + sync.best = td.func() + td.when(sync.best()).thenResolve({ + les: { status: { headNum: BigInt(2) } }, + latest: () => { return { number: BigInt(2), hash: () => new Uint8Array(0), } - }) as any + }, + } as any) let count = 0 BlockFetcher.prototype.fetch = vi.fn(async () => { if (count < 2) { From 9a6ae272836094474f9ed2a8bcf07293573d47d6 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 11:21:39 +0200 Subject: [PATCH 19/44] Client: Fix snapsync.spec.ts tests --- packages/client/test/sync/snapsync.spec.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/client/test/sync/snapsync.spec.ts b/packages/client/test/sync/snapsync.spec.ts index 2b5077ff1f..62f6f7a422 100644 --- a/packages/client/test/sync/snapsync.spec.ts +++ b/packages/client/test/sync/snapsync.spec.ts @@ -83,11 +83,23 @@ describe('[SnapSynchronizer]', async () => { snap: {}, eth: { status: { bestHash: '0xaa' }, getBlockHeaders: getBlockHeaders1 }, inbound: false, + latest: () => { + return { + number: BigInt(1), + hash: () => new Uint8Array(0), + } + }, }, { snap: {}, eth: { status: { bestHash: '0xbb' }, getBlockHeaders: getBlockHeaders2 }, inbound: false, + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, }, ] ;(sync as any).pool = { peers } From 5c42d73031e802f59cc9cb6163c3187ea5a94cc3 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 11:41:22 +0200 Subject: [PATCH 20/44] Client: Fix fetcher tests --- packages/client/test/sync/fetcher/accountfetcher.spec.ts | 5 +++++ packages/client/test/sync/fetcher/blockfetcher.spec.ts | 2 ++ packages/client/test/sync/fetcher/bytecodefetcher.spec.ts | 1 + packages/client/test/sync/fetcher/fetcher.spec.ts | 2 +- packages/client/test/sync/fetcher/headerfetcher.spec.ts | 1 + .../client/test/sync/fetcher/reverseblockfetcher.spec.ts | 1 + packages/client/test/sync/fetcher/storagefetcher.spec.ts | 4 ++++ packages/client/test/sync/fetcher/trienodefetcher.spec.ts | 2 ++ 8 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/client/test/sync/fetcher/accountfetcher.spec.ts b/packages/client/test/sync/fetcher/accountfetcher.spec.ts index 507094524e..c697ec49c4 100644 --- a/packages/client/test/sync/fetcher/accountfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/accountfetcher.spec.ts @@ -184,6 +184,7 @@ describe('[AccountFetcher]', async () => { snap: { getAccountRange: vi.fn() }, id: 'random', address: 'random', + latest: vi.fn(), } const partialResult: any = [ [ @@ -243,6 +244,7 @@ describe('[AccountFetcher]', async () => { }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) @@ -269,6 +271,7 @@ describe('[AccountFetcher]', async () => { snap: { getAccountRange: mockedGetAccountRange }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } @@ -307,6 +310,7 @@ describe('[AccountFetcher]', async () => { snap: { getAccountRange: mockedGetAccountRange }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } @@ -352,6 +356,7 @@ describe('[AccountFetcher]', async () => { snap: { getAccountRange: mockedGetAccountRange }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } const results = await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/blockfetcher.spec.ts b/packages/client/test/sync/fetcher/blockfetcher.spec.ts index 7bb2bd73c1..27d125ed61 100644 --- a/packages/client/test/sync/fetcher/blockfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/blockfetcher.spec.ts @@ -179,6 +179,7 @@ describe('[BlockFetcher]', async () => { }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) @@ -227,6 +228,7 @@ describe('[BlockFetcher]', async () => { }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } const resp = await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/bytecodefetcher.spec.ts b/packages/client/test/sync/fetcher/bytecodefetcher.spec.ts index 654a776664..d4a6a4dc9f 100644 --- a/packages/client/test/sync/fetcher/bytecodefetcher.spec.ts +++ b/packages/client/test/sync/fetcher/bytecodefetcher.spec.ts @@ -140,6 +140,7 @@ describe('[ByteCodeFetcher]', async () => { snap: { getByteCodes: mockedGetByteCodes }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } const results = await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/fetcher.spec.ts b/packages/client/test/sync/fetcher/fetcher.spec.ts index 4081f46fbf..a0d7b875fb 100644 --- a/packages/client/test/sync/fetcher/fetcher.spec.ts +++ b/packages/client/test/sync/fetcher/fetcher.spec.ts @@ -66,7 +66,7 @@ describe('should handle expiration', async () => { timeout: 5, }) const job = { index: 0 } - const peer = { idle: true } + const peer = { idle: true, latest: vi.fn() } fetcher.peer = vi.fn().mockReturnValue(() => peer) fetcher.request = vi.fn().mockImplementationOnce(async (job, peer) => { await new Promise((resolve) => { diff --git a/packages/client/test/sync/fetcher/headerfetcher.spec.ts b/packages/client/test/sync/fetcher/headerfetcher.spec.ts index 9e08f4782d..9afb5d7825 100644 --- a/packages/client/test/sync/fetcher/headerfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/headerfetcher.spec.ts @@ -84,6 +84,7 @@ describe('[HeaderFetcher]', async () => { les: { getBlockHeaders: vi.fn() }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/reverseblockfetcher.spec.ts b/packages/client/test/sync/fetcher/reverseblockfetcher.spec.ts index e85f6fb73c..a911342f1c 100644 --- a/packages/client/test/sync/fetcher/reverseblockfetcher.spec.ts +++ b/packages/client/test/sync/fetcher/reverseblockfetcher.spec.ts @@ -169,6 +169,7 @@ describe('[ReverseBlockFetcher]', async () => { }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/storagefetcher.spec.ts b/packages/client/test/sync/fetcher/storagefetcher.spec.ts index 30d368e3d4..64eb1ad79f 100644 --- a/packages/client/test/sync/fetcher/storagefetcher.spec.ts +++ b/packages/client/test/sync/fetcher/storagefetcher.spec.ts @@ -299,6 +299,7 @@ describe('[StorageFetcher]', async () => { snap: { getStorageRanges: mockedGetStorageRanges }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) @@ -347,6 +348,7 @@ describe('[StorageFetcher]', async () => { snap: { getStorageRanges: mockedGetStorageRanges }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } @@ -390,6 +392,7 @@ describe('[StorageFetcher]', async () => { snap: { getStorageRanges: mockedGetStorageRanges }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, task } @@ -446,6 +449,7 @@ describe('[StorageFetcher]', async () => { snap: { getStorageRanges: mockedGetStorageRanges }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } let results = await fetcher.request(job as any) diff --git a/packages/client/test/sync/fetcher/trienodefetcher.spec.ts b/packages/client/test/sync/fetcher/trienodefetcher.spec.ts index d4b1202801..1551716746 100644 --- a/packages/client/test/sync/fetcher/trienodefetcher.spec.ts +++ b/packages/client/test/sync/fetcher/trienodefetcher.spec.ts @@ -109,6 +109,7 @@ describe('[TrieNodeFetcher]', async () => { }, id: 'random', address: 'random', + latest: vi.fn(), } const job = { peer, partialResult, task } await fetcher.request(job as any) @@ -148,6 +149,7 @@ describe('[TrieNodeFetcher]', async () => { snap: { getTrieNodes: mockedGetTrieNodes }, id: 'random', address: 'random', + latest: vi.fn(), } const task = { pathStrings: [''], From 9e60a092146b2be05faf4e5533c9a45f8d6fe214 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 11:49:48 +0200 Subject: [PATCH 21/44] Client: Fix eth syncing.spec.ts tests --- packages/client/test/rpc/eth/syncing.spec.ts | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/client/test/rpc/eth/syncing.spec.ts b/packages/client/test/rpc/eth/syncing.spec.ts index c773b06094..de0d06c0fc 100644 --- a/packages/client/test/rpc/eth/syncing.spec.ts +++ b/packages/client/test/rpc/eth/syncing.spec.ts @@ -42,9 +42,13 @@ describe(method, () => { const manager = createManager(client) const rpcServer = startRPC(manager.getMethods()) const rpc = getRpcClient(rpcServer) - const synchronizer = client.services[0].synchronizer! - synchronizer.best = td.func() - td.when(synchronizer.best()).thenResolve('peer') + const sync = client.services[0].synchronizer! + sync.best = td.func() + td.when(sync.best()).thenResolve({ + latest: () => { + return + }, + } as any) client.config.synchronized = false assert.equal(client.config.synchronized, false, 'not synchronized yet') @@ -60,11 +64,16 @@ describe(method, () => { const manager = createManager(client) const rpcServer = startRPC(manager.getMethods()) const rpc = getRpcClient(rpcServer) - const synchronizer = client.services[0].synchronizer as FullSynchronizer - synchronizer.best = td.func() - synchronizer.latest = td.func() - td.when(synchronizer.best()).thenResolve('peer') - td.when(synchronizer.latest('peer' as any)).thenResolve({ number: BigInt(2) }) + const sync = client.services[0].synchronizer as FullSynchronizer + sync.best = td.func() + td.when(sync.best()).thenResolve({ + latest: () => { + return { + number: BigInt(2), + hash: () => new Uint8Array(0), + } + }, + } as any) client.config.synchronized = false assert.equal(client.config.synchronized, false, 'not synchronized yet') From d6c2a6c62ea0b2966dfd3593d38a5bd393a55194 Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 13:20:38 +0200 Subject: [PATCH 22/44] Client: Fix integration tests --- packages/client/src/net/peer/peer.ts | 7 ++++--- packages/client/test/integration/mocks/mockpeer.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/client/src/net/peer/peer.ts b/packages/client/src/net/peer/peer.ts index 947b998944..a814ed9af3 100644 --- a/packages/client/src/net/peer/peer.ts +++ b/packages/client/src/net/peer/peer.ts @@ -117,9 +117,10 @@ export abstract class Peer extends EventEmitter { if (latest !== undefined) { const height = latest.number if ( - this.config.syncTargetHeight === undefined || - this.config.syncTargetHeight === BIGINT_0 || - this.config.syncTargetHeight < latest.number + height > BIGINT_0 && + (this.config.syncTargetHeight === undefined || + this.config.syncTargetHeight === BIGINT_0 || + this.config.syncTargetHeight < latest.number) ) { this.config.syncTargetHeight = height this.config.logger.info(`New sync target height=${height} hash=${short(latest.hash())}`) diff --git a/packages/client/test/integration/mocks/mockpeer.ts b/packages/client/test/integration/mocks/mockpeer.ts index 182fb47dd1..cf935afde4 100644 --- a/packages/client/test/integration/mocks/mockpeer.ts +++ b/packages/client/test/integration/mocks/mockpeer.ts @@ -11,6 +11,7 @@ import { createStream } from './network' import type { PeerOptions } from '../../../src/net/peer' import type { MockServer } from './mockserver' import type { RemoteStream } from './network' +import type { BlockHeader } from '@ethereumjs/block' // TypeScript doesn't have support yet for ReturnType // with generic types, so this wrapper is used as a helper. @@ -40,6 +41,13 @@ export class MockPeer extends Peer { this.config.events.emit(Event.PEER_CONNECTED, this) } + async latest(): Promise { + if (this.eth !== undefined) { + this.eth.updatedBestHeader = undefined + } + return super.latest() + } + async accept(server: MockServer) { if (this.connected) { return From 58cb2b4eaff75d8c2c6b6e5c934bc5c6456b75ea Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 13:39:53 +0200 Subject: [PATCH 23/44] Client: Backup lightsync integration tests (lightsync not supported anymore) --- .../integration/{lightsync.spec.ts => lightsync.spec.backup.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/client/test/integration/{lightsync.spec.ts => lightsync.spec.backup.ts} (100%) diff --git a/packages/client/test/integration/lightsync.spec.ts b/packages/client/test/integration/lightsync.spec.backup.ts similarity index 100% rename from packages/client/test/integration/lightsync.spec.ts rename to packages/client/test/integration/lightsync.spec.backup.ts From c53fa4ba605ac152188a9ad1a731db0af9bfda3e Mon Sep 17 00:00:00 2001 From: Holger Drewes Date: Tue, 16 Apr 2024 13:41:18 +0200 Subject: [PATCH 24/44] Lightsync tests deprecation note --- .../client/test/integration/lightsync.spec.ts | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 packages/client/test/integration/lightsync.spec.ts diff --git a/packages/client/test/integration/lightsync.spec.ts b/packages/client/test/integration/lightsync.spec.ts new file mode 100644 index 0000000000..2630630f93 --- /dev/null +++ b/packages/client/test/integration/lightsync.spec.ts @@ -0,0 +1,100 @@ +import { assert, describe, it } from 'vitest' + +import { SyncMode } from '../../src/config' +import { Event } from '../../src/types' + +import { destroy, setup, wait } from './util' + +/** + * Lightsync is not actively supported anymore. + * + * Tests are kept for now until a more finalized decision is taken + * on the topic. // 2024-04-16 + */ +describe( + 'should sync headers', + async () => { + const [remoteServer, remoteService] = await setup({ + location: '127.0.0.2', + height: 20, + syncmode: SyncMode.Full, + }) + const [localServer, localService] = await setup({ + location: '127.0.0.1', + height: 0, + syncmode: SyncMode.Light, + }) + await localService.synchronizer!.stop() + await localServer.discover('remotePeer1', '127.0.0.2') + localService.config.events.on(Event.SYNC_SYNCHRONIZED, async () => { + it('should sync', () => { + assert.equal(localService.chain.headers.height, BigInt(20), 'synced') + }) + await destroy(localServer, localService) + await destroy(remoteServer, remoteService) + }) + await localService.synchronizer!.start() + }, + { timeout: 30000 } +) + +describe( + 'should not sync with stale peers', + async () => { + const [remoteServer, remoteService] = await setup({ + location: '127.0.0.2', + height: 9, + syncmode: SyncMode.Full, + }) + const [localServer, localService] = await setup({ + location: '127.0.0.1', + height: 10, + syncmode: SyncMode.Light, + }) + localService.config.events.on(Event.SYNC_SYNCHRONIZED, async () => { + throw new Error('synced with a stale peer') + }) + await localServer.discover('remotePeer', '127.0.0.2') + await wait(100) + await destroy(localServer, localService) + await destroy(remoteServer, remoteService) + it('should not sync', async () => { + assert.ok('did not sync') + }) + }, + { timeout: 30000 } +) + +describe( + 'should sync with best peer', + async () => { + const [remoteServer1, remoteService1] = await setup({ + location: '127.0.0.2', + height: 9, + syncmode: SyncMode.Full, + }) + const [remoteServer2, remoteService2] = await setup({ + location: '127.0.0.3', + height: 10, + syncmode: SyncMode.Full, + }) + const [localServer, localService] = await setup({ + location: '127.0.0.1', + height: 0, + syncmode: SyncMode.Light, + }) + await localService.synchronizer!.stop() + await localServer.discover('remotePeer1', '127.0.0.2') + await localServer.discover('remotePeer2', '127.0.0.3') + localService.config.events.on(Event.SYNC_SYNCHRONIZED, async () => { + it('should sync with best peer', async () => { + assert.equal(localService.chain.headers.height, BigInt(10), 'synced with best peer') + }) + await destroy(localServer, localService) + await destroy(remoteServer1, remoteService1) + await destroy(remoteServer2, remoteService2) + }) + await localService.synchronizer!.start() + }, + { timeout: 30000 } +) From fe9a4ecd324e3945b8c68320a9dbf6b9d44d9b74 Mon Sep 17 00:00:00 2001 From: acolytec3 <17355484+acolytec3@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:28:05 -0400 Subject: [PATCH 25/44] Make lightsync tests run (but fail) --- .../client/test/integration/lightsync.spec.ts | 26 ++++++------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/packages/client/test/integration/lightsync.spec.ts b/packages/client/test/integration/lightsync.spec.ts index 2630630f93..6508181544 100644 --- a/packages/client/test/integration/lightsync.spec.ts +++ b/packages/client/test/integration/lightsync.spec.ts @@ -11,9 +11,8 @@ import { destroy, setup, wait } from './util' * Tests are kept for now until a more finalized decision is taken * on the topic. // 2024-04-16 */ -describe( - 'should sync headers', - async () => { +describe('light sync integration tests', () => { + it('should sync headers', async () => { const [remoteServer, remoteService] = await setup({ location: '127.0.0.2', height: 20, @@ -34,13 +33,9 @@ describe( await destroy(remoteServer, remoteService) }) await localService.synchronizer!.start() - }, - { timeout: 30000 } -) + }, 30000) -describe( - 'should not sync with stale peers', - async () => { + it('should not sync with stale peers', async () => { const [remoteServer, remoteService] = await setup({ location: '127.0.0.2', height: 9, @@ -61,13 +56,9 @@ describe( it('should not sync', async () => { assert.ok('did not sync') }) - }, - { timeout: 30000 } -) + }, 30000) -describe( - 'should sync with best peer', - async () => { + it('should sync with best peer', async () => { const [remoteServer1, remoteService1] = await setup({ location: '127.0.0.2', height: 9, @@ -95,6 +86,5 @@ describe( await destroy(remoteServer2, remoteService2) }) await localService.synchronizer!.start() - }, - { timeout: 30000 } -) + }, 30000) +}) From d8028eb0fe65a28b27f6e6f28058b782834cf920 Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Wed, 17 Apr 2024 18:24:01 -0700 Subject: [PATCH 26/44] Receive peer updates only in request function of fetchers --- .../client/src/service/fullethereumservice.ts | 7 +---- packages/client/src/sync/snapsync.ts | 30 ++++++++----------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index eeeea114aa..c1521bdc73 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -243,12 +243,7 @@ export class FullEthereumService extends Service { * vm execution */ async buildHeadState(): Promise { - // If during building FCU identifies a new root/height to sync to, evalueate new height with sync - // strategy and initiate state healing and fetcher root updates for snap sync - if (this.building) { - const peer = await this.snapsync?.best() - void this.snapsync?.syncWithPeer(peer) - } + if (this.building) return this.building = true try { diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index b93a9a4395..bd3657eade 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -206,7 +206,10 @@ export class SnapSynchronizer extends Synchronizer { this.config.logger.info(`New sync target height=${height} hash=${bytesToHex(latest.hash())}`) } - if (this.config.syncTargetHeight <= latest.number + this.config.snapAvailabilityDepth) { + if ( + (this.fetcher === null || this.fetcher.syncErrored !== undefined) && + this.config.syncTargetHeight <= latest.number + this.config.snapAvailabilityDepth + ) { if ((this.fetcherDoneFlags.snapTargetHeight ?? BIGINT_0) < latest.number) { this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot @@ -222,22 +225,15 @@ export class SnapSynchronizer extends Synchronizer { : 'previous fetcher errored=' + this.fetcher.syncErrored?.message }` ) - - if (this.fetcher === null || this.fetcher.syncErrored !== undefined) { - this.fetcher = new AccountFetcher({ - config: this.config, - pool: this.pool, - stateManager: this.execution.vm.stateManager as DefaultStateManager, - root: stateRoot, - // This needs to be determined from the current state of the MPT dump - first: BigInt(0), - fetcherDoneFlags: this.fetcherDoneFlags, - }) - } else { - // update root - this.config.logger.info(`UPDATING ROOTS`) - this.fetcher?.updateStateRoot(stateRoot) - } + this.fetcher = new AccountFetcher({ + config: this.config, + pool: this.pool, + stateManager: this.execution.vm.stateManager as DefaultStateManager, + root: stateRoot, + // This needs to be determined from the current state of the MPT dump + first: BigInt(0), + fetcherDoneFlags: this.fetcherDoneFlags, + }) } else { return false } From 52d1f6e21188c44b2770baee0c870cbfe9b129bc Mon Sep 17 00:00:00 2001 From: Indigo Alpha Date: Wed, 17 Apr 2024 21:14:41 -0700 Subject: [PATCH 27/44] Update fetcher roots on new latestHeader --- .../client/src/sync/fetcher/accountfetcher.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 614115b13f..da724fa2fe 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -1,3 +1,4 @@ +import type { BlockHeader } from '@ethereumjs/block' import { DefaultStateManager } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' import { @@ -100,6 +101,7 @@ export class AccountFetcher extends Fetcher this.accountTrie = this.stateManager['_getAccountTrie']() this.debug = createDebugLogger('client:AccountFetcher') + this.debug('dbg101') this.storageFetcher = new StorageFetcher({ config: this.config, @@ -376,7 +378,22 @@ export class AccountFetcher extends Fetcher // TODOs: // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool - await peer?.latest() + this.debug('dbg100') + const latestHeader = await peer?.latest() + if (latestHeader !== undefined) { + const latestStateRoot = (latestHeader as BlockHeader).stateRoot + this.debug(this.root) + this.debug(bytesToHex(latestStateRoot)) + if (!equalsBytes(this.root, latestStateRoot)) { + this.debug( + `updating fetcher roots from ${bytesToHex(this.root)} over to ${bytesToHex( + latestStateRoot + )}` + ) + this.updateStateRoot(latestStateRoot) + } + } + const origin = this.getOrigin(job) const limit = this.getLimit(job) From d469538460928d70e1d43b77eee974044e93e51c Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 9 Jul 2024 15:07:31 -0700 Subject: [PATCH 28/44] Latest changes for integrating updated roots from latest function --- .../client/src/service/fullethereumservice.ts | 32 ++++++++++++++++++- .../client/src/sync/fetcher/accountfetcher.ts | 2 ++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index 94268bd9d4..a78bc19c51 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -11,6 +11,7 @@ import { LesProtocol } from '../net/protocol/lesprotocol.js' import { SnapProtocol } from '../net/protocol/snapprotocol.js' import { BeaconSynchronizer, FullSynchronizer, SnapSynchronizer } from '../sync/index.js' import { Event } from '../types.js' +import { wait } from '../util/wait.js' import { Service, type ServiceOptions } from './service.js' import { Skeleton } from './skeleton.js' @@ -243,7 +244,36 @@ export class FullEthereumService extends Service { * vm execution */ async buildHeadState(): Promise { - if (this.building) return + if (this.building) { + if (this.snapsync !== undefined) { + // discover best peer + let peer = await this.snapsync.best() + let numAttempts = 1 + while (!peer && this.snapsync.opened) { + this.snapsync.config.logger.debug(`Waiting for best peer (attempt #${numAttempts})`) + await wait(5000) + peer = await this.snapsync.best() + numAttempts += 1 + } + + if (peer === undefined) throw new Error('Unable to find a peer to sync with') + const latest = await peer.latest() + + if (latest === undefined || latest.stateRoot === undefined) { + // TODO if latest is undefined, latest known root is assumed to not yet have been updated? + return + } + + // TODO only update root if it is a new one compared to what's already being used + const fetcher = this.snapsync.fetcher + if (fetcher === null) { + return + } + fetcher.updateStateRoot(latest!.stateRoot as Uint8Array) + } + + return + } this.building = true try { diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 8fe597189a..400ffbfdfa 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -278,6 +278,8 @@ export class AccountFetcher extends Fetcher } break case TrieNodeFetcher: + // TODO update to check if heal phase completed successfully and then continue with next + // healing phase fetcherDoneFlags.trieNodeFetcher.done = true break } From 87ee2aa237815a1b4a8d9db14aca4606bb5df7f4 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 9 Jul 2024 16:54:22 -0700 Subject: [PATCH 29/44] fix linting issue --- packages/client/src/sync/fetcher/accountfetcher.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 400ffbfdfa..9f58163570 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -1,4 +1,3 @@ -import type { BlockHeader } from '@ethereumjs/block' import { DefaultStateManager } from '@ethereumjs/statemanager' import { Trie } from '@ethereumjs/trie' import { From 50fb30b09aae11a181449d30af68a42372155eec Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 26 Jul 2024 10:34:47 -0700 Subject: [PATCH 30/44] Move over to getting latest in request function of relevant fetchers --- .../client/src/service/fullethereumservice.ts | 32 +-------- .../client/src/sync/fetcher/accountfetcher.ts | 72 ++++++++++++------- .../client/src/sync/fetcher/storagefetcher.ts | 34 ++++++--- .../src/sync/fetcher/trienodefetcher.ts | 27 ++++--- packages/client/src/sync/snapsync.ts | 1 + 5 files changed, 91 insertions(+), 75 deletions(-) diff --git a/packages/client/src/service/fullethereumservice.ts b/packages/client/src/service/fullethereumservice.ts index a78bc19c51..94268bd9d4 100644 --- a/packages/client/src/service/fullethereumservice.ts +++ b/packages/client/src/service/fullethereumservice.ts @@ -11,7 +11,6 @@ import { LesProtocol } from '../net/protocol/lesprotocol.js' import { SnapProtocol } from '../net/protocol/snapprotocol.js' import { BeaconSynchronizer, FullSynchronizer, SnapSynchronizer } from '../sync/index.js' import { Event } from '../types.js' -import { wait } from '../util/wait.js' import { Service, type ServiceOptions } from './service.js' import { Skeleton } from './skeleton.js' @@ -244,36 +243,7 @@ export class FullEthereumService extends Service { * vm execution */ async buildHeadState(): Promise { - if (this.building) { - if (this.snapsync !== undefined) { - // discover best peer - let peer = await this.snapsync.best() - let numAttempts = 1 - while (!peer && this.snapsync.opened) { - this.snapsync.config.logger.debug(`Waiting for best peer (attempt #${numAttempts})`) - await wait(5000) - peer = await this.snapsync.best() - numAttempts += 1 - } - - if (peer === undefined) throw new Error('Unable to find a peer to sync with') - const latest = await peer.latest() - - if (latest === undefined || latest.stateRoot === undefined) { - // TODO if latest is undefined, latest known root is assumed to not yet have been updated? - return - } - - // TODO only update root if it is a new one compared to what's already being used - const fetcher = this.snapsync.fetcher - if (fetcher === null) { - return - } - fetcher.updateStateRoot(latest!.stateRoot as Uint8Array) - } - - return - } + if (this.building) return this.building = true try { diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 9f58163570..519ab58c85 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -46,6 +46,8 @@ export interface AccountFetcherOptions extends FetcherOptions { /** Root hash of the account trie to serve */ root: Uint8Array + height: bigint + /** The origin to start account fetcher from (including), by default starts from 0 (0x0000...) */ first: bigint /** The range to eventually, by default should be set at BIGINT_2 ** BigInt(256) + BIGINT_1 - first */ @@ -73,6 +75,7 @@ export class AccountFetcher extends Fetcher accountTrie: Trie root: Uint8Array + height: bigint highestKnownHash: Uint8Array | undefined /** The origin to start account fetcher from (including), by default starts from 0 (0x0000...) */ @@ -93,6 +96,7 @@ export class AccountFetcher extends Fetcher this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFecherDoneFlags() this.root = options.root + this.height = options.height this.first = options.first this.count = options.count ?? BIGINT_2EXP256 - this.first @@ -100,12 +104,12 @@ export class AccountFetcher extends Fetcher this.accountTrie = this.stateManager['_getAccountTrie']() this.debug = createDebugLogger('client:AccountFetcher') - this.debug('dbg101') this.storageFetcher = new StorageFetcher({ config: this.config, pool: this.pool, - root: this.root, + root: this.fetcherDoneFlags.snapTargetRoot!, + height: this.height, storageRequests: [], first: BIGINT_1, destroyWhenDone: false, @@ -123,7 +127,8 @@ export class AccountFetcher extends Fetcher this.trieNodeFetcher = new TrieNodeFetcher({ config: this.config, pool: this.pool, - root: this.root, + root: this.fetcherDoneFlags.snapTargetRoot!, + height: this.height, stateManager: this.stateManager, destroyWhenDone: false, fetcherDoneFlags: this.fetcherDoneFlags, @@ -138,9 +143,9 @@ export class AccountFetcher extends Fetcher const limit = this.getLimit(syncRange) this.debug( - `Account fetcher instantiated root=${short(this.root)} origin=${short(origin)} limit=${short( - limit - )} destroyWhenDone=${this.destroyWhenDone}` + `Account fetcher instantiated root=${short( + this.fetcherDoneFlags.snapTargetRoot! + )} origin=${short(origin)} limit=${short(limit)} destroyWhenDone=${this.destroyWhenDone}` ) } @@ -288,15 +293,21 @@ export class AccountFetcher extends Fetcher accountFetcher.done && storageFetcher.done && byteCodeFetcher.done && trieNodeFetcher.done this.config.superMsg( - `snapFetchersCompletion root=${short(this.root)} accountsRoot=${short( - fetcherDoneFlags.stateRoot ?? 'na' - )} done=${this.fetcherDoneFlags.done} accountsDone=${accountFetcher.done} storageDone=${ - storageFetcher.done - } byteCodesDone=${byteCodeFetcher.done} trieNodesDone=${trieNodeFetcher.done}` + `snapFetchersCompletion root=${short( + this.fetcherDoneFlags.snapTargetRoot! + )} accountsRoot=${short(fetcherDoneFlags.stateRoot ?? 'na')} done=${ + this.fetcherDoneFlags.done + } accountsDone=${accountFetcher.done} storageDone=${storageFetcher.done} byteCodesDone=${ + byteCodeFetcher.done + } trieNodesDone=${trieNodeFetcher.done}` ) if (this.fetcherDoneFlags.done) { - this.config.events.emit(Event.SYNC_SNAPSYNC_COMPLETE, this.root, this.stateManager) + this.config.events.emit( + Event.SYNC_SNAPSYNC_COMPLETE, + this.fetcherDoneFlags.snapTargetRoot!, + this.stateManager + ) } } @@ -379,10 +390,24 @@ export class AccountFetcher extends Fetcher // TODOs: // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool - await peer?.latest() + const latest = await peer?.latest() + if (latest !== undefined && latest.stateRoot !== undefined) { + // TODO currently doing this check and update both in account and trienode fetchers since they + // could be running independently of eachother in some cases + const currentHeight = this.height + const newestHeight = latest.number + if (newestHeight - currentHeight >= 1) { + this.fetcherDoneFlags.snapTargetHeight = latest.number + this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot + this.fetcherDoneFlags.snapTargetHash = latest.hash() + + // TODO do we need to set this one since it's a duplicate of snapTargetRoot? + this.fetcherDoneFlags.stateRoot = latest.stateRoot + } + } + const origin = this.getOrigin(job) const limit = this.getLimit(job) - if (this.highestKnownHash && compareBytes(limit, this.highestKnownHash) < 0) { // skip this job and don't rerequest it if it's limit is lower than the highest known key hash this.debug(`skipping request with limit lower than highest known hash`) @@ -390,7 +415,7 @@ export class AccountFetcher extends Fetcher } const rangeResult = await peer!.snap!.getAccountRange({ - root: this.root, + root: this.fetcherDoneFlags.snapTargetRoot!, origin, limit, bytes: BigInt(this.config.maxRangeBytes), @@ -407,7 +432,7 @@ export class AccountFetcher extends Fetcher if (rangeResult.proof.length > 0) { try { const isMissingRightRange = await Trie.verifyRangeProof( - this.root, + this.fetcherDoneFlags.snapTargetRoot!, origin, null, [], @@ -434,7 +459,11 @@ export class AccountFetcher extends Fetcher try { // verifyRangeProof will also verify validate there are no missed states between origin and // response data - const isMissingRightRange = await this.verifyRangeProof(this.root, origin, rangeResult) + const isMissingRightRange = await this.verifyRangeProof( + this.fetcherDoneFlags.snapTargetRoot!, + origin, + rangeResult + ) // Check if there is any pending data to be synced to the right let completed: boolean @@ -586,15 +615,6 @@ export class AccountFetcher extends Fetcher return tasks } - updateStateRoot(stateRoot: Uint8Array) { - this.root = stateRoot - - this.storageFetcher.updateStateRoot(stateRoot) - - // TODO trieNodeFetcher needs to be constantly healing while other fetchers work - this.trieNodeFetcher.updateStateRoot(stateRoot) - } - nextTasks(): void { if ( this.in.length === 0 && diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index d59c848fdb..bc3be98f48 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -45,6 +45,8 @@ export interface StorageFetcherOptions extends FetcherOptions { /** Root hash of the account trie to serve */ root: Uint8Array + height: bigint + /** Storage requests to fetch */ storageRequests?: StorageRequest[] @@ -70,6 +72,7 @@ export type JobTask = { export class StorageFetcher extends Fetcher { protected debug: Debugger root: Uint8Array + height: bigint stateManager: DefaultStateManager fetcherDoneFlags: SnapFetcherDoneFlags @@ -89,6 +92,7 @@ export class StorageFetcher extends Fetcher allow to at least place in Fetcher.next() // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool - await peer?.latest() + const latest = await peer?.latest() + if (latest !== undefined && latest.stateRoot !== undefined) { + // TODO currently doing this check and update both in account and trienode fetchers since they + // could be running independently of eachother in some cases + const currentHeight = this.height + const newestHeight = latest.number + if (newestHeight - currentHeight >= 1) { + this.fetcherDoneFlags.snapTargetHeight = latest.number + this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot + this.fetcherDoneFlags.snapTargetHash = latest.hash() + + // TODO do we need to set this one since it's a duplicate of snapTargetRoot? + this.fetcherDoneFlags.stateRoot = latest.stateRoot + } + } const origin = this.getOrigin(job) const limit = this.getLimit(job) - this.debug(`requested root: ${bytesToHex(this.root)}`) + this.debug(`requested root: ${bytesToHex(this.fetcherDoneFlags.snapTargetRoot!)}`) this.debug(`requested origin: ${bytesToHex(origin)}`) this.debug(`requested limit: ${bytesToHex(limit)}`) this.debug( @@ -250,7 +268,7 @@ export class StorageFetcher extends Fetcher req.accountHash), origin, limit, @@ -617,10 +635,6 @@ export class StorageFetcher extends Fetcher stateManager?: DefaultStateManager @@ -70,6 +71,7 @@ type NodeRequestData = { export class TrieNodeFetcher extends Fetcher { protected debug: Debugger root: Uint8Array + height: bigint stateManager: DefaultStateManager fetcherDoneFlags: SnapFetcherDoneFlags @@ -102,6 +104,7 @@ export class TrieNodeFetcher extends Fetcher constructor(options: TrieNodeFetcherOptions) { super(options) this.root = options.root + this.height = options.height this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFecherDoneFlags() this.pathToNodeRequestData = new OrderedMap() this.requestedNodeToPath = new Map() @@ -119,7 +122,7 @@ export class TrieNodeFetcher extends Fetcher // will always start with root node as first set of node requests this.pathToNodeRequestData.setElement('', { requested: false, - nodeHash: bytesToHex(this.root), + nodeHash: bytesToHex(this.fetcherDoneFlags.snapTargetRoot!), nodeParentHash: '', // root node does not have a parent } as NodeRequestData) @@ -149,12 +152,24 @@ export class TrieNodeFetcher extends Fetcher // TODOs: // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool - await peer?.latest() + const latest = await peer?.latest() + if (latest !== undefined && latest.stateRoot !== undefined) { + const currentHeight = this.height + const newestHeight = latest.number + if (newestHeight - currentHeight >= 1) { + this.fetcherDoneFlags.snapTargetHeight = latest.number + this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot + this.fetcherDoneFlags.snapTargetHash = latest.hash() + + // TODO do we need to set this one since it's a duplicate of snapTargetRoot? + this.fetcherDoneFlags.stateRoot = latest.stateRoot + } + } const { paths, pathStrings } = task const rangeResult = await peer!.snap!.getTrieNodes({ - root: this.root, + root: this.fetcherDoneFlags.snapTargetRoot!, paths, bytes: BigInt(this.config.maxRangeBytes), }) @@ -385,7 +400,7 @@ export class TrieNodeFetcher extends Fetcher this.debug( `Stored accountTrie with root actual=${bytesToHex( this.accountTrie.root() - )} expected=${bytesToHex(this.root)}` + )} expected=${bytesToHex(this.fetcherDoneFlags.snapTargetRoot!)}` ) } } catch (e) { @@ -447,10 +462,6 @@ export class TrieNodeFetcher extends Fetcher return tasks } - updateStateRoot(stateRoot: Uint8Array) { - this.root = stateRoot - } - nextTasks(): void { try { if (this.in.length === 0) { diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 69162ed194..00790ea9b7 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -230,6 +230,7 @@ export class SnapSynchronizer extends Synchronizer { pool: this.pool, stateManager: this.execution.vm.stateManager as DefaultStateManager, root: stateRoot, + height: height, // This needs to be determined from the current state of the MPT dump first: BigInt(0), fetcherDoneFlags: this.fetcherDoneFlags, From b6c02c41b2e6f1f21477894c28efeb65dc59b483 Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 26 Jul 2024 10:41:15 -0700 Subject: [PATCH 31/44] Remove root field from fetcher classes that use the root in the flags object --- packages/client/src/sync/fetcher/accountfetcher.ts | 6 +----- packages/client/src/sync/fetcher/storagefetcher.ts | 6 +----- packages/client/src/sync/fetcher/trienodefetcher.ts | 5 ++--- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 519ab58c85..4e07fcfc27 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -43,9 +43,7 @@ type AccountDataResponse = AccountData[] & { completed?: boolean } * @memberof module:sync/fetcher */ export interface AccountFetcherOptions extends FetcherOptions { - /** Root hash of the account trie to serve */ - root: Uint8Array - + // height of block being targeted for snap sync height: bigint /** The origin to start account fetcher from (including), by default starts from 0 (0x0000...) */ @@ -74,7 +72,6 @@ export class AccountFetcher extends Fetcher stateManager: DefaultStateManager accountTrie: Trie - root: Uint8Array height: bigint highestKnownHash: Uint8Array | undefined @@ -95,7 +92,6 @@ export class AccountFetcher extends Fetcher super(options) this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFecherDoneFlags() - this.root = options.root this.height = options.height this.first = options.first this.count = options.count ?? BIGINT_2EXP256 - this.first diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index bc3be98f48..4caba9b3d9 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -42,9 +42,7 @@ export type StorageRequest = { * @memberof module:sync/fetcher */ export interface StorageFetcherOptions extends FetcherOptions { - /** Root hash of the account trie to serve */ - root: Uint8Array - + // height of block being targeted for snap sync height: bigint /** Storage requests to fetch */ @@ -71,7 +69,6 @@ export type JobTask = { export class StorageFetcher extends Fetcher { protected debug: Debugger - root: Uint8Array height: bigint stateManager: DefaultStateManager fetcherDoneFlags: SnapFetcherDoneFlags @@ -91,7 +88,6 @@ export class StorageFetcher extends Fetcher stateManager?: DefaultStateManager @@ -70,7 +69,8 @@ type NodeRequestData = { export class TrieNodeFetcher extends Fetcher { protected debug: Debugger - root: Uint8Array + + // height of block being targeted for snap sync height: bigint stateManager: DefaultStateManager @@ -103,7 +103,6 @@ export class TrieNodeFetcher extends Fetcher */ constructor(options: TrieNodeFetcherOptions) { super(options) - this.root = options.root this.height = options.height this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFecherDoneFlags() this.pathToNodeRequestData = new OrderedMap() From 1cd93152b95a62e3b47358f8df2d115b796fb9e7 Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 26 Jul 2024 11:12:25 -0700 Subject: [PATCH 32/44] Remove unused parameter inputs --- packages/client/src/sync/fetcher/accountfetcher.ts | 2 -- packages/client/src/sync/snapsync.ts | 1 - 2 files changed, 3 deletions(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 4e07fcfc27..ab4252fa35 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -104,7 +104,6 @@ export class AccountFetcher extends Fetcher this.storageFetcher = new StorageFetcher({ config: this.config, pool: this.pool, - root: this.fetcherDoneFlags.snapTargetRoot!, height: this.height, storageRequests: [], first: BIGINT_1, @@ -123,7 +122,6 @@ export class AccountFetcher extends Fetcher this.trieNodeFetcher = new TrieNodeFetcher({ config: this.config, pool: this.pool, - root: this.fetcherDoneFlags.snapTargetRoot!, height: this.height, stateManager: this.stateManager, destroyWhenDone: false, diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 00790ea9b7..291b3a8265 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -229,7 +229,6 @@ export class SnapSynchronizer extends Synchronizer { config: this.config, pool: this.pool, stateManager: this.execution.vm.stateManager as DefaultStateManager, - root: stateRoot, height: height, // This needs to be determined from the current state of the MPT dump first: BigInt(0), From 9c8495f2bf68e4af74ab05dc66bbcf3dcb245908 Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 26 Jul 2024 11:58:05 -0700 Subject: [PATCH 33/44] Fix import and usage of proof function --- packages/client/src/sync/fetcher/accountfetcher.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 5917f8fe7d..71c7e29984 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -33,7 +33,7 @@ import type { AccountData } from '../../net/protocol/snapprotocol.js' import type { FetcherOptions } from './fetcher.js' import type { StorageRequest } from './storagefetcher.js' import type { Job, SnapFetcherDoneFlags } from './types.js' -import { Trie } from '@ethereumjs/trie' +import type { Trie } from '@ethereumjs/trie' import type { Debugger } from 'debug' type AccountDataResponse = AccountData[] & { completed?: boolean } @@ -425,7 +425,7 @@ export class AccountFetcher extends Fetcher // check zero-element proof if (rangeResult.proof.length > 0) { try { - const isMissingRightRange = await Trie.verifyRangeProof( + const isMissingRightRange = await verifyTrieRangeProof( this.fetcherDoneFlags.snapTargetRoot!, origin, null, From 20db364c4938b25d354c22f444969447b48731b7 Mon Sep 17 00:00:00 2001 From: Amir Date: Mon, 29 Jul 2024 10:52:36 -0700 Subject: [PATCH 34/44] Do not set fetcherDoneFlags.stateRoot on FCU root updates --- packages/client/src/sync/fetcher/accountfetcher.ts | 3 --- packages/client/src/sync/fetcher/storagefetcher.ts | 3 --- packages/client/src/sync/fetcher/trienodefetcher.ts | 3 --- 3 files changed, 9 deletions(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 71c7e29984..5d70faa378 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -394,9 +394,6 @@ export class AccountFetcher extends Fetcher this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() - - // TODO do we need to set this one since it's a duplicate of snapTargetRoot? - this.fetcherDoneFlags.stateRoot = latest.stateRoot } } diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index c823b31084..7f9c0ab387 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -231,9 +231,6 @@ export class StorageFetcher extends Fetcher this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() - - // TODO do we need to set this one since it's a duplicate of snapTargetRoot? - this.fetcherDoneFlags.stateRoot = latest.stateRoot } } From edd086615e62ffe24c09b613412fe663e0ab0c8c Mon Sep 17 00:00:00 2001 From: Amir Date: Mon, 29 Jul 2024 14:12:43 -0700 Subject: [PATCH 35/44] Update storageFetcher to track root and height of each request created on a more granular level --- .../client/src/sync/fetcher/storagefetcher.ts | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index 7f9c0ab387..0cc4556083 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -26,6 +26,7 @@ import type { Job, SnapFetcherDoneFlags } from './types.js' import type { Debugger } from 'debug' const TOTAL_RANGE_END = BIGINT_2 ** BIGINT_256 - BIGINT_1 +const LOOKBACK_WINDOW = 1 type StorageDataResponse = StorageData[][] & { completed?: boolean } @@ -34,6 +35,8 @@ export type StorageRequest = { storageRoot: Uint8Array first: bigint count: bigint + stateRoot: Uint8Array // needed for verifying request if targetRoot has been updated since the creation of this request + height: bigint // needed for checking if request is still safely within stateRoot-lookback-window before requesting to avoid being banned by peer } /** @@ -64,6 +67,8 @@ export interface StorageFetcherOptions extends FetcherOptions { export type JobTask = { storageRequests: StorageRequest[] multi: boolean + stateRoot: Uint8Array // needed for verifying request if targetRoot has been updated since the creation of this request + height: bigint // needed for checking if request is still safely within stateRoot-lookback-window before requesting to avoid being banned by peer } export class StorageFetcher extends Fetcher { @@ -227,7 +232,7 @@ export class StorageFetcher extends Fetcher= 1) { + if (newestHeight - currentHeight >= LOOKBACK_WINDOW) { this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() @@ -259,8 +264,17 @@ export class StorageFetcher extends Fetcher= + LOOKBACK_WINDOW + ) { + // skip request if we are close to being outside of lookback range to avoid getting banned + this.debug(`skipping request that is close to being outside of lookback range`) + return Object.assign([], [{ skipped: true }], { completed: true }) + } + const rangeResult = await peer!.snap!.getStorageRanges({ - root: this.fetcherDoneFlags.snapTargetRoot!, + root: task.stateRoot, accounts: task.storageRequests.map((req) => req.accountHash), origin, limit, @@ -398,6 +412,8 @@ export class StorageFetcher extends Fetcher Date: Tue, 30 Jul 2024 13:46:44 -0700 Subject: [PATCH 36/44] Add constant for lookback window and debug --- packages/client/src/config.ts | 4 +++ .../client/src/sync/fetcher/accountfetcher.ts | 3 +- .../client/src/sync/fetcher/storagefetcher.ts | 5 ++-- .../src/sync/fetcher/trienodefetcher.ts | 30 +++++++++++++++++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/client/src/config.ts b/packages/client/src/config.ts index 74fa734339..1d614dc9dd 100644 --- a/packages/client/src/config.ts +++ b/packages/client/src/config.ts @@ -326,6 +326,7 @@ export interface ConfigOptions { pruneEngineCache?: boolean snapAvailabilityDepth?: bigint snapTransitionSafeDepth?: bigint + snapLookbackWindow?: number /** * Save account keys preimages in the meta db (default: false) @@ -393,6 +394,7 @@ export class Config { // distance from head at which we can safely transition from a synced snapstate to vmexecution // randomly kept it at 5 for fast testing purposes but ideally should be >=32 slots public static readonly SNAP_TRANSITION_SAFE_DEPTH = BigInt(5) + public static readonly SNAP_LOOKBACK_WINDOW = 2 public readonly logger: Logger public readonly syncmode: SyncMode @@ -441,6 +443,7 @@ export class Config { public readonly engineNewpayloadMaxTxsExecute: number public readonly snapAvailabilityDepth: bigint public readonly snapTransitionSafeDepth: bigint + public readonly snapLookbackWindow: number public readonly prefixStorageTrieKeys: boolean // Defaulting to false as experimental as of now @@ -535,6 +538,7 @@ export class Config { this.snapAvailabilityDepth = options.snapAvailabilityDepth ?? Config.SNAP_AVAILABILITY_DEPTH this.snapTransitionSafeDepth = options.snapTransitionSafeDepth ?? Config.SNAP_TRANSITION_SAFE_DEPTH + this.snapLookbackWindow = options.snapLookbackWindow ?? Config.SNAP_LOOKBACK_WINDOW this.prefixStorageTrieKeys = options.prefixStorageTrieKeys ?? true this.enableSnapSync = options.enableSnapSync ?? false diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 5d70faa378..e11d2a39e6 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -390,10 +390,11 @@ export class AccountFetcher extends Fetcher // could be running independently of eachother in some cases const currentHeight = this.height const newestHeight = latest.number - if (newestHeight - currentHeight >= 1) { + if (newestHeight - currentHeight >= this.config.snapLookbackWindow) { this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() + this.height = newestHeight } } diff --git a/packages/client/src/sync/fetcher/storagefetcher.ts b/packages/client/src/sync/fetcher/storagefetcher.ts index 0cc4556083..c3c4c60dd7 100644 --- a/packages/client/src/sync/fetcher/storagefetcher.ts +++ b/packages/client/src/sync/fetcher/storagefetcher.ts @@ -26,7 +26,6 @@ import type { Job, SnapFetcherDoneFlags } from './types.js' import type { Debugger } from 'debug' const TOTAL_RANGE_END = BIGINT_2 ** BIGINT_256 - BIGINT_1 -const LOOKBACK_WINDOW = 1 type StorageDataResponse = StorageData[][] & { completed?: boolean } @@ -232,7 +231,7 @@ export class StorageFetcher extends Fetcher= LOOKBACK_WINDOW) { + if (newestHeight - currentHeight >= this.config.snapLookbackWindow) { this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() @@ -266,7 +265,7 @@ export class StorageFetcher extends Fetcher= - LOOKBACK_WINDOW + this.config.snapLookbackWindow ) { // skip request if we are close to being outside of lookback range to avoid getting banned this.debug(`skipping request that is close to being outside of lookback range`) diff --git a/packages/client/src/sync/fetcher/trienodefetcher.ts b/packages/client/src/sync/fetcher/trienodefetcher.ts index 88c0776b88..a143ffe06d 100644 --- a/packages/client/src/sync/fetcher/trienodefetcher.ts +++ b/packages/client/src/sync/fetcher/trienodefetcher.ts @@ -120,7 +120,7 @@ export class TrieNodeFetcher extends Fetcher // will always start with root node as first set of node requests this.pathToNodeRequestData.setElement('', { requested: false, - nodeHash: bytesToHex(this.fetcherDoneFlags.snapTargetRoot!), + nodeHash: bytesToHex(this.fetcherDoneFlags.snapTargetRoot!), // TODO this target could change, have to update tfn to clear and recreate root request on targetRoot changing nodeParentHash: '', // root node does not have a parent } as NodeRequestData) @@ -154,10 +154,25 @@ export class TrieNodeFetcher extends Fetcher if (latest !== undefined && latest.stateRoot !== undefined) { const currentHeight = this.height const newestHeight = latest.number - if (newestHeight - currentHeight >= 1) { + if (newestHeight - currentHeight >= this.config.snapLookbackWindow) { + this.debug('dbg810: updating roots') + // update latest height and root this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() + this.height = newestHeight + + // clear any tasks or requests and create new request for newest root set to + this.clear() + this.pathToNodeRequestData = new OrderedMap() + this.requestedNodeToPath = new Map() + this.fetchedAccountNodes = new Map() + this.nodeCount = 0 + this.pathToNodeRequestData.setElement('', { + requested: false, + nodeHash: bytesToHex(this.fetcherDoneFlags.snapTargetRoot!), // TODO this target could change, have to update tfn to clear and recreate root request on targetRoot changing + nodeParentHash: '', // root node does not have a parent + } as NodeRequestData) } } @@ -169,6 +184,14 @@ export class TrieNodeFetcher extends Fetcher bytes: BigInt(this.config.maxRangeBytes), }) + this.debug(`dbg800: trienodefetcher.ts`) + this.debug(`root: ${bytesToHex(this.fetcherDoneFlags.snapTargetRoot!)}`) + this.debug( + `paths requested: ${paths.map((valArr, _) => { + return valArr.map((val, _) => bytesToHex(val)) + })}`, + ) + // Response is valid, but check if peer is signalling that it does not have // the requested data. For trie node range queries that means the peer is not // yet synced. @@ -237,6 +260,9 @@ export class TrieNodeFetcher extends Fetcher let unknownChildNodeCount = 0 let hasStorageComponent = false + this.debug(`dbg804: trienodefetcher.ts`) + this.debug(`nodeData: ${node}`) + // get all children of received node if (node instanceof BranchNode) { const children = (node as BranchNode).getChildren() From 3e8a1e49cc1b878a5500c0c532cbe915dc50e826 Mon Sep 17 00:00:00 2001 From: harkamal Date: Fri, 6 Sep 2024 22:30:52 +0530 Subject: [PATCH 37/44] debug, fix and get the snapsync sim working on dencun --- packages/blockchain/src/blockchain.ts | 8 +++ packages/client/src/net/server/rlpxserver.ts | 2 +- packages/client/src/sync/snapsync.ts | 2 +- packages/client/test/sim/configs/mainnet.json | 1 + packages/client/test/sim/simutils.ts | 53 +++++++++++++++---- packages/client/test/sim/single-run.sh | 2 +- packages/client/test/sim/snapsync.md | 2 +- packages/client/test/sim/snapsync.spec.ts | 9 +++- 8 files changed, 64 insertions(+), 15 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index a9a45d740e..e2f6de1079 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -1272,6 +1272,13 @@ export class Blockchain implements BlockchainInterface { number: 0, stateRoot, withdrawalsRoot: common.isActivatedEIP(4895) ? KECCAK256_RLP : undefined, + ...(common.isActivatedEIP(4844) + ? { + blobGasUsed: 0, + excessBlobGas: 0, + parentBeaconBlockRoot: new Uint8Array(32), + } + : undefined), } if (common.consensusType() === 'poa') { if (common.genesis().extraData) { @@ -1282,6 +1289,7 @@ export class Blockchain implements BlockchainInterface { header.extraData = concatBytes(new Uint8Array(32), new Uint8Array(65)) } } + return createBlockFromBlockData( { header, withdrawals: common.isActivatedEIP(4895) ? [] : undefined }, { common }, diff --git a/packages/client/src/net/server/rlpxserver.ts b/packages/client/src/net/server/rlpxserver.ts index f041ff7a5c..34fdcdff28 100644 --- a/packages/client/src/net/server/rlpxserver.ts +++ b/packages/client/src/net/server/rlpxserver.ts @@ -65,7 +65,7 @@ export class RlpxServer extends Server { super(options) // As of now, the devp2p dpt server listens on the ip4 protocol by default and hence the ip in the // bootnode needs to be of ip4 by default - this.ip = options.config.extIP ?? '0.0.0.0' + this.ip = options.config.extIP ?? '127.0.0.1' this.discovery = options.config.discV4 || options.config.discDns this.clientFilter = options.clientFilter ?? [ 'go1.5', diff --git a/packages/client/src/sync/snapsync.ts b/packages/client/src/sync/snapsync.ts index 7e5ff092fd..92a4add023 100644 --- a/packages/client/src/sync/snapsync.ts +++ b/packages/client/src/sync/snapsync.ts @@ -229,7 +229,7 @@ export class SnapSynchronizer extends Synchronizer { config: this.config, pool: this.pool, stateManager: this.execution.vm.stateManager as DefaultStateManager, - height: height, + height, // This needs to be determined from the current state of the MPT dump first: BigInt(0), fetcherDoneFlags: this.fetcherDoneFlags, diff --git a/packages/client/test/sim/configs/mainnet.json b/packages/client/test/sim/configs/mainnet.json index 321a864e18..d58d77ed2c 100644 --- a/packages/client/test/sim/configs/mainnet.json +++ b/packages/client/test/sim/configs/mainnet.json @@ -13,6 +13,7 @@ "londonBlock": 0, "mergeForkBlock": 0, "shanghaiTime": 0, + "cancunTime": 0, "terminalTotalDifficulty": 1, "terminalTotalDifficultyPassed": true }, diff --git a/packages/client/test/sim/simutils.ts b/packages/client/test/sim/simutils.ts index 2ddba64f10..cd25699ecc 100644 --- a/packages/client/test/sim/simutils.ts +++ b/packages/client/test/sim/simutils.ts @@ -488,19 +488,48 @@ export async function setupEngineUpdateRelay(client: EthereumClient, peerBeaconU } } - const playUpdate = async (payload: any, finalizedBlockHash: string, version: string) => { - if (version !== 'capella') { - throw Error('only capella replay supported yet') - } + const playUpdate = async ( + payload: any, + { + finalizedBlockHash, + parentBeaconBlockRoot, + blobVersionedHashes, + }: { + finalizedBlockHash: string + parentBeaconBlockRoot?: string + blobVersionedHashes?: string[] + }, + version: string, + ) => { + let newPayloadParams, newPayloadMethod, fcuMethod const fcUState = { headBlockHash: payload.blockHash, safeBlockHash: finalizedBlockHash, finalizedBlockHash, } + + if (version === 'capella') { + newPayloadParams = [payload] + newPayloadMethod = 'engine_newPayloadV2' + fcuMethod = 'engine_forkchoiceUpdatedV2' + } else if (version === 'deneb') { + if (parentBeaconBlockRoot === undefined || blobVersionedHashes === undefined) { + throw Error( + `parentBeaconBlockRoot=undefined or blobVersionedHashes=undefined for CL fork=${version}`, + ) + } + + newPayloadMethod = 'engine_newPayloadV3' + fcuMethod = 'engine_forkchoiceUpdatedV3' + newPayloadParams = [payload, blobVersionedHashes, parentBeaconBlockRoot] + } else { + throw Error(`playUpdate not implemented for CL fork=${version}`) + } + console.log('playUpdate', fcUState) try { - const newPayloadRes = await engineMethods['engine_newPayloadV2']([payload]) + const newPayloadRes = await engineMethods[newPayloadMethod](newPayloadParams) if ( newPayloadRes.status === undefined || !['SYNCING', 'VALID', 'ACCEPTED'].includes(newPayloadRes.status) @@ -510,7 +539,7 @@ export async function setupEngineUpdateRelay(client: EthereumClient, peerBeaconU ) } - const fcuRes = await engineMethods['engine_forkchoiceUpdatedV2']([fcUState]) + const fcuRes = await engineMethods[fcuMethod]([fcUState]) if ( fcuRes.payloadStatus === undefined || !['SYNCING', 'VALID', 'ACCEPTED'].includes(newPayloadRes.status) @@ -543,13 +572,19 @@ export async function setupEngineUpdateRelay(client: EthereumClient, peerBeaconU } const beaconHead = await (await fetch(`${peerBeaconUrl}/eth/v2/beacon/blocks/head`)).json() - const payload = executionPayloadFromBeaconPayload( beaconHead.data.message.body.execution_payload, ) const finalizedBlockHash = beaconFinalized.data.finalized_header.execution.block_hash - - await playUpdate(payload, finalizedBlockHash, beaconHead.version) + const parentBeaconBlockRoot = beaconHead.data.message.parent_root + // blobVersionedHashes should be correctly calculated from commitments + const blobVersionedHashes: string[] = [] + + await playUpdate( + payload, + { finalizedBlockHash, parentBeaconBlockRoot, blobVersionedHashes }, + beaconHead.version, + ) } catch (e) { console.log('update fetch error', e) updateState('ERRORED') diff --git a/packages/client/test/sim/single-run.sh b/packages/client/test/sim/single-run.sh index 50b5852f08..92cc6e069b 100755 --- a/packages/client/test/sim/single-run.sh +++ b/packages/client/test/sim/single-run.sh @@ -39,7 +39,7 @@ then echo "geth requires NETWORKID to be passed in env, exiting..." exit; fi; - ELCLIENT_IMAGE="ethereum/client-go:v1.12.2" + ELCLIENT_IMAGE="ethereum/client-go:v1.14.8" echo "ELCLIENT=$ELCLIENT using ELCLIENT_IMAGE=$ELCLIENT_IMAGE NETWORKID=$NETWORKID" ;; *) diff --git a/packages/client/test/sim/snapsync.md b/packages/client/test/sim/snapsync.md index ce71305420..be4284a000 100644 --- a/packages/client/test/sim/snapsync.md +++ b/packages/client/test/sim/snapsync.md @@ -16,7 +16,7 @@ Note: All commands should be run from the `client` package directory root (so so 1. Start external geth client: ```bash -NETWORK=mainnet NETWORKID=1337903 ELCLIENT=geth EXTRA_CL_PARAMS="--params.CAPELLA_FORK_EPOCH 0" DATADIR=/usr/app/ethereumjs/packages/client/data test/sim/single-run.sh +NETWORK=mainnet NETWORKID=1337903 ELCLIENT=geth EXTRA_CL_PARAMS="--params.CAPELLA_FORK_EPOCH 0 --params.DENEB_FORK_EPOCH 0" DATADIR=/usr/app/ethereumjs/packages/client/data test/sim/single-run.sh ``` 2. (optional) Add some txs/state to geth diff --git a/packages/client/test/sim/snapsync.spec.ts b/packages/client/test/sim/snapsync.spec.ts index 89ae5e2422..7b2269f853 100644 --- a/packages/client/test/sim/snapsync.spec.ts +++ b/packages/client/test/sim/snapsync.spec.ts @@ -13,6 +13,7 @@ import { assert, describe, it } from 'vitest' import { Config } from '../../src/config.js' import { getLogger } from '../../src/logging.js' import { Event } from '../../src/types.js' +import { parseMultiaddrs } from '../../src/util/index.js' import { createInlineClient, @@ -55,7 +56,7 @@ export async function runTx(data: PrefixedHexString | '', to?: PrefixedHexString describe('simple mainnet test run', async () => { if (process.env.EXTRA_CL_PARAMS === undefined) { - process.env.EXTRA_CL_PARAMS = '--params.CAPELLA_FORK_EPOCH 0' + process.env.EXTRA_CL_PARAMS = '--params.CAPELLA_FORK_EPOCH 0 --params.DENEB_FORK_EPOCH 0' } // Better add it as a option in startnetwork process.env.NETWORKID = `${common.chainId()}` @@ -74,6 +75,7 @@ describe('simple mainnet test run', async () => { const nodeInfo = (await client.request('admin_nodeInfo', [])).result assert.ok(nodeInfo.enode !== undefined, 'fetched enode for peering') + console.log({ peerEnode: nodeInfo.enode }) console.log(`Waiting for network to start...`) try { @@ -151,7 +153,9 @@ describe('simple mainnet test run', async () => { assert.ok(ejsClient !== null, 'ethereumjs client started') const enode = ejsClient!.server()!.getRlpxInfo().enode + console.log({ enode }) const res = await client.request('admin_addPeer', [enode]) + console.log(res) assert.equal(res.result, true, 'successfully requested Geth add EthereumJS as peer') const peerConnectTimeout = new Promise((_resolve, reject) => setTimeout(reject, 10000)) @@ -159,6 +163,7 @@ describe('simple mainnet test run', async () => { await Promise.race([peerConnectedPromise, peerConnectTimeout]) assert.ok(true, 'connected to geth peer') } catch (e) { + console.log(e) assert.fail('could not connect to geth peer in 10 seconds') } }, @@ -251,7 +256,7 @@ async function createSnapClient( const logger = getLogger({ logLevel: 'debug' }) const config = new Config({ common, - bootnodes, + bootnodes: parseMultiaddrs(bootnodes), multiaddrs: [], logger, accountCache: 10000, From 9a3b14cf3126bd17fb26d3130f6f08edc6fa5324 Mon Sep 17 00:00:00 2001 From: Amir Date: Thu, 12 Sep 2024 18:29:28 -0700 Subject: [PATCH 38/44] Add heal cycle repeat flag --- packages/client/src/config.ts | 2 +- .../client/src/sync/fetcher/accountfetcher.ts | 4 ++++ .../client/src/sync/fetcher/trienodefetcher.ts | 17 ++++++++++++++--- packages/client/src/sync/fetcher/types.ts | 4 ++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/client/src/config.ts b/packages/client/src/config.ts index 1d614dc9dd..e6a5975107 100644 --- a/packages/client/src/config.ts +++ b/packages/client/src/config.ts @@ -394,7 +394,7 @@ export class Config { // distance from head at which we can safely transition from a synced snapstate to vmexecution // randomly kept it at 5 for fast testing purposes but ideally should be >=32 slots public static readonly SNAP_TRANSITION_SAFE_DEPTH = BigInt(5) - public static readonly SNAP_LOOKBACK_WINDOW = 2 + public static readonly SNAP_LOOKBACK_WINDOW = 1 public readonly logger: Logger public readonly syncmode: SyncMode diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index e11d2a39e6..5ca7b10cb9 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -278,6 +278,10 @@ export class AccountFetcher extends Fetcher case TrieNodeFetcher: // TODO update to check if heal phase completed successfully and then continue with next // healing phase + if (fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount > 0) { + fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount-- + break + } fetcherDoneFlags.trieNodeFetcher.done = true break } diff --git a/packages/client/src/sync/fetcher/trienodefetcher.ts b/packages/client/src/sync/fetcher/trienodefetcher.ts index a143ffe06d..7663ebfbfb 100644 --- a/packages/client/src/sync/fetcher/trienodefetcher.ts +++ b/packages/client/src/sync/fetcher/trienodefetcher.ts @@ -102,7 +102,6 @@ export class TrieNodeFetcher extends Fetcher */ constructor(options: TrieNodeFetcherOptions) { super(options) - this.height = options.height this.fetcherDoneFlags = options.fetcherDoneFlags ?? getInitFecherDoneFlags() this.pathToNodeRequestData = new OrderedMap() this.requestedNodeToPath = new Map() @@ -151,16 +150,27 @@ export class TrieNodeFetcher extends Fetcher // 1. Properly rewrite Fetcher with async/await -> allow to at least place in Fetcher.next() // 2. Properly implement ETH request IDs -> allow to call on non-idle in Peer Pool const latest = await peer?.latest() + // console.log('dbg850') if (latest !== undefined && latest.stateRoot !== undefined) { - const currentHeight = this.height + const currentHeight = this.fetcherDoneFlags.trieNodeFetcher.currentHeight const newestHeight = latest.number + // console.log(currentHeight) + // console.log(newestHeight) + // console.log(this.config.snapLookbackWindow) + // console.log(newestHeight - currentHeight >= this.config.snapLookbackWindow) if (newestHeight - currentHeight >= this.config.snapLookbackWindow) { this.debug('dbg810: updating roots') // update latest height and root this.fetcherDoneFlags.snapTargetHeight = latest.number this.fetcherDoneFlags.snapTargetRoot = latest.stateRoot this.fetcherDoneFlags.snapTargetHash = latest.hash() - this.height = newestHeight + this.fetcherDoneFlags.trieNodeFetcher.currentHeight = newestHeight + + // console.log('dbg811') + // console.log(latest.number) + // console.log(bytesToHex(latest.stateRoot)) + // console.log(bytesToHex(latest.hash())) + // console.log(latest.toJSON()) // clear any tasks or requests and create new request for newest root set to this.clear() @@ -332,6 +342,7 @@ export class TrieNodeFetcher extends Fetcher await this.accountTrie.lookupNode(childNode.nodeHash as Uint8Array) } } catch (e) { + // console.log('dbg901') // if error is thrown, than the node is unknown and should be queued for fetching unknownChildNodeCount++ const { parentAccountHash } = this.pathToNodeRequestData.getElementByKey( diff --git a/packages/client/src/sync/fetcher/types.ts b/packages/client/src/sync/fetcher/types.ts index 8004360450..bfbde0cd7e 100644 --- a/packages/client/src/sync/fetcher/types.ts +++ b/packages/client/src/sync/fetcher/types.ts @@ -39,6 +39,8 @@ export type SnapFetcherDoneFlags = { first: bigint count: bigint done: boolean + healCycleRepeatCount: number // used in AccountFetcher to determine how many times to repeat heal cycles + currentHeight: bigint } stateRoot?: Uint8Array } @@ -70,6 +72,8 @@ export function getInitFecherDoneFlags(): SnapFetcherDoneFlags { first: BigInt(0), count: BigInt(0), done: false, + healCycleRepeatCount: Number(process.env.HEAL_CYCLE_REPEATS ?? '0'), + currentHeight: BigInt(0), }, } } From 939582938525bece4d4bc02400e86db389cc51c8 Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 13 Sep 2024 12:52:51 -0700 Subject: [PATCH 39/44] Make repeats accurate --- packages/client/src/sync/fetcher/accountfetcher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/sync/fetcher/accountfetcher.ts b/packages/client/src/sync/fetcher/accountfetcher.ts index 5ca7b10cb9..147ea07bfe 100644 --- a/packages/client/src/sync/fetcher/accountfetcher.ts +++ b/packages/client/src/sync/fetcher/accountfetcher.ts @@ -278,7 +278,7 @@ export class AccountFetcher extends Fetcher case TrieNodeFetcher: // TODO update to check if heal phase completed successfully and then continue with next // healing phase - if (fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount > 0) { + if (fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount > 1) { fetcherDoneFlags.trieNodeFetcher.healCycleRepeatCount-- break } From a73b65974f3477daef756f67d8d556fa27ed6bca Mon Sep 17 00:00:00 2001 From: Amir Date: Fri, 13 Sep 2024 12:53:02 -0700 Subject: [PATCH 40/44] Remove unused field --- packages/client/src/sync/fetcher/trienodefetcher.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/client/src/sync/fetcher/trienodefetcher.ts b/packages/client/src/sync/fetcher/trienodefetcher.ts index 7663ebfbfb..ef41fbc49b 100644 --- a/packages/client/src/sync/fetcher/trienodefetcher.ts +++ b/packages/client/src/sync/fetcher/trienodefetcher.ts @@ -69,9 +69,6 @@ type NodeRequestData = { export class TrieNodeFetcher extends Fetcher { protected debug: Debugger - // height of block being targeted for snap sync - height: bigint - stateManager: DefaultStateManager fetcherDoneFlags: SnapFetcherDoneFlags accountTrie: Trie From b5ede5ef302633852ecb2b146ae771c7da3221ed Mon Sep 17 00:00:00 2001 From: Amir Date: Mon, 14 Oct 2024 15:19:52 -0700 Subject: [PATCH 41/44] Setup connect script for debugging devp2p payloads with live client --- packages/client/test/sim/configs/pectra3.json | 926 ++++++++++++++++++ packages/client/test/sim/connect.spec.ts | 83 ++ 2 files changed, 1009 insertions(+) create mode 100644 packages/client/test/sim/configs/pectra3.json create mode 100644 packages/client/test/sim/connect.spec.ts diff --git a/packages/client/test/sim/configs/pectra3.json b/packages/client/test/sim/configs/pectra3.json new file mode 100644 index 0000000000..150b9b484d --- /dev/null +++ b/packages/client/test/sim/configs/pectra3.json @@ -0,0 +1,926 @@ +{ + "config": { + "chainId": 7011893082, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0, + "mergeNetsplitBlock": 0, + "terminalTotalDifficulty": 0, + "terminalTotalDifficultyPassed": true, + "shanghaiTime": 0, + "cancunTime": 0, + "depositContractAddress": "0x4242424242424242424242424242424242424242", + "pragueTime": 1726058628 + }, + "alloc": { + "0x0000000000000000000000000000000000000000": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000001": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000003": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000004": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000005": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000006": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000007": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000008": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000009": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000000f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000010": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000011": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000012": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000013": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000014": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000015": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000016": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000017": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000018": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000019": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000001f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000020": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000021": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000022": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000023": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000024": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000025": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000026": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000027": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000028": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000029": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000002f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000030": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000031": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000032": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000033": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000034": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000035": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000036": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000037": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000038": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000039": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000003f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000040": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000041": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000042": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000043": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000044": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000045": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000046": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000047": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000048": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000049": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000004f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000050": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000051": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000052": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000053": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000054": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000055": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000056": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000057": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000058": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000059": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000005f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000060": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000061": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000062": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000063": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000064": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000065": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000066": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000067": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000068": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000069": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000006f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000070": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000071": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000072": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000073": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000074": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000075": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000076": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000077": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000078": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000079": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000007f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000080": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000081": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000082": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000083": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000084": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000085": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000086": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000087": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000088": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000089": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000008f": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000090": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000091": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000092": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000093": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000094": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000095": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000096": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000097": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000098": { + "balance": "1" + }, + "0x0000000000000000000000000000000000000099": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009a": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009b": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009c": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009d": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009e": { + "balance": "1" + }, + "0x000000000000000000000000000000000000009f": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000a9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000aa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ab": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ac": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ad": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ae": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000af": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000b9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ba": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000be": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000bf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000c9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ca": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ce": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000cf": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000d9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000da": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000db": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000dd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000de": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000df": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000e9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ea": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000eb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ec": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ed": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ee": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ef": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f0": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f1": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f2": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f3": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f4": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f5": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f6": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f7": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f8": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000f9": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fa": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fb": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fc": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fd": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000fe": { + "balance": "1" + }, + "0x00000000000000000000000000000000000000ff": { + "balance": "1" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a2646970667358221220dceca8706b29e917dacf25fceef95acac8d90d765ac926663ce4096195952b6164736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + }, + "0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02": { + "balance": "0", + "nonce": "1", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500" + }, + "0x0aae40965e6800cd9b1f4b05ff21581047e3f91e": { + "balance": "0", + "nonce": "1", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe1460575767ffffffffffffffff5f3511605357600143035f3511604b575f35612000014311604b57611fff5f3516545f5260205ff35b5f5f5260205ff35b5f5ffd5b5f35611fff60014303165500" + }, + "0x00A3ca265EBcb825B45F985A16CEFB49958cE017": { + "balance": "0", + "nonce": "1", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe146090573615156028575f545f5260205ff35b366038141561012e5760115f54600182026001905f5b5f82111560595781019083028483029004916001019190603e565b90939004341061012e57600154600101600155600354806003026004013381556001015f3581556001016020359055600101600355005b6003546002548082038060101160a4575060105b5f5b81811460dd5780604c02838201600302600401805490600101805490600101549160601b83528260140152906034015260010160a6565b910180921460ed579060025560f8565b90505f6002555f6003555b5f548061049d141561010757505f5b60015460028282011161011c5750505f610122565b01600290035b5f555f600155604c025ff35b5f5ffd", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000000000000000000000000000000000000000049d" + } + }, + "0x00b42dbF2194e931E80326D950320f7d9Dbeac02": { + "balance": "0", + "nonce": "1", + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe146098573615156028575f545f5260205ff35b36606014156101445760115f54600182026001905f5b5f82111560595781019083028483029004916001019190603e565b90939004341061014457600154600101600155600354806004026004013381556001015f35815560010160203581556001016040359055600101600355005b6003546002548082038060011160ac575060015b5f5b81811460f15780607402838201600402600401805490600101805490600101805490600101549260601b84529083601401528260340152906054015260010160ae565b9101809214610103579060025561010e565b90505f6002555f6003555b5f548061049d141561011d57505f5b6001546001828201116101325750505f610138565b01600190035b5f555f6001556074025ff35b5f5ffd", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000000000000000000000000000000000000000049d" + } + }, + "0x454b0EA7d8aD3C56D0CF2e44Ed97b2Feab4D7AF2": { + "balance": "1000000000000000000000000000" + }, + "0xd3248BA3E5492D767F8e427Cb9C7B9D5C3972D7B": { + "balance": "1000000000000000000000000000" + }, + "0xAD01b55d7c3448B8899862eb335FBb17075d8DE2": { + "balance": "1000000000000000000000000000" + }, + "0x7e454a14B8e7528465eeF86f0DC1da4f235d9D79": { + "balance": "1000000000000000000000000000" + }, + "0x7a40026A3b9A41754a95EeC8c92C6B99886f440C": { + "balance": "1000000000000000000000000000" + }, + "0x8c4D8CDD1f474510Dd70D66F2785a3a38a29AC1A": { + "balance": "1000000000000000000000000000" + }, + "0xfC7360b3b28cf4204268A8354dbEc60720d155D2": { + "balance": "1000000000000000000000000000" + }, + "0x2F7626bBDb8c0f9071bC98046Ef6fDed2167F97F": { + "balance": "1000000000000000000000000000" + }, + "0x752CE31Dec0dde7D1563CdF6438d892De2D4FBee": { + "balance": "1000000000000000000000000000" + }, + "0x455f42d91096c4Aa708D7Cbcb2DC499dE89C402c": { + "balance": "1000000000000000000000000000" + }, + "0x85154341488732D57a97F54AB9706Bc4B71B8636": { + "balance": "1000000000000000000000000000" + }, + "0x6a9CcA73d4Ff3a249fa778C7651f4Df8B9fFa0Df": { + "balance": "1000000000000000000000000000" + }, + "0xee2d0567AAe8080CA269b7908F4aF8BBb59A6804": { + "balance": "1000000000000000000000000000" + }, + "0xDd8D4027078a471816e4Ef7F69aFc0A5d2947dDc": { + "balance": "1000000000000000000000000000" + }, + "0x20466E9A67f299F6056bE52A50ea324FA6Bd05D5": { + "balance": "1000000000000000000000000000" + }, + "0x03F24BB0C9cfb30217Ff992A36ae9230F2A1697f": { + "balance": "1000000000000000000000000000" + }, + "0x032d8372C519c3927b87BDe4479E846a81EF2d10": { + "balance": "1000000000000000000000000000" + }, + "0xF863DF14954df73804b3150F3754a8F98CBB1D0d": { + "balance": "1000000000000000000000000000" + }, + "0xbe918A6aef1920F3706E23d153146aA6C5982620": { + "balance": "1000000000000000000000000000" + }, + "0xA0c7edA3CE474BC670A11EA9537cBEfd36331123": { + "balance": "1000000000000000000000000000" + }, + "0xF03b43BeB861044492Eb43E247bEE2AC6C80c651": { + "balance": "1000000000000000000000000000" + } + }, + "coinbase": "0x0000000000000000000000000000000000000000", + "difficulty": "0x01", + "extraData": "", + "gasLimit": "0x17d7840", + "nonce": "0x1234", + "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "1726057800" +} diff --git a/packages/client/test/sim/connect.spec.ts b/packages/client/test/sim/connect.spec.ts new file mode 100644 index 0000000000..5cf2d9e89c --- /dev/null +++ b/packages/client/test/sim/connect.spec.ts @@ -0,0 +1,83 @@ +import { createCommonFromGethGenesis } from '@ethereumjs/common' +import { DPT, ETH, RLPx, SNAP } from '@ethereumjs/devp2p' +import { bytesToHex, hexToBytes, parseGethGenesisState, privateToAddress } from '@ethereumjs/util' +import { assert, describe, it } from 'vitest' + +import pectra3Json from './configs/pectra3.json' + +// import { +// createInlineClient, +// filterKeywords, +// filterOutWords, +// runTxHelper, +// setupEngineUpdateRelay, +// startNetwork, +// waitForELStart, +// } from './simutils.js' + +import type { Peer } from '@ethereumjs/devp2p' +const STATIC_ID = hexToBytes('0x' + '0'.repeat(64)) + +let rlpx: RLPx + +describe('simple mainnet test run', async () => { + it.skipIf(false)( + 'connect', + async () => { + try { + const common = createCommonFromGethGenesis(pectra3Json, { chain: 'pectra3' }) + // const customGenesisState = parseGethGenesisState(networkJSON) + + const privateKey = hexToBytes( + '0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d', + ) + const peerHost = '0.0.0.0' // IP address of the devp2p server + const peerPort = 30303 // devp2p server port + + rlpx = new RLPx(privateKey, { + dpt: new DPT(privateKey, { + // Disable discovery, only connect to peers manually + shouldFindNeighbours: false, + }), + maxPeers: 25, + capabilities: [ETH.eth65, ETH.eth64, SNAP.snap], + listenPort: null, + common, + }) + + await rlpx.connect({ + id: STATIC_ID, + address: peerHost, + tcpPort: peerPort, + }) + rlpx.events.on('peer:added', (peer: Peer) => { + console.log('Connected to peer', peer) + // Send payload to the peer here + // const payload = Buffer.from([0x01, 0x02, 0x03]) // Example payload + // peer.sendMessage(protocol, messageCode, payload) + }) + + rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { + console.log('Disconnected from peer', reasonCode, disconnectMessage) + }) + } catch (error) { + console.log(error) + } + }, + 60_000, + ) + + it('network cleanup', async () => { + try { + rlpx.disconnect(STATIC_ID) + assert.ok(true, 'network cleaned') + } catch (e) { + assert.fail('network not cleaned properly') + } + }, 60_000) +}) + +process.on('uncaughtException', (err, origin) => { + console.log({ err, origin }) + process.exit() +}) From b2f471a766af9a6480c0cbadc2cdd3a9807089e8 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 15 Oct 2024 11:12:50 -0700 Subject: [PATCH 42/44] Get connection established --- packages/client/test/sim/connect.spec.ts | 60 +++++++++++++++--------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/packages/client/test/sim/connect.spec.ts b/packages/client/test/sim/connect.spec.ts index 5cf2d9e89c..cc5c6df6b2 100644 --- a/packages/client/test/sim/connect.spec.ts +++ b/packages/client/test/sim/connect.spec.ts @@ -15,8 +15,10 @@ import pectra3Json from './configs/pectra3.json' // waitForELStart, // } from './simutils.js' -import type { Peer } from '@ethereumjs/devp2p' -const STATIC_ID = hexToBytes('0x' + '0'.repeat(64)) +import { Peer } from '@ethereumjs/devp2p' +const STATIC_ID = hexToBytes( + '0x' + 'a9faec679fb6d5a68dc7f0301e4e13f35747f74d25e80edbc8768c24f43b128b', +) let rlpx: RLPx @@ -31,35 +33,49 @@ describe('simple mainnet test run', async () => { const privateKey = hexToBytes( '0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d', ) - const peerHost = '0.0.0.0' // IP address of the devp2p server + const peerHost = '127.0.0.1' // IP address of the devp2p server const peerPort = 30303 // devp2p server port - + const dpt = new DPT(privateKey, { + // Disable discovery, only connect to peers manually + shouldFindNeighbours: false, + }) rlpx = new RLPx(privateKey, { - dpt: new DPT(privateKey, { - // Disable discovery, only connect to peers manually - shouldFindNeighbours: false, - }), + dpt, maxPeers: 25, capabilities: [ETH.eth65, ETH.eth64, SNAP.snap], listenPort: null, common, }) - await rlpx.connect({ - id: STATIC_ID, - address: peerHost, - tcpPort: peerPort, - }) - rlpx.events.on('peer:added', (peer: Peer) => { - console.log('Connected to peer', peer) - // Send payload to the peer here - // const payload = Buffer.from([0x01, 0x02, 0x03]) // Example payload - // peer.sendMessage(protocol, messageCode, payload) - }) + await dpt + .addPeer({ address: peerHost, udpPort: peerPort, tcpPort: peerPort }) + .then((peer) => { + // return rlpx.connect({ + // id: peer.id, + // address: peer.address, + // tcpPort: peer.tcpPort, + // udpPort: peer.tcpPort + // }) + // console.log(peer) + // console.log(bytesToHex(peer.id)) + console.log(rlpx) + // const p = rlpx._peers.get(bytesToHex(peer.id).slice(2)) + // console.log(p) + rlpx.events.on('peer:added', (peer: Peer) => { + console.log('Connected to peer', peer) + // Send payload to the peer here + // const payload = Buffer.from([0x01, 0x02, 0x03]) // Example payload + // peer.sendMessage(protocol, messageCode, payload) + }) - rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { - console.log('Disconnected from peer', reasonCode, disconnectMessage) - }) + rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { + console.log('Disconnected from peer', reasonCode, disconnectMessage) + }) + }) + .catch((err) => { + console.log(`error on connection to local node: ${err.stack ?? err}`) + rlpx.destroy() + }) } catch (error) { console.log(error) } From 15f890632d67351ca81059d8619a2078eb4f9820 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 15 Oct 2024 13:34:55 -0700 Subject: [PATCH 43/44] Add capability to send message to live peer --- packages/client/test/sim/connect.spec.ts | 124 +++++++++++------------ 1 file changed, 57 insertions(+), 67 deletions(-) diff --git a/packages/client/test/sim/connect.spec.ts b/packages/client/test/sim/connect.spec.ts index cc5c6df6b2..68f606f3de 100644 --- a/packages/client/test/sim/connect.spec.ts +++ b/packages/client/test/sim/connect.spec.ts @@ -1,86 +1,76 @@ import { createCommonFromGethGenesis } from '@ethereumjs/common' -import { DPT, ETH, RLPx, SNAP } from '@ethereumjs/devp2p' -import { bytesToHex, hexToBytes, parseGethGenesisState, privateToAddress } from '@ethereumjs/util' +import { DPT, ETH, LES, RLPx, SNAP } from '@ethereumjs/devp2p' +import { hexToBytes } from '@ethereumjs/util' import { assert, describe, it } from 'vitest' +import { Peer } from '@ethereumjs/devp2p' import pectra3Json from './configs/pectra3.json' +import { Protocol } from '../../src/net/protocol/protocol.js' -// import { -// createInlineClient, -// filterKeywords, -// filterOutWords, -// runTxHelper, -// setupEngineUpdateRelay, -// startNetwork, -// waitForELStart, -// } from './simutils.js' - -import { Peer } from '@ethereumjs/devp2p' const STATIC_ID = hexToBytes( '0x' + 'a9faec679fb6d5a68dc7f0301e4e13f35747f74d25e80edbc8768c24f43b128b', ) +const _getTrieNodesRLP = + '0xeb01a0aa3cd09df0b7c0efbd473200c6db3117b51b68af7a5523334db0208d05e1729ec4c180c180834c4b40' let rlpx: RLPx +const getPeerAddr = (peer: Peer) => `${peer['_socket'].remoteAddress}:${peer['_socket'].remotePort}` describe('simple mainnet test run', async () => { - it.skipIf(false)( - 'connect', - async () => { - try { - const common = createCommonFromGethGenesis(pectra3Json, { chain: 'pectra3' }) - // const customGenesisState = parseGethGenesisState(networkJSON) + try { + const common = createCommonFromGethGenesis(pectra3Json, { chain: 'pectra3' }) + const privateKey = hexToBytes( + '0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d', + ) + const peerHost = '127.0.0.1' // IP address of the devp2p server + const peerPort = 30303 // devp2p server port + const dpt = new DPT(privateKey, { + // Disable discovery, only connect to peers manually + shouldFindNeighbours: false, + refreshInterval: 30000, + endpoint: { address: peerHost, udpPort: peerPort, tcpPort: peerPort }, + }) + dpt.events.on('error', (err) => console.error(`DPT error: ${err}`)) + rlpx = new RLPx(privateKey, { + dpt, + maxPeers: 25, + capabilities: [SNAP.snap, LES.les4], + common, + }) + rlpx.events.on('error', (err) => console.error(`RLPx error: ${err.stack ?? err}`)) - const privateKey = hexToBytes( - '0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d', - ) - const peerHost = '127.0.0.1' // IP address of the devp2p server - const peerPort = 30303 // devp2p server port - const dpt = new DPT(privateKey, { - // Disable discovery, only connect to peers manually - shouldFindNeighbours: false, + await dpt + .addPeer({ address: peerHost, udpPort: peerPort, tcpPort: peerPort }) + .then((peer) => { + rlpx.events.on('peer:added', (peer: Peer) => { + // Send payload to the peer here + const addr = getPeerAddr(peer) + console.log(peer.getProtocols()) + const snap = peer.getProtocols()[0] + // snap.sendMessage(SNAP.MESSAGE_CODES.GET_TRIE_NODES, _getTrieNodesRLP) + // snap.sendMessage(SNAP.MESSAGE_CODES.TRIE_NODES, []) + // const requests: { headers: BlockHeader[]; bodies: any[] } = { headers: [], bodies: [] } + // const clientId = peer.getHelloMessage().clientId + // console.log(`Add peer: ${addr} ${clientId} (snap${snap.getVersion()}) (total: ${rlpx.getPeers().length})`,) }) - rlpx = new RLPx(privateKey, { - dpt, - maxPeers: 25, - capabilities: [ETH.eth65, ETH.eth64, SNAP.snap], - listenPort: null, - common, + rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { + console.log('Disconnected from peer', reasonCode, disconnectMessage) }) - - await dpt - .addPeer({ address: peerHost, udpPort: peerPort, tcpPort: peerPort }) - .then((peer) => { - // return rlpx.connect({ - // id: peer.id, - // address: peer.address, - // tcpPort: peer.tcpPort, - // udpPort: peer.tcpPort - // }) - // console.log(peer) - // console.log(bytesToHex(peer.id)) - console.log(rlpx) - // const p = rlpx._peers.get(bytesToHex(peer.id).slice(2)) - // console.log(p) - rlpx.events.on('peer:added', (peer: Peer) => { - console.log('Connected to peer', peer) - // Send payload to the peer here - // const payload = Buffer.from([0x01, 0x02, 0x03]) // Example payload - // peer.sendMessage(protocol, messageCode, payload) - }) - - rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { - console.log('Disconnected from peer', reasonCode, disconnectMessage) - }) - }) - .catch((err) => { - console.log(`error on connection to local node: ${err.stack ?? err}`) - rlpx.destroy() - }) - } catch (error) { - console.log(error) - } + rlpx._connectToPeer(peer) + }) + .catch((err) => { + console.log(`error on connection to local node: ${err.stack ?? err}`) + rlpx.destroy() + }) + } catch (error) { + console.log(error) + } + it.skipIf(false)( + 'connect', + async () => { + console.log('dbg200') }, - 60_000, + 0, ) it('network cleanup', async () => { From a17fe93f013735ed1fe062e9cb07f0badc271c00 Mon Sep 17 00:00:00 2001 From: Amir Date: Thu, 17 Oct 2024 16:17:23 -0700 Subject: [PATCH 44/44] Add payload encoding to connect script --- packages/client/test/sim/connect.spec.ts | 57 +++++++++++++++++------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/packages/client/test/sim/connect.spec.ts b/packages/client/test/sim/connect.spec.ts index 68f606f3de..d6713fab96 100644 --- a/packages/client/test/sim/connect.spec.ts +++ b/packages/client/test/sim/connect.spec.ts @@ -1,23 +1,25 @@ import { createCommonFromGethGenesis } from '@ethereumjs/common' -import { DPT, ETH, LES, RLPx, SNAP } from '@ethereumjs/devp2p' +import { DPT, LES, RLPx, SNAP } from '@ethereumjs/devp2p' import { hexToBytes } from '@ethereumjs/util' import { assert, describe, it } from 'vitest' -import { Peer } from '@ethereumjs/devp2p' +import { Peer, Protocol } from '@ethereumjs/devp2p' import pectra3Json from './configs/pectra3.json' -import { Protocol } from '../../src/net/protocol/protocol.js' +import { Config } from '../../src/index.js' +import { Chain } from '../../src/blockchain/index.js' +import { SnapProtocol } from '../../src/net/protocol/index.js' const STATIC_ID = hexToBytes( '0x' + 'a9faec679fb6d5a68dc7f0301e4e13f35747f74d25e80edbc8768c24f43b128b', ) -const _getTrieNodesRLP = - '0xeb01a0aa3cd09df0b7c0efbd473200c6db3117b51b68af7a5523334db0208d05e1729ec4c180c180834c4b40' - let rlpx: RLPx -const getPeerAddr = (peer: Peer) => `${peer['_socket'].remoteAddress}:${peer['_socket'].remotePort}` describe('simple mainnet test run', async () => { try { + const config = new Config({ accountCache: 10000, storageCache: 1000 }) + const chain = await Chain.create({ config }) + const p = new SnapProtocol({ config, chain }) + const common = createCommonFromGethGenesis(pectra3Json, { chain: 'pectra3' }) const privateKey = hexToBytes( '0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d', @@ -44,14 +46,42 @@ describe('simple mainnet test run', async () => { .then((peer) => { rlpx.events.on('peer:added', (peer: Peer) => { // Send payload to the peer here - const addr = getPeerAddr(peer) - console.log(peer.getProtocols()) - const snap = peer.getProtocols()[0] - // snap.sendMessage(SNAP.MESSAGE_CODES.GET_TRIE_NODES, _getTrieNodesRLP) + const snap: Protocol = peer.getProtocols()[0] as any + const reqId = BigInt(1) + const root = hexToBytes( + '0x04157502e6177a76ca4dbf7784e5ec1a926049db6a91e13efb70a095a72a45d9', + ) + const paths = [[hexToBytes('0x')]] + const bytes = BigInt(5000000) + + const payload = p.encode( + p.messages.filter((message) => message.name === 'GetTrieNodes')[0], + { + reqId, + root, + paths, + bytes, + }, + ) + + snap.sendMessage(SNAP.MESSAGE_CODES.GET_TRIE_NODES, payload) // snap.sendMessage(SNAP.MESSAGE_CODES.TRIE_NODES, []) // const requests: { headers: BlockHeader[]; bodies: any[] } = { headers: [], bodies: [] } // const clientId = peer.getHelloMessage().clientId // console.log(`Add peer: ${addr} ${clientId} (snap${snap.getVersion()}) (total: ${rlpx.getPeers().length})`,) + + // snap.events.once('TrieNodes', (status: devp2p.SNAP.) => { + // const msg = [ + // Uint8Array.from([]), + // [ + // bytesToInt(status['headNum']), + // Uint8Array.from([1]), + // Uint8Array.from([]), + // Uint8Array.from([1]), + // ], + // ] + // snap.sendMessage(devp2p.LES.MESSAGE_CODES.GET_BLOCK_HEADERS, msg) + // }) }) rlpx.events.on('peer:removed', (peer, reasonCode, disconnectMessage) => { console.log('Disconnected from peer', reasonCode, disconnectMessage) @@ -82,8 +112,3 @@ describe('simple mainnet test run', async () => { } }, 60_000) }) - -process.on('uncaughtException', (err, origin) => { - console.log({ err, origin }) - process.exit() -})