From b23bf62045ce779ce57487fd644e3b8246583bc3 Mon Sep 17 00:00:00 2001 From: Ford Date: Tue, 18 Oct 2022 09:59:49 -0700 Subject: [PATCH] indexer-common: Support provider connection for any EVM network - If ethereumNetwork name not provided fallback to 'any', to support non-standard networks with unknown names like arbitrum-goerli, etc... - Move network provider connection establishment to a shared indexer-common function for usage by indexer-agent and indexer-service --- packages/indexer-agent/src/commands/start.ts | 108 +++--------------- packages/indexer-common/src/network.ts | 89 +++++++++++++++ .../indexer-service/src/commands/start.ts | 95 ++------------- 3 files changed, 117 insertions(+), 175 deletions(-) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 244d99edb..0ebd02b07 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -33,7 +33,7 @@ import { } from '@graphprotocol/indexer-common' import { startAgent } from '../agent' import { Indexer } from '../indexer' -import { providers, Wallet } from 'ethers' +import { Wallet } from 'ethers' import { startCostModelAutomation } from '../cost' import { createSyncingServer } from '../syncing-server' import { monitorEthBalance } from '../utils' @@ -53,7 +53,7 @@ export default { description: 'Ethereum network', type: 'string', required: false, - default: 'mainnet', + default: 'any', group: 'Ethereum', }) .option('ethereum-polling-interval', { @@ -581,107 +581,31 @@ export default { await sequelize.sync() logger.info(`Successfully synced database models`) - logger.info(`Connect to Ethereum`, { - provider: argv.ethereum, - network: argv.ethereumNetwork, - }) - - let providerUrl - try { - providerUrl = new URL(argv.ethereum) - } catch (err) { - logger.fatal(`Invalid Ethereum URL`, { - err: indexerError(IndexerErrorCode.IE002, err), - url: argv.ethereum, - }) - process.exit(1) - return - } - - const ethProviderMetrics = { - requests: new metrics.client.Counter({ - name: 'eth_provider_requests', - help: 'Ethereum provider requests', - registers: [metrics.registry], - labelNames: ['method'], - }), - } - - if (providerUrl.password && providerUrl.protocol == 'http:') { - logger.warn( - 'Ethereum endpoint does not use HTTPS, your authentication credentials may not be secure', - ) - } - - // Prevent passing empty basicAuth info - let username - let password - if (providerUrl.username == '' && providerUrl.password == '') { - username = undefined - password = undefined - } else { - username = providerUrl.username - password = providerUrl.password - } - - const ethereumProvider = new providers.StaticJsonRpcProvider( - { - url: providerUrl.toString(), - user: username, - password: password, - allowInsecureAuthentication: true, - }, + const networkProvider = await Network.provider( + logger, + metrics, argv.ethereumNetwork, + argv.ethereum, + argv.ethereumPollingInterval, ) - ethereumProvider.pollingInterval = argv.ethereumPollingInterval - - ethereumProvider.on('debug', info => { - if (info.action === 'response') { - ethProviderMetrics.requests.inc({ - method: info.request.method, - }) - - logger.trace('Ethereum request', { - method: info.request.method, - params: info.request.params, - response: info.response, - }) - } - }) - - ethereumProvider.on('network', (newNetwork, oldNetwork) => { - logger.trace('Ethereum network change', { - oldNetwork: oldNetwork, - newNetwork: newNetwork, - }) - }) - - const networkMeta = await ethereumProvider.getNetwork() - logger.info(`Connected to Ethereum`, { - provider: ethereumProvider.connection.url, - pollingInterval: ethereumProvider.pollingInterval, - network: await ethereumProvider.detectNetwork(), - }) + const networkMeta = await networkProvider.getNetwork() logger.info(`Connect wallet`, { - network: ethereumProvider.network.name, - chainId: ethereumProvider.network.chainId, + network: networkMeta.name, + chainId: networkMeta.chainId, }) let wallet = Wallet.fromMnemonic(argv.mnemonic) - wallet = wallet.connect(ethereumProvider) + wallet = wallet.connect(networkProvider) logger.info(`Connected wallet`) logger.info(`Connect to contracts`, { network: networkMeta.name, chainId: networkMeta.chainId, - providerNetworkChainID: ethereumProvider.network.chainId, + providerNetworkChainID: networkProvider.network.chainId, }) let contracts = undefined try { - contracts = await connectContracts( - wallet, - ethereumProvider.network.chainId, - ) + contracts = await connectContracts(wallet, networkMeta.chainId) } catch (err) { logger.error( `Failed to connect to contracts, please ensure you are using the intended Ethereum network`, @@ -731,7 +655,7 @@ export default { const maxGasFee = argv.baseFeeGasMax || argv.gasPriceMax const network = await Network.create( logger, - ethereumProvider, + networkProvider, contracts, wallet, indexerAddress, @@ -772,7 +696,7 @@ export default { logger.child({ component: 'NetworkMonitor' }), indexingStatusResolver, networkSubgraph, - ethereumProvider, + networkProvider, epochSubgraph, ) @@ -847,7 +771,7 @@ export default { startCostModelAutomation({ logger, - ethereum: ethereumProvider, + ethereum: networkProvider, contracts: network.contracts, indexerManagement: indexerManagementClient, injectDai: argv.injectDai, diff --git a/packages/indexer-common/src/network.ts b/packages/indexer-common/src/network.ts index 27346b7d7..06522a132 100644 --- a/packages/indexer-common/src/network.ts +++ b/packages/indexer-common/src/network.ts @@ -3,6 +3,7 @@ import { Eventual, formatGRT, Logger, + Metrics, mutable, NetworkContracts, SubgraphDeploymentID, @@ -154,6 +155,94 @@ export class Network { ) } + static async provider( + logger: Logger, + metrics: Metrics, + networkName: string, + networkURL: string, + pollingInterval: number, + ): Promise { + logger.info(`Connect to Network chain`, { + provider: networkURL, + }) + + let providerUrl + try { + providerUrl = new URL(networkURL) + } catch (err) { + logger.fatal(`Invalid Network provider URL`, { + err: indexerError(IndexerErrorCode.IE002, err), + url: networkURL, + }) + process.exit(1) + } + + const ethProviderMetrics = { + requests: new metrics.client.Counter({ + name: 'eth_provider_requests', + help: 'Ethereum provider requests', + registers: [metrics.registry], + labelNames: ['method'], + }), + } + + if (providerUrl.password && providerUrl.protocol == 'http:') { + logger.warn( + 'Network endpoint does not use HTTPS, your authentication credentials may not be secure', + ) + } + + let username + let password + if (providerUrl.username == '' && providerUrl.password == '') { + username = undefined + password = undefined + } else { + username = providerUrl.username + password = providerUrl.password + } + + const networkProvider = new providers.StaticJsonRpcProvider( + { + url: providerUrl.toString(), + user: username, + password: password, + allowInsecureAuthentication: true, + }, + networkName, + ) + networkProvider.pollingInterval = pollingInterval + + networkProvider.on('debug', (info) => { + if (info.action === 'response') { + ethProviderMetrics.requests.inc({ + method: info.request.method, + }) + + logger.trace('Network request', { + method: info.request.method, + params: info.request.params, + response: info.response, + }) + } + }) + + networkProvider.on('network', (newNetwork, oldNetwork) => { + logger.trace('Network change', { + oldNetwork: oldNetwork, + newNetwork: newNetwork, + }) + }) + + logger.info(`Connected to network`, { + provider: networkProvider.connection.url, + pollingInterval: networkProvider.pollingInterval, + network: await networkProvider.detectNetwork(), + }) + + return networkProvider + } + // TODO: Move to NetworkMonitor async claimableAllocations(disputableEpoch: number): Promise { try { diff --git a/packages/indexer-service/src/commands/start.ts b/packages/indexer-service/src/commands/start.ts index b8f5fa11c..f46eb45cb 100644 --- a/packages/indexer-service/src/commands/start.ts +++ b/packages/indexer-service/src/commands/start.ts @@ -1,7 +1,7 @@ import path from 'path' import readPackage from 'read-pkg' import { Argv } from 'yargs' -import { BigNumber, providers, Wallet } from 'ethers' +import { BigNumber, Wallet } from 'ethers' import { connectContracts, @@ -19,6 +19,7 @@ import { indexerError, IndexerErrorCode, IndexingStatusResolver, + Network, NetworkSubgraph, registerIndexerErrorMetrics, } from '@graphprotocol/indexer-common' @@ -43,7 +44,7 @@ export default { description: 'Ethereum network', type: 'string', required: false, - default: 'mainnet', + default: 'any', group: 'Ethereum', }) .option('ethereum-polling-interval', { @@ -276,7 +277,7 @@ export default { // the agent and assume the models are up to date in the service. logger.info('Successfully connected to database') - logger.info(`Connect to network`) + logger.info(`Connect to network subgraph`) const indexingStatusResolver = new IndexingStatusResolver({ logger, statusEndpoint: argv.graphNodeStatusEndpoint, @@ -292,88 +293,16 @@ export default { } : undefined, }) - logger.info(`Successfully connected to network`) + logger.info(`Successfully connected to network subgraph`) - logger.info('Connecting to Ethereum', { - provider: argv.ethereum, - network: argv.ethereumNetwork, - }) - let providerUrl - try { - providerUrl = new URL(argv.ethereum) - } catch (err) { - logger.critical(`Invalid Ethereum URL`, { - err: indexerError(IndexerErrorCode.IE002, err), - url: argv.ethereum, - }) - process.exit(1) - return - } - - const web3ProviderMetrics = { - requests: new metrics.client.Counter({ - name: 'eth_provider_requests', - help: 'Ethereum provider requests', - registers: [metrics.registry], - labelNames: ['method'], - }), - } - - if (providerUrl.password && providerUrl.protocol == 'http:') { - logger.warn( - 'Ethereum endpoint does not use HTTPS, your authentication credentials may not be secure', - ) - } - - // Prevent passing empty basicAuth info - let username - let password - if (providerUrl.username == '' && providerUrl.password == '') { - username = undefined - password = undefined - } else { - username = providerUrl.username - password = providerUrl.password - } - - const ethereumProvider = new providers.StaticJsonRpcProvider( - { - url: providerUrl.toString(), - user: username, - password: password, - allowInsecureAuthentication: true, - }, + const networkProvider = await Network.provider( + logger, + metrics, argv.ethereumNetwork, + argv.ethereum, + argv.ethereumPollingInterval, ) - ethereumProvider.pollingInterval = argv.ethereumPollingInterval - - ethereumProvider.on('debug', info => { - if (info.action === 'response') { - web3ProviderMetrics.requests.inc({ - method: info.request.method, - }) - - logger.trace('Ethereum request', { - method: info.request.method, - params: info.request.params, - response: info.response, - }) - } - }) - - ethereumProvider.on('network', (newNetwork, oldNetwork) => { - logger.trace('Ethereum network change', { - oldNetwork: oldNetwork, - newNetwork: newNetwork, - }) - }) - - const network = await ethereumProvider.getNetwork() - logger.info('Successfully connected to Ethereum', { - provider: ethereumProvider.connection.url, - pollingInterval: ethereumProvider.pollingInterval, - network: await ethereumProvider.detectNetwork(), - }) + const network = await networkProvider.getNetwork() logger.info('Connect to contracts', { network: network.name, @@ -382,7 +311,7 @@ export default { let contracts = undefined try { - contracts = await connectContracts(ethereumProvider, network.chainId) + contracts = await connectContracts(networkProvider, network.chainId) } catch (error) { logger.error( `Failed to connect to contracts, please ensure you are using the intended Ethereum Network`,