diff --git a/docs/networks/arbitrum-one.md b/docs/networks/arbitrum-one.md index b3a666fa5..4e2702398 100644 --- a/docs/networks/arbitrum-one.md +++ b/docs/networks/arbitrum-one.md @@ -38,17 +38,19 @@ Other network contracts can be found in [graphprotocol/contracts](https://github ### Indexer Agent | Environment Variable | CLI Argument | Value | -| ------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +|---------------------------------------------|---------------------------------| ------------------------------------------------------------------------------------------------------------------------- | | `INDEXER_AGENT_ETHEREUM` | `--ethereum` | An Arbitrum mainnet node/provider | | `INDEXER_AGENT_INDEXER_ADDRESS` | `--indexer-address` | Ethereum address of mainnet indexer | | `INDEXER_AGENT_INDEXER_GEO_COORDINATES` | `--indexer-geo-coordinates` | Geo coordinates of mainnet indexer infrastructure | | `INDEXER_AGENT_MNEMONIC` | `--mnemonic` | Ethereum mnemonic for mainnet operator | -| `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2FDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3DDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp) | -| `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT` | `--network-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp` | | `INDEXER_AGENT_GATEWAY_ENDPOINT` | `--gateway-endpoint` | `https://gateway-arbitrum.network.thegraph.com/` | | `INDEXER_AGENT_GAS_PRICE_MAX` | `--gas-price-max` | `50` | +| `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2FDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3DDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp) | +| `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT` | `--network-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp` | +| `INDEXER_AGENT_EPOCH_SUBGRAPH_DEPLOYMENT` | `--epoch-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2F4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3D4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB) | | `INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT` | `--epoch-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB` | -| `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT` | `--tap-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1` | +| `INDEXER_AGENT_TAP_SUBGRAPH_DEPLOYMENT` | `--tap-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2F4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3D4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1) | +| `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT` | `--tap-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1` | In order to avoid collecting or claiming query fees below a certain threshold diff --git a/docs/networks/arbitrum-sepolia.md b/docs/networks/arbitrum-sepolia.md index 8266b9aab..f4c378cee 100644 --- a/docs/networks/arbitrum-sepolia.md +++ b/docs/networks/arbitrum-sepolia.md @@ -44,15 +44,17 @@ testnet (for now) are Mainnet subgraphs. This means: ### Indexer Agent | Environment Variable | CLI Argument | Value | -| ------------------------------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +|---------------------------------------------|---------------------------------| ----------------------------------------------------------------------------------------------------------------------- | | `INDEXER_AGENT_ETHEREUM` | `--ethereum` | An Arbitrum Sepolia node/provider | | `INDEXER_AGENT_INDEXER_ADDRESS` | `--indexer-address` | Ethereum address of testnet indexer | | `INDEXER_AGENT_INDEXER_GEO_COORDINATES` | `--indexer-geo-coordinates` | Geo coordinates of testnet indexer infrastructure | | `INDEXER_AGENT_MNEMONIC` | `--mnemonic` | Ethereum mnemonic for testnet operator | +| `INDEXER_AGENT_GATEWAY_ENDPOINT` | `--gateway-endpoint` | `https://gateway-arbitrum.testnet.thegraph.com/` | | `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | `QmXnGVrg6DvscnvJd86aHAPLGyGrkM17weMrAsFAEMmQLL` | | `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT` | `--network-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/3xQHhMudr1oh69ut36G2mbzpYmYxwqCeU6wwqyCDCnqV` | -| `INDEXER_AGENT_GATEWAY_ENDPOINT` | `--gateway-endpoint` | `https://gateway-arbitrum.testnet.thegraph.com/` | +| `INDEXER_AGENT_EPOCH_SUBGRAPH_DEPLOYMENT` | `--epoch-subgraph-deployment` | `QmNNn48AU8CZei8sYLXrXABMHkiXAuQDvFaZHYYPryKEvF` | | `INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT` | `--epoch-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS` | +| `INDEXER_AGENT_TAP_SUBGRAPH_DEPLOYMENT` | `--tap-subgraph-deployment` | `QmUiLdbsk6c51UMdcNBxsP3KadJpkmp6a3k2NCprR4ZFeM` | | `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT` | `--tap-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/7ubx365MiqBH5iUz6XWXWT8PTof5BVAyEzdb8m17RvbD` | In order to avoid collecting or claiming query fees below a certain threshold diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index abf1c5dde..6771896fd 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -231,7 +231,7 @@ export class Agent { async ({ network, operator }: NetworkAndOperator) => { try { await operator.ensureGlobalIndexingRule() - await this.ensureNetworkSubgraphIsIndexing(network) + await this.ensureAllSubgraphsIndexing(network) await network.register() } catch (err) { this.logger.critical( @@ -1241,38 +1241,56 @@ export class Agent { ) } - // TODO: This could be a initialization check inside Network.create() once/if the Indexer Service - // uses Network instances. - async ensureNetworkSubgraphIsIndexing(network: Network) { + // TODO: After indexer-service deprecation: Move to be an initialization check inside Network.create() + async ensureSubgraphIndexing(deployment: string, networkIdentifier: string) { + try { + // TODO: Check both the local deployment and the external subgraph endpoint + // Make sure the subgraph is being indexed + await this.graphNode.ensure( + `indexer-agent/${deployment.slice(-10)}`, + new SubgraphDeploymentID(deployment), + ) + + // Validate if the Network Subgraph belongs to the current provider's network. + // This check must be performed after we ensure the Network Subgraph is being indexed. + await validateProviderNetworkIdentifier( + networkIdentifier, + deployment, + this.graphNode, + this.logger, + ) + } catch (e) { + this.logger.warn( + 'Failed to deploy and validate Network Subgraph on index-nodes. Will use external subgraph endpoint instead', + e, + ) + } + } + async ensureAllSubgraphsIndexing(network: Network) { + // Network subgraph if ( network.specification.subgraphs.networkSubgraph.deployment !== undefined ) { - try { - // TODO: Check both the local deployment and the external subgraph endpoint - // Make sure the network subgraph is being indexed - await this.graphNode.ensure( - `indexer-agent/${network.specification.subgraphs.networkSubgraph.deployment.slice( - -10, - )}`, - new SubgraphDeploymentID( - network.specification.subgraphs.networkSubgraph.deployment, - ), - ) - - // Validate if the Network Subgraph belongs to the current provider's network. - // This check must be performed after we ensure the Network Subgraph is being indexed. - await validateProviderNetworkIdentifier( - network.specification.networkIdentifier, - network.specification.subgraphs.networkSubgraph.deployment, - this.graphNode, - this.logger, - ) - } catch (e) { - this.logger.warn( - 'Failed to deploy and validate Network Subgraph on index-nodes. Will use external subgraph endpoint instead', - e, - ) - } + await this.ensureSubgraphIndexing( + network.specification.subgraphs.networkSubgraph.deployment, + network.specification.networkIdentifier, + ) + } + // Epoch subgraph + if ( + network.specification.subgraphs.epochSubgraph.deployment !== undefined + ) { + await this.ensureSubgraphIndexing( + network.specification.subgraphs.epochSubgraph.deployment, + network.specification.networkIdentifier, + ) + } + // TAP subgraph + if (network.specification.subgraphs.tapSubgraph?.deployment !== undefined) { + await this.ensureSubgraphIndexing( + network.specification.subgraphs.tapSubgraph.deployment, + network.specification.networkIdentifier, + ) } } } diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 08dc0a5ff..f22943ddb 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -140,7 +140,7 @@ export const start = { }, }) .option('network-subgraph-deployment', { - description: 'Network subgraph deployment', + description: 'Network subgraph deployment (for local hosting)', array: false, type: 'string', group: 'Network Subgraph', @@ -151,6 +151,12 @@ export const start = { type: 'string', group: 'Network Subgraph', }) + .option('tap-subgraph-deployment', { + description: 'TAP subgraph deployment (for local hosting)', + array: false, + type: 'string', + group: 'TAP Subgraph', + }) .option('tap-subgraph-endpoint', { description: 'Endpoint to query the tap subgraph from', array: false, @@ -163,6 +169,12 @@ export const start = { default: false, group: 'Network Subgraph', }) + .option('epoch-subgraph-deployment', { + description: 'Epoch subgraph deployment (for local hosting)', + array: false, + type: 'string', + group: 'Network Subgraph', + }) .option('epoch-subgraph-endpoint', { description: 'Endpoint to query the epoch block oracle subgraph from', array: false, @@ -370,11 +382,11 @@ export async function createNetworkSpecification( url: argv.networkSubgraphEndpoint, }, epochSubgraph: { - // TODO: We should consider indexing the Epoch Subgraph, similar - // to how we currently do it for the Network Subgraph. + deployment: argv.epochSubgraphDeployment, url: argv.epochSubgraphEndpoint, }, tapSubgraph: { + deployment: argv.tapSubgraphDeployment, url: argv.tapSubgraphEndpoint, }, } diff --git a/packages/indexer-agent/src/syncing-server.ts b/packages/indexer-agent/src/syncing-server.ts index 4be36ef10..2c08da48b 100644 --- a/packages/indexer-agent/src/syncing-server.ts +++ b/packages/indexer-agent/src/syncing-server.ts @@ -7,13 +7,13 @@ import { Logger } from '@graphprotocol/common-ts' import { parse } from 'graphql' import { NetworkMapped, - NetworkSubgraph, + SubgraphClient, resolveChainId, } from '@graphprotocol/indexer-common' export interface CreateSyncingServerOptions { logger: Logger - networkSubgraphs: NetworkMapped + networkSubgraphs: NetworkMapped port: number } diff --git a/packages/indexer-common/src/__tests__/subgraph.test.ts b/packages/indexer-common/src/__tests__/subgraph.test.ts index fb85fcb19..a5585f54a 100644 --- a/packages/indexer-common/src/__tests__/subgraph.test.ts +++ b/packages/indexer-common/src/__tests__/subgraph.test.ts @@ -5,7 +5,7 @@ import { ProviderInterface, SubgraphQueryInterface, } from '../subgraphs' -import { QueryResult } from '../network-subgraph' +import { QueryResult } from '../subgraph-client' import gql from 'graphql-tag' import { mergeSelectionSets } from '../utils' diff --git a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts index b7552fd58..cbddb3fc5 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts @@ -2,7 +2,7 @@ import { Address, Eventual, createLogger, createMetrics } from '@graphprotocol/c import { Allocation, AllocationsResponse, - NetworkSubgraph, + SubgraphClient, QueryFeeModels, QueryResult, TapCollector, @@ -11,7 +11,6 @@ import { TransactionManager, } from '@graphprotocol/indexer-common' import { NetworkContracts as TapContracts } from '@semiotic-labs/tap-contracts-bindings' -import { TAPSubgraph } from '../../tap-subgraph' import { NetworkSpecification } from 'indexer-common/src/network-specification' import { createMockAllocation } from '../../indexer-management/__tests__/helpers.test' import { getContractAddress } from 'ethers/lib/utils' @@ -155,10 +154,10 @@ const setup = () => { const tapSubgraph = { query: mockQueryTapSubgraph, - } as unknown as TAPSubgraph + } as unknown as SubgraphClient const networkSubgraph = { query: mockQueryNetworkSubgraph, - } as unknown as NetworkSubgraph + } as unknown as SubgraphClient tapCollector = TapCollector.create({ logger, diff --git a/packages/indexer-common/src/allocations/escrow-accounts.ts b/packages/indexer-common/src/allocations/escrow-accounts.ts index 06784a54b..1d126fd94 100644 --- a/packages/indexer-common/src/allocations/escrow-accounts.ts +++ b/packages/indexer-common/src/allocations/escrow-accounts.ts @@ -1,5 +1,5 @@ import { Address, toAddress } from '@graphprotocol/common-ts' -import { TAPSubgraph } from '../tap-subgraph' +import { SubgraphClient } from '../subgraph-client' import gql from 'graphql-tag' type U256 = bigint @@ -44,7 +44,7 @@ export class EscrowAccounts { } export const getEscrowAccounts = async ( - tapSubgraph: TAPSubgraph, + tapSubgraph: SubgraphClient, indexer: Address, ): Promise => { const result = await tapSubgraph.query( diff --git a/packages/indexer-common/src/allocations/query-fees.ts b/packages/indexer-common/src/allocations/query-fees.ts index 53fff3e66..5e92cceed 100644 --- a/packages/indexer-common/src/allocations/query-fees.ts +++ b/packages/indexer-common/src/allocations/query-fees.ts @@ -20,12 +20,12 @@ import { ensureAllocationSummary, TransactionManager, specification as spec, + SubgraphClient, } from '..' import { DHeap } from '@thi.ng/heaps' import { BigNumber, BigNumberish, Contract } from 'ethers' import { Op } from 'sequelize' import pReduce from 'p-reduce' -import { NetworkSubgraph } from '../network-subgraph' // Receipts are collected with a delay of 20 minutes after // the corresponding allocation was closed @@ -71,7 +71,7 @@ export interface AllocationReceiptCollectorOptions { allocations: Eventual models: QueryFeeModels networkSpecification: spec.NetworkSpecification - networkSubgraph: NetworkSubgraph + networkSubgraph: SubgraphClient } export interface ReceiptCollector { @@ -94,7 +94,7 @@ export class AllocationReceiptCollector implements ReceiptCollector { declare voucherRedemptionBatchThreshold: BigNumber declare voucherRedemptionMaxBatchSize: number declare protocolNetwork: string - declare networkSubgraph: NetworkSubgraph + declare networkSubgraph: SubgraphClient // eslint-disable-next-line @typescript-eslint/no-empty-function -- Private constructor to prevent direct instantiation private constructor() {} diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 7628cfa78..9e89db446 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -26,8 +26,7 @@ import { } from '..' import { BigNumber } from 'ethers' import pReduce from 'p-reduce' -import { TAPSubgraph } from '../tap-subgraph' -import { NetworkSubgraph, QueryResult } from '../network-subgraph' +import { SubgraphClient, QueryResult } from '../subgraph-client' import gql from 'graphql-tag' import { getEscrowAccounts } from './escrow-accounts' @@ -52,8 +51,8 @@ interface TapCollectorOptions { allocations: Eventual models: QueryFeeModels networkSpecification: spec.NetworkSpecification - tapSubgraph: TAPSubgraph - networkSubgraph: NetworkSubgraph + tapSubgraph: SubgraphClient + networkSubgraph: SubgraphClient } interface ValidRavs { @@ -107,8 +106,8 @@ export class TapCollector { declare allocations: Eventual declare ravRedemptionThreshold: BigNumber declare protocolNetwork: string - declare tapSubgraph: TAPSubgraph - declare networkSubgraph: NetworkSubgraph + declare tapSubgraph: SubgraphClient + declare networkSubgraph: SubgraphClient declare finalityTime: number declare indexerAddress: Address diff --git a/packages/indexer-common/src/allocations/types.ts b/packages/indexer-common/src/allocations/types.ts index cc7728a48..b31c6c71f 100644 --- a/packages/indexer-common/src/allocations/types.ts +++ b/packages/indexer-common/src/allocations/types.ts @@ -1,5 +1,5 @@ import { BigNumber } from 'ethers' -import { NetworkSubgraph, SubgraphDeployment } from '@graphprotocol/indexer-common' +import { SubgraphClient, SubgraphDeployment } from '@graphprotocol/indexer-common' import { Logger, Address } from '@graphprotocol/common-ts' @@ -31,7 +31,7 @@ export enum AllocationStatus { export interface MonitorEligibleAllocationsOptions { indexer: Address logger: Logger - networkSubgraph: NetworkSubgraph + networkSubgraph: SubgraphClient interval: number protocolNetwork: string } diff --git a/packages/indexer-common/src/epoch-subgraph.ts b/packages/indexer-common/src/epoch-subgraph.ts deleted file mode 100644 index e6c9241b7..000000000 --- a/packages/indexer-common/src/epoch-subgraph.ts +++ /dev/null @@ -1,63 +0,0 @@ -import axios, { AxiosInstance, AxiosResponse } from 'axios' -import { DocumentNode, print } from 'graphql' -import { CombinedError } from '@urql/core' -import { QueryResult } from './network-subgraph' -import { Logger } from '@graphprotocol/common-ts' -import { SubgraphFreshnessChecker } from './subgraphs' -export class EpochSubgraph { - endpointClient: AxiosInstance - freshnessChecker: SubgraphFreshnessChecker - logger: Logger - endpoint: string - - constructor( - endpoint: string, - freshnessChecker: SubgraphFreshnessChecker, - logger: Logger, - ) { - this.endpoint = endpoint - this.endpointClient = axios.create({ - baseURL: endpoint, - headers: { 'content-type': 'application/json' }, - - // Don't parse responses as JSON - responseType: 'text', - - // Don't transform responses - transformResponse: (data) => data, - }) - this.freshnessChecker = freshnessChecker - this.logger = logger - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async checkedQuery( - query: DocumentNode, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - variables?: Record, - ): Promise> { - return this.freshnessChecker.checkedQuery(this, query, variables) - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async query( - query: DocumentNode, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - variables?: Record, - ): Promise> { - const response = await this.endpointClient.post('', { - query: print(query), - variables, - }) - const data = JSON.parse(response.data) - this.logger.trace('Epoch Subgraph query', { data }) - if (data.errors) { - return { error: new CombinedError({ graphQLErrors: data.errors }) } - } - return data - } - - async queryRaw(body: string): Promise { - return await this.endpointClient.post('', body) - } -} diff --git a/packages/indexer-common/src/index.ts b/packages/indexer-common/src/index.ts index 0090cad0d..e760232d6 100644 --- a/packages/indexer-common/src/index.ts +++ b/packages/indexer-common/src/index.ts @@ -1,16 +1,15 @@ export * from './actions' export * from './allocations' export * from './async-cache' -export * from './epoch-subgraph' export * from './errors' export * from './indexer-management' export * from './graph-node' export * from './operator' export * from './network' -export * from './network-subgraph' export * from './query-fees' export * from './rules' export * from './subgraphs' +export * from './subgraph-client' export * from './transactions' export * from './types' export * from './utils' diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index 8d4351cc2..c97df0722 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -23,12 +23,11 @@ import { literal, Op, Sequelize } from 'sequelize' import { Allocation, AllocationStatus, - EpochSubgraph, indexerError, IndexerErrorCode, GraphNode, NetworkMonitor, - NetworkSubgraph, + SubgraphClient, resolveChainAlias, resolveChainId, SubgraphDeployment, @@ -48,8 +47,8 @@ let models: IndexerManagementModels let ethereum: ethers.providers.BaseProvider let contracts: NetworkContracts let graphNode: GraphNode -let networkSubgraph: NetworkSubgraph -let epochSubgraph: EpochSubgraph +let networkSubgraph: SubgraphClient +let epochSubgraph: SubgraphClient let networkMonitor: NetworkMonitor let logger: Logger @@ -84,18 +83,22 @@ const setupMonitor = async () => { ) const INDEXER_TEST_API_KEY: string = process.env['INDEXER_TEST_API_KEY'] || '' - networkSubgraph = await NetworkSubgraph.create({ + + networkSubgraph = await SubgraphClient.create({ + name: 'NetworkSubgraph', logger, endpoint: `https://gateway-arbitrum.network.thegraph.com/api/${INDEXER_TEST_API_KEY}/subgraphs/id/3xQHhMudr1oh69ut36G2mbzpYmYxwqCeU6wwqyCDCnqV`, deployment: undefined, subgraphFreshnessChecker, }) - epochSubgraph = new EpochSubgraph( - `https://gateway-arbitrum.network.thegraph.com/api/${INDEXER_TEST_API_KEY}/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS`, - subgraphFreshnessChecker, + epochSubgraph = await SubgraphClient.create({ + name: 'EpochSubgraph', logger, - ) + endpoint: `https://gateway-arbitrum.network.thegraph.com/api/${INDEXER_TEST_API_KEY}/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS`, + subgraphFreshnessChecker, + }) + graphNode = new GraphNode( logger, 'http://test-admin-endpoint.xyz', diff --git a/packages/indexer-common/src/indexer-management/monitor.ts b/packages/indexer-common/src/indexer-management/monitor.ts index 09c284822..5b259d3a4 100644 --- a/packages/indexer-common/src/indexer-management/monitor.ts +++ b/packages/indexer-common/src/indexer-management/monitor.ts @@ -6,7 +6,6 @@ import { indexerError, IndexerErrorCode, GraphNode, - NetworkSubgraph, parseGraphQLAllocation, parseGraphQLEpochs, parseGraphQLSubgraphDeployment, @@ -14,7 +13,6 @@ import { SubgraphDeployment, SubgraphVersion, NetworkEpoch, - EpochSubgraph, BlockPointer, resolveChainId, resolveChainAlias, @@ -37,6 +35,7 @@ import { providers, utils, Wallet } from 'ethers' import pRetry, { Options } from 'p-retry' import { IndexerOptions } from '../network-specification' import pMap from 'p-map' +import { SubgraphClient } from '../subgraph-client' // The new read only Network class export class NetworkMonitor { @@ -46,9 +45,9 @@ export class NetworkMonitor { private indexerOptions: IndexerOptions, private logger: Logger, private graphNode: GraphNode, - private networkSubgraph: NetworkSubgraph, + private networkSubgraph: SubgraphClient, private ethereum: providers.BaseProvider, - private epochSubgraph: EpochSubgraph, + private epochSubgraph: SubgraphClient, ) {} async currentEpochNumber(): Promise { @@ -992,7 +991,7 @@ Please submit an issue at https://github.com/graphprotocol/block-oracle/issues/n async monitorNetworkPauses( logger: Logger, contracts: NetworkContracts, - networkSubgraph: NetworkSubgraph, + networkSubgraph: SubgraphClient, ): Promise> { // eslint-disable-next-line @typescript-eslint/no-unused-vars const initialPauseValue = await contracts.controller.paused().catch((_) => { diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index 385c9a424..62c753da8 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -24,8 +24,8 @@ import { IndexerManagementResolverContext, IndexingDecisionBasis, IndexingRuleAttributes, - NetworkSubgraph, ReallocateAllocationResult, + SubgraphClient, SubgraphIdentifierType, uniqueAllocationID, } from '@graphprotocol/indexer-common' @@ -173,7 +173,7 @@ const ALLOCATION_QUERIES = { async function queryAllocations( logger: Logger, - networkSubgraph: NetworkSubgraph, + networkSubgraph: SubgraphClient, variables: { indexer: Address allocation: Address | null diff --git a/packages/indexer-common/src/network.ts b/packages/indexer-common/src/network.ts index d955c494d..ec73f84c8 100644 --- a/packages/indexer-common/src/network.ts +++ b/packages/indexer-common/src/network.ts @@ -12,36 +12,33 @@ import { connectContracts as connectTapContracts, NetworkContracts as TapContracts, } from '@semiotic-labs/tap-contracts-bindings' +import { providers, Wallet } from 'ethers' +import { strict as assert } from 'assert' +import geohash from 'ngeohash' +import pRetry, { Options } from 'p-retry' + import { INDEXER_ERROR_MESSAGES, indexerError, IndexerErrorCode, - NetworkSubgraph, + SubgraphClient, TransactionManager, specification as spec, GraphNode, - EpochSubgraph, NetworkMonitor, AllocationReceiptCollector, SubgraphFreshnessChecker, monitorEligibleAllocations, } from '.' -import { providers, Wallet } from 'ethers' -import { strict as assert } from 'assert' -import geohash from 'ngeohash' - -import pRetry, { Options } from 'p-retry' import { resolveChainId } from './indexer-management' import { monitorEthBalance } from './utils' import { QueryFeeModels } from './query-fees' import { readFileSync } from 'fs' - -import { TAPSubgraph } from './tap-subgraph' import { TapCollector } from './allocations/tap-collector' export class Network { logger: Logger - networkSubgraph: NetworkSubgraph + networkSubgraph: SubgraphClient contracts: NetworkContracts wallet: Wallet networkProvider: providers.StaticJsonRpcProvider @@ -57,7 +54,7 @@ export class Network { logger: Logger, contracts: NetworkContracts, wallet: Wallet, - networkSubgraph: NetworkSubgraph, + networkSubgraph: SubgraphClient, networkProvider: providers.StaticJsonRpcProvider, transactionManager: TransactionManager, networkMonitor: NetworkMonitor, @@ -122,7 +119,8 @@ export class Network { ? new SubgraphDeploymentID(specification.subgraphs.networkSubgraph.deployment) : undefined - const networkSubgraph = await NetworkSubgraph.create({ + const networkSubgraph = await SubgraphClient.create({ + name: 'NetworkSubgraph', logger, endpoint: specification.subgraphs.networkSubgraph.url, deployment: @@ -143,13 +141,24 @@ export class Network { Infinity, ) - let tapSubgraph: TAPSubgraph | undefined = undefined - if (specification.subgraphs.tapSubgraph && specification.subgraphs.tapSubgraph.url) { - tapSubgraph = new TAPSubgraph( - specification.subgraphs.tapSubgraph!.url!, - tapSubgraphFreshnessChecker, - logger.child({ component: 'TAPSubgraph' }), - ) + let tapSubgraph: SubgraphClient | undefined = undefined + if (specification.subgraphs.tapSubgraph) { + const tapSubgraphDeploymentId = specification.subgraphs.tapSubgraph.deployment + ? new SubgraphDeploymentID(specification.subgraphs.tapSubgraph.deployment) + : undefined + tapSubgraph = await SubgraphClient.create({ + name: 'TapSubgraph', + logger, + deployment: + tapSubgraphDeploymentId !== undefined + ? { + graphNode, + deployment: tapSubgraphDeploymentId, + } + : undefined, + endpoint: specification.subgraphs.tapSubgraph!.url, + subgraphFreshnessChecker: tapSubgraphFreshnessChecker, + }) } // * ----------------------------------------------------------------------- @@ -189,16 +198,22 @@ export class Network { Infinity, ) - const epochSubgraph = new EpochSubgraph( - /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- - * Accept the non-null `url` property of the Epoch Subgraph, as it has - * already been validated during parsing. Once indexing is supported, - * initialize it in the same way as the NetworkSubgraph - */ - specification.subgraphs.epochSubgraph.url!, - epochSubgraphFreshnessChecker, - logger.child({ component: 'EpochSubgraph' }), - ) + const epochSubgraphDeploymentId = specification.subgraphs.epochSubgraph.deployment + ? new SubgraphDeploymentID(specification.subgraphs.epochSubgraph.deployment) + : undefined + const epochSubgraph = await SubgraphClient.create({ + name: 'EpochSubgraph', + logger, + deployment: + epochSubgraphDeploymentId !== undefined + ? { + graphNode, + deployment: epochSubgraphDeploymentId, + } + : undefined, + endpoint: specification.subgraphs.epochSubgraph.url, + subgraphFreshnessChecker: epochSubgraphFreshnessChecker, + }) // * ----------------------------------------------------------------------- // * Network Monitor diff --git a/packages/indexer-common/src/network-subgraph.ts b/packages/indexer-common/src/subgraph-client.ts similarity index 79% rename from packages/indexer-common/src/network-subgraph.ts rename to packages/indexer-common/src/subgraph-client.ts index ff8b4766d..d5bc2dd23 100644 --- a/packages/indexer-common/src/network-subgraph.ts +++ b/packages/indexer-common/src/subgraph-client.ts @@ -6,8 +6,9 @@ import { BlockPointer, IndexingError } from './types' import { GraphNode } from './graph-node' import { SubgraphFreshnessChecker } from './subgraphs' -export interface NetworkSubgraphCreateOptions { +export interface SubgraphCreateOptions { logger: Logger + name: string endpoint?: string deployment?: { graphNode: GraphNode @@ -24,7 +25,8 @@ interface DeploymentStatus { fatalError?: IndexingError } -interface NetworkSubgraphOptions { +interface SubgraphOptions { + name: string logger: Logger endpoint?: string deployment?: { @@ -40,14 +42,15 @@ export type QueryResult = Pick< 'error' | 'data' | 'extensions' > -export class NetworkSubgraph { +export class SubgraphClient { + name: string logger: Logger freshnessChecker: SubgraphFreshnessChecker | undefined endpointClient?: AxiosInstance - /** Endpoint URL for the Network Subgraph Endpoint from the config */ - private networkSubgraphConfigEndpoint?: string - /** Endpoint URL for the Network Subgraph Endpoint from the deployment */ - private networkSubgraphDeploymentEndpoint?: string + /** Endpoint URL for the Subgraph Endpoint from the config */ + private subgraphConfigEndpoint?: string + /** Endpoint URL for the Subgraph Endpoint from the deployment */ + private subgraphDeploymentEndpoint?: string endpoint?: string public readonly deployment?: { @@ -56,12 +59,14 @@ export class NetworkSubgraph { endpointClient: AxiosInstance } - private constructor(options: NetworkSubgraphOptions) { + private constructor(options: SubgraphOptions) { + this.name = options.name this.logger = options.logger this.freshnessChecker = options.subgraphFreshnessChecker - this.networkSubgraphConfigEndpoint = options.endpoint - this.networkSubgraphDeploymentEndpoint = - options.deployment?.graphNode.getQueryEndpoint(options.deployment.id.ipfsHash) + this.subgraphConfigEndpoint = options.endpoint + this.subgraphDeploymentEndpoint = options.deployment?.graphNode.getQueryEndpoint( + options.deployment.id.ipfsHash, + ) if (options.endpoint) { this.endpointClient = axios.create({ @@ -74,7 +79,7 @@ export class NetworkSubgraph { // Don't transform responses transformResponse: (data) => data, }) - this.endpoint = this.networkSubgraphConfigEndpoint + this.endpoint = this.subgraphConfigEndpoint } if (options.deployment) { @@ -89,23 +94,24 @@ export class NetworkSubgraph { status, endpointClient: graphNodeEndpointClient, } - this.endpoint = this.networkSubgraphDeploymentEndpoint + this.endpoint = this.subgraphDeploymentEndpoint } } public static async create({ logger: parentLogger, + name, endpoint, deployment, subgraphFreshnessChecker, - }: NetworkSubgraphCreateOptions): Promise { + }: SubgraphCreateOptions): Promise { // Either an endpoint or a deployment needs to be provided; the CLI // validation should already guarantee that but we're asserting this again // here, just to be on the safe side console.assert(endpoint || deployment) const logger = parentLogger.child({ - component: 'NetworkSubgraph', + component: name, endpoint, deployment: deployment?.deployment.ipfsHash, }) @@ -120,6 +126,7 @@ export class NetworkSubgraph { if (deployment) { const status = await monitorDeployment({ + name, logger, graphNode: deployment.graphNode, deployment: deployment.deployment, @@ -132,22 +139,23 @@ export class NetworkSubgraph { } } - // Create the network subgraph instance - const networkSubgraph = new NetworkSubgraph({ + // Create the subgraph instance + const subgraph = new SubgraphClient({ + name, logger, endpoint, deployment: deploymentInfo, subgraphFreshnessChecker, }) - // If we don't have a network subgraph endpoint configured, we + // If we don't have a subgraph endpoint configured, we // need to wait until the deployment is synced if (!endpoint) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion await deploymentInfo!.status.filter((status) => status.synced).value() } - return networkSubgraph + return subgraph } private async getClient(): Promise { @@ -156,24 +164,24 @@ export class NetworkSubgraph { const healthy = status.synced && status.health === 'healthy' if (healthy) { - this.logger.trace('Use own deployment for network subgraph query') - this.endpoint = this.networkSubgraphDeploymentEndpoint + this.logger.trace(`Use own deployment for ${this.name} query`) + this.endpoint = this.subgraphDeploymentEndpoint return this.deployment.endpointClient } else if (this.endpointClient) { - this.logger.trace('Use provided endpoint for network subgraph query') - this.endpoint = this.networkSubgraphConfigEndpoint + this.logger.trace(`Use provided endpoint for ${this.name} query`) + this.endpoint = this.subgraphConfigEndpoint return this.endpointClient } else { // We have no endpoint and our deployment is not synced or unhealthy; // there's no way to proceed from here, so crash this.logger.critical( - `No network subgraph deployment endpoint provided and network subgraph deployment is unhealthy`, + `No ${this.name} deployment endpoint provided and ${this.name} deployment is unhealthy`, ) process.exit(1) } } else { - this.logger.trace('Use provided endpoint for network subgraph query') - this.endpoint = this.networkSubgraphConfigEndpoint + this.logger.trace(`Use provided endpoint for ${this.name} query`) + this.endpoint = this.subgraphConfigEndpoint // eslint-disable-next-line @typescript-eslint/no-non-null-assertion return this.endpointClient! } @@ -217,10 +225,12 @@ export class NetworkSubgraph { } const monitorDeployment = async ({ + name, logger, graphNode, deployment, }: { + name: string logger: Logger graphNode: GraphNode deployment: SubgraphDeploymentID @@ -235,7 +245,7 @@ const monitorDeployment = async ({ return timer(60_000).reduce(async (lastStatus) => { try { - logger.trace(`Checking the network subgraph deployment status`) + logger.trace(`Checking the ${name} deployment status`) const indexingStatuses = await graphNode.indexingStatus([deployment]) const indexingStatus = indexingStatuses.pop() @@ -253,7 +263,7 @@ const monitorDeployment = async ({ // If failed for the first time, log an error if (!lastStatus || (!lastStatus.fatalError && status.fatalError)) { - logger.error(`Failed to index network subgraph deployment`, { + logger.error(`Failed to index ${name} deployment`, { err: status.fatalError, latestBlock: status.latestBlock, }) @@ -271,8 +281,8 @@ const monitorDeployment = async ({ const syncedPercent = ((100 * latestBlock) / chainHeadBlock).toFixed(2) - logger.info( - `Network subgraph is synced ${syncedPercent}% (block #${latestBlock} of #${chainHeadBlock})`, + logger.debug( + `${name} is synced ${syncedPercent}% (block #${latestBlock} of #${chainHeadBlock})`, ) } diff --git a/packages/indexer-common/src/subgraphs.ts b/packages/indexer-common/src/subgraphs.ts index 78aaa9074..9ffb3fb06 100644 --- a/packages/indexer-common/src/subgraphs.ts +++ b/packages/indexer-common/src/subgraphs.ts @@ -9,7 +9,7 @@ import { } from './indexer-management' import { DocumentNode, print } from 'graphql' import gql from 'graphql-tag' -import { QueryResult } from './network-subgraph' +import { QueryResult } from './subgraph-client' import { mergeSelectionSets, sleep } from './utils' export enum SubgraphIdentifierType { diff --git a/packages/indexer-common/src/tap-subgraph.ts b/packages/indexer-common/src/tap-subgraph.ts deleted file mode 100644 index 0264c18f8..000000000 --- a/packages/indexer-common/src/tap-subgraph.ts +++ /dev/null @@ -1,63 +0,0 @@ -import axios, { AxiosInstance, AxiosResponse } from 'axios' -import { DocumentNode, print } from 'graphql' -import { CombinedError } from '@urql/core' -import { QueryResult } from './network-subgraph' -import { Logger } from '@graphprotocol/common-ts' -import { SubgraphFreshnessChecker } from './subgraphs' -export class TAPSubgraph { - endpointClient: AxiosInstance - freshnessChecker: SubgraphFreshnessChecker - logger: Logger - - constructor( - endpoint: string, - freshnessChecker: SubgraphFreshnessChecker, - logger: Logger, - ) { - this.endpointClient = axios.create({ - baseURL: endpoint, - headers: { 'content-type': 'application/json' }, - - // Don't parse responses as JSON - responseType: 'text', - - // Don't transform responses - transformResponse: (data) => data, - }) - this.freshnessChecker = freshnessChecker - this.logger = logger - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async checkedQuery( - query: DocumentNode, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - variables?: Record, - ): Promise> { - return this.freshnessChecker.checkedQuery(this, query, variables) - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async query( - query: DocumentNode, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - variables?: Record, - ): Promise> { - // magic solution for jest tests https://stackoverflow.com/questions/69976411/jest-tlswrap-open-handle-error-using-simple-node-postgres-pool-query-fixed-wit/70012434#70012434 - await Promise.resolve(process.nextTick(Boolean)) - const response = await this.endpointClient.post('', { - query: print(query), - variables, - }) - const data = JSON.parse(response.data) - this.logger.trace('TAP Subgraph query', { data }) - if (data.errors) { - return { error: new CombinedError({ graphQLErrors: data.errors }) } - } - return data - } - - async queryRaw(body: string): Promise { - return await this.endpointClient.post('', body) - } -} diff --git a/packages/indexer-common/src/transactions.ts b/packages/indexer-common/src/transactions.ts index 9a73c08c9..0e557ecef 100644 --- a/packages/indexer-common/src/transactions.ts +++ b/packages/indexer-common/src/transactions.ts @@ -20,7 +20,7 @@ import delay from 'delay' import { TransactionMonitoring } from './network-specification' import { IndexerError, indexerError, IndexerErrorCode } from './errors' import { TransactionConfig, TransactionType } from './types' -import { NetworkSubgraph } from './network-subgraph' +import { SubgraphClient } from './subgraph-client' import gql from 'graphql-tag' export class TransactionManager { @@ -321,7 +321,7 @@ export class TransactionManager { async monitorNetworkPauses( logger: Logger, contracts: NetworkContracts, - networkSubgraph: NetworkSubgraph, + networkSubgraph: SubgraphClient, ): Promise> { return timer(60_000) .reduce( diff --git a/packages/indexer-service/src/commands/start.ts b/packages/indexer-service/src/commands/start.ts index 60abe9a15..e28cdabd7 100644 --- a/packages/indexer-service/src/commands/start.ts +++ b/packages/indexer-service/src/commands/start.ts @@ -25,7 +25,7 @@ import { GraphNode, monitorEligibleAllocations, Network, - NetworkSubgraph, + SubgraphClient, registerIndexerErrorMetrics, resolveChainId, validateProviderNetworkIdentifier, @@ -338,7 +338,8 @@ export default { const networkIdentifier = await networkProvider.getNetwork() const protocolNetwork = resolveChainId(networkIdentifier.chainId) - const networkSubgraph = await NetworkSubgraph.create({ + const networkSubgraph = await SubgraphClient.create({ + name: 'NetworkSubgraph', logger, endpoint: argv.networkSubgraphEndpoint, deployment: argv.networkSubgraphDeployment diff --git a/packages/indexer-service/src/server/__tests__/server.test.ts b/packages/indexer-service/src/server/__tests__/server.test.ts index a74dd8fc1..b075f39d5 100644 --- a/packages/indexer-service/src/server/__tests__/server.test.ts +++ b/packages/indexer-service/src/server/__tests__/server.test.ts @@ -29,7 +29,7 @@ import { IndexerManagementClient, IndexerManagementModels, monitorEligibleAllocations, - NetworkSubgraph, + SubgraphClient, QueryFeeModels, getTestProvider, GraphNode, @@ -48,7 +48,7 @@ let models: IndexerManagementModels let queryFeeModels: QueryFeeModels let address: string let contracts: NetworkContracts -let networkSubgraph: NetworkSubgraph +let networkSubgraph: SubgraphClient let client: IndexerManagementClient let receiptManager: AllocationReceiptManager @@ -67,7 +67,8 @@ const setup = async () => { const queryEndpoint = 'http://127.0.0.1:8000/' const INDEXER_TEST_API_KEY: string = process.env['INDEXER_TEST_API_KEY'] || '' - networkSubgraph = await NetworkSubgraph.create({ + networkSubgraph = await SubgraphClient.create({ + name: 'NetworkSubgraph', logger, endpoint: `https://gateway-arbitrum.network.thegraph.com/api/${INDEXER_TEST_API_KEY}/subgraphs/name/graphprotocol/graph-network-arbitrum-sepolia`, deployment: undefined, diff --git a/packages/indexer-service/src/server/index.ts b/packages/indexer-service/src/server/index.ts index 68db157cd..95f936810 100644 --- a/packages/indexer-service/src/server/index.ts +++ b/packages/indexer-service/src/server/index.ts @@ -16,7 +16,7 @@ import { indexerError, IndexerErrorCode, IndexerManagementClient, - NetworkSubgraph, + SubgraphClient, } from '@graphprotocol/indexer-common' import { createCostServer } from './cost' import { createOperatorServer } from './operator' @@ -36,7 +36,7 @@ export interface ServerOptions { dependencies: { [key: string]: string } } operatorPublicKey: string - networkSubgraph: NetworkSubgraph + networkSubgraph: SubgraphClient networkSubgraphAuthToken: string | undefined serveNetworkSubgraph: boolean infoRateLimit: number