*: Add support for local querying of Epoch and TAP subgraphs

docs/networks/arbitrum-one.md

@@ -38,17 +38,19 @@ Other network contracts can be found in [graphprotocol/contracts](https://github
 ### Indexer Agent
 
 | Environment Variable                        | CLI Argument                    | Value                                                                                                                     |
-| ------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
+|---------------------------------------------|---------------------------------| ------------------------------------------------------------------------------------------------------------------------- |
 | `INDEXER_AGENT_ETHEREUM`                    | `--ethereum`                    | An Arbitrum mainnet node/provider                                                                                         |
 | `INDEXER_AGENT_INDEXER_ADDRESS`             | `--indexer-address`             | Ethereum address of mainnet indexer                                                                                       |
 | `INDEXER_AGENT_INDEXER_GEO_COORDINATES`     | `--indexer-geo-coordinates`     | Geo coordinates of mainnet indexer infrastructure                                                                         |
 | `INDEXER_AGENT_MNEMONIC`                    | `--mnemonic`                    | Ethereum mnemonic for mainnet operator                                                                                    |
-| `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2FDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3DDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp)   |
-| `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT`   | `--network-subgraph-endpoint`   | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp`   |
 | `INDEXER_AGENT_GATEWAY_ENDPOINT`            | `--gateway-endpoint`            | `https://gateway-arbitrum.network.thegraph.com/`                                                                          |
 | `INDEXER_AGENT_GAS_PRICE_MAX`               | `--gas-price-max`               | `50`                                                                                                                      |
+| `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2FDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3DDZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp)   |
+| `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT`   | `--network-subgraph-endpoint`   | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/DZz4kDTdmzWLWsV373w2bSmoar3umKKH9y82SUKr5qmp`   |
+| `INDEXER_AGENT_EPOCH_SUBGRAPH_DEPLOYMENT`   | `--epoch-subgraph-deployment`   | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2F4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3D4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB)   |
 | `INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT`     | `--epoch-subgraph-endpoint`     | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4KFYqUWRTZQ9gn7GPHC6YQ2q15chJfVrX43ezYcwkgxB`   |
-| `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT` | `--tap-subgraph-endpoint` | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1` |
+| `INDEXER_AGENT_TAP_SUBGRAPH_DEPLOYMENT`     | `--tap-subgraph-deployment`     | [![Dynamic JSON Badge](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fthegraph.com%2Fexplorer%2F_next%2Fdata%2F5PBypsdmUEy39BcWLsyp0%2Fsubgraphs%2F4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1.json%3Fview%3DAbout%26chain%3Darbitrum-one%26id%3D4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1&query=%24.pageProps.subgraph.currentVersion.subgraphDeployment.ipfsHash&label=deployment-id)](https://thegraph.com/explorer/subgraphs/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1)   |
+| `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT`       | `--tap-subgraph-endpoint`       | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/4sukbNVTzGELnhdnpyPqsf1QqtzNHEYKKmJkgaT8z6M1` |
 
 
 In order to avoid collecting or claiming query fees below a certain threshold

docs/networks/arbitrum-sepolia.md

@@ -44,15 +44,17 @@ testnet (for now) are Mainnet subgraphs. This means:
 ### Indexer Agent
 
 | Environment Variable                        | CLI Argument                    | Value                                                                                                                   |
-| ------------------------------------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+|---------------------------------------------|---------------------------------| ----------------------------------------------------------------------------------------------------------------------- |
 | `INDEXER_AGENT_ETHEREUM`                    | `--ethereum`                    | An Arbitrum Sepolia node/provider                                                                                       |
 | `INDEXER_AGENT_INDEXER_ADDRESS`             | `--indexer-address`             | Ethereum address of testnet indexer                                                                                     |
 | `INDEXER_AGENT_INDEXER_GEO_COORDINATES`     | `--indexer-geo-coordinates`     | Geo coordinates of testnet indexer infrastructure                                                                       |
 | `INDEXER_AGENT_MNEMONIC`                    | `--mnemonic`                    | Ethereum mnemonic for testnet operator                                                                                  |
+| `INDEXER_AGENT_GATEWAY_ENDPOINT`            | `--gateway-endpoint`            | `https://gateway-arbitrum.testnet.thegraph.com/`                                                                        |
 | `INDEXER_AGENT_NETWORK_SUBGRAPH_DEPLOYMENT` | `--network-subgraph-deployment` | `QmXnGVrg6DvscnvJd86aHAPLGyGrkM17weMrAsFAEMmQLL`                                                                        |
 | `INDEXER_AGENT_NETWORK_SUBGRAPH_ENDPOINT`   | `--network-subgraph-endpoint`   | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/3xQHhMudr1oh69ut36G2mbzpYmYxwqCeU6wwqyCDCnqV` |
-| `INDEXER_AGENT_GATEWAY_ENDPOINT`            | `--gateway-endpoint`            | `https://gateway-arbitrum.testnet.thegraph.com/`                                                                        |
+| `INDEXER_AGENT_EPOCH_SUBGRAPH_DEPLOYMENT`   | `--epoch-subgraph-deployment`   | `QmNNn48AU8CZei8sYLXrXABMHkiXAuQDvFaZHYYPryKEvF`                                                                        |
 | `INDEXER_AGENT_EPOCH_SUBGRAPH_ENDPOINT`     | `--epoch-subgraph-endpoint`     | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/BhnsdeZihU4SuokxZMLF4FQBVJ3jgtZf6v51gHvz3bSS` |
+| `INDEXER_AGENT_TAP_SUBGRAPH_DEPLOYMENT`     | `--tap-subgraph-deployment`     | `QmUiLdbsk6c51UMdcNBxsP3KadJpkmp6a3k2NCprR4ZFeM`                                                                        |
 | `INDEXER_AGENT_TAP_SUBGRAPH_ENDPOINT`       | `--tap-subgraph-endpoint`       | `https://gateway-arbitrum.network.thegraph.com/api/[api-key]/subgraphs/id/7ubx365MiqBH5iUz6XWXWT8PTof5BVAyEzdb8m17RvbD` |
 
 In order to avoid collecting or claiming query fees below a certain threshold

packages/indexer-agent/src/agent.ts

@@ -231,7 +231,7 @@ export class Agent {
       async ({ network, operator }: NetworkAndOperator) => {
         try {
           await operator.ensureGlobalIndexingRule()
-          await this.ensureNetworkSubgraphIsIndexing(network)
+          await this.ensureAllSubgraphsIndexing(network)
           await network.register()
         } catch (err) {
           this.logger.critical(
@@ -1241,38 +1241,56 @@ export class Agent {
     )
   }
 
-  // TODO: This could be a initialization check inside Network.create() once/if the Indexer Service
-  // uses Network instances.
-  async ensureNetworkSubgraphIsIndexing(network: Network) {
+  // TODO: After indexer-service deprecation: Move to be an initialization check inside Network.create()
+  async ensureSubgraphIndexing(deployment: string, networkIdentifier: string) {
+    try {
+      // TODO: Check both the local deployment and the external subgraph endpoint
+      // Make sure the subgraph is being indexed
+      await this.graphNode.ensure(
+        `indexer-agent/${deployment.slice(-10)}`,
+        new SubgraphDeploymentID(deployment),
+      )
+
+      // Validate if the Network Subgraph belongs to the current provider's network.
+      // This check must be performed after we ensure the Network Subgraph is being indexed.
+      await validateProviderNetworkIdentifier(
+        networkIdentifier,
+        deployment,
+        this.graphNode,
+        this.logger,
+      )
+    } catch (e) {
+      this.logger.warn(
+        'Failed to deploy and validate Network Subgraph on index-nodes. Will use external subgraph endpoint instead',
+        e,
+      )
+    }
+  }
+  async ensureAllSubgraphsIndexing(network: Network) {
+    // Network subgraph
     if (
       network.specification.subgraphs.networkSubgraph.deployment !== undefined
     ) {
-      try {
-        // TODO: Check both the local deployment and the external subgraph endpoint
-        // Make sure the network subgraph is being indexed
-        await this.graphNode.ensure(
-          `indexer-agent/${network.specification.subgraphs.networkSubgraph.deployment.slice(
-            -10,
-          )}`,
-          new SubgraphDeploymentID(
-            network.specification.subgraphs.networkSubgraph.deployment,
-          ),
-        )
-
-        // Validate if the Network Subgraph belongs to the current provider's network.
-        // This check must be performed after we ensure the Network Subgraph is being indexed.
-        await validateProviderNetworkIdentifier(
-          network.specification.networkIdentifier,
-          network.specification.subgraphs.networkSubgraph.deployment,
-          this.graphNode,
-          this.logger,
-        )
-      } catch (e) {
-        this.logger.warn(
-          'Failed to deploy and validate Network Subgraph on index-nodes. Will use external subgraph endpoint instead',
-          e,
-        )
-      }
+      await this.ensureSubgraphIndexing(
+        network.specification.subgraphs.networkSubgraph.deployment,
+        network.specification.networkIdentifier,
+      )
+    }
+    // Epoch subgraph
+    if (
+      network.specification.subgraphs.epochSubgraph.deployment !== undefined
+    ) {
+      await this.ensureSubgraphIndexing(
+        network.specification.subgraphs.epochSubgraph.deployment,
+        network.specification.networkIdentifier,
+      )
+    }
+    // TAP subgraph
+    if (network.specification.subgraphs.tapSubgraph?.deployment !== undefined) {
+      await this.ensureSubgraphIndexing(
+        network.specification.subgraphs.tapSubgraph.deployment,
+        network.specification.networkIdentifier,
+      )
     }
   }
 }

packages/indexer-agent/src/commands/start.ts

@@ -140,7 +140,7 @@ export const start = {
         },
       })
       .option('network-subgraph-deployment', {
-        description: 'Network subgraph deployment',
+        description: 'Network subgraph deployment (for local hosting)',
         array: false,
         type: 'string',
         group: 'Network Subgraph',
@@ -151,6 +151,12 @@ export const start = {
         type: 'string',
         group: 'Network Subgraph',
       })
+      .option('tap-subgraph-deployment', {
+        description: 'TAP subgraph deployment (for local hosting)',
+        array: false,
+        type: 'string',
+        group: 'TAP Subgraph',
+      })
       .option('tap-subgraph-endpoint', {
         description: 'Endpoint to query the tap subgraph from',
         array: false,
@@ -163,6 +169,12 @@ export const start = {
         default: false,
         group: 'Network Subgraph',
       })
+      .option('epoch-subgraph-deployment', {
+        description: 'Epoch subgraph deployment (for local hosting)',
+        array: false,
+        type: 'string',
+        group: 'Network Subgraph',
+      })
       .option('epoch-subgraph-endpoint', {
         description: 'Endpoint to query the epoch block oracle subgraph from',
         array: false,
@@ -370,11 +382,11 @@ export async function createNetworkSpecification(
       url: argv.networkSubgraphEndpoint,
     },
     epochSubgraph: {
-      // TODO: We should consider indexing the Epoch Subgraph, similar
-      // to how we currently do it for the Network Subgraph.
+      deployment: argv.epochSubgraphDeployment,
       url: argv.epochSubgraphEndpoint,
     },
     tapSubgraph: {
+      deployment: argv.tapSubgraphDeployment,
       url: argv.tapSubgraphEndpoint,
     },
   }

packages/indexer-agent/src/syncing-server.ts

@@ -7,13 +7,13 @@ import { Logger } from '@graphprotocol/common-ts'
 import { parse } from 'graphql'
 import {
   NetworkMapped,
-  NetworkSubgraph,
+  SubgraphClient,
   resolveChainId,
 } from '@graphprotocol/indexer-common'
 
 export interface CreateSyncingServerOptions {
   logger: Logger
-  networkSubgraphs: NetworkMapped<NetworkSubgraph>
+  networkSubgraphs: NetworkMapped<SubgraphClient>
   port: number
 }
 

packages/indexer-common/src/__tests__/subgraph.test.ts

@@ -5,7 +5,7 @@ import {
   ProviderInterface,
   SubgraphQueryInterface,
 } from '../subgraphs'
-import { QueryResult } from '../network-subgraph'
+import { QueryResult } from '../subgraph-client'
 import gql from 'graphql-tag'
 import { mergeSelectionSets } from '../utils'
 

packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts

@@ -2,7 +2,7 @@ import { Address, Eventual, createLogger, createMetrics } from '@graphprotocol/c
 import {
   Allocation,
   AllocationsResponse,
-  NetworkSubgraph,
+  SubgraphClient,
   QueryFeeModels,
   QueryResult,
   TapCollector,
@@ -11,7 +11,6 @@ import {
   TransactionManager,
 } from '@graphprotocol/indexer-common'
 import { NetworkContracts as TapContracts } from '@semiotic-labs/tap-contracts-bindings'
-import { TAPSubgraph } from '../../tap-subgraph'
 import { NetworkSpecification } from 'indexer-common/src/network-specification'
 import { createMockAllocation } from '../../indexer-management/__tests__/helpers.test'
 import { getContractAddress } from 'ethers/lib/utils'
@@ -155,10 +154,10 @@ const setup = () => {
 
     const tapSubgraph = {
       query: mockQueryTapSubgraph,
-    } as unknown as TAPSubgraph
+    } as unknown as SubgraphClient
     const networkSubgraph = {
       query: mockQueryNetworkSubgraph,
-    } as unknown as NetworkSubgraph
+    } as unknown as SubgraphClient
 
     tapCollector = TapCollector.create({
       logger,

packages/indexer-common/src/allocations/escrow-accounts.ts

@@ -1,5 +1,5 @@
 import { Address, toAddress } from '@graphprotocol/common-ts'
-import { TAPSubgraph } from '../tap-subgraph'
+import { SubgraphClient } from '../subgraph-client'
 import gql from 'graphql-tag'
 
 type U256 = bigint
@@ -44,7 +44,7 @@ export class EscrowAccounts {
 }
 
 export const getEscrowAccounts = async (
-  tapSubgraph: TAPSubgraph,
+  tapSubgraph: SubgraphClient,
   indexer: Address,
 ): Promise<EscrowAccounts> => {
   const result = await tapSubgraph.query<EscrowAccountResponse>(

packages/indexer-common/src/allocations/query-fees.ts

@@ -20,12 +20,12 @@ import {
   ensureAllocationSummary,
   TransactionManager,
   specification as spec,
+  SubgraphClient,
 } from '..'
 import { DHeap } from '@thi.ng/heaps'
 import { BigNumber, BigNumberish, Contract } from 'ethers'
 import { Op } from 'sequelize'
 import pReduce from 'p-reduce'
-import { NetworkSubgraph } from '../network-subgraph'
 
 // Receipts are collected with a delay of 20 minutes after
 // the corresponding allocation was closed
@@ -71,7 +71,7 @@ export interface AllocationReceiptCollectorOptions {
   allocations: Eventual<Allocation[]>
   models: QueryFeeModels
   networkSpecification: spec.NetworkSpecification
-  networkSubgraph: NetworkSubgraph
+  networkSubgraph: SubgraphClient
 }
 
 export interface ReceiptCollector {
@@ -94,7 +94,7 @@ export class AllocationReceiptCollector implements ReceiptCollector {
   declare voucherRedemptionBatchThreshold: BigNumber
   declare voucherRedemptionMaxBatchSize: number
   declare protocolNetwork: string
-  declare networkSubgraph: NetworkSubgraph
+  declare networkSubgraph: SubgraphClient
 
   // eslint-disable-next-line @typescript-eslint/no-empty-function -- Private constructor to prevent direct instantiation
   private constructor() {}

packages/indexer-common/src/allocations/tap-collector.ts

@@ -26,8 +26,7 @@ import {
 } from '..'
 import { BigNumber } from 'ethers'
 import pReduce from 'p-reduce'
-import { TAPSubgraph } from '../tap-subgraph'
-import { NetworkSubgraph, QueryResult } from '../network-subgraph'
+import { SubgraphClient, QueryResult } from '../subgraph-client'
 import gql from 'graphql-tag'
 import { getEscrowAccounts } from './escrow-accounts'
 
@@ -52,8 +51,8 @@ interface TapCollectorOptions {
   allocations: Eventual<Allocation[]>
   models: QueryFeeModels
   networkSpecification: spec.NetworkSpecification
-  tapSubgraph: TAPSubgraph
-  networkSubgraph: NetworkSubgraph
+  tapSubgraph: SubgraphClient
+  networkSubgraph: SubgraphClient
 }
 
 interface ValidRavs {
@@ -107,8 +106,8 @@ export class TapCollector {
   declare allocations: Eventual<Allocation[]>
   declare ravRedemptionThreshold: BigNumber
   declare protocolNetwork: string
-  declare tapSubgraph: TAPSubgraph
-  declare networkSubgraph: NetworkSubgraph
+  declare tapSubgraph: SubgraphClient
+  declare networkSubgraph: SubgraphClient
   declare finalityTime: number
   declare indexerAddress: Address
 

packages/indexer-common/src/allocations/types.ts

@@ -1,5 +1,5 @@
 import { BigNumber } from 'ethers'
-import { NetworkSubgraph, SubgraphDeployment } from '@graphprotocol/indexer-common'
+import { SubgraphClient, SubgraphDeployment } from '@graphprotocol/indexer-common'
 
 import { Logger, Address } from '@graphprotocol/common-ts'
 
@@ -31,7 +31,7 @@ export enum AllocationStatus {
 export interface MonitorEligibleAllocationsOptions {
   indexer: Address
   logger: Logger
-  networkSubgraph: NetworkSubgraph
+  networkSubgraph: SubgraphClient
   interval: number
   protocolNetwork: string
 }