diff --git a/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts b/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts index abac0a88b2..1cfa51e46f 100644 --- a/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts +++ b/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts @@ -62,3 +62,27 @@ export async function getVaultsPnl( return result.rows; } + +export async function getLatestVaultPnl(): Promise { + const result: { + rows: PnlTicksFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT DISTINCT ON ("subaccountId") + "id", + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "blockHeight", + "blockTime" + FROM ${VAULT_HOURLY_PNL_VIEW} + ORDER BY "subaccountId", "blockTime" DESC; + `, + ) as unknown as { + rows: PnlTicksFromDatabase[], + }; + + return result.rows; +} diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 4035552057..03a453daff 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -24,6 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; import { DateTime, Settings } from 'luxon'; import Big from 'big.js'; import config from '../../../../src/config'; +import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl'; describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; @@ -131,6 +132,7 @@ describe('vault-controller#V4', () => { await dbHelpers.clearData(); await VaultPnlTicksView.refreshDailyView(); await VaultPnlTicksView.refreshHourlyView(); + clearVaultStartPnl(); config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev; config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev; config.VAULT_PNL_START_DATE = vaultPnlStartDatePrev; @@ -653,6 +655,7 @@ describe('vault-controller#V4', () => { } await VaultPnlTicksView.refreshDailyView(); await VaultPnlTicksView.refreshHourlyView(); + await startVaultStartPnlCache(); return createdTicks; } diff --git a/indexer/services/comlink/src/caches/vault-start-pnl.ts b/indexer/services/comlink/src/caches/vault-start-pnl.ts new file mode 100644 index 0000000000..44e25c5d63 --- /dev/null +++ b/indexer/services/comlink/src/caches/vault-start-pnl.ts @@ -0,0 +1,33 @@ +import { NodeEnv } from '@dydxprotocol-indexer/base'; +import { + PnlTicksFromDatabase, + PnlTicksTable, +} from '@dydxprotocol-indexer/postgres'; +import _ from 'lodash'; + +import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers'; +import { VaultMapping } from '../types'; + +let vaultStartPnl: PnlTicksFromDatabase[] = []; + +export async function startVaultStartPnlCache(): Promise { + const vaultMapping: VaultMapping = await getVaultMapping(); + vaultStartPnl = await PnlTicksTable.getLatestPnlTick( + _.keys(vaultMapping), + // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't + // created exactly on the hour. + getVaultPnlStartDate().plus({ minutes: 10 }), + ); +} + +export function getVaultStartPnl(): PnlTicksFromDatabase[] { + return vaultStartPnl; +} + +export function clearVaultStartPnl(): void { + if (process.env.NODE_ENV !== NodeEnv.TEST) { + throw Error('cannot clear vault start pnl cache outside of test environment'); + } + + vaultStartPnl = []; +} diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 4df7f35fe4..ee77fe4ec8 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -1,7 +1,6 @@ -import { logger, stats } from '@dydxprotocol-indexer/base'; +import { stats } from '@dydxprotocol-indexer/base'; import { PnlTicksFromDatabase, - PnlTicksTable, perpetualMarketRefresher, PerpetualMarketFromDatabase, USDC_ASSET_ID, @@ -22,7 +21,6 @@ import { BlockFromDatabase, FundingIndexUpdatesTable, PnlTickInterval, - VaultTable, VaultFromDatabase, MEGAVAULT_SUBACCOUNT_ID, TransferFromDatabase, @@ -42,10 +40,13 @@ import { } from 'tsoa'; import { getReqRateLimiter } from '../../../caches/rate-limiters'; +import { getVaultStartPnl } from '../../../caches/vault-start-pnl'; import config from '../../../config'; import { aggregateHourlyPnlTicks, getSubaccountResponse, + getVaultMapping, + getVaultPnlStartDate, handleControllerError, } from '../../../lib/helpers'; import { rateLimiterMiddleware } from '../../../lib/rate-limit'; @@ -108,7 +109,7 @@ class VaultController extends Controller { getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), - getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)), + getLatestPnlTick(_.values(vaultSubaccounts)), getFirstMainVaultTransferDateTime(), ]); stats.timing( @@ -162,7 +163,7 @@ class VaultController extends Controller { getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), - getLatestPnlTicks(_.keys(vaultSubaccounts)), + getLatestPnlTicks(), ]); const latestTicksBySubaccountId: Dictionary = _.keyBy( latestTicks, @@ -348,27 +349,13 @@ async function getVaultSubaccountPnlTicks( windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds } - const [ - pnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - VaultPnlTicksView.getVaultsPnl( - resolution, - windowSeconds, - getVaultPnlStartDate(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); + const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + resolution, + windowSeconds, + getVaultPnlStartDate(), + ); - return adjustVaultPnlTicks(pnlTicks, adjustByPnlTicks); + return adjustVaultPnlTicks(pnlTicks, getVaultStartPnl()); } async function getVaultPositions( @@ -559,60 +546,26 @@ function getPnlTicksWithCurrentTick( return pnlTicks.concat([currentTick]); } -export async function getLatestPnlTicks( - vaultSubaccountIds: string[], -): Promise { - const [ - latestPnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - DateTime.now().toUTC(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); +export async function getLatestPnlTicks(): Promise { + const latestPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getLatestVaultPnl(); const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks( latestPnlTicks, - adjustByPnlTicks, + getVaultStartPnl(), ); return adjustedPnlTicks; } export async function getLatestPnlTick( - vaultSubaccountIds: string[], vaults: VaultFromDatabase[], ): Promise { - const [ - pnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - VaultPnlTicksView.getVaultsPnl( - PnlTickInterval.hour, - config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60, - getVaultPnlStartDate(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); + const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + PnlTickInterval.hour, + config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60, + getVaultPnlStartDate(), + ); const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks( pnlTicks, - adjustByPnlTicks, + getVaultStartPnl(), ); // Aggregate and get pnl tick closest to the hour const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( @@ -802,41 +755,4 @@ function adjustVaultPnlTicks( }); } -async function getVaultMapping(): Promise { - const vaults: VaultFromDatabase[] = await VaultTable.findAll( - {}, - [], - {}, - ); - const vaultMapping: VaultMapping = _.zipObject( - vaults.map((vault: VaultFromDatabase): string => { - return SubaccountTable.uuid(vault.address, 0); - }), - vaults, - ); - const validVaultMapping: VaultMapping = {}; - for (const subaccountId of _.keys(vaultMapping)) { - const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId( - vaultMapping[subaccountId].clobPairId, - ); - if (perpetual === undefined) { - logger.warning({ - at: 'VaultController#getVaultPositions', - message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + - 'perpetual market.', - subaccountId, - }); - continue; - } - validVaultMapping[subaccountId] = vaultMapping[subaccountId]; - } - return validVaultMapping; -} - -function getVaultPnlStartDate(): DateTime { - const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC(); - return startDate; -} - export default router; diff --git a/indexer/services/comlink/src/index.ts b/indexer/services/comlink/src/index.ts index 584fe8edff..114fb5a713 100644 --- a/indexer/services/comlink/src/index.ts +++ b/indexer/services/comlink/src/index.ts @@ -5,6 +5,7 @@ import { } from '@dydxprotocol-indexer/base'; import { perpetualMarketRefresher, liquidityTierRefresher } from '@dydxprotocol-indexer/postgres'; +import { startVaultStartPnlCache } from './caches/vault-start-pnl'; import config from './config'; import IndexV4 from './controllers/api/index-v4'; import { connect as connectToRedis } from './helpers/redis/redis-controller'; @@ -42,6 +43,8 @@ async function start() { ]); wrapBackgroundTask(perpetualMarketRefresher.start(), true, 'startUpdatePerpetualMarkets'); wrapBackgroundTask(liquidityTierRefresher.start(), true, 'startUpdateLiquidityTiers'); + // Initialize cache for vault start PnL + await startVaultStartPnlCache(); await connectToRedis(); logger.info({ diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index b072ca4b0d..97ee94915d 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -24,6 +24,7 @@ import { AssetFromDatabase, AssetColumns, MarketColumns, + VaultFromDatabase, VaultTable, perpetualMarketRefresher, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; import express from 'express'; @@ -47,6 +48,7 @@ import { PerpetualPositionWithFunding, Risk, SubaccountResponseObject, + VaultMapping, } from '../types'; import { ZERO, ZERO_USDC_POSITION } from './constants'; import { InvalidParamError, NotFoundError } from './errors'; @@ -720,3 +722,42 @@ export function aggregateHourlyPnlTicks( }; }); } + +/* ------- VAULT HELPERS ------- */ + +export async function getVaultMapping(): Promise { + const vaults: VaultFromDatabase[] = await VaultTable.findAll( + {}, + [], + {}, + ); + const vaultMapping: VaultMapping = _.zipObject( + vaults.map((vault: VaultFromDatabase): string => { + return SubaccountTable.uuid(vault.address, 0); + }), + vaults, + ); + const validVaultMapping: VaultMapping = {}; + for (const subaccountId of _.keys(vaultMapping)) { + const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId( + vaultMapping[subaccountId].clobPairId, + ); + if (perpetual === undefined) { + logger.warning({ + at: 'get-vault-mapping', + message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + + 'perpetual market.', + subaccountId, + }); + continue; + } + validVaultMapping[subaccountId] = vaultMapping[subaccountId]; + } + return validVaultMapping; +} + +export function getVaultPnlStartDate(): DateTime { + const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC(); + return startDate; +} diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index fc83a92e29..30e540d4b0 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -24,6 +24,7 @@ import { TradeType, TradingRewardAggregationPeriod, TransferType, + VaultFromDatabase, } from '@dydxprotocol-indexer/postgres'; import { RedisOrder } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -691,6 +692,10 @@ export interface MegavaultHistoricalPnlRequest { export interface VaultsHistoricalPnlRequest extends MegavaultHistoricalPnlRequest {} +export interface VaultMapping { + [subaccountId: string]: VaultFromDatabase, +} + /* ------- Affiliates Types ------- */ export interface AffiliateMetadataRequest{ address: string,