Skip to content

Commit

Permalink
Improve vault start pnl query. (backport #2664) (#2665)
Browse files Browse the repository at this point in the history
Co-authored-by: vincentwschau <[email protected]>
  • Loading branch information
mergify[bot] and vincentwschau authored Dec 17, 2024
1 parent cea6c05 commit 58c0475
Show file tree
Hide file tree
Showing 7 changed files with 130 additions and 105 deletions.
24 changes: 24 additions & 0 deletions indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,27 @@ export async function getVaultsPnl(

return result.rows;
}

export async function getLatestVaultPnl(): Promise<PnlTicksFromDatabase[]> {
const result: {
rows: PnlTicksFromDatabase[],
} = await knexReadReplica.getConnection().raw(
`
SELECT DISTINCT ON ("subaccountId")
"id",
"subaccountId",
"equity",
"totalPnl",
"netTransfers",
"createdAt",
"blockHeight",
"blockTime"
FROM ${VAULT_HOURLY_PNL_VIEW}
ORDER BY "subaccountId", "blockTime" DESC;
`,
) as unknown as {
rows: PnlTicksFromDatabase[],
};

return result.rows;
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers';
import { DateTime, Settings } from 'luxon';
import Big from 'big.js';
import config from '../../../../src/config';
import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl';

describe('vault-controller#V4', () => {
const latestBlockHeight: string = '25';
Expand Down Expand Up @@ -131,6 +132,7 @@ describe('vault-controller#V4', () => {
await dbHelpers.clearData();
await VaultPnlTicksView.refreshDailyView();
await VaultPnlTicksView.refreshHourlyView();
clearVaultStartPnl();
config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev;
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev;
config.VAULT_PNL_START_DATE = vaultPnlStartDatePrev;
Expand Down Expand Up @@ -653,6 +655,7 @@ describe('vault-controller#V4', () => {
}
await VaultPnlTicksView.refreshDailyView();
await VaultPnlTicksView.refreshHourlyView();
await startVaultStartPnlCache();

return createdTicks;
}
Expand Down
33 changes: 33 additions & 0 deletions indexer/services/comlink/src/caches/vault-start-pnl.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { NodeEnv } from '@dydxprotocol-indexer/base';
import {
PnlTicksFromDatabase,
PnlTicksTable,
} from '@dydxprotocol-indexer/postgres';
import _ from 'lodash';

import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers';
import { VaultMapping } from '../types';

let vaultStartPnl: PnlTicksFromDatabase[] = [];

export async function startVaultStartPnlCache(): Promise<void> {
const vaultMapping: VaultMapping = await getVaultMapping();
vaultStartPnl = await PnlTicksTable.getLatestPnlTick(
_.keys(vaultMapping),
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
);
}

export function getVaultStartPnl(): PnlTicksFromDatabase[] {
return vaultStartPnl;
}

export function clearVaultStartPnl(): void {
if (process.env.NODE_ENV !== NodeEnv.TEST) {
throw Error('cannot clear vault start pnl cache outside of test environment');
}

vaultStartPnl = [];
}
126 changes: 21 additions & 105 deletions indexer/services/comlink/src/controllers/api/v4/vault-controller.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { logger, stats } from '@dydxprotocol-indexer/base';
import { stats } from '@dydxprotocol-indexer/base';
import {
PnlTicksFromDatabase,
PnlTicksTable,
perpetualMarketRefresher,
PerpetualMarketFromDatabase,
USDC_ASSET_ID,
Expand All @@ -22,7 +21,6 @@ import {
BlockFromDatabase,
FundingIndexUpdatesTable,
PnlTickInterval,
VaultTable,
VaultFromDatabase,
MEGAVAULT_SUBACCOUNT_ID,
TransferFromDatabase,
Expand All @@ -42,10 +40,13 @@ import {
} from 'tsoa';

import { getReqRateLimiter } from '../../../caches/rate-limiters';
import { getVaultStartPnl } from '../../../caches/vault-start-pnl';
import config from '../../../config';
import {
aggregateHourlyPnlTicks,
getSubaccountResponse,
getVaultMapping,
getVaultPnlStartDate,
handleControllerError,
} from '../../../lib/helpers';
import { rateLimiterMiddleware } from '../../../lib/rate-limit';
Expand Down Expand Up @@ -108,7 +109,7 @@ class VaultController extends Controller {
getVaultPositions(vaultSubaccounts),
BlockTable.getLatest(),
getMainSubaccountEquity(),
getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)),
getLatestPnlTick(_.values(vaultSubaccounts)),
getFirstMainVaultTransferDateTime(),
]);
stats.timing(
Expand Down Expand Up @@ -162,7 +163,7 @@ class VaultController extends Controller {
getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)),
getVaultPositions(vaultSubaccounts),
BlockTable.getLatest(),
getLatestPnlTicks(_.keys(vaultSubaccounts)),
getLatestPnlTicks(),
]);
const latestTicksBySubaccountId: Dictionary<PnlTicksFromDatabase> = _.keyBy(
latestTicks,
Expand Down Expand Up @@ -348,27 +349,13 @@ async function getVaultSubaccountPnlTicks(
windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds
}

const [
pnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
VaultPnlTicksView.getVaultsPnl(
resolution,
windowSeconds,
getVaultPnlStartDate(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
resolution,
windowSeconds,
getVaultPnlStartDate(),
);

return adjustVaultPnlTicks(pnlTicks, adjustByPnlTicks);
return adjustVaultPnlTicks(pnlTicks, getVaultStartPnl());
}

async function getVaultPositions(
Expand Down Expand Up @@ -559,60 +546,26 @@ function getPnlTicksWithCurrentTick(
return pnlTicks.concat([currentTick]);
}

export async function getLatestPnlTicks(
vaultSubaccountIds: string[],
): Promise<PnlTicksFromDatabase[]> {
const [
latestPnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
DateTime.now().toUTC(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
export async function getLatestPnlTicks(): Promise<PnlTicksFromDatabase[]> {
const latestPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getLatestVaultPnl();
const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
latestPnlTicks,
adjustByPnlTicks,
getVaultStartPnl(),
);
return adjustedPnlTicks;
}

export async function getLatestPnlTick(
vaultSubaccountIds: string[],
vaults: VaultFromDatabase[],
): Promise<PnlTicksFromDatabase | undefined> {
const [
pnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
VaultPnlTicksView.getVaultsPnl(
PnlTickInterval.hour,
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
getVaultPnlStartDate(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
PnlTickInterval.hour,
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
getVaultPnlStartDate(),
);
const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
pnlTicks,
adjustByPnlTicks,
getVaultStartPnl(),
);
// Aggregate and get pnl tick closest to the hour
const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks(
Expand Down Expand Up @@ -802,41 +755,4 @@ function adjustVaultPnlTicks(
});
}

async function getVaultMapping(): Promise<VaultMapping> {
const vaults: VaultFromDatabase[] = await VaultTable.findAll(
{},
[],
{},
);
const vaultMapping: VaultMapping = _.zipObject(
vaults.map((vault: VaultFromDatabase): string => {
return SubaccountTable.uuid(vault.address, 0);
}),
vaults,
);
const validVaultMapping: VaultMapping = {};
for (const subaccountId of _.keys(vaultMapping)) {
const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
.getPerpetualMarketFromClobPairId(
vaultMapping[subaccountId].clobPairId,
);
if (perpetual === undefined) {
logger.warning({
at: 'VaultController#getVaultPositions',
message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
'perpetual market.',
subaccountId,
});
continue;
}
validVaultMapping[subaccountId] = vaultMapping[subaccountId];
}
return validVaultMapping;
}

function getVaultPnlStartDate(): DateTime {
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
return startDate;
}

export default router;
3 changes: 3 additions & 0 deletions indexer/services/comlink/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
} from '@dydxprotocol-indexer/base';
import { perpetualMarketRefresher, liquidityTierRefresher } from '@dydxprotocol-indexer/postgres';

import { startVaultStartPnlCache } from './caches/vault-start-pnl';
import config from './config';
import IndexV4 from './controllers/api/index-v4';
import { connect as connectToRedis } from './helpers/redis/redis-controller';
Expand Down Expand Up @@ -42,6 +43,8 @@ async function start() {
]);
wrapBackgroundTask(perpetualMarketRefresher.start(), true, 'startUpdatePerpetualMarkets');
wrapBackgroundTask(liquidityTierRefresher.start(), true, 'startUpdateLiquidityTiers');
// Initialize cache for vault start PnL
await startVaultStartPnlCache();

await connectToRedis();
logger.info({
Expand Down
41 changes: 41 additions & 0 deletions indexer/services/comlink/src/lib/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
AssetFromDatabase,
AssetColumns,
MarketColumns,
VaultFromDatabase, VaultTable, perpetualMarketRefresher,
} from '@dydxprotocol-indexer/postgres';
import Big from 'big.js';
import express from 'express';
Expand All @@ -47,6 +48,7 @@ import {
PerpetualPositionWithFunding,
Risk,
SubaccountResponseObject,
VaultMapping,
} from '../types';
import { ZERO, ZERO_USDC_POSITION } from './constants';
import { InvalidParamError, NotFoundError } from './errors';
Expand Down Expand Up @@ -720,3 +722,42 @@ export function aggregateHourlyPnlTicks(
};
});
}

/* ------- VAULT HELPERS ------- */

export async function getVaultMapping(): Promise<VaultMapping> {
const vaults: VaultFromDatabase[] = await VaultTable.findAll(
{},
[],
{},
);
const vaultMapping: VaultMapping = _.zipObject(
vaults.map((vault: VaultFromDatabase): string => {
return SubaccountTable.uuid(vault.address, 0);
}),
vaults,
);
const validVaultMapping: VaultMapping = {};
for (const subaccountId of _.keys(vaultMapping)) {
const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
.getPerpetualMarketFromClobPairId(
vaultMapping[subaccountId].clobPairId,
);
if (perpetual === undefined) {
logger.warning({
at: 'get-vault-mapping',
message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
'perpetual market.',
subaccountId,
});
continue;
}
validVaultMapping[subaccountId] = vaultMapping[subaccountId];
}
return validVaultMapping;
}

export function getVaultPnlStartDate(): DateTime {
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
return startDate;
}
5 changes: 5 additions & 0 deletions indexer/services/comlink/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
TradeType,
TradingRewardAggregationPeriod,
TransferType,
VaultFromDatabase,
} from '@dydxprotocol-indexer/postgres';
import { RedisOrder } from '@dydxprotocol-indexer/v4-protos';
import Big from 'big.js';
Expand Down Expand Up @@ -691,6 +692,10 @@ export interface MegavaultHistoricalPnlRequest {

export interface VaultsHistoricalPnlRequest extends MegavaultHistoricalPnlRequest {}

export interface VaultMapping {
[subaccountId: string]: VaultFromDatabase,
}

/* ------- Affiliates Types ------- */
export interface AffiliateMetadataRequest{
address: string,
Expand Down

0 comments on commit 58c0475

Please sign in to comment.