From 0a2134617f1ef6b14d2eccb2e9509a2ce885e549 Mon Sep 17 00:00:00 2001 From: Matt Rice Date: Sun, 10 Sep 2023 18:23:11 -0400 Subject: [PATCH 1/4] fix: handle early deposits Signed-off-by: Matt Rice --- src/clients/BundleDataClient.ts | 22 +++++++++++++++++++++- src/dataworker/Dataworker.ts | 20 ++++++++++---------- src/dataworker/DataworkerUtils.ts | 13 +++++++++++++ src/dataworker/PoolRebalanceUtils.ts | 24 ++++++++++++++++++++++++ src/utils/DepositUtils.ts | 28 ++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/clients/BundleDataClient.ts b/src/clients/BundleDataClient.ts index ff792ef40..6e65685b3 100644 --- a/src/clients/BundleDataClient.ts +++ b/src/clients/BundleDataClient.ts @@ -20,6 +20,7 @@ import { flattenAndFilterUnfilledDepositsByOriginChain, updateUnfilledDepositsWithMatchedDeposit, getUniqueDepositsInRange, + getUniqueEarlyDepositsInRange, queryHistoricalDepositForFill, } from "../utils"; import { Clients } from "../common"; @@ -31,6 +32,8 @@ import { } from "../dataworker/DataworkerUtils"; import { getWidestPossibleExpectedBlockRange, isChainDisabled } from "../dataworker/PoolRebalanceUtils"; import { clients } from "@across-protocol/sdk-v2"; +// eslint-disable-next-line node/no-missing-import +import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; const { refundRequestIsValid, isUBAActivatedAtBlock } = clients; type DataCacheValue = { @@ -38,6 +41,7 @@ type DataCacheValue = { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; + earlyDeposits: FundsDepositedEvent[]; }; type DataCache = Record; @@ -175,6 +179,7 @@ export class BundleDataClient { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; + earlyDeposits: FundsDepositedEvent[]; }> { const mainnetStartBlock = getBlockRangeForChain( blockRangesForChains, @@ -197,6 +202,7 @@ export class BundleDataClient { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; + earlyDeposits: FundsDepositedEvent[]; }> { const key = JSON.stringify(blockRangesForChains); @@ -222,6 +228,7 @@ export class BundleDataClient { const deposits: DepositWithBlock[] = []; const allValidFills: FillWithBlock[] = []; const allInvalidFills: FillWithBlock[] = []; + const earlyDeposits: FundsDepositedEvent[] = []; // Save refund in-memory for validated fill. const addRefundForValidFill = ( @@ -359,6 +366,19 @@ export class BundleDataClient { ) ); + // TODO: replace this logic with something more clear where all deposits can be queried at once, + // but separated into early and not after the initial filter/query. + earlyDeposits.push( + ...getUniqueEarlyDepositsInRange( + blockRangesForChains, + Number(originChainId), + Number(destinationChainId), + this.chainIdListForBundleEvaluationBlockNumbers, + originClient, + earlyDeposits + ) + ); + const blockRangeForChain = getBlockRangeForChain( blockRangesForChains, Number(destinationChainId), @@ -432,7 +452,7 @@ export class BundleDataClient { }); } - this.loadDataCache[key] = { fillsToRefund, deposits, unfilledDeposits, allValidFills }; + this.loadDataCache[key] = { fillsToRefund, deposits, unfilledDeposits, allValidFills, earlyDeposits }; return this.loadDataFromCache(key); } diff --git a/src/dataworker/Dataworker.ts b/src/dataworker/Dataworker.ts index e0c91dea3..74856e154 100644 --- a/src/dataworker/Dataworker.ts +++ b/src/dataworker/Dataworker.ts @@ -48,6 +48,8 @@ import { import _ from "lodash"; import { spokePoolClientsToProviders } from "../common"; import * as sdk from "@across-protocol/sdk-v2"; +// eslint-disable-next-line node/no-missing-import +import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; // Internal error reasons for labeling a pending root bundle as "invalid" that we don't want to submit a dispute // for. These errors are due to issues with the dataworker configuration, instead of with the pending root @@ -167,10 +169,8 @@ export class Dataworker { spokePoolClients: SpokePoolClientsByChain, latestMainnetBlock?: number ): Promise { - const { fillsToRefund, deposits, allValidFills, unfilledDeposits } = await this.clients.bundleDataClient.loadData( - blockRangesForChains, - spokePoolClients - ); + const { fillsToRefund, deposits, allValidFills, unfilledDeposits, earlyDeposits } = + await this.clients.bundleDataClient.loadData(blockRangesForChains, spokePoolClients); const mainnetBundleEndBlock = getBlockRangeForChain( blockRangesForChains, @@ -193,6 +193,7 @@ export class Dataworker { allValidFills, allValidFillsInRange, unfilledDeposits, + earlyDeposits, true ); } @@ -492,12 +493,8 @@ export class Dataworker { logData = false ): Promise { const timerStart = Date.now(); - const { fillsToRefund, deposits, allValidFills, unfilledDeposits } = await this.clients.bundleDataClient._loadData( - blockRangesForProposal, - spokePoolClients, - false, - logData - ); + const { fillsToRefund, deposits, allValidFills, unfilledDeposits, earlyDeposits } = + await this.clients.bundleDataClient._loadData(blockRangesForProposal, spokePoolClients, false, logData); const allValidFillsInRange = getFillsInRange( allValidFills, blockRangesForProposal, @@ -520,6 +517,7 @@ export class Dataworker { allValidFills, allValidFillsInRange, unfilledDeposits, + earlyDeposits, true ); const relayerRefundRoot = _buildRelayerRefundRoot( @@ -2226,6 +2224,7 @@ export class Dataworker { allValidFills: FillWithBlock[], allValidFillsInRange: FillWithBlock[], unfilledDeposits: UnfilledDeposit[], + earlyDeposits: FundsDepositedEvent[], logSlowFillExcessData = false ): Promise { const key = JSON.stringify(blockRangesForChains); @@ -2241,6 +2240,7 @@ export class Dataworker { allValidFills, allValidFillsInRange, unfilledDeposits, + earlyDeposits, this.clients, spokePoolClients, this.chainIdListForBundleEvaluationBlockNumbers, diff --git a/src/dataworker/DataworkerUtils.ts b/src/dataworker/DataworkerUtils.ts index 1d823d175..ede105fac 100644 --- a/src/dataworker/DataworkerUtils.ts +++ b/src/dataworker/DataworkerUtils.ts @@ -38,12 +38,15 @@ import { initializeRunningBalancesFromRelayerRepayments, subtractExcessFromPreviousSlowFillsFromRunningBalances, updateRunningBalanceForDeposit, + updateRunningBalanceForEarlyDeposit, } from "./PoolRebalanceUtils"; import { getAmountToReturnForRelayerRefundLeaf, sortRefundAddresses, sortRelayerRefundLeaves, } from "./RelayerRefundUtils"; +// eslint-disable-next-line node/no-missing-import +import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; export const { getImpliedBundleBlockRanges, getBlockRangeForChain, getBlockForChain } = utils; export function getEndBlockBuffers( @@ -328,6 +331,7 @@ export async function _buildPoolRebalanceRoot( allValidFills: FillWithBlock[], allValidFillsInRange: FillWithBlock[], unfilledDeposits: UnfilledDeposit[], + earlyDeposits: FundsDepositedEvent[], clients: DataworkerClients, spokePoolClients: SpokePoolClientsByChain, chainIdListForBundleEvaluationBlockNumbers: number[], @@ -383,6 +387,15 @@ export async function _buildPoolRebalanceRoot( updateRunningBalanceForDeposit(runningBalances, clients.hubPoolClient, deposit, deposit.amount.mul(toBN(-1))); }); + earlyDeposits.forEach((earlyDeposit) => { + updateRunningBalanceForEarlyDeposit( + runningBalances, + clients.hubPoolClient, + earlyDeposit, + earlyDeposit.args.amount.mul(toBN(-1)) + ); + }); + // Add to the running balance value from the last valid root bundle proposal for {chainId, l1Token} // combination if found. addLastRunningBalance(latestMainnetBlock, runningBalances, clients.hubPoolClient); diff --git a/src/dataworker/PoolRebalanceUtils.ts b/src/dataworker/PoolRebalanceUtils.ts index e2ef9b8e6..d6dadd037 100644 --- a/src/dataworker/PoolRebalanceUtils.ts +++ b/src/dataworker/PoolRebalanceUtils.ts @@ -1,3 +1,5 @@ +// eslint-disable-next-line node/no-missing-import +import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; import { ConfigStoreClient, HubPoolClient, SpokePoolClient } from "../clients"; import { Clients } from "../common"; import * as interfaces from "../interfaces"; @@ -77,6 +79,28 @@ export function updateRunningBalanceForDeposit( updateRunningBalance(runningBalances, deposit.originChainId, l1TokenCounterpart, updateAmount); } +export function updateRunningBalanceForEarlyDeposit( + runningBalances: interfaces.RunningBalances, + hubPoolClient: HubPoolClient, + deposit: FundsDepositedEvent, + updateAmount: BigNumber +): void { + const l1TokenCounterpart = hubPoolClient.getL1TokenCounterpartAtBlock( + Number(deposit.args.originChainId.toString()), + deposit.args.originToken, + // TODO: this must be handled s.t. it doesn't depend on when this is run. + // For now, tokens do not change their mappings often, so this will work, but + // to keep the system resilient, this must be updated. + hubPoolClient.latestBlockNumber + ); + updateRunningBalance( + runningBalances, + Number(deposit.args.originChainId.toString()), + l1TokenCounterpart, + updateAmount + ); +} + export function addLastRunningBalance( latestMainnetBlock: number, runningBalances: interfaces.RunningBalances, diff --git a/src/utils/DepositUtils.ts b/src/utils/DepositUtils.ts index 0d64bec88..06d706905 100644 --- a/src/utils/DepositUtils.ts +++ b/src/utils/DepositUtils.ts @@ -4,6 +4,8 @@ import { SpokePoolClient } from "../clients"; import { assign, toBN, isFirstFillForDeposit } from "./"; import { getBlockRangeForChain } from "../dataworker/DataworkerUtils"; import { utils } from "@across-protocol/sdk-v2"; +// eslint-disable-next-line node/no-missing-import +import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; const { validateFillForDeposit } = utils; export function getDepositPath(deposit: Deposit): string { @@ -93,6 +95,32 @@ export function getUniqueDepositsInRange( ) as DepositWithBlock[]; } +export function getUniqueEarlyDepositsInRange( + blockRangesForChains: number[][], + originChain: number, + destinationChain: number, + chainIdListForBundleEvaluationBlockNumbers: number[], + originClient: SpokePoolClient, + existingUniqueDeposits: FundsDepositedEvent[] +): FundsDepositedEvent[] { + const originChainBlockRange = getBlockRangeForChain( + blockRangesForChains, + originChain, + chainIdListForBundleEvaluationBlockNumbers + ); + return (originClient["earlyDeposits"] as unknown as FundsDepositedEvent[]).filter( + (deposit: FundsDepositedEvent) => + deposit.blockNumber <= originChainBlockRange[1] && + deposit.blockNumber >= originChainBlockRange[0] && + deposit.args.destinationChainId.toString() === destinationChain.toString() && + !existingUniqueDeposits.some( + (existingDeposit) => + existingDeposit.args.originChainId.toString() === deposit.args.originChainId.toString() && + existingDeposit.args.depositId.toString() === deposit.args.depositId.toString() + ) + ); +} + export function isDepositSpedUp(deposit: Deposit): boolean { return deposit.speedUpSignature !== undefined && deposit.newRelayerFeePct !== undefined; } From 7b44370cc66e5a6a1b215fd9f713ec4f468df4af Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Sun, 10 Sep 2023 19:10:37 -0400 Subject: [PATCH 2/4] fix: resolve block error --- test/Dataworker.loadData.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/Dataworker.loadData.ts b/test/Dataworker.loadData.ts index d6aede977..7cb4c347b 100644 --- a/test/Dataworker.loadData.ts +++ b/test/Dataworker.loadData.ts @@ -37,7 +37,7 @@ import { import { spokePoolClientsToProviders } from "../src/common"; import { Dataworker } from "../src/dataworker/Dataworker"; // Tested -import { DepositWithBlock, Fill } from "../src/interfaces"; +import { Deposit, DepositWithBlock, Fill } from "../src/interfaces"; import { MAX_UINT_VAL, getRealizedLpFeeForFills, getRefundForFills, toBN } from "../src/utils"; let spokePool_1: Contract, erc20_1: Contract, spokePool_2: Contract, erc20_2: Contract; @@ -103,10 +103,13 @@ describe("Dataworker: Load data used in all functions", async function () { deposits: [], fillsToRefund: {}, allValidFills: [], + earlyDeposits: [], }); }); describe("Computing refunds for bundles", function () { - let fill1: Fill, deposit1; + let fill1: Fill; + let deposit1: Deposit; + beforeEach(async function () { await updateAllClients(); @@ -579,7 +582,7 @@ describe("Dataworker: Load data used in all functions", async function () { erc20_1, depositor, relayer, - { ...deposit2, realizedLpFeePct: deposit2.realizedLpFeePct.div(toBN(2)) }, + { ...deposit2, realizedLpFeePct: deposit2.realizedLpFeePct?.div(toBN(2)) }, 0.25 ); // Note: This fill has identical deposit data to fill2 except for the destination token being different From b9f6ab293e8ae369beb489a52190dd77c480524c Mon Sep 17 00:00:00 2001 From: Matt Rice Date: Sun, 10 Sep 2023 19:21:14 -0400 Subject: [PATCH 3/4] WIP Signed-off-by: Matt Rice --- src/dataworker/DataworkerUtils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/dataworker/DataworkerUtils.ts b/src/dataworker/DataworkerUtils.ts index ede105fac..3b3322150 100644 --- a/src/dataworker/DataworkerUtils.ts +++ b/src/dataworker/DataworkerUtils.ts @@ -392,7 +392,11 @@ export async function _buildPoolRebalanceRoot( runningBalances, clients.hubPoolClient, earlyDeposit, - earlyDeposit.args.amount.mul(toBN(-1)) + // TODO: fix this. + // Because cloneDeep drops the non-array elements of args, we have to use the index rather than the name. + // As a fix, earlyDeposits should be treated similarly to other events and transformed at ingestion time + // into a type that is more digestable rather than a raw event. + earlyDeposit.args[0].mul(toBN(-1)) ); }); From 7477f4cdcfbb9b3d19445e35f2251e4daa334fee Mon Sep 17 00:00:00 2001 From: james-a-morris Date: Sun, 10 Sep 2023 19:23:03 -0400 Subject: [PATCH 4/4] fix: :art: Import from primary directory This change imports from the correct directory rather than import from the dist directly. --- src/clients/BundleDataClient.ts | 12 +++++------- src/dataworker/Dataworker.ts | 4 +--- src/dataworker/DataworkerUtils.ts | 6 ++---- src/dataworker/PoolRebalanceUtils.ts | 5 ++--- src/utils/DepositUtils.ts | 12 +++++------- 5 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/clients/BundleDataClient.ts b/src/clients/BundleDataClient.ts index 6e65685b3..3d74d4704 100644 --- a/src/clients/BundleDataClient.ts +++ b/src/clients/BundleDataClient.ts @@ -31,9 +31,7 @@ import { prettyPrintSpokePoolEvents, } from "../dataworker/DataworkerUtils"; import { getWidestPossibleExpectedBlockRange, isChainDisabled } from "../dataworker/PoolRebalanceUtils"; -import { clients } from "@across-protocol/sdk-v2"; -// eslint-disable-next-line node/no-missing-import -import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; +import { clients, typechain } from "@across-protocol/sdk-v2"; const { refundRequestIsValid, isUBAActivatedAtBlock } = clients; type DataCacheValue = { @@ -41,7 +39,7 @@ type DataCacheValue = { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; - earlyDeposits: FundsDepositedEvent[]; + earlyDeposits: typechain.FundsDepositedEvent[]; }; type DataCache = Record; @@ -179,7 +177,7 @@ export class BundleDataClient { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; - earlyDeposits: FundsDepositedEvent[]; + earlyDeposits: typechain.FundsDepositedEvent[]; }> { const mainnetStartBlock = getBlockRangeForChain( blockRangesForChains, @@ -202,7 +200,7 @@ export class BundleDataClient { fillsToRefund: FillsToRefund; allValidFills: FillWithBlock[]; deposits: DepositWithBlock[]; - earlyDeposits: FundsDepositedEvent[]; + earlyDeposits: typechain.FundsDepositedEvent[]; }> { const key = JSON.stringify(blockRangesForChains); @@ -228,7 +226,7 @@ export class BundleDataClient { const deposits: DepositWithBlock[] = []; const allValidFills: FillWithBlock[] = []; const allInvalidFills: FillWithBlock[] = []; - const earlyDeposits: FundsDepositedEvent[] = []; + const earlyDeposits: typechain.FundsDepositedEvent[] = []; // Save refund in-memory for validated fill. const addRefundForValidFill = ( diff --git a/src/dataworker/Dataworker.ts b/src/dataworker/Dataworker.ts index 74856e154..0f9bc6dde 100644 --- a/src/dataworker/Dataworker.ts +++ b/src/dataworker/Dataworker.ts @@ -48,8 +48,6 @@ import { import _ from "lodash"; import { spokePoolClientsToProviders } from "../common"; import * as sdk from "@across-protocol/sdk-v2"; -// eslint-disable-next-line node/no-missing-import -import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; // Internal error reasons for labeling a pending root bundle as "invalid" that we don't want to submit a dispute // for. These errors are due to issues with the dataworker configuration, instead of with the pending root @@ -2224,7 +2222,7 @@ export class Dataworker { allValidFills: FillWithBlock[], allValidFillsInRange: FillWithBlock[], unfilledDeposits: UnfilledDeposit[], - earlyDeposits: FundsDepositedEvent[], + earlyDeposits: sdk.typechain.FundsDepositedEvent[], logSlowFillExcessData = false ): Promise { const key = JSON.stringify(blockRangesForChains); diff --git a/src/dataworker/DataworkerUtils.ts b/src/dataworker/DataworkerUtils.ts index ede105fac..16dd6df7b 100644 --- a/src/dataworker/DataworkerUtils.ts +++ b/src/dataworker/DataworkerUtils.ts @@ -1,4 +1,4 @@ -import { utils } from "@across-protocol/sdk-v2"; +import { utils, typechain } from "@across-protocol/sdk-v2"; import { SpokePoolClient } from "../clients"; import { spokesThatHoldEthAndWeth } from "../common/Constants"; import { CONTRACT_ADDRESSES } from "../common/ContractAddresses"; @@ -45,8 +45,6 @@ import { sortRefundAddresses, sortRelayerRefundLeaves, } from "./RelayerRefundUtils"; -// eslint-disable-next-line node/no-missing-import -import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; export const { getImpliedBundleBlockRanges, getBlockRangeForChain, getBlockForChain } = utils; export function getEndBlockBuffers( @@ -331,7 +329,7 @@ export async function _buildPoolRebalanceRoot( allValidFills: FillWithBlock[], allValidFillsInRange: FillWithBlock[], unfilledDeposits: UnfilledDeposit[], - earlyDeposits: FundsDepositedEvent[], + earlyDeposits: typechain.FundsDepositedEvent[], clients: DataworkerClients, spokePoolClients: SpokePoolClientsByChain, chainIdListForBundleEvaluationBlockNumbers: number[], diff --git a/src/dataworker/PoolRebalanceUtils.ts b/src/dataworker/PoolRebalanceUtils.ts index d6dadd037..8be708a54 100644 --- a/src/dataworker/PoolRebalanceUtils.ts +++ b/src/dataworker/PoolRebalanceUtils.ts @@ -1,5 +1,4 @@ -// eslint-disable-next-line node/no-missing-import -import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; +import { typechain } from "@across-protocol/sdk-v2"; import { ConfigStoreClient, HubPoolClient, SpokePoolClient } from "../clients"; import { Clients } from "../common"; import * as interfaces from "../interfaces"; @@ -82,7 +81,7 @@ export function updateRunningBalanceForDeposit( export function updateRunningBalanceForEarlyDeposit( runningBalances: interfaces.RunningBalances, hubPoolClient: HubPoolClient, - deposit: FundsDepositedEvent, + deposit: typechain.FundsDepositedEvent, updateAmount: BigNumber ): void { const l1TokenCounterpart = hubPoolClient.getL1TokenCounterpartAtBlock( diff --git a/src/utils/DepositUtils.ts b/src/utils/DepositUtils.ts index 06d706905..517448b8b 100644 --- a/src/utils/DepositUtils.ts +++ b/src/utils/DepositUtils.ts @@ -3,9 +3,7 @@ import { Deposit, DepositWithBlock, Fill, UnfilledDeposit, UnfilledDepositsForOr import { SpokePoolClient } from "../clients"; import { assign, toBN, isFirstFillForDeposit } from "./"; import { getBlockRangeForChain } from "../dataworker/DataworkerUtils"; -import { utils } from "@across-protocol/sdk-v2"; -// eslint-disable-next-line node/no-missing-import -import { FundsDepositedEvent } from "@across-protocol/sdk-v2/dist/typechain"; +import { utils, typechain } from "@across-protocol/sdk-v2"; const { validateFillForDeposit } = utils; export function getDepositPath(deposit: Deposit): string { @@ -101,15 +99,15 @@ export function getUniqueEarlyDepositsInRange( destinationChain: number, chainIdListForBundleEvaluationBlockNumbers: number[], originClient: SpokePoolClient, - existingUniqueDeposits: FundsDepositedEvent[] -): FundsDepositedEvent[] { + existingUniqueDeposits: typechain.FundsDepositedEvent[] +): typechain.FundsDepositedEvent[] { const originChainBlockRange = getBlockRangeForChain( blockRangesForChains, originChain, chainIdListForBundleEvaluationBlockNumbers ); - return (originClient["earlyDeposits"] as unknown as FundsDepositedEvent[]).filter( - (deposit: FundsDepositedEvent) => + return (originClient["earlyDeposits"] as unknown as typechain.FundsDepositedEvent[]).filter( + (deposit: typechain.FundsDepositedEvent) => deposit.blockNumber <= originChainBlockRange[1] && deposit.blockNumber >= originChainBlockRange[0] && deposit.args.destinationChainId.toString() === destinationChain.toString() &&