From f71a95384f42aca71817dba40e9f565b0f8c66c8 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 18 Dec 2024 18:16:56 +0100 Subject: [PATCH 1/5] common: add limit for fetching ravs Signed-off-by: Gustavo Inacio --- packages/indexer-common/src/allocations/tap-collector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 7628cfa78..ab6b01383 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -328,6 +328,7 @@ export class TapCollector { private async pendingRAVs(): Promise { return await this.models.receiptAggregateVouchers.findAll({ where: { last: true, final: false }, + limit: 100, }) } From 22d7dea33b33f44e6c96b03a825f8fa6ebf1679a Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 18 Dec 2024 18:17:13 +0100 Subject: [PATCH 2/5] common: fix problem of failing ravs stopping execution Signed-off-by: Gustavo Inacio --- packages/indexer-common/src/allocations/tap-collector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index ab6b01383..242913067 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -569,7 +569,7 @@ export class TapCollector { logger.error(`Failed to redeem RAV`, { err: indexerError(IndexerErrorCode.IE055, err), }) - return + continue } stopTimer() } From c9f2fd0c539821b9ab3a16ca8f9b3b563ac60ed9 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 18 Dec 2024 18:18:16 +0100 Subject: [PATCH 3/5] common: fix marking already redeemed ravs Signed-off-by: Gustavo Inacio --- .../src/allocations/tap-collector.ts | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 242913067..d3b70ba67 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -335,16 +335,22 @@ export class TapCollector { private async filterAndUpdateRavs( ravsLastNotFinal: ReceiptAggregateVoucher[], ): Promise { + // look for all transactions for that includes senderaddress[] and allocations[] const tapSubgraphResponse = await this.findTransactionsForRavs(ravsLastNotFinal) - const redeemedRavsNotOnOurDatabase = tapSubgraphResponse.transactions.filter( - (tx) => - !ravsLastNotFinal.find( - (rav) => - toAddress(rav.senderAddress) === toAddress(tx.sender.id) && - toAddress(rav.allocationId) === toAddress(tx.allocationID), - ), - ) + // get a list of transacations for ravs marked as not redeemed in our database + const redeemedRavsNotOnOurDatabase = tapSubgraphResponse.transactions.filter((tx) => { + // check if exists in the list sent + !!ravsLastNotFinal.find( + (rav) => + // rav has the same sender address as tx + toAddress(rav.senderAddress) === toAddress(tx.sender.id) && + // rav has the same allocation id as tx + toAddress(rav.allocationId) === toAddress(tx.allocationID) && + // rav was not redeemed + !rav.redeemedAt, + ) + }) // for each transaction that is not redeemed on our database // but was redeemed on the blockchain, update it to redeemed @@ -360,7 +366,9 @@ export class TapCollector { // Filter unfinalized RAVS fetched from DB, keeping RAVs that have not yet been redeemed on-chain const nonRedeemedRavs = ravsLastNotFinal + // get all ravs that were marked as redeemed in our database .filter((rav) => !!rav.redeemedAt) + // get all ravs that wasn't possible to find the transaction .filter( (rav) => !tapSubgraphResponse.transactions.find( @@ -396,6 +404,14 @@ export class TapCollector { let lastId = '' const transactions: TapTransaction[] = [] + const unfinalizedRavsAllocationIds = [ + ...new Set(ravs.map((value) => toAddress(value.allocationId).toLowerCase())), + ] + + const senderAddresses = [ + ...new Set(ravs.map((value) => toAddress(value.senderAddress).toLowerCase())), + ] + for (;;) { let block: { hash: string } | undefined = undefined if (meta?.block?.hash) { @@ -445,12 +461,8 @@ export class TapCollector { lastId, pageSize: PAGE_SIZE, block, - unfinalizedRavsAllocationIds: ravs.map((value) => - toAddress(value.allocationId).toLowerCase(), - ), - senderAddresses: ravs.map((value) => - toAddress(value.senderAddress).toLowerCase(), - ), + unfinalizedRavsAllocationIds, + senderAddresses, }, ) From 9d34c191d86fdb943ffda159eb3e2be96f0ed718 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 18 Dec 2024 18:52:38 +0100 Subject: [PATCH 4/5] common: add test for mark ravs in transactions as redeemed Signed-off-by: Gustavo Inacio --- .../src/allocations/__tests__/tap.test.ts | 49 +++++++++++++++ .../src/allocations/tap-collector.ts | 61 +++++++++++-------- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/packages/indexer-common/src/allocations/__tests__/tap.test.ts b/packages/indexer-common/src/allocations/__tests__/tap.test.ts index f35569f0e..64707cdbc 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap.test.ts @@ -281,6 +281,55 @@ describe('TAP', () => { ]) }) + test('should mark ravs as redeemed via `markRavsInTransactionsAsRedeemed`', async () => { + const nowSecs = Math.floor(Date.now() / 1000) + const transactions = { + transactions: [ + { + id: 'test', + allocationID: ALLOCATION_ID_2.toString().toLowerCase().replace('0x', ''), + timestamp: nowSecs, + sender: { + id: SENDER_ADDRESS_3.toString().toLowerCase().replace('0x', ''), + }, + }, + ], + _meta: { + block: { + timestamp: nowSecs, + hash: 'test', + }, + }, + } + + const rav2 = { + allocationId: ALLOCATION_ID_2, + last: true, + final: false, + timestampNs: 1709067401177959664n, + valueAggregate: 20000000000000n, + signature: SIGNATURE, + senderAddress: SENDER_ADDRESS_3, + redeemedAt: null, + } + await queryFeeModels.receiptAggregateVouchers.create(rav2) + let ravs = await tapCollector['pendingRAVs']() + await tapCollector['markRavsInTransactionsAsRedeemed'](transactions, ravs) + const redeemedRavs = await queryFeeModels.receiptAggregateVouchers.findAll({ + where: { + last: true, + final: false, + redeemedAt: { + [Op.ne]: null, + }, + }, + }) + // Expect redeemed rav to be returned here + expect(redeemedRavs).toEqual([ + expect.objectContaining({ ...rav2, redeemedAt: nowSecs }), + ]) + }) + test('should mark ravs as final via `markRavsAsFinal`', async () => { // we have a redeemed non-final rav in our database const nowSecs = Math.floor(Date.now() / 1000) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index d3b70ba67..486b11a81 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -338,31 +338,8 @@ export class TapCollector { // look for all transactions for that includes senderaddress[] and allocations[] const tapSubgraphResponse = await this.findTransactionsForRavs(ravsLastNotFinal) - // get a list of transacations for ravs marked as not redeemed in our database - const redeemedRavsNotOnOurDatabase = tapSubgraphResponse.transactions.filter((tx) => { - // check if exists in the list sent - !!ravsLastNotFinal.find( - (rav) => - // rav has the same sender address as tx - toAddress(rav.senderAddress) === toAddress(tx.sender.id) && - // rav has the same allocation id as tx - toAddress(rav.allocationId) === toAddress(tx.allocationID) && - // rav was not redeemed - !rav.redeemedAt, - ) - }) - - // for each transaction that is not redeemed on our database - // but was redeemed on the blockchain, update it to redeemed - if (redeemedRavsNotOnOurDatabase.length > 0) { - for (const rav of redeemedRavsNotOnOurDatabase) { - await this.markRavAsRedeemed( - toAddress(rav.allocationID), - toAddress(rav.sender.id), - rav.timestamp, - ) - } - } + // check for redeemed ravs in tx list but not marked as redeemed in our database + this.markRavsInTransactionsAsRedeemed(tapSubgraphResponse, ravsLastNotFinal) // Filter unfinalized RAVS fetched from DB, keeping RAVs that have not yet been redeemed on-chain const nonRedeemedRavs = ravsLastNotFinal @@ -397,6 +374,40 @@ export class TapCollector { }) } + public async markRavsInTransactionsAsRedeemed( + tapSubgraphResponse: TapSubgraphResponse, + ravsLastNotFinal: ReceiptAggregateVoucher[], + ) { + // get a list of transactions for ravs marked as not redeemed in our database + const redeemedRavsNotOnOurDatabase = tapSubgraphResponse.transactions + // get only the transactions that exists, this prevents errors marking as redeemed + // transactions for different senders with the same allocation id + .filter((tx) => { + // check if exists in the ravsLastNotFinal list + !!ravsLastNotFinal.find( + (rav) => + // rav has the same sender address as tx + toAddress(rav.senderAddress) === toAddress(tx.sender.id) && + // rav has the same allocation id as tx + toAddress(rav.allocationId) === toAddress(tx.allocationID) && + // rav was marked as not redeemed in the db + !rav.redeemedAt, + ) + }) + + // for each transaction that is not redeemed on our database + // but was redeemed on the blockchain, update it to redeemed + if (redeemedRavsNotOnOurDatabase.length > 0) { + for (const rav of redeemedRavsNotOnOurDatabase) { + await this.markRavAsRedeemed( + toAddress(rav.allocationID), + toAddress(rav.sender.id), + rav.timestamp, + ) + } + } + } + public async findTransactionsForRavs( ravs: ReceiptAggregateVoucher[], ): Promise { From 00c1a4059c406973fe199761f8f27026888ee719 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 18 Dec 2024 19:18:40 +0100 Subject: [PATCH 5/5] common: fix return in filter and to_timestamp query Signed-off-by: Gustavo Inacio --- packages/indexer-common/src/allocations/__tests__/tap.test.ts | 4 ++-- packages/indexer-common/src/allocations/tap-collector.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/indexer-common/src/allocations/__tests__/tap.test.ts b/packages/indexer-common/src/allocations/__tests__/tap.test.ts index 64707cdbc..b096a9220 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap.test.ts @@ -313,7 +313,7 @@ describe('TAP', () => { redeemedAt: null, } await queryFeeModels.receiptAggregateVouchers.create(rav2) - let ravs = await tapCollector['pendingRAVs']() + const ravs = await tapCollector['pendingRAVs']() await tapCollector['markRavsInTransactionsAsRedeemed'](transactions, ravs) const redeemedRavs = await queryFeeModels.receiptAggregateVouchers.findAll({ where: { @@ -326,7 +326,7 @@ describe('TAP', () => { }) // Expect redeemed rav to be returned here expect(redeemedRavs).toEqual([ - expect.objectContaining({ ...rav2, redeemedAt: nowSecs }), + expect.objectContaining({ ...rav2, redeemedAt: new Date(nowSecs * 1000) }), ]) }) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 486b11a81..40f83c4f3 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -384,7 +384,7 @@ export class TapCollector { // transactions for different senders with the same allocation id .filter((tx) => { // check if exists in the ravsLastNotFinal list - !!ravsLastNotFinal.find( + return !!ravsLastNotFinal.find( (rav) => // rav has the same sender address as tx toAddress(rav.senderAddress) === toAddress(tx.sender.id) && @@ -695,7 +695,7 @@ export class TapCollector { // https://github.com/sequelize/sequelize/issues/7664 (bug been open for 7 years no fix yet or ever) const query = ` UPDATE scalar_tap_ravs - SET redeemed_at = ${timestamp ? timestamp : 'NOW()'} + SET redeemed_at = ${timestamp ? `to_timestamp(${timestamp})` : 'NOW()'} WHERE allocation_id = '${allocationId .toString() .toLowerCase()