From ea04f42bbc25f6eb21e405a6a01e3d1515f6f1ef Mon Sep 17 00:00:00 2001 From: yanguoyu <841185308@qq.com> Date: Fri, 17 Nov 2023 10:31:42 +0800 Subject: [PATCH 1/3] fix: Fix light client sync slow when fetch much transactions. --- .../sync/light-connector.ts | 23 +++++++++++++++++-- .../src/block-sync-renderer/sync/queue.ts | 5 +++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts index 272c510bf3..3ea2fb05ec 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts @@ -1,7 +1,7 @@ import { BI } from '@ckb-lumos/bi' import { Subject } from 'rxjs' import { queue, QueueObject } from 'async' -import { HexString, QueryOptions } from '@ckb-lumos/base' +import type { HexString, QueryOptions, TransactionWithStatus } from '@ckb-lumos/base' import { Indexer as CkbIndexer, CellCollector } from '@ckb-lumos/ckb-indexer' import logger from '../../utils/logger' import { Address } from '../../models/address' @@ -18,6 +18,7 @@ import AssetAccountInfo from '../../models/asset-account-info' import { DepType } from '../../models/chain/cell-dep' import { molecule } from '@ckb-lumos/codec' import { blockchain } from '@ckb-lumos/base' +import type { Base } from '@ckb-lumos/rpc/lib/Base' interface SyncQueueParam { script: CKBComponents.Script @@ -232,7 +233,9 @@ export default class LightConnector extends Connector { }) return } - this.transactionsSubject.next({ txHashes: result.txs.map(v => v.txHash), params: syncProgress.hash }) + const txHashes = result.txs.map(v => v.txHash) + await this.fetchPreviousOutputs(txHashes) + this.transactionsSubject.next({ txHashes, params: syncProgress.hash }) this.syncInQueue.set(syncProgress.hash, { blockStartNumber: result.lastCursor === '0x' ? parseInt(blockRange[1]) : parseInt(blockRange[0]), blockEndNumber: parseInt(blockRange[1]), @@ -240,6 +243,22 @@ export default class LightConnector extends Connector { }) } + private async fetchPreviousOutputs(txHashes: string[]) { + const transactions = await this.lightRpc + .createBatchRequest<'getTransaction', string[], TransactionWithStatus[]>(txHashes.map(v => ['getTransaction', v])) + .exec() + const previousTxHashes = new Set() + transactions.forEach(tx => { + tx.transaction.inputs.forEach(input => { + const previousTxHash = input.previousOutput!.txHash + if (previousTxHash !== `0x${'0'.repeat(64)}`) { + previousTxHashes.add(previousTxHash) + } + }) + }) + await this.lightRpc.createBatchRequest([...previousTxHashes].map(v => ['fetchTransaction' as keyof Base, v])).exec() + } + private async collectLiveCellsByScript(query: LumosCellQuery) { const { lock, type, data } = query if (!lock && !type) { diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts index 3d3bb0cea1..66b9cbc4a9 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts @@ -171,7 +171,10 @@ export default class Queue { }, 1) const drainFetchTxQueue = new Promise((resolve, reject) => { - fetchTxQueue.error(reject) + fetchTxQueue.error(err => { + fetchTxQueue.kill() + reject(err) + }) fetchTxQueue.drain(() => resolve(0)) }) From 95dedd8d6ae3404fa20a53ee91e3a847ac3731fa Mon Sep 17 00:00:00 2001 From: yanguoyu <841185308@qq.com> Date: Fri, 17 Nov 2023 15:38:15 +0800 Subject: [PATCH 2/3] fix: Add message send to avoid the tx db has not submit finish yet. --- .../src/block-sync-renderer/sync/light-connector.ts | 6 +++--- .../neuron-wallet/src/block-sync-renderer/sync/queue.ts | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts index 3ea2fb05ec..5e546d673b 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts @@ -248,14 +248,14 @@ export default class LightConnector extends Connector { .createBatchRequest<'getTransaction', string[], TransactionWithStatus[]>(txHashes.map(v => ['getTransaction', v])) .exec() const previousTxHashes = new Set() - transactions.forEach(tx => { - tx.transaction.inputs.forEach(input => { + transactions + .flatMap(tx => tx.transaction.inputs) + .forEach(input => { const previousTxHash = input.previousOutput!.txHash if (previousTxHash !== `0x${'0'.repeat(64)}`) { previousTxHashes.add(previousTxHash) } }) - }) await this.lightRpc.createBatchRequest([...previousTxHashes].map(v => ['fetchTransaction' as keyof Base, v])).exec() } diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts index 66b9cbc4a9..e53df531d4 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts @@ -90,6 +90,7 @@ export default class Queue { while (true) { try { await this.#checkAndSave(txHashes) + process.send?.({ channel: 'tx-db-changed' }) break } catch (error) { logger.error('retry saving transactions in 2 seconds due to error:', error) From 985b6dfeca3aa23cac8042a4e21850837fb74ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=A5=E5=9B=BD=E5=AE=87?= <841185308@qq.com> Date: Mon, 20 Nov 2023 14:28:13 +0800 Subject: [PATCH 3/3] Add annotation Co-authored-by: homura --- .../src/block-sync-renderer/sync/light-connector.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts index 5e546d673b..46a9ad4d59 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/light-connector.ts @@ -252,6 +252,7 @@ export default class LightConnector extends Connector { .flatMap(tx => tx.transaction.inputs) .forEach(input => { const previousTxHash = input.previousOutput!.txHash + // exclude the cell base transaction in a block if (previousTxHash !== `0x${'0'.repeat(64)}`) { previousTxHashes.add(previousTxHash) }