Skip to content

Commit

Permalink
perf: Optimize synchronization logic for multiple wallets in the ligh…
Browse files Browse the repository at this point in the history 
…t client. (#3160)

perf: Optimize synchronization logic for multiple wallets in a light client node

Co-authored-by: Chen Yu <[email protected]>
  • Loading branch information
@yanguoyu @Keith-CY
yanguoyu and Keith-CY authored May 28, 2024
1 parent 810ff2f commit 493d07b
Show file tree
Hide file tree
Showing 2 changed files with 190 additions and 28 deletions.
53 changes: 33 additions & 20 deletions packages/neuron-wallet/src/block-sync-renderer/sync/light-synchronizer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import NetworksService from '../../services/networks'
import { getConnection } from '../../database/chain/connection'

const unpackGroup = molecule.vector(blockchain.OutPoint)
const THRESHOLD_BLOCK_NUMBER_IN_DIFF_WALLET = 100_000

export default class LightSynchronizer extends Synchronizer {
private lightRpc: LightRPC
Expand Down Expand Up @@ -167,20 +168,32 @@ export default class LightSynchronizer extends Synchronizer {
await getConnection('light').manager.save(updatedSyncProgress, { chunk: 100 })
}

private static async getWalletsSyncedMinBlockNumber() {
const walletMinBlockNumber = await SyncProgressService.getWalletMinLocalSavedBlockNumber()
const wallets = await WalletService.getInstance().getAll()
return wallets.reduce<Record<string, number>>(
(pre, cur) => ({
...pre,
[cur.id]: Math.max(parseInt(cur.startBlockNumber ?? '0x0'), walletMinBlockNumber?.[cur.id] ?? 0),
}),
{}
)
}

private async initSyncProgress(appendScripts: AppendScript[] = []) {
if (!this.addressMetas.length && !appendScripts.length) {
return
}
const existSyncArgses = await SyncProgressService.getExistingSyncArgses()
const syncScripts = await this.lightRpc.getScripts()
const retainedSyncScripts = syncScripts.filter(v => existSyncArgses.has(v.script.args))
const existSyncscripts: Record<string, LightScriptFilter> = {}
const existSyncScripts: Record<string, LightScriptFilter> = {}
retainedSyncScripts.forEach(v => {
existSyncscripts[scriptToHash(v.script)] = v
existSyncScripts[scriptToHash(v.script)] = v
})
const walletMinBlockNumber = await LightSynchronizer.getWalletsSyncedMinBlockNumber()
const currentWalletId = WalletService.getInstance().getCurrent()?.id
const allScripts = this.addressMetas
.filter(v => (currentWalletId ? v.walletId === currentWalletId : true))
.map(addressMeta => {
const lockScripts = [
addressMeta.generateDefaultLockScript(),
Expand All @@ -194,32 +207,32 @@ export default class LightSynchronizer extends Synchronizer {
}))
})
.flat()
const walletMinBlockNumber = await SyncProgressService.getWalletMinLocalSavedBlockNumber()
const wallets = await WalletService.getInstance().getAll()
const walletStartBlockMap = wallets.reduce<Record<string, string | undefined>>(
(pre, cur) => ({ ...pre, [cur.id]: cur.startBlockNumber }),
{}
)
.filter(v => {
return (
!currentWalletId ||
v.walletId === currentWalletId ||
walletMinBlockNumber[v.walletId] >
walletMinBlockNumber[currentWalletId] - THRESHOLD_BLOCK_NUMBER_IN_DIFF_WALLET
)
})
const otherTypeSyncBlockNumber = await SyncProgressService.getOtherTypeSyncBlockNumber()
const addScripts = [
...allScripts
.filter(
v =>
!existSyncscripts[scriptToHash(v.script)] ||
+existSyncscripts[scriptToHash(v.script)].blockNumber < +(walletStartBlockMap[v.walletId] ?? 0)
)
.map(v => {
const blockNumber = Math.max(
parseInt(walletStartBlockMap[v.walletId] ?? '0x0'),
walletMinBlockNumber?.[v.walletId] ?? 0
.filter(v => {
const scriptHash = scriptToHash(v.script)
return (
!existSyncScripts[scriptHash] ||
+existSyncScripts[scriptHash].blockNumber < walletMinBlockNumber[v.walletId]
)
})
.map(v => {
return {
...v,
blockNumber: `0x${blockNumber.toString(16)}`,
blockNumber: `0x${walletMinBlockNumber[v.walletId].toString(16)}`,
}
}),
...appendScripts
.filter(v => !existSyncscripts[scriptToHash(v.script)])
.filter(v => !existSyncScripts[scriptToHash(v.script)])
.map(v => ({
...v,
blockNumber: `0x${(otherTypeSyncBlockNumber[scriptToHash(v.script)] ?? 0).toString(16)}`,
Expand Down
165 changes: 157 additions & 8 deletions packages/neuron-wallet/tests/block-sync-renderer/light-synchronizer.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const schedulerWaitMock = jest.fn()
const getMultisigConfigForLightMock = jest.fn()
const walletGetCurrentMock = jest.fn()
const walletGetAllMock = jest.fn()
const updateBlockStartNumberMock = jest.fn()

function mockReset() {
getSyncStatusMock.mockReset()
Expand All @@ -46,6 +47,8 @@ function mockReset() {
removeByHashesAndAddressType.mockReset()
walletGetCurrentMock.mockReset()
walletGetAllMock.mockReset()

updateBlockStartNumberMock.mockReset()
}

jest.mock('../../src/services/sync-progress', () => {
Expand Down Expand Up @@ -138,6 +141,7 @@ describe('test light synchronizer', () => {
})
it('when syncing script in the local DB', async () => {
getScriptsMock.mockResolvedValue([{ script, blockNumber: '0xaa' }])
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getExistingSyncArgsesMock.mockResolvedValue(new Set([script.args]))
const addressMeta = AddressMeta.fromObject({
walletId: 'walletId',
Expand Down Expand Up @@ -187,6 +191,7 @@ describe('test light synchronizer', () => {
})
it('when syncing script not in the local DB', async () => {
getScriptsMock.mockResolvedValue([{ script, blockNumber: '0xaa' }])
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getExistingSyncArgsesMock.mockResolvedValue(new Set())
const addressMeta = AddressMeta.fromObject({
walletId: 'walletId',
Expand Down Expand Up @@ -256,6 +261,7 @@ describe('test light synchronizer', () => {
addressType: 0,
blake160: script.args,
})
walletGetAllMock.mockReturnValue([{ id: 'walletId' }])
getWalletMinLocalSavedBlockNumberMock.mockResolvedValue({ walletId: 170 })
const connect = new LightSynchronizer([addressMeta], '')
//@ts-ignore
Expand Down Expand Up @@ -367,6 +373,152 @@ describe('test light synchronizer', () => {
)
expect(setScriptsMock).toHaveBeenLastCalledWith([], 'delete')
})
it('when other wallet min synced block number near the current wallet', async () => {
walletGetCurrentMock.mockReturnValue({ id: 'walletId1' })
walletGetAllMock.mockReturnValue([
{ id: 'walletId1', startBlockNumber: '0xaa' },
{ id: 'walletId2', startBlockNumber: '0xab' },
])
const addressMeta1 = AddressMeta.fromObject({
walletId: 'walletId1',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script.args,
})
const script2: Script = {
args: '0x403f0d4e833b2a8d372772a63facaa310dfeef93',
codeHash: '0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8',
hashType: 'type',
}
const getScriptsResult = [{ script: script2, blockNumber: '0xaa', scriptType: 'lock' }]
getScriptsMock.mockResolvedValue(getScriptsResult)
getExistingSyncArgsesMock.mockResolvedValue(new Set([script2.args]))
const addressMeta2 = AddressMeta.fromObject({
walletId: 'walletId2',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script2.args,
})
const connect = new LightSynchronizer([addressMeta1, addressMeta2], '')
//@ts-ignore
await connect.initSyncProgress()
expect(setScriptsMock).toHaveBeenNthCalledWith(
1,
[
...[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaa',
})),
...[
addressMeta2.generateDefaultLockScript().toSDK(),
addressMeta2.generateACPLockScript().toSDK(),
addressMeta2.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta2.walletId,
blockNumber: '0xab',
})),
],
'partial'
)
expect(setScriptsMock).toHaveBeenLastCalledWith([], 'delete')
expect(initSyncProgressMock).toBeCalledWith([
...[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaa',
})),
...[
addressMeta2.generateDefaultLockScript().toSDK(),
addressMeta2.generateACPLockScript().toSDK(),
addressMeta2.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta2.walletId,
blockNumber: '0xab',
})),
])
expect(updateSyncProgressFlagMock).toBeCalledWith(['walletId1', 'walletId2'])
})
it('when other wallet min synced block number bigger than the current wallet', async () => {
walletGetCurrentMock.mockReturnValue({ id: 'walletId1' })
walletGetAllMock.mockReturnValue([
{ id: 'walletId1', startBlockNumber: '0xaaaa0' },
{ id: 'walletId2', startBlockNumber: '0xab' },
])
const addressMeta1 = AddressMeta.fromObject({
walletId: 'walletId1',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script.args,
})
const script2: Script = {
args: '0x403f0d4e833b2a8d372772a63facaa310dfeef93',
codeHash: '0x9bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce8',
hashType: 'type',
}
const getScriptsResult = [{ script: script2, blockNumber: '0xab', scriptType: 'lock' }]
getScriptsMock.mockResolvedValue(getScriptsResult)
getExistingSyncArgsesMock.mockResolvedValue(new Set([script2.args]))
const addressMeta2 = AddressMeta.fromObject({
walletId: 'walletId2',
address,
path: '',
addressIndex: 10,
addressType: 0,
blake160: script2.args,
})
const connect = new LightSynchronizer([addressMeta1, addressMeta2], '')
//@ts-ignore
await connect.initSyncProgress()
expect(setScriptsMock).toHaveBeenNthCalledWith(
1,
[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaaaa0',
})),
'partial'
)
expect(setScriptsMock).toHaveBeenLastCalledWith(getScriptsResult, 'delete')
expect(initSyncProgressMock).toBeCalledWith(
[
addressMeta1.generateDefaultLockScript().toSDK(),
addressMeta1.generateACPLockScript().toSDK(),
addressMeta1.generateLegacyACPLockScript().toSDK(),
].map(script => ({
script,
scriptType: 'lock',
walletId: addressMeta1.walletId,
blockNumber: '0xaaaa0',
}))
)
expect(updateSyncProgressFlagMock).toBeCalledWith(['walletId1', 'walletId2'])
})
})

describe('test initSync', () => {
Expand Down Expand Up @@ -428,16 +580,10 @@ describe('test light synchronizer', () => {
})

describe('#notifyCurrentBlockNumberProcessed', () => {
const synchronizer = new LightSynchronizer([], '')
const updateBlockStartNumberMock = jest.fn()
beforeAll(() => {
it('last process block number finish', async () => {
const synchronizer = new LightSynchronizer([], '')
// @ts-ignore private property
synchronizer.updateBlockStartNumber = updateBlockStartNumberMock
})
beforeEach(() => {
updateBlockStartNumberMock.mockReset()
})
it('last process block number finish', async () => {
// @ts-ignore private property
synchronizer.processingBlockNumber = '0xaa'
getCurrentWalletMinSyncedBlockNumberMock.mockResolvedValueOnce(100)
Expand All @@ -447,6 +593,9 @@ describe('test light synchronizer', () => {
expect(updateBlockStartNumberMock).toBeCalledWith(100)
})
it('not last process block number finish', async () => {
const synchronizer = new LightSynchronizer([], '')
// @ts-ignore private property
synchronizer.updateBlockStartNumber = updateBlockStartNumberMock
// @ts-ignore private property
synchronizer.processingBlockNumber = undefined
await synchronizer.notifyCurrentBlockNumberProcessed('0xaa')
Expand Down

4 comments on commit 493d07b

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Packaging for test is done in 9263483423

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Packaging for test is done in 9263484079

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Packaging for test is done in 9264161611

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Packaging for test is done in 9264130453

Please sign in to comment.