Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle token changes from logs rather than querying each token #1465

Draft
wants to merge 3 commits into
base: develop
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 144 additions & 0 deletions main/externalData/balances/logs/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import { toTokenId } from '../../../../resources/domain/balance'
import { hexZeroPad } from '@ethersproject/bytes'
import { BigNumber } from '@ethersproject/bignumber'
import log from 'electron-log'
import { TokenDefinition } from 'nebula/dist/ipfs/manifest/tokens'
import { BytesLike, formatUnits } from 'ethers/lib/utils'
import type EthereumProvider from 'ethereum-provider'
import { erc20Interface } from '../../../../resources/contracts'

export enum LogTopic {
TRANSFER = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
WITHDRAWAL = '0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65',
DEPOSIT = '0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c'
}

type TokenId = string
type Address = string
type AccountBalances = Record<TokenId, Balance>

export interface Log {
address: Address
blockHash: string
blockNumber: string
data: string
logIndex: string
removed: boolean
topics: string[]
transactionHash: string
transactionIndex: string
}

type TokensDict = Record<TokenId, TokenDefinition>

const toTokenDict = (definitions: TokenDefinition[]) =>
definitions.reduce((tokens: TokensDict, token) => {
const { address, chainId } = token
tokens[toTokenId({ address, chainId: parseInt(chainId) })] = token
return tokens
}, {})

export class LogProcessor {
private balances: AccountBalances = {}
private ownerPadded

private handlers: Record<LogTopic, (log: Log, tokensDict: TokensDict) => void> = {
[LogTopic.TRANSFER]: this.handleTransfer.bind(this),
[LogTopic.WITHDRAWAL]: this.handleWithdrawal.bind(this),
[LogTopic.DEPOSIT]: this.handleDeposit.bind(this)
}

private async getTokenBalance(token: TokenDefinition) {
const functionData = erc20Interface.encodeFunctionData('balanceOf', [this.owner])

const response: BytesLike = await this.provider.request({
method: 'eth_call',
chainId: '0x' + Number(token.chainId).toString(16),
params: [{ to: token.address, value: '0x0', data: functionData }, 'latest']
})

return BigNumber.from(response)._hex
}

private async processDelta(tokenId: TokenId, delta: BigNumber, tokens: TokensDict) {
const existing = this.balances[tokenId]
const tokenDefinition = tokens[tokenId]
if (!existing && !tokenDefinition) {
log.warn('Unsupported Token', { tokenId, chainId: this.chainId })
return
}

const balance = existing ? delta.add(existing.balance)._hex : await this.getTokenBalance(tokenDefinition)

const { decimals } = tokenDefinition || balance

this.balances[tokenId] = {
...existing,
...tokenDefinition,
chainId: this.chainId,
balance,
displayBalance: formatUnits(balance, decimals)
}
}

private async handleTransfer(log: Log, tokens: TokensDict) {
if (parseInt(log.blockNumber, 16) <= this.lastProcessedBlock) return
const [, fromPadded, toPadded] = log.topics
const tokenId = toTokenId({ address: log.address, chainId: this.chainId })
const value = BigNumber.from(log.data)

let delta = BigNumber.from(0)
if (fromPadded === this.ownerPadded) delta = delta.add(value.mul(-1))
if (toPadded === this.ownerPadded) delta = delta.add(value)

await this.processDelta(tokenId, value, tokens)
}

private async handleWithdrawal(log: Log, tokens: TokensDict) {
const [, addressPadded] = log.topics
if (addressPadded !== this.ownerPadded) return

const tokenId = toTokenId({ address: log.address, chainId: this.chainId })
await this.processDelta(tokenId, BigNumber.from(log.data).mul(-1), tokens)
}

private async handleDeposit(log: Log, tokens: TokensDict) {
const [, addressPadded] = log.topics
if (addressPadded !== this.ownerPadded) return

const tokenId = toTokenId({ address: log.address, chainId: this.chainId })
await this.processDelta(tokenId, BigNumber.from(log.data), tokens)
}

private async handle(eventLog: Log, tokensDict: TokensDict) {
const logBlock = parseInt(eventLog.blockNumber, 16)
log.info('Processing logs', {
lastProcessed: this.lastProcessedBlock,
logBlock,
process: logBlock > this.lastProcessedBlock
})

return logBlock > this.lastProcessedBlock
? this.handlers[eventLog.topics[0] as LogTopic](eventLog, tokensDict)
: new Promise((r) => r(null))
}

public async process(logs: Log[], latestBlock: number, tokens: TokenDefinition[]) {
log.info('Processing logs', { latestBlock, owner: this.owner })
const tokensDict = toTokenDict(tokens)
await Promise.all(logs.map((log) => this.handle(log, tokensDict)))
this.lastProcessedBlock = latestBlock
return Object.values(this.balances)
}

constructor(
private owner: Address,
balances: Balance[],
public lastProcessedBlock: number,
private chainId: number,
private provider: EthereumProvider
) {
balances.forEach((balance) => (this.balances[toTokenId(balance)] = balance))
this.ownerPadded = hexZeroPad(owner, 32)
}
}
82 changes: 71 additions & 11 deletions main/externalData/balances/scan.ts
Original file line number Diff line number Diff line change
@@ -3,13 +3,18 @@ import { BigNumber as EthersBigNumber } from '@ethersproject/bignumber'
import { Interface } from '@ethersproject/abi'
import { addHexPrefix } from '@ethereumjs/util'
import log from 'electron-log'
import { hexZeroPad, BytesLike } from '@ethersproject/bytes'

import multicall, { Call, supportsChain as multicallSupportsChain } from '../../multicall'
import erc20TokenAbi from './erc-20-abi'
import { groupByChain, TokensByChain } from './reducers'

import type { BytesLike } from '@ethersproject/bytes'
import type EthereumProvider from 'ethereum-provider'
import { Log, LogProcessor, LogTopic } from './logs'

//TODO: move the log processing outside of the scanning system - on startup seed the balances and then get logs for each block // at a polling interval
const toLogProcessorKey = (owner: Address, chainId: number) => `${chainId}:${owner}`

const logProcessors: Record<string, LogProcessor> = {}

const erc20Interface = new Interface(erc20TokenAbi)

@@ -41,6 +46,43 @@ function createBalance(rawBalance: string, decimals: number): ExternalBalance {
}

export default function (eth: EthereumProvider) {
async function getLatestBlock(chainId: number) {
const blockNumber: string = await eth.request({
method: 'eth_blockNumber',
params: [],
chainId: addHexPrefix(chainId.toString(16))
})
return parseInt(blockNumber)
}

async function getTransferLogs(address: string, chainId: number, fromBlock: number): Promise<Log[]> {
const logs = (await Promise.all([
eth.request({
method: 'eth_getLogs',
params: [
{
fromBlock: '0x' + fromBlock.toString(16),
toBlock: 'latest',
topics: [[LogTopic.TRANSFER, LogTopic.DEPOSIT, LogTopic.WITHDRAWAL], [hexZeroPad(address, 32)]]
}
],
chainId: addHexPrefix(chainId.toString(16))
}),
eth.request({
method: 'eth_getLogs',
params: [
{
fromBlock: '0x' + fromBlock.toString(16),
toBlock: 'latest',
topics: [[LogTopic.TRANSFER], [], [hexZeroPad(address, 32)]]
}
],
chainId: addHexPrefix(chainId.toString(16))
})
])) as [Log[], Log[]]
return logs.flat()
}

function balanceCalls(owner: string, tokens: TokenDefinition[]): Call<EthersBigNumber, ExternalBalance>[] {
return tokens.map((token) => ({
target: token.address,
@@ -89,10 +131,11 @@ export default function (eth: EthereumProvider) {
try {
const rawBalance = await getTokenBalance(token, owner)

return {
const balance = {
...token,
...createBalance(rawBalance, token.decimals)
}
return balance
} catch (e) {
log.warn(`could not load balance for token with address ${token.address}`, e)
return undefined
@@ -109,16 +152,18 @@ export default function (eth: EthereumProvider) {

const results = await multicall(chainId, eth).batchCall(calls)

return results.reduce((acc, result, i) => {
const balances = results.reduce((acc, result, i) => {
if (result.success) {
acc.push({
const balance = {
...tokens[i],
...result.returnValues[0]
})
}
acc.push(balance)
}

return acc
}, [] as Balance[])
return balances
}

return {
@@ -131,12 +176,27 @@ export default function (eth: EthereumProvider) {
const tokensByChain = tokens.reduce(groupByChain, {} as TokensByChain)

const tokenBalances = await Promise.all(
Object.entries(tokensByChain).map(([chain, tokens]) => {
Object.entries(tokensByChain).map(async ([chain, tokens]) => {
const chainId = parseInt(chain)

return multicallSupportsChain(chainId)
? getTokenBalancesFromMulticall(owner, tokens, chainId)
: getTokenBalancesFromContracts(owner, tokens)
const latestBlock = await getLatestBlock(chainId)
const logProcessorKey = toLogProcessorKey(owner, chainId)
const logProcessor = logProcessors[logProcessorKey]
if (logProcessor) {
try {
const logs = await getTransferLogs(owner, chainId, logProcessor.lastProcessedBlock)
return logProcessor.process(logs, latestBlock, tokens)
} catch (error) {
log.warn('Unable to update balances using eth_getLogs', { chainId })
}
}

const balances = multicallSupportsChain(chainId)
? await getTokenBalancesFromMulticall(owner, tokens, chainId)
: await getTokenBalancesFromContracts(owner, tokens)

logProcessors[logProcessorKey] = new LogProcessor(owner, balances, latestBlock, chainId, eth)

return balances
})
)