From ce7bbd66b4efdbf8b632bfde663329e7fc3df023 Mon Sep 17 00:00:00 2001 From: Paul Miller Date: Sat, 11 May 2024 19:03:04 +0000 Subject: [PATCH] Add ArchiveNodeProvider --- README.md | 26 +++++----- package.json | 11 ++--- src/net/{tx.ts => archive.ts} | 4 +- src/net/index.ts | 92 ++++------------------------------- src/net/provider.ts | 89 +++++++++++++++++++++++++++++++++ test/net.test.js | 26 ++++++---- 6 files changed, 132 insertions(+), 116 deletions(-) rename src/net/{tx.ts => archive.ts} (99%) create mode 100644 src/net/provider.ts diff --git a/README.md b/README.md index dfcbd0d..3eae2f4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Minimal library for Ethereum transactions, addresses and smart contracts. - ðŸ”ŧ Tree-shaking-friendly: use only what's necessary, other code won't be included - 🔍 Reliable: 150MB of test vectors from EIPs, ethers and viem - ✍ïļ Create, sign and decode transactions using human-readable hints -- 🌍 Fetch historical transactions and token balances from an archive node +- 🌍 Fetch balances and history from an archive node - 🆎 Call smart contracts: Chainlink and Uniswap APIs are included - ðŸĶš Typescript-friendly ABI, RLP and SSZ decoding - ðŸŠķ 1200 lines for core functionality @@ -27,7 +27,7 @@ If you don't like NPM, a standalone [eth-signer.js](https://github.com/paulmillr - [Transactions: create, sign](#create-and-sign-transactions) - [Addresses: create, checksum](#create-and-checksum-addresses) - [Generate random wallet](#generate-random-keys-and-addresses) -- [Fetch balances and history using RPC](#fetch-balances-and-history-using-rpc) +- [Fetch balances and history from an archive node](#fetch-balances-and-history-from-an-archive-node) - [Call smart contracts](#call-smart-contracts) - [Fetch Chainlink oracle prices](#fetch-chainlink-oracle-prices) - [Swap tokens with Uniswap](#swap-tokens-with-uniswap) @@ -89,23 +89,21 @@ console.log(random.privateKey, random.address); // 0x26d930712fd2f612a107A70fd0Ad79b777cD87f6 ``` -### Fetch balances and history using RPC +### Fetch balances and history from an archive node ```ts -import { TxProvider, calcTransfersDiff } from 'micro-eth-signer/net/tx'; -const txp = TxProvider(fetchProvider(fetch)); +import { ArchiveNodeProvider, FetchProvider, calcTransfersDiff } from 'micro-eth-signer/net'; +const prov = new ArchiveNodeProvider(new FetchProvider(globalThis.fetch)); // use built-in fetch() const addr = '0x26d930712fd2f612a107A70fd0Ad79b777cD87f6'; -const _2 = await txp.height(); -const _3 = await txp.transfers(addr); -const _4 = await txp.allowances(addr); -const _5 = await txp.tokenBalances(addr); -// High-level methods are `height`, `unspent`, `transfers`, `allowances` and `tokenBalances`. -// -// Low-level methods are `blockInfo`, `internalTransactions`, `ethLogs`, `tokenTransfers`, `wethTransfers`, -// `tokenInfo` and `txInfo`. +for (let method of ['height', 'unspent', 'transfers', 'allowances', 'tokenBalances']) { + console.log(method, await prov[method](addr)); +} +// Other available methods: +// `blockInfo`, `internalTransactions`, `ethLogs`, +// `tokenTransfers`, `wethTransfers`, `tokenInfo` and `txInfo` ``` -An archive node is required. Reth has 100-block window limit, which means it's too slow for now. +Erigon is preferred as archive node software. Reth has 100-block window limit, which means it's too slow for now. ### Call smart contracts diff --git a/package.json b/package.json index 7c407d4..0117543 100644 --- a/package.json +++ b/package.json @@ -3,9 +3,9 @@ "version": "0.8.1", "description": "Minimal library for Ethereum transactions, addresses and smart contracts", "files": ["abi", "esm", "net", "src", "*.js", "*.d.ts", "*.js.map", "*.d.ts.map"], - "main": "lib/index.js", - "module": "lib/esm/index.js", - "types": "lib/index.d.ts", + "main": "index.js", + "module": "esm/index.js", + "types": "index.d.ts", "dependencies": { "@noble/curves": "~1.4.0", "@noble/hashes": "~1.4.0", @@ -42,11 +42,6 @@ "import": "./esm/net/index.js", "default": "./net/index.js" }, - "./net/tx": { - "types": "./net/tx.d.ts", - "import": "./esm/net/tx.js", - "default": "./net/tx.js" - }, "./rlp": { "types": "./net/rlp.d.ts", "import": "./esm/net/rlp.js", diff --git a/src/net/tx.ts b/src/net/archive.ts similarity index 99% rename from src/net/tx.ts rename to src/net/archive.ts index 89b25d2..b3bcf8f 100644 --- a/src/net/tx.ts +++ b/src/net/archive.ts @@ -7,7 +7,7 @@ import { ContractInfo, createContract, events, ERC20, WETH } from '../abi/index. Methods to fetch list of transactions from any ETH node RPC. It should be easy. However, this is sparta^W ethereum, so, prepare to suffer. -The network is not directly called: `TxProvider#rpc` calls `Web3Provider`. +The network is not directly called: `ArchiveNodeProvider#rpc` calls `Web3Provider`. - There is no simple & fast API inside nodes, all external API create their own namespace for this - API is different between nodes: erigon uses streaming, other nodes use pagination @@ -335,7 +335,7 @@ function validateLogOpts(opts: Record) { * Low-level methods are `blockInfo`, `internalTransactions`, `ethLogs`, `tokenTransfers`, `wethTransfers`, * `tokenInfo` and `txInfo`. */ -export class TxProvider { +export class ArchiveNodeProvider { constructor(private provider: Web3Provider) {} // The low-level place where network calls are done diff --git a/src/net/index.ts b/src/net/index.ts index 07d8bd7..ed74c64 100644 --- a/src/net/index.ts +++ b/src/net/index.ts @@ -1,89 +1,17 @@ import Chainlink from './chainlink.js'; import ENS from './ens.js'; +import FetchProvider from './provider.js'; +import { ArchiveNodeProvider, calcTransfersDiff } from './archive.js'; import UniswapV2 from './uniswap-v2.js'; import UniswapV3 from './uniswap-v3.js'; -import { Web3Provider, Web3CallArgs, hexToNumber } from '../utils.js'; // There are many low level APIs inside which are not exported yet. -export { Chainlink, ENS, UniswapV2, UniswapV3 }; - -// There is a lot features required for network to make this useful -// This is attempt to create them via small composable wrappers -export type FetchFn = ( - url: string, - opt?: Record -) => Promise<{ json: () => Promise }>; -type Headers = Record; -type JsonFn = (url: string, headers: Headers, body: unknown) => Promise; -type PromiseCb = { - resolve: (value: T | PromiseLike) => void; - reject: (reason?: any) => void; -}; - -function getJSONUsingFetch(fn: FetchFn): JsonFn { - return async (url: string, headers: Headers = {}, body: unknown) => { - const res = await fn(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', ...headers }, - body: JSON.stringify(body), - }); - return await res.json(); - }; -} - -// Unsafe. TODO: inspect for race conditions and bugs. -function limitParallel(jsonFn: JsonFn, limit: number): JsonFn { - let cur = 0; - const queue: ({ url: string; headers: Headers; body: unknown } & PromiseCb)[] = []; - const process = () => { - if (cur >= limit) return; - const next = queue.shift(); - if (!next) return; - try { - jsonFn(next.url, next.headers, next.body) - .then(next.resolve) - .catch(next.reject) - .finally(() => { - cur--; - process(); - }); - } catch (e) { - next.reject(e); - cur--; - } - cur++; - }; - return (url, headers, body) => { - return new Promise((resolve, reject) => { - queue.push({ url, headers, body, resolve, reject }); - process(); - }); - }; -} - -type NetworkOpts = { - limitParallel?: number; -}; - -export const FetchProvider = ( - fetch: FetchFn, - url: string, - headers: Headers = {}, - opts: NetworkOpts = {} -): Web3Provider => { - let fn = getJSONUsingFetch(fetch); - if (opts.limitParallel) fn = limitParallel(fn, opts.limitParallel); - const jsonrpc = async (method: string, ...params: any[]) => { - const json = await fn(url, headers, { jsonrpc: '2.0', id: 0, method, params }); - if (json && json.error) - throw new Error(`FetchProvider(${json.error.code}): ${json.error.message || json.error}`); - return json.result; - }; - return { - ethCall: (args: Web3CallArgs, tag = 'latest') => - jsonrpc('eth_call', args, tag) as Promise, - estimateGas: async (args: Web3CallArgs, tag = 'latest') => - hexToNumber(await jsonrpc('eth_estimateGas', args, tag)), - call: (method: string, ...args: any[]) => jsonrpc(method, ...args), - }; +export { + ArchiveNodeProvider, + calcTransfersDiff, + Chainlink, + ENS, + FetchProvider, + UniswapV2, + UniswapV3, }; diff --git a/src/net/provider.ts b/src/net/provider.ts new file mode 100644 index 0000000..95c857b --- /dev/null +++ b/src/net/provider.ts @@ -0,0 +1,89 @@ +import { Web3Provider, Web3CallArgs, hexToNumber } from '../utils.js'; + +export type FetchFn = ( + url: string, + opt?: Record +) => Promise<{ json: () => Promise }>; +type Headers = Record; +type NetworkOpts = { + concurrencyLimit?: number; + headers?: Headers; +}; +type PromiseCb = { + resolve: (value: T | PromiseLike) => void; + reject: (reason?: any) => void; +}; + +export default class FetchProvider implements Web3Provider { + private concurrencyLimit: number; + private currentlyFetching: number; + private headers: Headers; + constructor( + private fetchFunction: FetchFn, + readonly rpcUrl: string, + options: NetworkOpts = {} + ) { + this.concurrencyLimit = options.concurrencyLimit == null ? 0 : options.concurrencyLimit; + this.currentlyFetching = 0; + this.headers = options.headers || {}; + if (typeof this.headers !== 'object') throw new Error('invalid headers: expected object'); + } + private async fetchJson(body: unknown) { + const url = this.rpcUrl; + const args = { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...this.headers }, + body: JSON.stringify(body), + }; + const res = await this.fetchFunction(url, args); + return res.json(); + } + private addToFetchQueue(body: unknown): Promise { + if (this.concurrencyLimit === 0) return this.fetchJson(body); + const queue: ({ body: unknown } & PromiseCb)[] = []; + const process = () => { + if (this.currentlyFetching >= this.concurrencyLimit) return; + const next = queue.shift(); + if (!next) return; + try { + this.fetchJson(next.body) + .then(next.resolve) + .catch(next.reject) + .finally(() => { + this.currentlyFetching--; + process(); + }); + } catch (e) { + next.reject(e); + this.currentlyFetching--; + } + this.currentlyFetching++; + }; + return new Promise((resolve, reject) => { + queue.push({ body, resolve, reject }); + process(); + }); + } + private async rpc(method: string, ...params: any[]): Promise { + const body = { + jsonrpc: '2.0', + id: 0, + method, + params, + }; + const json = await this.addToFetchQueue(body); + if (json && json.error) + throw new Error(`FetchProvider(${json.error.code}): ${json.error.message || json.error}`); + return json.result; + } + + ethCall(args: Web3CallArgs, tag = 'latest') { + return this.rpc('eth_call', args, tag); + } + async estimateGas(args: Web3CallArgs, tag = 'latest') { + return hexToNumber(await this.rpc('eth_estimateGas', args, tag)); + } + call(method: string, ...args: any[]) { + return this.rpc(method, ...args); + } +} diff --git a/test/net.test.js b/test/net.test.js index d78cb32..e71bcc4 100644 --- a/test/net.test.js +++ b/test/net.test.js @@ -1,9 +1,15 @@ -import { deepStrictEqual, throws } from 'node:assert'; +import { deepStrictEqual } from 'node:assert'; import { describe, should } from 'micro-should'; import { tokenFromSymbol } from '../esm/abi/index.js'; -import { FetchProvider, ENS, Chainlink, UniswapV3 } from '../esm/net/index.js'; +import { + ArchiveNodeProvider, + calcTransfersDiff, + FetchProvider, + ENS, + Chainlink, + UniswapV3, +} from '../esm/net/index.js'; import { ethDecimal, numberTo0xHex } from '../esm/utils.js'; -import * as netTx from '../esm/net/tx.js'; // These real network responses from real nodes, captured by fetchReplay import { default as NET_TX_REPLAY } from './vectors/rpc/transactions.js'; import { default as NET_ENS_REPLAY } from './vectors/rpc/ens.js'; @@ -58,7 +64,7 @@ const fetchReplay = (logs, offline = true) => { describe('Network', () => { should('ENS', async () => { const replay = fetchReplay(NET_ENS_REPLAY); - const provider = FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); + const provider = new FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); const ens = new ENS(provider); const vitalikAddr = await ens.nameToAddress('vitalik.eth'); const vitalikName = await ens.addressToName(vitalikAddr); @@ -69,7 +75,7 @@ describe('Network', () => { should('Chainlink', async () => { const replay = fetchReplay(NET_CHAINLINK_REPLAY); - const provider = FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); + const provider = new FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); const chainlink = new Chainlink(provider); const btcPrice = await chainlink.coinPrice('BTC'); deepStrictEqual(btcPrice, 69369.10271); @@ -78,7 +84,7 @@ describe('Network', () => { should('UniswapV3', async () => { const replay = fetchReplay(NET_UNISWAP_REPLAY); - const provider = FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); + const provider = new FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); const univ3 = new UniswapV3(provider); // Actual code const vitalikAddr = '0xd8da6bf26964af9d7eed9e03e53415d37aa96045'; @@ -102,7 +108,7 @@ describe('Network', () => { should('estimateGas', async () => { const replay = fetchReplay(NET_ESTIMATE_GAS_REPLAY); - const provider = FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); + const provider = new FetchProvider(REAL_NETWORK ? fetch : replay, NODE_URL, NODE_HEADERS); const gasLimit = await provider.estimateGas({ from: '0xd8da6bf26964af9d7eed9e03e53415d37aa96045', to: '0xe592427a0aece92de3edee1f18e0157c05861564', @@ -118,8 +124,8 @@ describe('Network', () => { // Perfect for tests: only has a few transactions and provides different types of txs. const addr = '0x6994eCe772cC4aBb5C9993c065a34C94544A4087'; const replay = fetchReplay(NET_TX_REPLAY, true); - const provider = FetchProvider(REAL_NETWORK ? fetch : replay, 'NODE_URL', {}); - const tx = new netTx.TxProvider(provider); + const provider = new FetchProvider(REAL_NETWORK ? fetch : replay, 'NODE_URL', {}); + const tx = new ArchiveNodeProvider(provider); // Blocks deepStrictEqual(await tx.blockInfo(15_010_733), NET_TX_VECTORS.block); // Internal transactions sanity @@ -168,7 +174,7 @@ describe('Network', () => { const transfers = (await tx.transfers(addr)).map((i) => ({ ...i, info: undefined })); deepStrictEqual(transfers, NET_TX_VECTORS.transfers); - const diff = netTx.calcTransfersDiff(transfers); + const diff = calcTransfersDiff(transfers); const diffLast = diff[diff.length - 1]; // From etherscan // 0.000130036071955215