Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

trie: use Uint8Array as value type in DB #3067

Merged
merged 18 commits into from
Oct 27, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions packages/block/src/block.ts
Original file line number Diff line number Diff line change
@@ -48,9 +48,12 @@ export class Block {
public readonly transactions: TypedTransaction[] = []
public readonly uncleHeaders: BlockHeader[] = []
public readonly withdrawals?: Withdrawal[]
public readonly txTrie = new Trie()
public readonly common: Common

private cache: {
txTrieRoot?: Uint8Array
} = {}

/**
* Returns the withdrawals trie root for array of Withdrawal.
* @param wts array of Withdrawal to compute the root of
@@ -439,9 +442,8 @@ export class Block {
/**
* Generates transaction trie for validation.
*/
async genTxTrie(): Promise<void> {
const { transactions, txTrie } = this
await Block.genTransactionsTrieRoot(transactions, txTrie)
async genTxTrie(): Promise<Uint8Array> {
return Block.genTransactionsTrieRoot(this.transactions, new Trie())
}

/**
@@ -456,10 +458,10 @@ export class Block {
return result
}

if (equalsBytes(this.txTrie.root(), KECCAK256_RLP)) {
await this.genTxTrie()
if (this.cache.txTrieRoot === undefined) {
this.cache.txTrieRoot = await this.genTxTrie()
}
result = equalsBytes(this.txTrie.root(), this.header.transactionsTrie)
result = equalsBytes(this.cache.txTrieRoot, this.header.transactionsTrie)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice. 🙂👍

return result
}

9 changes: 9 additions & 0 deletions packages/client/bin/cli.ts
Original file line number Diff line number Diff line change
@@ -334,6 +334,14 @@ const args: ClientOpts = yargs(hideBin(process.argv))
deprecated:
'Support for `--prefixStorageTrieKeys=false` is temporary. Please sync new instances with `prefixStorageTrieKeys` enabled',
})
.options('useStringValueTrieDB', {
describe:
'Use string values in the trie DB. This is old behavior, new behavior uses Uint8Arrays in the DB (more performant)',
boolean: true,
default: false,
deprecated:
'Usage of old DBs which uses string-values is temporary. Please sync new instances without this option.',
})
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No big deal, but I think we should rather use StateDB as a terminoloy for something like this in the future.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, good point, so it is not being confused with a blocks DB or anything 😄

.option('txLookupLimit', {
describe:
'Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)',
@@ -876,6 +884,7 @@ async function run() {
disableBeaconSync: args.disableBeaconSync,
forceSnapSync: args.forceSnapSync,
prefixStorageTrieKeys: args.prefixStorageTrieKeys,
useStringValueTrieDB: args.useStringValueTrieDB,
txLookupLimit: args.txLookupLimit,
pruneEngineCache: args.pruneEngineCache,
})
8 changes: 8 additions & 0 deletions packages/client/src/config.ts
Original file line number Diff line number Diff line change
@@ -68,6 +68,12 @@ export interface ConfigOptions {
*/
prefixStorageTrieKeys?: boolean

/**
* A temporary option to offer backward compatibility with already-synced databases that stores
* trie items as `string`, instead of the more performant `Uint8Array`
*/
useStringValueTrieDB?: boolean

/**
* Provide a custom VM instance to process blocks
*
@@ -427,6 +433,7 @@ export class Config {
// Just a development only flag, will/should be removed
public readonly disableSnapSync: boolean = false
public readonly prefixStorageTrieKeys: boolean
public readonly useStringValueTrieDB: boolean

public synchronized: boolean
/** lastSyncDate in ms */
@@ -507,6 +514,7 @@ export class Config {
this.disableBeaconSync = options.disableBeaconSync ?? false
this.forceSnapSync = options.forceSnapSync ?? false
this.prefixStorageTrieKeys = options.prefixStorageTrieKeys ?? true
this.useStringValueTrieDB = options.useStringValueTrieDB ?? false

// Start it off as synchronized if this is configured to mine or as single node
this.synchronized = this.isSingleNode ?? this.mine
5 changes: 4 additions & 1 deletion packages/client/src/execution/vmexecution.ts
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ import { ConsensusType, Hardfork } from '@ethereumjs/common'
import { getGenesis } from '@ethereumjs/genesis'
import { CacheType, DefaultStateManager } from '@ethereumjs/statemanager'
import { Trie } from '@ethereumjs/trie'
import { BIGINT_0, BIGINT_1, Lock, bytesToHex, equalsBytes } from '@ethereumjs/util'
import { BIGINT_0, BIGINT_1, Lock, ValueEncoding, bytesToHex, equalsBytes } from '@ethereumjs/util'
import { VM } from '@ethereumjs/vm'

import { Event } from '../types'
@@ -57,6 +57,9 @@ export class VMExecution extends Execution {
db: new LevelDB(this.stateDB),
useKeyHashing: true,
cacheSize: this.config.trieCache,
valueEncoding: options.config.useStringValueTrieDB
? ValueEncoding.String
: ValueEncoding.Bytes,
})

this.config.logger.info(`Initializing account cache size=${this.config.accountCache}`)
1 change: 1 addition & 0 deletions packages/client/src/types.ts
Original file line number Diff line number Diff line change
@@ -155,6 +155,7 @@ export interface ClientOpts {
disableBeaconSync?: boolean
forceSnapSync?: boolean
prefixStorageTrieKeys?: boolean
useStringValueTrieDB?: boolean
txLookupLimit?: number
startBlock?: number
isSingleNode?: boolean
59 changes: 39 additions & 20 deletions packages/trie/src/db/checkpoint.ts
Original file line number Diff line number Diff line change
@@ -7,16 +7,17 @@ import {
import { LRUCache } from 'lru-cache'

import type { Checkpoint, CheckpointDBOpts } from '../types.js'
import type { BatchDBOp, DB, DelBatch, PutBatch } from '@ethereumjs/util'
import type { BatchDBOp, DB, EncodingOpts } from '@ethereumjs/util'

/**
* DB is a thin wrapper around the underlying levelup db,
* which validates inputs and sets encoding type.
*/
export class CheckpointDB implements DB {
public checkpoints: Checkpoint[]
public db: DB<string, string>
public db: DB<string, string | Uint8Array>
public readonly cacheSize: number
private readonly valueEncoding: ValueEncoding

// Starting with lru-cache v8 undefined and null are not allowed any more
// as cache values. At the same time our design works well, since undefined
@@ -27,7 +28,7 @@ export class CheckpointDB implements DB {
// be some not so clean workaround.
//
// (note that @ts-ignore doesn't work since stripped on declaration (.d.ts) files)
protected _cache?: LRUCache<string, any>
protected _cache?: LRUCache<string, Uint8Array>
// protected _cache?: LRUCache<string, Uint8Array | undefined>

_stats = {
@@ -49,6 +50,7 @@ export class CheckpointDB implements DB {
constructor(opts: CheckpointDBOpts) {
this.db = opts.db
this.cacheSize = opts.cacheSize ?? 0
this.valueEncoding = opts.valueEncoding ?? ValueEncoding.String
// Roots of trie at the moment of checkpoint
this.checkpoints = []

@@ -152,38 +154,44 @@ export class CheckpointDB implements DB {
}
}
// Nothing has been found in diff cache, look up from disk
const valueHex = await this.db.get(keyHex, {
const value = await this.db.get(keyHex, {
keyEncoding: KeyEncoding.String,
valueEncoding: ValueEncoding.String,
valueEncoding: this.valueEncoding,
})
this._stats.db.reads += 1
if (valueHex !== undefined) {
if (value !== undefined) {
this._stats.db.hits += 1
}
const value = valueHex !== undefined ? unprefixedHexToBytes(valueHex) : undefined
this._cache?.set(keyHex, value)
const returnValue =
value !== undefined
? value instanceof Uint8Array
? value
: unprefixedHexToBytes(<string>value)
: undefined
this._cache?.set(keyHex, returnValue)
if (this.hasCheckpoints()) {
// Since we are a checkpoint, put this value in diff cache,
// so future `get` calls will not look the key up again from disk.
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value)
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, returnValue)
}

return value
return returnValue
}

/**
* @inheritDoc
*/
async put(key: Uint8Array, value: Uint8Array): Promise<void> {
const keyHex = bytesToUnprefixedHex(key)
const valueHex = bytesToUnprefixedHex(value)
if (this.hasCheckpoints()) {
// put value in diff cache
this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(keyHex, value)
} else {
await this.db.put(keyHex, valueHex, {
const valuePut =
this.valueEncoding === ValueEncoding.Bytes ? value : bytesToUnprefixedHex(value)
await this.db.put(keyHex, <any>valuePut, {
keyEncoding: KeyEncoding.String,
valueEncoding: ValueEncoding.String,
valueEncoding: this.valueEncoding,
})
this._stats.db.writes += 1

@@ -230,16 +238,23 @@ export class CheckpointDB implements DB {
}
} else {
const convertedOps = opStack.map((op) => {
const convertedOp = {
const convertedOp: {
key: string
value: Uint8Array | string | undefined
type: 'put' | 'del'
opts?: EncodingOpts
} = {
key: bytesToUnprefixedHex(op.key),
value: op.type === 'put' ? bytesToUnprefixedHex(op.value) : undefined,
value: op.type === 'put' ? op.value : undefined,
type: op.type,
opts: op.opts,
opts: { ...op.opts, ...{ valueEncoding: this.valueEncoding } },
}
if (op.type === 'put' && this.valueEncoding === ValueEncoding.String) {
convertedOp.value = bytesToUnprefixedHex(<Uint8Array>convertedOp.value)
}
if (op.type === 'put') return convertedOp as PutBatch<string, string>
else return convertedOp as DelBatch<string>
return convertedOp
})
await this.db.batch(convertedOps)
await this.db.batch(<any>convertedOps)
}
}

@@ -266,7 +281,11 @@ export class CheckpointDB implements DB {
* @inheritDoc
*/
shallowCopy(): CheckpointDB {
return new CheckpointDB({ db: this.db, cacheSize: this.cacheSize })
return new CheckpointDB({
db: this.db,
cacheSize: this.cacheSize,
valueEncoding: this.valueEncoding,
})
}

open() {
45 changes: 35 additions & 10 deletions packages/trie/src/trie.ts
Original file line number Diff line number Diff line change
@@ -61,6 +61,7 @@ export class Trie {
useRootPersistence: false,
useNodePruning: false,
cacheSize: 0,
valueEncoding: ValueEncoding.String,
}

/** The root for an empty trie */
@@ -84,8 +85,21 @@ export class Trie {
* Note: in most cases, the static {@link Trie.create} constructor should be used. It uses the same API but provides sensible defaults
*/
constructor(opts?: TrieOpts) {
let valueEncoding: ValueEncoding
if (opts !== undefined) {
// Sanity check: can only set valueEncoding if a db is provided
// The valueEncoding defaults to `Bytes` if no DB is provided (use a MapDB in memory)
if (opts?.valueEncoding !== undefined && opts.db === undefined) {
throw new Error('`valueEncoding` can only be set if a `db` is provided')
}
this._opts = { ...this._opts, ...opts }

valueEncoding =
opts.db !== undefined ? opts.valueEncoding ?? ValueEncoding.String : ValueEncoding.Bytes
} else {
// No opts are given, so create a MapDB later on
// Use `Bytes` for ValueEncoding
valueEncoding = ValueEncoding.Bytes
}

this.DEBUG = process.env.DEBUG?.includes('ethjs') === true
@@ -99,7 +113,7 @@ export class Trie {
}
: (..._: any) => {}

this.database(opts?.db ?? new MapDB<string, string>())
this.database(opts?.db ?? new MapDB<string, Uint8Array>(), valueEncoding)

this.EMPTY_TRIE_ROOT = this.hash(RLP_EMPTY_STRING)
this._hashLen = this.EMPTY_TRIE_ROOT.length
@@ -121,6 +135,9 @@ export class Trie {
static async create(opts?: TrieOpts) {
let key = ROOT_DB_KEY

const encoding =
opts?.valueEncoding === ValueEncoding.Bytes ? ValueEncoding.Bytes : ValueEncoding.String

if (opts?.useKeyHashing === true) {
key = (opts?.useKeyHashingFunction ?? keccak256)(ROOT_DB_KEY) as Uint8Array
}
@@ -130,29 +147,37 @@ export class Trie {

if (opts?.db !== undefined && opts?.useRootPersistence === true) {
if (opts?.root === undefined) {
const rootHex = await opts?.db.get(bytesToUnprefixedHex(key), {
const root = await opts?.db.get(bytesToUnprefixedHex(key), {
keyEncoding: KeyEncoding.String,
valueEncoding: ValueEncoding.String,
valueEncoding: encoding,
})
opts.root = rootHex !== undefined ? unprefixedHexToBytes(rootHex) : undefined
if (typeof root === 'string') {
opts.root = unprefixedHexToBytes(root)
} else {
opts.root = root
}
} else {
await opts?.db.put(bytesToUnprefixedHex(key), bytesToUnprefixedHex(opts.root), {
keyEncoding: KeyEncoding.String,
valueEncoding: ValueEncoding.String,
})
await opts?.db.put(
bytesToUnprefixedHex(key),
<any>(encoding === ValueEncoding.Bytes ? opts.root : bytesToUnprefixedHex(opts.root)),
{
keyEncoding: KeyEncoding.String,
valueEncoding: encoding,
}
)
}
}

return new Trie(opts)
}

database(db?: DB<string, string>) {
database(db?: DB<string, string | Uint8Array>, valueEncoding?: ValueEncoding) {
if (db !== undefined) {
if (db instanceof CheckpointDB) {
throw new Error('Cannot pass in an instance of CheckpointDB')
}

this._db = new CheckpointDB({ db, cacheSize: this._opts.cacheSize })
this._db = new CheckpointDB({ db, cacheSize: this._opts.cacheSize, valueEncoding })
}

return this._db
16 changes: 13 additions & 3 deletions packages/trie/src/types.ts
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ import { utf8ToBytes } from '@ethereumjs/util'

import type { BranchNode, ExtensionNode, LeafNode } from './node/index.js'
import type { WalkController } from './util/walkController.js'
import type { DB } from '@ethereumjs/util'
import type { DB, ValueEncoding } from '@ethereumjs/util'

export type TrieNode = BranchNode | ExtensionNode | LeafNode

@@ -27,7 +27,7 @@ export interface TrieOpts {
/**
* A database instance.
*/
db?: DB<string, string>
db?: DB<string, string | Uint8Array>

/**
* A `Uint8Array` for the root of a previously stored trie
@@ -61,6 +61,11 @@ export interface TrieOpts {
*/
keyPrefix?: Uint8Array

/**
* ValueEncoding of the database (the values which are `put`/`get` in the db are of this type). Defaults to `string`
*/
valueEncoding?: ValueEncoding

/**
* Store the root inside the database after every `write` operation
*/
@@ -97,7 +102,12 @@ export interface CheckpointDBOpts {
/**
* A database instance.
*/
db: DB<string, string>
db: DB<string, string | Uint8Array>

/**
* ValueEncoding of the database (the values which are `put`/`get` in the db are of this type). Defaults to `string`
*/
valueEncoding?: ValueEncoding

/**
* Cache size (default: 0)
Loading