diff --git a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts index 7c782e6edf9..cf414d6df7f 100644 --- a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts @@ -150,7 +150,7 @@ describe('e2e_p2p_network', () => { numTxs: number, ): Promise => { const rpcConfig = getRpcConfig(); - const pxeService = await createPXEService(node, rpcConfig, {}, true); + const pxeService = await createPXEService(node, rpcConfig, true); const keyPair = ConstantKeyPair.random(new Grumpkin()); const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(keyPair.getPrivateKey(), Fr.random()); diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 978147ab5e0..3ac20b8339f 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -151,7 +151,7 @@ export async function setupPXEService( logger: DebugLogger; }> { const pxeServiceConfig = getPXEServiceConfig(); - const pxe = await createPXEService(aztecNode, pxeServiceConfig, {}, useLogSuffix); + const pxe = await createPXEService(aztecNode, pxeServiceConfig, useLogSuffix); const wallets = await createAccounts(pxe, numberOfAccounts); diff --git a/yarn-project/foundation/src/serialize/buffer_reader.ts b/yarn-project/foundation/src/serialize/buffer_reader.ts index 1b8749d6358..b54a2d1b52b 100644 --- a/yarn-project/foundation/src/serialize/buffer_reader.ts +++ b/yarn-project/foundation/src/serialize/buffer_reader.ts @@ -31,8 +31,16 @@ export class BufferReader { * @param bufferOrReader - A Buffer or BufferReader to initialize the BufferReader. * @returns An instance of BufferReader. */ - public static asReader(bufferOrReader: Buffer | BufferReader) { - return Buffer.isBuffer(bufferOrReader) ? new BufferReader(bufferOrReader) : bufferOrReader; + public static asReader(bufferOrReader: Uint8Array | Buffer | BufferReader): BufferReader { + if (bufferOrReader instanceof BufferReader) { + return bufferOrReader; + } + + const buf = Buffer.isBuffer(bufferOrReader) + ? bufferOrReader + : Buffer.from(bufferOrReader.buffer, bufferOrReader.byteOffset, bufferOrReader.byteLength); + + return new BufferReader(buf); } /** diff --git a/yarn-project/key-store/package.json b/yarn-project/key-store/package.json index bbcdabc0de8..a19af464fd2 100644 --- a/yarn-project/key-store/package.json +++ b/yarn-project/key-store/package.json @@ -32,6 +32,7 @@ "dependencies": { "@aztec/circuits.js": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/types": "workspace:^", "tslib": "^2.4.0" }, diff --git a/yarn-project/key-store/src/test_key_store.ts b/yarn-project/key-store/src/test_key_store.ts index 865fb9d75ee..65d64fd0231 100644 --- a/yarn-project/key-store/src/test_key_store.ts +++ b/yarn-project/key-store/src/test_key_store.ts @@ -1,5 +1,6 @@ -import { GrumpkinPrivateKey } from '@aztec/circuits.js'; +import { GrumpkinPrivateKey, GrumpkinScalar, Point } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; +import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { KeyPair, KeyStore, PublicKey } from '@aztec/types'; import { ConstantKeyPair } from './key_pair.js'; @@ -9,30 +10,27 @@ import { ConstantKeyPair } from './key_pair.js'; * It should be utilized in testing scenarios where secure key management is not required, and ease-of-use is prioritized. */ export class TestKeyStore implements KeyStore { - private accounts: KeyPair[] = []; - constructor(private curve: Grumpkin) {} + #keys: AztecMap; - public addAccount(privKey: GrumpkinPrivateKey): PublicKey { - const keyPair = ConstantKeyPair.fromPrivateKey(this.curve, privKey); - - // check if private key has already been used - const account = this.accounts.find(a => a.getPublicKey().equals(keyPair.getPublicKey())); - if (account) { - return account.getPublicKey(); - } + constructor(private curve: Grumpkin, database: AztecKVStore) { + this.#keys = database.createMap('key_store'); + } - this.accounts.push(keyPair); + public async addAccount(privKey: GrumpkinPrivateKey): Promise { + const keyPair = ConstantKeyPair.fromPrivateKey(this.curve, privKey); + await this.#keys.setIfNotExists(keyPair.getPublicKey().toString(), keyPair.getPrivateKey().toBuffer()); return keyPair.getPublicKey(); } - public createAccount(): Promise { + public async createAccount(): Promise { const keyPair = ConstantKeyPair.random(this.curve); - this.accounts.push(keyPair); - return Promise.resolve(keyPair.getPublicKey()); + await this.#keys.set(keyPair.getPublicKey().toString(), keyPair.getPrivateKey().toBuffer()); + return keyPair.getPublicKey(); } public getAccounts(): Promise { - return Promise.resolve(this.accounts.map(a => a.getPublicKey())); + const range = Array.from(this.#keys.keys()); + return Promise.resolve(range.map(key => Point.fromString(key))); } public getAccountPrivateKey(pubKey: PublicKey): Promise { @@ -48,13 +46,13 @@ export class TestKeyStore implements KeyStore { * @param pubKey - The public key of the account to retrieve. * @returns The KeyPair object associated with the provided key. */ - private getAccount(pubKey: PublicKey) { - const account = this.accounts.find(a => a.getPublicKey().equals(pubKey)); - if (!account) { + private getAccount(pubKey: PublicKey): KeyPair { + const privKey = this.#keys.get(pubKey.toString()); + if (!privKey) { throw new Error( 'Unknown account.\nSee docs for context: https://docs.aztec.network/dev_docs/contracts/common_errors#unknown-contract-error', ); } - return account; + return ConstantKeyPair.fromPrivateKey(this.curve, GrumpkinScalar.fromBuffer(privKey)); } } diff --git a/yarn-project/key-store/tsconfig.json b/yarn-project/key-store/tsconfig.json index 1820488d409..76107a492b5 100644 --- a/yarn-project/key-store/tsconfig.json +++ b/yarn-project/key-store/tsconfig.json @@ -6,6 +6,9 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../kv-store" + }, { "path": "../circuits.js" }, diff --git a/yarn-project/kv-store/.eslintrc.cjs b/yarn-project/kv-store/.eslintrc.cjs new file mode 100644 index 00000000000..e659927475c --- /dev/null +++ b/yarn-project/kv-store/.eslintrc.cjs @@ -0,0 +1 @@ +module.exports = require('@aztec/foundation/eslint'); diff --git a/yarn-project/kv-store/README.md b/yarn-project/kv-store/README.md new file mode 100644 index 00000000000..37c15c72f12 --- /dev/null +++ b/yarn-project/kv-store/README.md @@ -0,0 +1,10 @@ +# KV Store + +The Aztec KV store is an implementation of a durable key-value database with a pluggable backend. THe only supported backend right now is LMDB by using the [`lmdb-js` package](https://github.com/kriszyp/lmdb-js). + +This package exports a number of primitive data structures that can be used to build domain-specific databases in each node component (e.g. a PXE database or an Archiver database). The data structures supported: + +- singleton - holds a single value. Great for when a value needs to be stored but it's not a collection (e.g. the latest block header or the length of an array) +- array - works like a normal in-memory JS array. It can't contain holes and it can be used as a stack (push-pop mechanics). +- map - a hashmap where keys can be numbers or strings +- multi-map - just like a map but each key holds multiple values. Can be used for indexing into other data structures diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json new file mode 100644 index 00000000000..8aea043378d --- /dev/null +++ b/yarn-project/kv-store/package.json @@ -0,0 +1,50 @@ +{ + "name": "@aztec/kv-store", + "version": "0.1.0", + "type": "module", + "exports": "./dest/index.js", + "scripts": { + "build": "yarn clean && tsc -b", + "build:dev": "tsc -b --watch", + "clean": "rm -rf ./dest .tsbuildinfo", + "formatting": "run -T prettier --check ./src && run -T eslint ./src", + "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests", + "start": "DEBUG='aztec:*' && node ./dest/bin/index.js" + }, + "inherits": [ + "../package.common.json" + ], + "jest": { + "preset": "ts-jest/presets/default-esm", + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" + }, + "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", + "rootDir": "./src", + "workerThreads": true + }, + "dependencies": { + "@aztec/foundation": "workspace:^", + "lmdb": "^2.9.1" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@types/jest": "^29.5.0", + "@types/node": "^18.7.23", + "jest": "^29.5.0", + "jest-mock-extended": "^3.0.3", + "ts-jest": "^29.1.0", + "ts-node": "^10.9.1", + "typescript": "^5.0.4" + }, + "files": [ + "dest", + "src", + "!*.test.*" + ], + "types": "./dest/index.d.ts", + "engines": { + "node": ">=18" + } +} diff --git a/yarn-project/kv-store/src/index.ts b/yarn-project/kv-store/src/index.ts new file mode 100644 index 00000000000..2a71333f9e6 --- /dev/null +++ b/yarn-project/kv-store/src/index.ts @@ -0,0 +1,5 @@ +export * from './interfaces/array.js'; +export * from './interfaces/map.js'; +export * from './interfaces/singleton.js'; +export * from './interfaces/store.js'; +export * from './lmdb/store.js'; diff --git a/yarn-project/kv-store/src/interfaces/array.ts b/yarn-project/kv-store/src/interfaces/array.ts new file mode 100644 index 00000000000..e2492204212 --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/array.ts @@ -0,0 +1,54 @@ +/** + * An array backed by a persistent store. Can not have any holes in it. + */ +export interface AztecArray { + /** + * The size of the array + */ + length: number; + + /** + * Pushes values to the end of the array + * @param vals - The values to push to the end of the array + * @returns The new length of the array + */ + push(...vals: T[]): Promise; + + /** + * Pops a value from the end of the array. + * @returns The value that was popped, or undefined if the array was empty + */ + pop(): Promise; + + /** + * Gets the value at the given index. Index can be in the range [-length, length - 1). + * If the index is negative, it will be treated as an offset from the end of the array. + * + * @param index - The index to get the value from + * @returns The value at the given index or undefined if the index is out of bounds + */ + at(index: number): T | undefined; + + /** + * Updates the value at the given index. Index can be in the range [-length, length - 1). + * @param index - The index to set the value at + * @param val - The value to set + * @returns Whether the value was set + */ + setAt(index: number, val: T): Promise; + + /** + * Iterates over the array with indexes. + */ + entries(): IterableIterator<[number, T]>; + + /** + * Iterates over the array. + */ + values(): IterableIterator; + + /** + * Iterates over the array. + */ + [Symbol.iterator](): IterableIterator; +} diff --git a/yarn-project/kv-store/src/interfaces/map.ts b/yarn-project/kv-store/src/interfaces/map.ts new file mode 100644 index 00000000000..8de773837b7 --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/map.ts @@ -0,0 +1,70 @@ +/** + * A map backed by a persistent store. + */ +export interface AztecMap { + /** + * Gets the value at the given key. + * @param key - The key to get the value from + */ + get(key: K): V | undefined; + + /** + * Checks if a key exists in the map. + * @param key - The key to check + * @returns True if the key exists, false otherwise + */ + has(key: K): boolean; + + /** + * Sets the value at the given key. + * @param key - The key to set the value at + * @param val - The value to set + */ + set(key: K, val: V): Promise; + + /** + * Sets the value at the given key if it does not already exist. + * @param key - The key to set the value at + * @param val - The value to set + */ + setIfNotExists(key: K, val: V): Promise; + + /** + * Deletes the value at the given key. + * @param key - The key to delete the value at + */ + delete(key: K): Promise; + + /** + * Iterates over the map's key-value entries + */ + entries(): IterableIterator<[K, V]>; + + /** + * Iterates over the map's values + */ + values(): IterableIterator; + + /** + * Iterates over the map's keys + */ + keys(): IterableIterator; +} + +/** + * A map backed by a persistent store that can have multiple values for a single key. + */ +export interface AztecMultiMap extends AztecMap { + /** + * Gets all the values at the given key. + * @param key - The key to get the values from + */ + getValues(key: K): IterableIterator; + + /** + * Deletes a specific value at the given key. + * @param key - The key to delete the value at + * @param val - The value to delete + */ + deleteValue(key: K, val: V): Promise; +} diff --git a/yarn-project/kv-store/src/interfaces/singleton.ts b/yarn-project/kv-store/src/interfaces/singleton.ts new file mode 100644 index 00000000000..43b34aa0ad8 --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/singleton.ts @@ -0,0 +1,20 @@ +/** + * Represents a singleton value in the database. + */ +export interface AztecSingleton { + /** + * Gets the value. + */ + get(): T | undefined; + + /** + * Sets the value. + * @param val - The new value + */ + set(val: T): Promise; + + /** + * Deletes the value. + */ + delete(): Promise; +} diff --git a/yarn-project/kv-store/src/interfaces/store.ts b/yarn-project/kv-store/src/interfaces/store.ts new file mode 100644 index 00000000000..d7ccfa3cd29 --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/store.ts @@ -0,0 +1,40 @@ +import { AztecArray } from './array.js'; +import { AztecMap, AztecMultiMap } from './map.js'; +import { AztecSingleton } from './singleton.js'; + +/** A key-value store */ +export interface AztecKVStore { + /** + * Creates a new map. + * @param name - The name of the map + * @returns The map + */ + createMap(name: string): AztecMap; + + /** + * Creates a new multi-map. + * @param name - The name of the multi-map + * @returns The multi-map + */ + createMultiMap(name: string): AztecMultiMap; + + /** + * Creates a new array. + * @param name - The name of the array + * @returns The array + */ + createArray(name: string): AztecArray; + + /** + * Creates a new singleton. + * @param name - The name of the singleton + * @returns The singleton + */ + createSingleton(name: string): AztecSingleton; + + /** + * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically. + * @param callback - The callback to execute in a transaction + */ + transaction>>(callback: () => T): Promise; +} diff --git a/yarn-project/kv-store/src/lmdb/array.test.ts b/yarn-project/kv-store/src/lmdb/array.test.ts new file mode 100644 index 00000000000..3058302e87f --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/array.test.ts @@ -0,0 +1,91 @@ +import { Database, open } from 'lmdb'; + +import { LmdbAztecArray } from './array.js'; + +describe('LmdbAztecArray', () => { + let db: Database; + let arr: LmdbAztecArray; + + beforeEach(() => { + db = open({} as any); + arr = new LmdbAztecArray(db, 'test'); + }); + + it('should be able to push and pop values', async () => { + await arr.push(1); + await arr.push(2); + await arr.push(3); + + expect(arr.length).toEqual(3); + expect(await arr.pop()).toEqual(3); + expect(await arr.pop()).toEqual(2); + expect(await arr.pop()).toEqual(1); + expect(await arr.pop()).toEqual(undefined); + }); + + it('should be able to get values by index', async () => { + await arr.push(1); + await arr.push(2); + await arr.push(3); + + expect(arr.at(0)).toEqual(1); + expect(arr.at(1)).toEqual(2); + expect(arr.at(2)).toEqual(3); + expect(arr.at(3)).toEqual(undefined); + expect(arr.at(-1)).toEqual(3); + expect(arr.at(-2)).toEqual(2); + expect(arr.at(-3)).toEqual(1); + expect(arr.at(-4)).toEqual(undefined); + }); + + it('should be able to set values by index', async () => { + await arr.push(1); + await arr.push(2); + await arr.push(3); + + expect(await arr.setAt(0, 4)).toEqual(true); + expect(await arr.setAt(1, 5)).toEqual(true); + expect(await arr.setAt(2, 6)).toEqual(true); + + expect(await arr.setAt(3, 7)).toEqual(false); + + expect(arr.at(0)).toEqual(4); + expect(arr.at(1)).toEqual(5); + expect(arr.at(2)).toEqual(6); + expect(arr.at(3)).toEqual(undefined); + + expect(await arr.setAt(-1, 8)).toEqual(true); + expect(await arr.setAt(-2, 9)).toEqual(true); + expect(await arr.setAt(-3, 10)).toEqual(true); + + expect(await arr.setAt(-4, 11)).toEqual(false); + + expect(arr.at(-1)).toEqual(8); + expect(arr.at(-2)).toEqual(9); + expect(arr.at(-3)).toEqual(10); + expect(arr.at(-4)).toEqual(undefined); + }); + + it('should be able to iterate over values', async () => { + await arr.push(1); + await arr.push(2); + await arr.push(3); + + expect([...arr.values()]).toEqual([1, 2, 3]); + expect([...arr.entries()]).toEqual([ + [0, 1], + [1, 2], + [2, 3], + ]); + }); + + it('should be able to restore state', async () => { + await arr.push(1); + await arr.push(2); + await arr.push(3); + + const arr2 = new LmdbAztecArray(db, 'test'); + expect(arr2.length).toEqual(3); + expect([...arr2.values()]).toEqual([...arr.values()]); + }); +}); diff --git a/yarn-project/kv-store/src/lmdb/array.ts b/yarn-project/kv-store/src/lmdb/array.ts new file mode 100644 index 00000000000..1da3676aac0 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/array.ts @@ -0,0 +1,109 @@ +import { Database, Key } from 'lmdb'; + +import { AztecArray } from '../interfaces/array.js'; +import { LmdbAztecSingleton } from './singleton.js'; + +/** The shape of a key that stores a value in an array */ +type ArrayIndexSlot = ['array', string, 'slot', number]; + +/** + * An persistent array backed by LMDB. + */ +export class LmdbAztecArray implements AztecArray { + #db: Database; + #name: string; + #length: LmdbAztecSingleton; + + constructor(db: Database, arrName: string) { + this.#name = arrName; + this.#length = new LmdbAztecSingleton(db, `${arrName}:meta:length`); + this.#db = db as Database; + } + + get length(): number { + return this.#length.get() ?? 0; + } + + push(...vals: T[]): Promise { + return this.#db.childTransaction(() => { + let length = this.length; + for (const val of vals) { + void this.#db.put(this.#slot(length), val); + length += 1; + } + + void this.#length.set(length); + + return length; + }); + } + + pop(): Promise { + return this.#db.childTransaction(() => { + const length = this.length; + if (length === 0) { + return undefined; + } + + const slot = this.#slot(length - 1); + const val = this.#db.get(slot) as T; + + void this.#db.remove(slot); + void this.#length.set(length - 1); + + return val; + }); + } + + at(index: number): T | undefined { + if (index < 0) { + index = this.length + index; + } + + // the Array API only accepts indexes in the range [-this.length, this.length) + // so if after normalizing the index is still out of range, return undefined + if (index < 0 || index >= this.length) { + return undefined; + } + + return this.#db.get(this.#slot(index)); + } + + setAt(index: number, val: T): Promise { + if (index < 0) { + index = this.length + index; + } + + if (index < 0 || index >= this.length) { + return Promise.resolve(false); + } + + return this.#db.put(this.#slot(index), val); + } + + *entries(): IterableIterator<[number, T]> { + const values = this.#db.getRange({ + start: this.#slot(0), + limit: this.length, + }); + + for (const { key, value } of values) { + const index = key[3]; + yield [index, value]; + } + } + + *values(): IterableIterator { + for (const [_, value] of this.entries()) { + yield value; + } + } + + [Symbol.iterator](): IterableIterator { + return this.values(); + } + + #slot(index: number): ArrayIndexSlot { + return ['array', this.#name, 'slot', index]; + } +} diff --git a/yarn-project/kv-store/src/lmdb/map.test.ts b/yarn-project/kv-store/src/lmdb/map.test.ts new file mode 100644 index 00000000000..5319e0a26c3 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/map.test.ts @@ -0,0 +1,72 @@ +import { Database, open } from 'lmdb'; + +import { LmdbAztecMap } from './map.js'; + +describe('LmdbAztecMap', () => { + let db: Database; + let map: LmdbAztecMap; + + beforeEach(() => { + db = open({ dupSort: true } as any); + map = new LmdbAztecMap(db, 'test'); + }); + + it('should be able to set and get values', async () => { + await map.set('foo', 'bar'); + await map.set('baz', 'qux'); + + expect(map.get('foo')).toEqual('bar'); + expect(map.get('baz')).toEqual('qux'); + expect(map.get('quux')).toEqual(undefined); + }); + + it('should be able to set values if they do not exist', async () => { + expect(await map.setIfNotExists('foo', 'bar')).toEqual(true); + expect(await map.setIfNotExists('foo', 'baz')).toEqual(false); + + expect(map.get('foo')).toEqual('bar'); + }); + + it('should be able to delete values', async () => { + await map.set('foo', 'bar'); + await map.set('baz', 'qux'); + + expect(await map.delete('foo')).toEqual(true); + + expect(map.get('foo')).toEqual(undefined); + expect(map.get('baz')).toEqual('qux'); + }); + + it('should be able to iterate over entries', async () => { + await map.set('foo', 'bar'); + await map.set('baz', 'qux'); + + expect([...map.entries()]).toEqual( + expect.arrayContaining([ + ['foo', 'bar'], + ['baz', 'qux'], + ]), + ); + }); + + it('should be able to iterate over values', async () => { + await map.set('foo', 'bar'); + await map.set('baz', 'qux'); + + expect([...map.values()]).toEqual(expect.arrayContaining(['bar', 'qux'])); + }); + + it('should be able to iterate over keys', async () => { + await map.set('foo', 'bar'); + await map.set('baz', 'qux'); + + expect([...map.keys()]).toEqual(expect.arrayContaining(['foo', 'baz'])); + }); + + it('should be able to get multiple values for a single key', async () => { + await map.set('foo', 'bar'); + await map.set('foo', 'baz'); + + expect([...map.getValues('foo')]).toEqual(['bar', 'baz']); + }); +}); diff --git a/yarn-project/kv-store/src/lmdb/map.ts b/yarn-project/kv-store/src/lmdb/map.ts new file mode 100644 index 00000000000..b883b809738 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/map.ts @@ -0,0 +1,88 @@ +import { Database, Key } from 'lmdb'; + +import { AztecMultiMap } from '../interfaces/map.js'; + +/** The slot where a key-value entry would be stored */ +type MapKeyValueSlot = ['map', string, 'slot', K]; + +/** + * A map backed by LMDB. + */ +export class LmdbAztecMap implements AztecMultiMap { + protected db: Database>; + protected name: string; + + constructor(rootDb: Database, mapName: string) { + this.name = mapName; + this.db = rootDb as Database>; + } + + close(): Promise { + return this.db.close(); + } + + get(key: K): V | undefined { + return this.db.get(this.#slot(key)) as V | undefined; + } + + *getValues(key: K): IterableIterator { + const values = this.db.getValues(this.#slot(key)); + for (const value of values) { + yield value; + } + } + + has(key: K): boolean { + return this.db.doesExist(this.#slot(key)); + } + + set(key: K, val: V): Promise { + return this.db.put(this.#slot(key), val); + } + + setIfNotExists(key: K, val: V): Promise { + const slot = this.#slot(key); + return this.db.ifNoExists(slot, () => { + void this.db.put(slot, val); + }); + } + + delete(key: K): Promise { + return this.db.remove(this.#slot(key)); + } + + async deleteValue(key: K, val: V): Promise { + await this.db.remove(this.#slot(key), val); + } + + *entries(): IterableIterator<[K, V]> { + const iterator = this.db.getRange({ + start: ['map', this.name, 'slot'], + }); + + for (const { key, value } of iterator) { + if (key[0] !== 'map' || key[1] !== this.name) { + break; + } + + const originalKey = key[3]; + yield [originalKey, value]; + } + } + + *values(): IterableIterator { + for (const [_, value] of this.entries()) { + yield value; + } + } + + *keys(): IterableIterator { + for (const [key, _] of this.entries()) { + yield key; + } + } + + #slot(key: K): MapKeyValueSlot { + return ['map', this.name, 'slot', key]; + } +} diff --git a/yarn-project/kv-store/src/lmdb/singleton.test.ts b/yarn-project/kv-store/src/lmdb/singleton.test.ts new file mode 100644 index 00000000000..de1eefae462 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/singleton.test.ts @@ -0,0 +1,25 @@ +import { open } from 'lmdb'; + +import { LmdbAztecSingleton } from './singleton.js'; + +describe('LmdbAztecSingleton', () => { + let singleton: LmdbAztecSingleton; + beforeEach(() => { + singleton = new LmdbAztecSingleton(open({} as any), 'test'); + }); + + it('returns undefined if the value is not set', () => { + expect(singleton.get()).toEqual(undefined); + }); + + it('should be able to set and get values', async () => { + expect(await singleton.set('foo')).toEqual(true); + expect(singleton.get()).toEqual('foo'); + }); + + it('overwrites the value if it is set again', async () => { + expect(await singleton.set('foo')).toEqual(true); + expect(await singleton.set('bar')).toEqual(true); + expect(singleton.get()).toEqual('bar'); + }); +}); diff --git a/yarn-project/kv-store/src/lmdb/singleton.ts b/yarn-project/kv-store/src/lmdb/singleton.ts new file mode 100644 index 00000000000..0fa4ffe69e4 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/singleton.ts @@ -0,0 +1,31 @@ +import { Database, Key } from 'lmdb'; + +import { AztecSingleton } from '../interfaces/singleton.js'; + +/** The slot where this singleton will store its value */ +type ValueSlot = ['singleton', string, 'value']; + +/** + * Stores a single value in LMDB. + */ +export class LmdbAztecSingleton implements AztecSingleton { + #db: Database; + #slot: ValueSlot; + + constructor(db: Database, name: string) { + this.#db = db as Database; + this.#slot = ['singleton', name, 'value']; + } + + get(): T | undefined { + return this.#db.get(this.#slot); + } + + set(val: T): Promise { + return this.#db.put(this.#slot, val); + } + + delete(): Promise { + return this.#db.remove(this.#slot); + } +} diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts new file mode 100644 index 00000000000..9ede111a875 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -0,0 +1,131 @@ +import { EthAddress } from '@aztec/foundation/eth-address'; +import { Logger, createDebugLogger } from '@aztec/foundation/log'; + +import { Database, Key, RootDatabase, open } from 'lmdb'; + +import { AztecArray } from '../interfaces/array.js'; +import { AztecMap, AztecMultiMap } from '../interfaces/map.js'; +import { AztecSingleton } from '../interfaces/singleton.js'; +import { AztecKVStore } from '../interfaces/store.js'; +import { LmdbAztecArray } from './array.js'; +import { LmdbAztecMap } from './map.js'; +import { LmdbAztecSingleton } from './singleton.js'; + +/** + * A key-value store backed by LMDB. + */ +export class AztecLmdbStore implements AztecKVStore { + #rootDb: RootDatabase; + #data: Database; + #multiMapData: Database; + #rollupAddress: AztecSingleton; + #log: Logger; + + constructor(rootDb: RootDatabase, log: Logger) { + this.#rootDb = rootDb; + this.#log = log; + + // big bucket to store all the data + this.#data = rootDb.openDB('data', { + encoding: 'msgpack', + keyEncoding: 'ordered-binary', + }); + + this.#multiMapData = rootDb.openDB('data_dup_sort', { + encoding: 'msgpack', + keyEncoding: 'ordered-binary', + dupSort: true, + }); + + this.#rollupAddress = this.createSingleton('rollupAddress'); + } + + /** + * Creates a new AztecKVStore backed by LMDB. The path to the database is optional. If not provided, + * the database will be stored in a temporary location and be deleted when the process exists. + * + * The `rollupAddress` passed is checked against what is stored in the database. If they do not match, + * the database is cleared before returning the store. This way data is not accidentally shared between + * different rollup instances. + * + * @param rollupAddress - The ETH address of the rollup contract + * @param path - A path on the disk to store the database. Optional + * @param log - A logger to use. Optional + * @returns The store + */ + static async create( + rollupAddress: EthAddress, + path?: string, + log = createDebugLogger('aztec:kv-store:lmdb'), + ): Promise { + log.info(`Opening LMDB database at ${path || 'temporary location'}`); + + const rootDb = open({ + path, + }); + + const db = new AztecLmdbStore(rootDb, log); + await db.#init(rollupAddress); + + return db; + } + + /** + * Creates a new AztecMap in the store. + * @param name - Name of the map + * @returns A new AztecMap + */ + createMap(name: string): AztecMap { + return new LmdbAztecMap(this.#data, name); + } + + /** + * Creates a new AztecMultiMap in the store. A multi-map stores multiple values for a single key automatically. + * @param name - Name of the map + * @returns A new AztecMultiMap + */ + createMultiMap(name: string): AztecMultiMap { + return new LmdbAztecMap(this.#multiMapData, name); + } + + /** + * Creates a new AztecArray in the store. + * @param name - Name of the array + * @returns A new AztecArray + */ + createArray(name: string): AztecArray { + return new LmdbAztecArray(this.#data, name); + } + + /** + * Creates a new AztecSingleton in the store. + * @param name - Name of the singleton + * @returns A new AztecSingleton + */ + createSingleton(name: string): AztecSingleton { + return new LmdbAztecSingleton(this.#data, name); + } + + /** + * Runs a callback in a transaction. + * @param callback - Function to execute in a transaction + * @returns A promise that resolves to the return value of the callback + */ + transaction(callback: () => T): Promise { + return this.#rootDb.transaction(callback); + } + + async #init(rollupAddress: EthAddress): Promise { + const storedRollupAddress = this.#rollupAddress.get(); + const rollupAddressString = rollupAddress.toString(); + + if (typeof storedRollupAddress === 'string' && rollupAddressString !== storedRollupAddress) { + this.#log.warn( + `Rollup address mismatch: expected ${rollupAddress}, found ${storedRollupAddress}. Clearing entire database...`, + ); + await this.#rootDb.clearAsync(); + } + + await this.#rollupAddress.set(rollupAddressString); + } +} diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json new file mode 100644 index 00000000000..63f8ab3e9f7 --- /dev/null +++ b/yarn-project/kv-store/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "..", + "compilerOptions": { + "outDir": "dest", + "rootDir": "src", + "tsBuildInfoFile": ".tsbuildinfo" + }, + "references": [ + { + "path": "../foundation" + } + ], + "include": ["src"] +} diff --git a/yarn-project/package.json b/yarn-project/package.json index 34945e2ad55..542e94b28e4 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -46,7 +46,8 @@ "scripts", "types", "world-state", - "yarn-project-base" + "yarn-project-base", + "kv-store" ], "prettier": "@aztec/foundation/prettier", "devDependencies": { diff --git a/yarn-project/pxe/package.json b/yarn-project/pxe/package.json index b426db2c3ed..b1892b3b1f3 100644 --- a/yarn-project/pxe/package.json +++ b/yarn-project/pxe/package.json @@ -29,7 +29,8 @@ "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" }, "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", - "rootDir": "./src" + "rootDir": "./src", + "workerThreads": true }, "dependencies": { "@aztec/acir-simulator": "workspace:^", @@ -37,6 +38,7 @@ "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/key-store": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/noir-compiler": "workspace:^", "@aztec/noir-protocol-circuits": "workspace:^", "@aztec/types": "workspace:^", diff --git a/yarn-project/pxe/src/config/index.ts b/yarn-project/pxe/src/config/index.ts index 8c511ce4fa0..e96a5e64519 100644 --- a/yarn-project/pxe/src/config/index.ts +++ b/yarn-project/pxe/src/config/index.ts @@ -12,17 +12,21 @@ export interface PXEServiceConfig { l2BlockPollingIntervalMS: number; /** L2 block to start scanning from */ l2StartingBlock: number; + + /** Where to store PXE data. If not set will store in memory */ + dataDirectory?: string; } /** * Creates an instance of PXEServiceConfig out of environment variables using sensible defaults for integration testing if not set. */ export function getPXEServiceConfig(): PXEServiceConfig { - const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK } = process.env; + const { PXE_BLOCK_POLLING_INTERVAL_MS, PXE_L2_STARTING_BLOCK, DATA_DIRECTORY } = process.env; return { l2BlockPollingIntervalMS: PXE_BLOCK_POLLING_INTERVAL_MS ? +PXE_BLOCK_POLLING_INTERVAL_MS : 1000, l2StartingBlock: PXE_L2_STARTING_BLOCK ? +PXE_L2_STARTING_BLOCK : INITIAL_L2_BLOCK_NUM, + dataDirectory: DATA_DIRECTORY, }; } diff --git a/yarn-project/pxe/src/contract_tree/index.ts b/yarn-project/pxe/src/contract_tree/index.ts index 2c3f9de54dc..8078e310f3d 100644 --- a/yarn-project/pxe/src/contract_tree/index.ts +++ b/yarn-project/pxe/src/contract_tree/index.ts @@ -93,12 +93,7 @@ export class ContractTree { const completeAddress = computeCompleteAddress(from, contractAddressSalt, root, constructorHash); - const contractDao: ContractDao = { - ...artifact, - completeAddress, - functions, - portalContract, - }; + const contractDao = new ContractDao(artifact, completeAddress, portalContract); const NewContractConstructor = { functionData, vkHash, diff --git a/yarn-project/pxe/src/database/index.ts b/yarn-project/pxe/src/database/index.ts index d1306dbafe0..35d4e000a20 100644 --- a/yarn-project/pxe/src/database/index.ts +++ b/yarn-project/pxe/src/database/index.ts @@ -1,2 +1,2 @@ -export * from './database.js'; +export * from './pxe_database.js'; export * from './memory_db.js'; diff --git a/yarn-project/pxe/src/database/kv_pxe_database.test.ts b/yarn-project/pxe/src/database/kv_pxe_database.test.ts new file mode 100644 index 00000000000..a9054af0719 --- /dev/null +++ b/yarn-project/pxe/src/database/kv_pxe_database.test.ts @@ -0,0 +1,15 @@ +import { EthAddress } from '@aztec/circuits.js'; +import { AztecLmdbStore } from '@aztec/kv-store'; + +import { KVPxeDatabase } from './kv_pxe_database.js'; +import { describePxeDatabase } from './pxe_database_test_suite.js'; + +describe('KVPxeDatabase', () => { + let database: KVPxeDatabase; + + beforeEach(async () => { + database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); + }); + + describePxeDatabase(() => database); +}); diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts new file mode 100644 index 00000000000..8e5e27c2b8b --- /dev/null +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -0,0 +1,288 @@ +import { AztecAddress, BlockHeader, CompleteAddress } from '@aztec/circuits.js'; +import { Fr } from '@aztec/foundation/fields'; +import { AztecArray, AztecKVStore, AztecMap, AztecMultiMap, AztecSingleton } from '@aztec/kv-store'; +import { ContractDao, MerkleTreeId, NoteFilter, PublicKey } from '@aztec/types'; + +import { NoteDao } from './note_dao.js'; +import { PxeDatabase } from './pxe_database.js'; + +/** Serialized structure of a block header */ +type SerializedBlockHeader = { + /** The tree roots when the block was created */ + roots: Record; + /** The hash of the global variables */ + globalVariablesHash: string; +}; + +/** + * A PXE database backed by LMDB. + */ +export class KVPxeDatabase implements PxeDatabase { + #blockHeader: AztecSingleton; + #addresses: AztecArray; + #addressIndex: AztecMap; + #authWitnesses: AztecMap; + #capsules: AztecArray; + #contracts: AztecMap; + #notes: AztecArray; + #nullifiedNotes: AztecMap; + #notesByContract: AztecMultiMap; + #notesByStorageSlot: AztecMultiMap; + #notesByTxHash: AztecMultiMap; + #notesByOwner: AztecMultiMap; + #db: AztecKVStore; + + constructor(db: AztecKVStore) { + this.#db = db; + + this.#addresses = db.createArray('addresses'); + this.#addressIndex = db.createMap('address_index'); + + this.#authWitnesses = db.createMap('auth_witnesses'); + this.#capsules = db.createArray('capsules'); + this.#blockHeader = db.createSingleton('block_header'); + this.#contracts = db.createMap('contracts'); + + this.#notes = db.createArray('notes'); + this.#nullifiedNotes = db.createMap('nullified_notes'); + + this.#notesByContract = db.createMultiMap('notes_by_contract'); + this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot'); + this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash'); + this.#notesByOwner = db.createMultiMap('notes_by_owner'); + } + + async addAuthWitness(messageHash: Fr, witness: Fr[]): Promise { + await this.#authWitnesses.set( + messageHash.toString(), + witness.map(w => w.toBuffer()), + ); + } + + getAuthWitness(messageHash: Fr): Promise { + const witness = this.#authWitnesses.get(messageHash.toString()); + return Promise.resolve(witness?.map(w => Fr.fromBuffer(w))); + } + + async addCapsule(capsule: Fr[]): Promise { + await this.#capsules.push(capsule.map(c => c.toBuffer())); + } + + async popCapsule(): Promise { + const val = await this.#capsules.pop(); + return val?.map(b => Fr.fromBuffer(b)); + } + + async addNote(note: NoteDao): Promise { + await this.addNotes([note]); + } + + async addNotes(notes: NoteDao[]): Promise { + const newLength = await this.#notes.push(...notes.map(note => note.toBuffer())); + for (const [index, note] of notes.entries()) { + const noteId = newLength - notes.length + index; + await Promise.all([ + this.#notesByContract.set(note.contractAddress.toString(), noteId), + this.#notesByStorageSlot.set(note.storageSlot.toString(), noteId), + this.#notesByTxHash.set(note.txHash.toString(), noteId), + this.#notesByOwner.set(note.publicKey.toString(), noteId), + ]); + } + } + + *#getAllNonNullifiedNotes(): IterableIterator { + for (const [index, serialized] of this.#notes.entries()) { + if (this.#nullifiedNotes.has(index)) { + continue; + } + + yield NoteDao.fromBuffer(serialized); + } + } + + async getNotes(filter: NoteFilter): Promise { + const publicKey: PublicKey | undefined = filter.owner + ? (await this.getCompleteAddress(filter.owner))?.publicKey + : undefined; + + const initialNoteIds = publicKey + ? this.#notesByOwner.getValues(publicKey.toString()) + : filter.txHash + ? this.#notesByTxHash.getValues(filter.txHash.toString()) + : filter.contractAddress + ? this.#notesByContract.getValues(filter.contractAddress.toString()) + : filter.storageSlot + ? this.#notesByStorageSlot.getValues(filter.storageSlot.toString()) + : undefined; + + if (!initialNoteIds) { + return Array.from(this.#getAllNonNullifiedNotes()); + } + + const result: NoteDao[] = []; + for (const noteId of initialNoteIds) { + const serializedNote = this.#notes.at(noteId); + if (!serializedNote) { + continue; + } + + const note = NoteDao.fromBuffer(serializedNote); + if (filter.contractAddress && !note.contractAddress.equals(filter.contractAddress)) { + continue; + } + + if (filter.txHash && !note.txHash.equals(filter.txHash)) { + continue; + } + + if (filter.storageSlot && !note.storageSlot.equals(filter.storageSlot!)) { + continue; + } + + if (publicKey && !note.publicKey.equals(publicKey)) { + continue; + } + + result.push(note); + } + + return result; + } + + removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise { + const nullifierSet = new Set(nullifiers.map(n => n.toString())); + return this.#db.transaction(() => { + const notesIds = this.#notesByOwner.getValues(account.toString()); + const nullifiedNotes: NoteDao[] = []; + + for (const noteId of notesIds) { + const note = NoteDao.fromBuffer(this.#notes.at(noteId)!); + if (nullifierSet.has(note.siloedNullifier.toString())) { + nullifiedNotes.push(note); + + void this.#nullifiedNotes.set(noteId, true); + void this.#notesByOwner.deleteValue(account.toString(), noteId); + void this.#notesByTxHash.deleteValue(note.txHash.toString(), noteId); + void this.#notesByContract.deleteValue(note.contractAddress.toString(), noteId); + void this.#notesByStorageSlot.deleteValue(note.storageSlot.toString(), noteId); + } + } + + return nullifiedNotes; + }); + } + + getTreeRoots(): Record { + const roots = this.#blockHeader.get()?.roots; + if (!roots) { + throw new Error(`Tree roots not set`); + } + + return { + [MerkleTreeId.ARCHIVE]: Fr.fromString(roots[MerkleTreeId.ARCHIVE]), + [MerkleTreeId.CONTRACT_TREE]: Fr.fromString(roots[MerkleTreeId.CONTRACT_TREE].toString()), + [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: Fr.fromString(roots[MerkleTreeId.L1_TO_L2_MESSAGES_TREE].toString()), + [MerkleTreeId.NOTE_HASH_TREE]: Fr.fromString(roots[MerkleTreeId.NOTE_HASH_TREE].toString()), + [MerkleTreeId.PUBLIC_DATA_TREE]: Fr.fromString(roots[MerkleTreeId.PUBLIC_DATA_TREE].toString()), + [MerkleTreeId.NULLIFIER_TREE]: Fr.fromString(roots[MerkleTreeId.NULLIFIER_TREE].toString()), + }; + } + + async setBlockHeader(blockHeader: BlockHeader): Promise { + await this.#blockHeader.set({ + globalVariablesHash: blockHeader.globalVariablesHash.toString(), + roots: { + [MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot.toString(), + [MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot.toString(), + [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot.toString(), + [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: blockHeader.l1ToL2MessagesTreeRoot.toString(), + [MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot.toString(), + [MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot.toString(), + }, + }); + } + + getBlockHeader(): BlockHeader { + const value = this.#blockHeader.get(); + if (!value) { + throw new Error(`Block header not set`); + } + + const blockHeader = new BlockHeader( + Fr.fromString(value.roots[MerkleTreeId.NOTE_HASH_TREE]), + Fr.fromString(value.roots[MerkleTreeId.NULLIFIER_TREE]), + Fr.fromString(value.roots[MerkleTreeId.CONTRACT_TREE]), + Fr.fromString(value.roots[MerkleTreeId.L1_TO_L2_MESSAGES_TREE]), + Fr.fromString(value.roots[MerkleTreeId.ARCHIVE]), + Fr.ZERO, // todo: private kernel vk tree root + Fr.fromString(value.roots[MerkleTreeId.PUBLIC_DATA_TREE]), + Fr.fromString(value.globalVariablesHash), + ); + + return blockHeader; + } + + addCompleteAddress(completeAddress: CompleteAddress): Promise { + return this.#db.transaction(() => { + const addressString = completeAddress.address.toString(); + const buffer = completeAddress.toBuffer(); + const existing = this.#addressIndex.get(addressString); + if (typeof existing === 'undefined') { + const index = this.#addresses.length; + void this.#addresses.push(buffer); + void this.#addressIndex.set(addressString, index); + + return true; + } else { + const existingBuffer = this.#addresses.at(existing); + + if (existingBuffer?.equals(buffer)) { + return false; + } + + throw new Error( + `Complete address with aztec address ${addressString} but different public key or partial key already exists in memory database`, + ); + } + }); + } + + getCompleteAddress(address: AztecAddress): Promise { + const index = this.#addressIndex.get(address.toString()); + if (typeof index === 'undefined') { + return Promise.resolve(undefined); + } + + const value = this.#addresses.at(index); + return Promise.resolve(value ? CompleteAddress.fromBuffer(value) : undefined); + } + + getCompleteAddresses(): Promise { + return Promise.resolve(Array.from(this.#addresses).map(v => CompleteAddress.fromBuffer(v))); + } + + estimateSize(): number { + const notesSize = Array.from(this.#getAllNonNullifiedNotes()).reduce((sum, note) => sum + note.getSize(), 0); + const authWitsSize = Array.from(this.#authWitnesses.values()).reduce( + (sum, value) => sum + value.length * Fr.SIZE_IN_BYTES, + 0, + ); + const addressesSize = this.#addresses.length * CompleteAddress.SIZE_IN_BYTES; + const treeRootsSize = Object.keys(MerkleTreeId).length * Fr.SIZE_IN_BYTES; + + return notesSize + treeRootsSize + authWitsSize + addressesSize; + } + + async addContract(contract: ContractDao): Promise { + await this.#contracts.set(contract.completeAddress.address.toString(), contract.toBuffer()); + } + + getContract(address: AztecAddress): Promise { + const contract = this.#contracts.get(address.toString()); + return Promise.resolve(contract ? ContractDao.fromBuffer(contract) : undefined); + } + + getContracts(): Promise { + return Promise.resolve(Array.from(this.#contracts.values()).map(c => ContractDao.fromBuffer(c))); + } +} diff --git a/yarn-project/pxe/src/database/memory_db.test.ts b/yarn-project/pxe/src/database/memory_db.test.ts index 077f705167a..f505efa4a79 100644 --- a/yarn-project/pxe/src/database/memory_db.test.ts +++ b/yarn-project/pxe/src/database/memory_db.test.ts @@ -2,6 +2,7 @@ import { AztecAddress, Fr } from '@aztec/circuits.js'; import { MemoryDB } from './memory_db.js'; import { randomNoteDao } from './note_dao.test.js'; +import { describePxeDatabase } from './pxe_database_test_suite.js'; describe('Memory DB', () => { let db: MemoryDB; @@ -10,6 +11,8 @@ describe('Memory DB', () => { db = new MemoryDB(); }); + describePxeDatabase(() => db); + describe('NoteDao', () => { const contractAddress = AztecAddress.random(); const storageSlot = Fr.random(); diff --git a/yarn-project/pxe/src/database/memory_db.ts b/yarn-project/pxe/src/database/memory_db.ts index 4f2849b42e5..641603b0210 100644 --- a/yarn-project/pxe/src/database/memory_db.ts +++ b/yarn-project/pxe/src/database/memory_db.ts @@ -5,8 +5,8 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { MerkleTreeId, NoteFilter } from '@aztec/types'; import { MemoryContractDatabase } from '../contract_database/index.js'; -import { Database } from './database.js'; import { NoteDao } from './note_dao.js'; +import { PxeDatabase } from './pxe_database.js'; /** * The MemoryDB class provides an in-memory implementation of a database to manage transactions and auxiliary data. @@ -14,7 +14,7 @@ import { NoteDao } from './note_dao.js'; * The class offers methods to add, fetch, and remove transaction records and auxiliary data based on various filters such as transaction hash, address, and storage slot. * As an in-memory database, the stored data will not persist beyond the life of the application instance. */ -export class MemoryDB extends MemoryContractDatabase implements Database { +export class MemoryDB extends MemoryContractDatabase implements PxeDatabase { private notesTable: NoteDao[] = []; private treeRoots: Record | undefined; private globalVariablesHash: Fr | undefined; @@ -43,7 +43,7 @@ export class MemoryDB extends MemoryContractDatabase implements Database { * @param messageHash - The message hash. * @returns A Promise that resolves to an array of field elements representing the auth witness. */ - public getAuthWitness(messageHash: Fr): Promise { + public getAuthWitness(messageHash: Fr): Promise { return Promise.resolve(this.authWitnesses[messageHash.toString()]); } @@ -113,9 +113,8 @@ export class MemoryDB extends MemoryContractDatabase implements Database { return roots; } - public setTreeRoots(roots: Record) { + private setTreeRoots(roots: Record) { this.treeRoots = roots; - return Promise.resolve(); } public getBlockHeader(): BlockHeader { @@ -135,9 +134,9 @@ export class MemoryDB extends MemoryContractDatabase implements Database { ); } - public async setBlockHeader(blockHeader: BlockHeader): Promise { + public setBlockHeader(blockHeader: BlockHeader): Promise { this.globalVariablesHash = blockHeader.globalVariablesHash; - await this.setTreeRoots({ + this.setTreeRoots({ [MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot, [MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot, [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot, @@ -145,6 +144,8 @@ export class MemoryDB extends MemoryContractDatabase implements Database { [MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot, [MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot, }); + + return Promise.resolve(); } public addCompleteAddress(completeAddress: CompleteAddress): Promise { @@ -154,8 +155,10 @@ export class MemoryDB extends MemoryContractDatabase implements Database { return Promise.resolve(false); } - throw new Error( - `Complete address with aztec address ${completeAddress.address.toString()} but different public key or partial key already exists in memory database`, + return Promise.reject( + new Error( + `Complete address with aztec address ${completeAddress.address.toString()} but different public key or partial key already exists in memory database`, + ), ); } this.addresses.push(completeAddress); diff --git a/yarn-project/pxe/src/database/database.ts b/yarn-project/pxe/src/database/pxe_database.ts similarity index 88% rename from yarn-project/pxe/src/database/database.ts rename to yarn-project/pxe/src/database/pxe_database.ts index 72f50a1c576..0c7f1551770 100644 --- a/yarn-project/pxe/src/database/database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -9,7 +9,7 @@ import { NoteDao } from './note_dao.js'; * A database interface that provides methods for retrieving, adding, and removing transactional data related to Aztec * addresses, storage slots, and nullifiers. */ -export interface Database extends ContractDatabase { +export interface PxeDatabase extends ContractDatabase { /** * Add a auth witness to the database. * @param messageHash - The message hash. @@ -22,7 +22,7 @@ export interface Database extends ContractDatabase { * @param messageHash - The message hash. * @returns A Promise that resolves to an array of field elements representing the auth witness. */ - getAuthWitness(messageHash: Fr): Promise; + getAuthWitness(messageHash: Fr): Promise; /** * Adding a capsule to the capsule dispenser. @@ -79,17 +79,6 @@ export interface Database extends ContractDatabase { */ getTreeRoots(): Record; - /** - * Set the tree roots for the Merkle trees in the database. - * This function updates the 'treeRoots' property of the instance - * with the provided 'roots' object containing MerkleTreeId and Fr pairs. - * Note that this will overwrite any existing tree roots in the database. - * - * @param roots - A Record object mapping MerkleTreeIds to their corresponding Fr root values. - * @returns A Promise that resolves when the tree roots have been successfully updated in the database. - */ - setTreeRoots(roots: Record): Promise; - /** * Retrieve the stored Block Header from the database. * The function returns a Promise that resolves to the Block Header. diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts new file mode 100644 index 00000000000..69eaff032b9 --- /dev/null +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -0,0 +1,220 @@ +import { AztecAddress, BlockHeader, CompleteAddress } from '@aztec/circuits.js'; +import { Fr, Point } from '@aztec/foundation/fields'; +import { MerkleTreeId, NoteFilter, randomTxHash } from '@aztec/types'; + +import { NoteDao } from './note_dao.js'; +import { randomNoteDao } from './note_dao.test.js'; +import { PxeDatabase } from './pxe_database.js'; + +/** + * A common test suite for a PXE database. + * @param getDatabase - A function that returns a database instance. + */ +export function describePxeDatabase(getDatabase: () => PxeDatabase) { + let database: PxeDatabase; + + beforeEach(() => { + database = getDatabase(); + }); + + describe('Database', () => { + describe('auth witnesses', () => { + it('stores and retrieves auth witnesses', async () => { + const messageHash = Fr.random(); + const witness = [Fr.random(), Fr.random()]; + + await database.addAuthWitness(messageHash, witness); + await expect(database.getAuthWitness(messageHash)).resolves.toEqual(witness); + }); + + it("returns undefined if it doesn't have auth witnesses for the message", async () => { + const messageHash = Fr.random(); + await expect(database.getAuthWitness(messageHash)).resolves.toBeUndefined(); + }); + + it.skip('refuses to overwrite auth witnesses for the same message', async () => { + const messageHash = Fr.random(); + const witness = [Fr.random(), Fr.random()]; + + await database.addAuthWitness(messageHash, witness); + await expect(database.addAuthWitness(messageHash, witness)).rejects.toThrow(); + }); + }); + + describe('capsules', () => { + it('stores and retrieves capsules', async () => { + const capsule = [Fr.random(), Fr.random()]; + + await database.addCapsule(capsule); + await expect(database.popCapsule()).resolves.toEqual(capsule); + }); + + it("returns undefined if it doesn't have capsules", async () => { + await expect(database.popCapsule()).resolves.toBeUndefined(); + }); + + it('behaves like a stack when storing capsules', async () => { + const capsule1 = [Fr.random(), Fr.random()]; + const capsule2 = [Fr.random(), Fr.random()]; + + await database.addCapsule(capsule1); + await database.addCapsule(capsule2); + await expect(database.popCapsule()).resolves.toEqual(capsule2); + await expect(database.popCapsule()).resolves.toEqual(capsule1); + }); + }); + + describe('notes', () => { + let owners: CompleteAddress[]; + let contractAddresses: AztecAddress[]; + let storageSlots: Fr[]; + let notes: NoteDao[]; + + const filteringTests: [() => NoteFilter, () => NoteDao[]][] = [ + [() => ({}), () => notes], + + [ + () => ({ contractAddress: contractAddresses[0] }), + () => notes.filter(note => note.contractAddress.equals(contractAddresses[0])), + ], + [() => ({ contractAddress: AztecAddress.random() }), () => []], + + [ + () => ({ storageSlot: storageSlots[0] }), + () => notes.filter(note => note.storageSlot.equals(storageSlots[0])), + ], + [() => ({ storageSlot: Fr.random() }), () => []], + + [() => ({ txHash: notes[0].txHash }), () => [notes[0]]], + [() => ({ txHash: randomTxHash() }), () => []], + + [() => ({ owner: owners[0].address }), () => notes.filter(note => note.publicKey.equals(owners[0].publicKey))], + + [ + () => ({ contractAddress: contractAddresses[0], storageSlot: storageSlots[0] }), + () => + notes.filter( + note => note.contractAddress.equals(contractAddresses[0]) && note.storageSlot.equals(storageSlots[0]), + ), + ], + [() => ({ contractAddress: contractAddresses[0], storageSlot: storageSlots[1] }), () => []], + ]; + + beforeEach(() => { + owners = Array.from({ length: 2 }).map(() => CompleteAddress.random()); + contractAddresses = Array.from({ length: 2 }).map(() => AztecAddress.random()); + storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); + + notes = Array.from({ length: 10 }).map((_, i) => + randomNoteDao({ + contractAddress: contractAddresses[i % contractAddresses.length], + storageSlot: storageSlots[i % storageSlots.length], + publicKey: owners[i % owners.length].publicKey, + }), + ); + }); + + beforeEach(async () => { + for (const owner of owners) { + await database.addCompleteAddress(owner); + } + }); + + it.each(filteringTests)('stores notes in bulk and retrieves notes', async (getFilter, getExpected) => { + await database.addNotes(notes); + await expect(database.getNotes(getFilter())).resolves.toEqual(getExpected()); + }); + + it.each(filteringTests)('stores notes one by one and retrieves notes', async (getFilter, getExpected) => { + for (const note of notes) { + await database.addNote(note); + } + await expect(database.getNotes(getFilter())).resolves.toEqual(getExpected()); + }); + + it('removes nullified notes', async () => { + const notesToNullify = notes.filter(note => note.publicKey.equals(owners[0].publicKey)); + const nullifiers = notesToNullify.map(note => note.siloedNullifier); + + await database.addNotes(notes); + + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].publicKey)).resolves.toEqual( + notesToNullify, + ); + await expect( + database.getNotes({ + owner: owners[0].address, + }), + ).resolves.toEqual([]); + await expect(database.getNotes({})).resolves.toEqual(notes.filter(note => !notesToNullify.includes(note))); + }); + }); + + describe('block header', () => { + it('stores and retrieves the block header', async () => { + const blockHeader = BlockHeader.random(); + blockHeader.privateKernelVkTreeRoot = Fr.zero(); + + await database.setBlockHeader(blockHeader); + expect(database.getBlockHeader()).toEqual(blockHeader); + }); + + it('retrieves the merkle tree roots from the block', async () => { + const blockHeader = BlockHeader.random(); + await database.setBlockHeader(blockHeader); + expect(database.getTreeRoots()).toEqual({ + [MerkleTreeId.NOTE_HASH_TREE]: blockHeader.noteHashTreeRoot, + [MerkleTreeId.NULLIFIER_TREE]: blockHeader.nullifierTreeRoot, + [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot, + [MerkleTreeId.L1_TO_L2_MESSAGES_TREE]: blockHeader.l1ToL2MessagesTreeRoot, + [MerkleTreeId.ARCHIVE]: blockHeader.archiveRoot, + [MerkleTreeId.PUBLIC_DATA_TREE]: blockHeader.publicDataTreeRoot, + }); + }); + + it('rejects getting merkle tree roots if no block set', () => { + expect(() => database.getTreeRoots()).toThrow(); + }); + }); + + describe('addresses', () => { + it('stores and retrieves addresses', async () => { + const address = CompleteAddress.random(); + await expect(database.addCompleteAddress(address)).resolves.toBe(true); + await expect(database.getCompleteAddress(address.address)).resolves.toEqual(address); + }); + + it('silently ignores an address it already knows about', async () => { + const address = CompleteAddress.random(); + await expect(database.addCompleteAddress(address)).resolves.toBe(true); + await expect(database.addCompleteAddress(address)).resolves.toBe(false); + }); + + it.skip('refuses to overwrite an address with a different public key', async () => { + const address = CompleteAddress.random(); + const otherAddress = new CompleteAddress(address.address, Point.random(), address.partialAddress); + + await database.addCompleteAddress(address); + await expect(database.addCompleteAddress(otherAddress)).rejects.toThrow(); + }); + + it('returns all addresses', async () => { + const addresses = Array.from({ length: 10 }).map(() => CompleteAddress.random()); + for (const address of addresses) { + await database.addCompleteAddress(address); + } + + const result = await database.getCompleteAddresses(); + expect(result).toEqual(expect.arrayContaining(addresses)); + }); + + it("returns an empty array if it doesn't have addresses", async () => { + expect(await database.getCompleteAddresses()).toEqual([]); + }); + + it("returns undefined if it doesn't have an address", async () => { + expect(await database.getCompleteAddress(CompleteAddress.random().address)).toBeUndefined(); + }); + }); + }); +} diff --git a/yarn-project/pxe/src/note_processor/note_processor.test.ts b/yarn-project/pxe/src/note_processor/note_processor.test.ts index d766a04c336..a317c24cddd 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.test.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.test.ts @@ -1,9 +1,10 @@ import { AcirSimulator } from '@aztec/acir-simulator'; -import { Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js'; +import { EthAddress, Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { pedersenHash } from '@aztec/foundation/crypto'; import { Point } from '@aztec/foundation/fields'; import { ConstantKeyPair } from '@aztec/key-store'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { AztecNode, FunctionL2Logs, @@ -21,7 +22,8 @@ import { import { jest } from '@jest/globals'; import { MockProxy, mock } from 'jest-mock-extended'; -import { Database, MemoryDB } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; +import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { NoteDao } from '../database/note_dao.js'; import { NoteProcessor } from './note_processor.js'; @@ -29,7 +31,7 @@ const TXS_PER_BLOCK = 4; describe('Note Processor', () => { let grumpkin: Grumpkin; - let database: Database; + let database: PxeDatabase; let aztecNode: ReturnType>; let addNotesSpy: any; let noteProcessor: NoteProcessor; @@ -114,8 +116,8 @@ describe('Note Processor', () => { owner = ConstantKeyPair.random(grumpkin); }); - beforeEach(() => { - database = new MemoryDB(); + beforeEach(async () => { + database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); addNotesSpy = jest.spyOn(database, 'addNotes'); aztecNode = mock(); diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts index 0f07830436f..436085df589 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.ts @@ -7,7 +7,7 @@ import { Timer } from '@aztec/foundation/timer'; import { AztecNode, KeyStore, L1NotePayload, L2BlockContext, L2BlockL2Logs } from '@aztec/types'; import { NoteProcessorStats } from '@aztec/types/stats'; -import { Database } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; import { NoteDao } from '../database/note_dao.js'; import { getAcirSimulator } from '../simulator/index.js'; @@ -45,7 +45,7 @@ export class NoteProcessor { */ public readonly publicKey: PublicKey, private keyStore: KeyStore, - private db: Database, + private db: PxeDatabase, private node: AztecNode, private startingBlock: number, private simulator = getAcirSimulator(db, node, keyStore), diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 04eddd7db48..f2942d8a800 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -1,25 +1,14 @@ import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { TestKeyStore } from '@aztec/key-store'; -import { AztecNode, KeyStore } from '@aztec/types'; +import { AztecLmdbStore } from '@aztec/kv-store'; +import { AztecNode } from '@aztec/types'; + +import { join } from 'path'; import { PXEServiceConfig } from '../config/index.js'; -import { Database, MemoryDB } from '../database/index.js'; +import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { PXEService } from './pxe_service.js'; -/** - * Optional information for creating an PXEService. - */ -interface CreatePXEServiceOptions { - /** - * A secure storage for cryptographic keys. - */ - keyStore?: KeyStore; - /** - * Storage for the PXE. - */ - db?: Database; -} - /** * Create and start an PXEService instance with the given AztecNode. * If no keyStore or database is provided, it will use TestKeyStore and MemoryDB as default values. @@ -33,7 +22,6 @@ interface CreatePXEServiceOptions { export async function createPXEService( aztecNode: AztecNode, config: PXEServiceConfig, - { keyStore, db }: CreatePXEServiceOptions = {}, useLogSuffix: string | boolean | undefined = undefined, ) { const logSuffix = @@ -43,10 +31,18 @@ export async function createPXEService( : undefined : useLogSuffix; - keyStore = keyStore || new TestKeyStore(new Grumpkin()); - db = db || new MemoryDB(logSuffix); + const pxeDbPath = config.dataDirectory ? join(config.dataDirectory, 'pxe_data') : undefined; + const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined; + const l1Contracts = await aztecNode.getL1ContractAddresses(); + + const keyStore = new TestKeyStore( + new Grumpkin(), + await AztecLmdbStore.create(l1Contracts.rollupAddress, keyStorePath), + ); + const db = new KVPxeDatabase(await AztecLmdbStore.create(l1Contracts.rollupAddress, pxeDbPath)); const server = new PXEService(keyStore, aztecNode, db, config, logSuffix); + await server.start(); return server; } diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 37f4eb7dcb3..f4cf6eb29d9 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -50,12 +50,11 @@ import { TxStatus, getNewContractPublicFunctions, isNoirCallStackUnresolved, - toContractDao, } from '@aztec/types'; import { PXEServiceConfig, getPackageInfo } from '../config/index.js'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; -import { Database } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; import { NoteDao } from '../database/note_dao.js'; import { KernelOracle } from '../kernel_oracle/index.js'; import { KernelProver } from '../kernel_prover/kernel_prover.js'; @@ -75,7 +74,7 @@ export class PXEService implements PXE { constructor( private keyStore: KeyStore, private node: AztecNode, - private db: Database, + private db: PxeDatabase, private config: PXEServiceConfig, logSuffix?: string, ) { @@ -128,7 +127,7 @@ export class PXEService implements PXE { const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(privKey, partialAddress); const wasAdded = await this.db.addCompleteAddress(completeAddress); if (wasAdded) { - const pubKey = this.keyStore.addAccount(privKey); + const pubKey = await this.keyStore.addAccount(privKey); this.synchronizer.addAccount(pubKey, this.keyStore, this.config.l2StartingBlock); this.log.info(`Registered account ${completeAddress.address.toString()}`); this.log.debug(`Registered account\n ${completeAddress.toReadableString()}`); @@ -178,7 +177,7 @@ export class PXEService implements PXE { } public async addContracts(contracts: DeployedContract[]) { - const contractDaos = contracts.map(c => toContractDao(c.artifact, c.completeAddress, c.portalContract)); + const contractDaos = contracts.map(c => new ContractDao(c.artifact, c.completeAddress, c.portalContract)); await Promise.all(contractDaos.map(c => this.db.addContract(c))); for (const contract of contractDaos) { const portalInfo = diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index e265d99a898..e55560280a0 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -2,19 +2,22 @@ import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { TestKeyStore } from '@aztec/key-store'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { AztecNode, INITIAL_L2_BLOCK_NUM, L2Tx, PXE, mockTx } from '@aztec/types'; import { MockProxy, mock } from 'jest-mock-extended'; -import { MemoryDB } from '../../database/memory_db.js'; +import { KVPxeDatabase } from '../../database/kv_pxe_database.js'; +import { PxeDatabase } from '../../database/pxe_database.js'; import { PXEServiceConfig } from '../../index.js'; import { PXEService } from '../pxe_service.js'; import { pxeTestSuite } from './pxe_test_suite.js'; -function createPXEService(): Promise { - const keyStore = new TestKeyStore(new Grumpkin()); +async function createPXEService(): Promise { + const kvStore = await AztecLmdbStore.create(EthAddress.random()); + const keyStore = new TestKeyStore(new Grumpkin(), kvStore); const node = mock(); - const db = new MemoryDB(); + const db = new KVPxeDatabase(kvStore); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM }; // Setup the relevant mocks @@ -39,13 +42,14 @@ pxeTestSuite('PXEService', createPXEService); describe('PXEService', () => { let keyStore: TestKeyStore; let node: MockProxy; - let db: MemoryDB; + let db: PxeDatabase; let config: PXEServiceConfig; - beforeEach(() => { - keyStore = new TestKeyStore(new Grumpkin()); + beforeEach(async () => { + const kvStore = await AztecLmdbStore.create(EthAddress.random()); + keyStore = new TestKeyStore(new Grumpkin(), kvStore); node = mock(); - db = new MemoryDB(); + db = new KVPxeDatabase(kvStore); config = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM }; }); diff --git a/yarn-project/pxe/src/simulator/index.ts b/yarn-project/pxe/src/simulator/index.ts index 0ffc34a35ef..64abb774d20 100644 --- a/yarn-project/pxe/src/simulator/index.ts +++ b/yarn-project/pxe/src/simulator/index.ts @@ -2,14 +2,14 @@ import { AcirSimulator } from '@aztec/acir-simulator'; import { KeyStore, StateInfoProvider } from '@aztec/types'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; -import { Database } from '../database/database.js'; +import { PxeDatabase } from '../database/pxe_database.js'; import { SimulatorOracle } from '../simulator_oracle/index.js'; /** * Helper method to create an instance of the acir simulator. */ export function getAcirSimulator( - db: Database, + db: PxeDatabase, stateInfoProvider: StateInfoProvider, keyStore: KeyStore, contractDataOracle?: ContractDataOracle, diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 2eb485fd287..4b3e21aee64 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -13,7 +13,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { KeyStore, L2Block, MerkleTreeId, NullifierMembershipWitness, StateInfoProvider } from '@aztec/types'; import { ContractDataOracle } from '../contract_data_oracle/index.js'; -import { Database } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; /** * A data oracle that provides information needed for simulating a transaction. @@ -21,7 +21,7 @@ import { Database } from '../database/index.js'; export class SimulatorOracle implements DBOracle { constructor( private contractDataOracle: ContractDataOracle, - private db: Database, + private db: PxeDatabase, private keyStore: KeyStore, private stateInfoProvider: StateInfoProvider, private log = createDebugLogger('aztec:pxe:simulator_oracle'), diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 95893f10810..5270b0d7d1e 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,22 +1,24 @@ -import { BlockHeader, CompleteAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js'; +import { BlockHeader, CompleteAddress, EthAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { TestKeyStore } from '@aztec/key-store'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { AztecNode, INITIAL_L2_BLOCK_NUM, L2Block, MerkleTreeId } from '@aztec/types'; import { MockProxy, mock } from 'jest-mock-extended'; import omit from 'lodash.omit'; -import { Database, MemoryDB } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; +import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { Synchronizer } from './synchronizer.js'; describe('Synchronizer', () => { let aztecNode: MockProxy; - let database: Database; + let database: PxeDatabase; let synchronizer: TestSynchronizer; let roots: Record; let blockHeader: BlockHeader; - beforeEach(() => { + beforeEach(async () => { blockHeader = BlockHeader.random(); roots = { [MerkleTreeId.CONTRACT_TREE]: blockHeader.contractTreeRoot, @@ -28,7 +30,7 @@ describe('Synchronizer', () => { }; aztecNode = mock(); - database = new MemoryDB(); + database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); synchronizer = new TestSynchronizer(aztecNode, database); }); @@ -102,9 +104,9 @@ describe('Synchronizer', () => { aztecNode.getBlockNumber.mockResolvedValueOnce(1); // Manually adding account to database so that we can call synchronizer.isAccountStateSynchronized - const keyStore = new TestKeyStore(new Grumpkin()); + const keyStore = new TestKeyStore(new Grumpkin(), await AztecLmdbStore.create(EthAddress.random())); const privateKey = GrumpkinScalar.random(); - keyStore.addAccount(privateKey); + await keyStore.addAccount(privateKey); const completeAddress = CompleteAddress.fromPrivateKeyAndPartialAddress(privateKey, Fr.random()); await database.addCompleteAddress(completeAddress); diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 1c557386c18..2346ae7d613 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -5,7 +5,7 @@ import { InterruptibleSleep } from '@aztec/foundation/sleep'; import { AztecNode, INITIAL_L2_BLOCK_NUM, KeyStore, L2BlockContext, L2BlockL2Logs, LogType } from '@aztec/types'; import { NoteProcessorCaughtUpStats } from '@aztec/types/stats'; -import { Database } from '../database/index.js'; +import { PxeDatabase } from '../database/index.js'; import { NoteProcessor } from '../note_processor/index.js'; /** @@ -25,7 +25,7 @@ export class Synchronizer { private log: DebugLogger; private noteProcessorsToCatchUp: NoteProcessor[] = []; - constructor(private node: AztecNode, private db: Database, logSuffix = '') { + constructor(private node: AztecNode, private db: PxeDatabase, logSuffix = '') { this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer'); } diff --git a/yarn-project/pxe/tsconfig.json b/yarn-project/pxe/tsconfig.json index 3106fcd762f..c282504efe6 100644 --- a/yarn-project/pxe/tsconfig.json +++ b/yarn-project/pxe/tsconfig.json @@ -9,6 +9,9 @@ { "path": "../acir-simulator" }, + { + "path": "../kv-store" + }, { "path": "../circuits.js" }, diff --git a/yarn-project/types/src/contract_dao.test.ts b/yarn-project/types/src/contract_dao.test.ts new file mode 100644 index 00000000000..5cfa8ea7e92 --- /dev/null +++ b/yarn-project/types/src/contract_dao.test.ts @@ -0,0 +1,54 @@ +import { CompleteAddress, EthAddress } from '@aztec/circuits.js'; +import { ABIParameterVisibility, ContractArtifact, FunctionSelector, FunctionType } from '@aztec/foundation/abi'; + +import { ContractDao } from './contract_dao.js'; +import { randomContractArtifact } from './mocks.js'; + +describe('ContractDao', () => { + it('serializes / deserializes correctly', () => { + const artifact = randomContractArtifact(); + const dao = new ContractDao(artifact, CompleteAddress.random(), EthAddress.random()); + + expect(ContractDao.fromBuffer(dao.toBuffer())).toEqual(dao); + }); + + it('extracts function data', () => { + const artifact: ContractArtifact = { + name: 'test', + functions: [ + { + name: 'bar', + functionType: FunctionType.SECRET, + isInternal: false, + parameters: [ + { + name: 'value', + type: { + kind: 'field', + }, + visibility: ABIParameterVisibility.PUBLIC, + }, + { + name: 'value', + type: { + kind: 'field', + }, + visibility: ABIParameterVisibility.SECRET, + }, + ], + returnTypes: [], + bytecode: '0af', + }, + ], + events: [], + }; + + const dao = new ContractDao(artifact, CompleteAddress.random(), EthAddress.random()); + + expect(dao.functions[0]).toEqual({ + ...artifact.functions[0], + // number representing bar((Field),Field) + selector: new FunctionSelector(4138634513), + }); + }); +}); diff --git a/yarn-project/types/src/contract_dao.ts b/yarn-project/types/src/contract_dao.ts index d07e2d66a58..10cb955d743 100644 --- a/yarn-project/types/src/contract_dao.ts +++ b/yarn-project/types/src/contract_dao.ts @@ -1,53 +1,67 @@ import { CompleteAddress, ContractFunctionDao } from '@aztec/circuits.js'; -import { ContractArtifact, FunctionSelector, FunctionType } from '@aztec/foundation/abi'; +import { ContractArtifact, DebugMetadata, EventAbi, FunctionSelector, FunctionType } from '@aztec/foundation/abi'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { prefixBufferWithLength } from '@aztec/foundation/serialize'; -import { EncodedContractFunction } from './contract_data.js'; +import { BufferReader, EncodedContractFunction } from './contract_data.js'; /** * A contract Data Access Object (DAO). * Contains the contract's address, portal contract address, and an array of ContractFunctionDao objects. * Each ContractFunctionDao object includes FunctionAbi data and the function selector buffer. */ -export interface ContractDao extends ContractArtifact { - /** - * The complete address representing the contract on L2. - */ - completeAddress: CompleteAddress; - /** - * The Ethereum address of the L1 contract serving as a bridge for cross-layer interactions. - */ - portalContract: EthAddress; - /** - * An array of contract functions with additional selector property. - */ - functions: ContractFunctionDao[]; -} +export class ContractDao implements ContractArtifact { + /** An array of contract functions with additional selector property. */ + public readonly functions: ContractFunctionDao[]; + constructor( + private contractArtifact: ContractArtifact, + /** The complete address representing the contract on L2. */ + public readonly completeAddress: CompleteAddress, + /** The Ethereum address of the L1 contract serving as a bridge for cross-layer interactions. */ + public readonly portalContract: EthAddress, + ) { + this.functions = contractArtifact.functions.map(f => ({ + ...f, + selector: FunctionSelector.fromNameAndParameters(f.name, f.parameters), + })); + } -/** - * Converts the given contract artifact into a ContractDao object that includes additional properties - * such as the address, portal contract, and function selectors. - * - * @param artifact - The contract artifact. - * @param completeAddress - The AztecAddress representing the contract's address. - * @param portalContract - The EthAddress representing the address of the associated portal contract. - * @returns A ContractDao object containing the provided information along with generated function selectors. - */ -export function toContractDao( - artifact: ContractArtifact, - completeAddress: CompleteAddress, - portalContract: EthAddress, -): ContractDao { - const functions = artifact.functions.map(f => ({ - ...f, - selector: FunctionSelector.fromNameAndParameters(f.name, f.parameters), - })); - return { - ...artifact, - completeAddress, - functions, - portalContract, - }; + get aztecNrVersion() { + return this.contractArtifact.aztecNrVersion; + } + + get name(): string { + return this.contractArtifact.name; + } + + get events(): EventAbi[] { + return this.contractArtifact.events; + } + + get debug(): DebugMetadata | undefined { + return this.contractArtifact.debug; + } + + toBuffer(): Buffer { + // the contract artifact was originally emitted to a JSON file by Noir + // should be safe to JSON.stringify it (i.e. it doesn't contain BigInts) + const contractArtifactJson = JSON.stringify(this.contractArtifact); + const buf = Buffer.concat([ + this.completeAddress.toBuffer(), + this.portalContract.toBuffer20(), + prefixBufferWithLength(Buffer.from(contractArtifactJson, 'utf-8')), + ]); + + return buf; + } + + static fromBuffer(buf: Uint8Array | BufferReader) { + const reader = BufferReader.asReader(buf); + const completeAddress = CompleteAddress.fromBuffer(reader); + const portalContract = new EthAddress(reader.readBytes(EthAddress.SIZE_IN_BYTES)); + const contractArtifact = JSON.parse(reader.readString()); + return new ContractDao(contractArtifact, completeAddress, portalContract); + } } /** diff --git a/yarn-project/types/src/keys/key_store.ts b/yarn-project/types/src/keys/key_store.ts index bce41ab0163..682aa89862e 100644 --- a/yarn-project/types/src/keys/key_store.ts +++ b/yarn-project/types/src/keys/key_store.ts @@ -18,7 +18,7 @@ export interface KeyStore { * @param privKey - The private key of the account. * @returns - The account's public key. */ - addAccount(privKey: GrumpkinPrivateKey): PublicKey; + addAccount(privKey: GrumpkinPrivateKey): Promise; /** * Retrieves the public keys of all accounts stored. diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 06a076f1cfe..8be9c238210 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -496,6 +496,7 @@ __metadata: dependencies: "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 @@ -508,6 +509,23 @@ __metadata: languageName: unknown linkType: soft +"@aztec/kv-store@workspace:^, @aztec/kv-store@workspace:kv-store": + version: 0.0.0-use.local + resolution: "@aztec/kv-store@workspace:kv-store" + dependencies: + "@aztec/foundation": "workspace:^" + "@jest/globals": ^29.5.0 + "@types/jest": ^29.5.0 + "@types/node": ^18.7.23 + jest: ^29.5.0 + jest-mock-extended: ^3.0.3 + lmdb: ^2.9.1 + ts-jest: ^29.1.0 + ts-node: ^10.9.1 + typescript: ^5.0.4 + languageName: unknown + linkType: soft + "@aztec/l1-artifacts@workspace:^, @aztec/l1-artifacts@workspace:l1-artifacts": version: 0.0.0-use.local resolution: "@aztec/l1-artifacts@workspace:l1-artifacts" @@ -715,6 +733,7 @@ __metadata: "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/key-store": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/noir-compiler": "workspace:^" "@aztec/noir-protocol-circuits": "workspace:^" "@aztec/types": "workspace:^"