From 880b7027a3e0a264e2a2bfb76918f65e47a59b82 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 10:44:45 +0000 Subject: [PATCH 01/18] wip abstracting leaf and preimage in indexed --- .../src/interfaces/indexed_tree.ts | 45 ++- yarn-project/merkle-tree/src/load_tree.ts | 5 +- yarn-project/merkle-tree/src/new_tree.ts | 17 +- .../src/sparse_tree/sparse_tree.test.ts | 5 +- .../standard_indexed_tree.ts | 275 +++++++++--------- .../test/standard_indexed_tree.test.ts | 33 ++- .../test/standard_indexed_tree_with_append.ts | 32 +- .../src/standard_tree/standard_tree.test.ts | 6 +- .../contracts/test_contract/src/interface.nr | 9 +- yarn-project/types/src/interfaces/index.ts | 4 +- .../types/src/interfaces/indexed_tree.ts | 36 +++ .../types/src/interfaces/leaf_data.ts | 17 -- .../types/src/interfaces/nullifier_tree.ts | 107 +++++++ .../types/src/interfaces/nullifier_witness.ts | 41 --- .../types/src/interfaces/state_provider.ts | 2 +- .../merkle_tree_operations_facade.ts | 14 +- .../src/world-state-db/merkle_tree_db.ts | 11 +- .../src/world-state-db/merkle_trees.ts | 19 +- 18 files changed, 412 insertions(+), 266 deletions(-) create mode 100644 yarn-project/types/src/interfaces/indexed_tree.ts delete mode 100644 yarn-project/types/src/interfaces/leaf_data.ts create mode 100644 yarn-project/types/src/interfaces/nullifier_tree.ts delete mode 100644 yarn-project/types/src/interfaces/nullifier_witness.ts diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 46c13f49bd9..17c66a9100d 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -1,16 +1,39 @@ -import { LeafData, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage, SiblingPath } from '@aztec/types'; -import { LowLeafWitnessData } from '../index.js'; import { AppendOnlyTree } from './append_only_tree.js'; +/* eslint-disable */ + +/** + * All of the data to be return during batch insertion. + */ +export interface LowLeafWitnessData { + /** + * Preimage of the low nullifier that proves non membership. + */ + leafData: IndexedTreeLeafPreimage; + /** + * Sibling path to prove membership of low nullifier. + */ + siblingPath: SiblingPath; + /** + * The index of low nullifier. + */ + index: bigint; +} + /** * The result of a batch insertion in an indexed merkle tree. */ -export interface BatchInsertionResult { +export interface BatchInsertionResult< + TreeHeight extends number, + SubtreeSiblingPathHeight extends number, + Leaf extends IndexedTreeLeaf, +> { /** * Data for the leaves to be updated when inserting the new ones. */ - lowLeavesWitnessData?: LowLeafWitnessData[]; + lowLeavesWitnessData?: LowLeafWitnessData[]; /** * Sibling path "pointing to" where the new subtree should be inserted into the tree. */ @@ -28,14 +51,14 @@ export interface BatchInsertionResult extends AppendOnlyTree { /** * Finds the index of the largest leaf whose value is less than or equal to the provided value. * @param newValue - The new value to be inserted into the tree. * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - findIndexOfPreviousValue( + findIndexOfPreviousKey( newValue: bigint, includeUncommitted: boolean, ): { @@ -50,12 +73,12 @@ export interface IndexedTree extends AppendOnlyTree { }; /** - * Gets the latest LeafData copy. - * @param index - Index of the leaf of which to obtain the LeafData copy. + * Gets the latest LeafPreimage copy. + * @param index - Index of the leaf of which to obtain the LeafPreimage copy. * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns A copy of the leaf data at the given index or undefined if the leaf was not found. + * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. */ - getLatestLeafDataCopy(index: number, includeUncommitted: boolean): LeafData | undefined; + getLatestLeafPreimageCopy(index: number, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined; /** * Batch insert multiple leaves into the tree. @@ -67,5 +90,5 @@ export interface IndexedTree extends AppendOnlyTree { leaves: Buffer[], subtreeHeight: SubtreeHeight, includeUncommitted: boolean, - ): Promise>; + ): Promise>; } diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts index baabe852735..7bbb85634aa 100644 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -13,14 +13,15 @@ import { TreeBase, decodeMeta } from './tree_base.js'; * @returns The newly created tree. */ export async function loadTree( - c: new (...args: any[]) => T, + c: (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, db: LevelUp, hasher: Hasher, name: string, ): Promise { const meta: Buffer = await db.get(name); const { root, depth, size } = decodeMeta(meta); - const tree = new c(db, hasher, name, depth, size, root); + + const tree = c(db, hasher, name, depth, size, root); await tree.initFromDb(); return tree; } diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts index f1cd4c2d3b5..819c0409c6c 100644 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -4,6 +4,19 @@ import { LevelUp } from 'levelup'; import { TreeBase } from './tree_base.js'; +/** + * Wraps a constructor in a regular builder + * @param clazz - The class to be instantiated. + * @returns A builder function. + */ +export function builder( + clazz: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => T, +) { + return (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => { + return new clazz(db, hasher, name, depth, size, root); + }; +} + /** * Creates a new tree. * @param c - The class of the tree to be instantiated. @@ -15,14 +28,14 @@ import { TreeBase } from './tree_base.js'; * @returns The newly created tree. */ export async function newTree( - c: new (...args: any[]) => T, + c: (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => T, db: LevelUp, hasher: Hasher, name: string, depth: number, prefilledSize = 1, ): Promise { - const tree = new c(db, hasher, name, depth, 0n, undefined); + const tree = c(db, hasher, name, depth, 0n); await tree.init(prefilledSize); return tree; } diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index bd4f1e6bd0b..7f1df12cc0f 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -7,6 +7,7 @@ import { default as levelup } from 'levelup'; import { INITIAL_LEAF, newTree } from '../index.js'; import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { loadTree } from '../load_tree.js'; +import { builder } from '../new_tree.js'; import { Pedersen } from '../pedersen.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; @@ -21,11 +22,11 @@ const createDb = async ( name: string, depth: number, ): Promise => { - return await newTree(SparseTree, levelUp, hasher, name, depth); + return await newTree(builder(SparseTree), levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string): Promise => { - return await loadTree(SparseTree, levelUp, hasher, name); + return await loadTree(builder(SparseTree), levelUp, hasher, name); }; const TEST_TREE_DEPTH = 3; diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index f9d44353fa9..b18e00275ae 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,92 +1,83 @@ import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { LeafData, SiblingPath } from '@aztec/types'; +import { Hasher, IndexedTreeLeaf, IndexedTreeLeafPreimage, SiblingPath } from '@aztec/types'; -import { BatchInsertionResult, IndexedTree } from '../interfaces/indexed_tree.js'; +import { LevelUp } from 'levelup'; + +import { BatchInsertionResult, IndexedTree, LowLeafWitnessData } from '../index.js'; import { TreeBase } from '../tree_base.js'; const log = createDebugLogger('aztec:standard-indexed-tree'); -const indexToKeyLeaf = (name: string, index: bigint) => { - return `${name}:leaf:${toBufferBE(index, 32).toString('hex')}`; -}; - -const keyLeafToIndex = (key: string): bigint => { - const index = key.split(':')[2]; - return toBigIntBE(Buffer.from(index, 'hex')); -}; +/* eslint-disable */ -const zeroLeaf: LeafData = { - value: 0n, - nextValue: 0n, - nextIndex: 0n, -}; - -/** - * All of the data to be return during batch insertion. - */ -export interface LowLeafWitnessData { - /** - * Preimage of the low nullifier that proves non membership. - */ - leafData: LeafData; - /** - * Sibling path to prove membership of low nullifier. - */ - siblingPath: SiblingPath; - /** - * The index of low nullifier. - */ - index: bigint; -} +// TODO /** * Pre-compute empty witness. * @param treeHeight - Height of tree for sibling path. * @returns An empty witness. */ -function getEmptyLowLeafWitness(treeHeight: N): LowLeafWitnessData { +function getEmptyLowLeafWitness( + treeHeight: N, + preimageFactory: Empty>, +): LowLeafWitnessData { return { - leafData: zeroLeaf, + leafData: preimageFactory.empty(), index: 0n, siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), }; } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const encodeTreeValue = (leafData: LeafData) => { - const valueAsBuffer = toBufferBE(leafData.value, 32); - const indexAsBuffer = toBufferBE(leafData.nextIndex, 32); - const nextValueAsBuffer = toBufferBE(leafData.nextValue, 32); - return Buffer.concat([valueAsBuffer, indexAsBuffer, nextValueAsBuffer]); +interface Empty { + empty(): T; +} + +interface DummyBuilder { + buildDummy(key: bigint): T; +} + +interface FromBuffer { + fromBuffer(buffer: Buffer): T; +} + +interface PreimageFactory { + fromLeaf(leaf: T, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage; +} + +const leafKeyToDbKey = (name: string, key: bigint) => { + return `${name}:leaf:${toBufferBE(key, 32).toString('hex')}`; }; -const decodeTreeValue = (buf: Buffer) => { - const value = toBigIntBE(buf.subarray(0, 32)); - const nextIndex = toBigIntBE(buf.subarray(32, 64)); - const nextValue = toBigIntBE(buf.subarray(64, 96)); - return { - value, - nextIndex, - nextValue, - } as LeafData; +const dbKeyToLeafKey = (key: string): bigint => { + const index = key.split(':')[2]; + return toBigIntBE(Buffer.from(index, 'hex')); }; -/** - * Indexed merkle tree. - */ -export class StandardIndexedTree extends TreeBase implements IndexedTree { - protected leaves: LeafData[] = []; - protected cachedLeaves: { [key: number]: LeafData } = {}; +export class StandardIndexedTree extends TreeBase implements IndexedTree { + protected leafPreimages: IndexedTreeLeafPreimage[] = []; + protected cachedLeafPreimages: { [key: number]: IndexedTreeLeafPreimage } = {}; + + public constructor( + db: LevelUp, + hasher: Hasher, + name: string, + depth: number, + size: bigint = 0n, + protected leafPreimageFactory: FromBuffer> & + Empty> & + PreimageFactory, + protected leafFactory: FromBuffer & DummyBuilder, + root?: Buffer, + ) { + super(db, hasher, name, depth, size, root); + } /** - * Appends the given leaves to the tree. - * @param _leaves - The leaves to append. - * @returns Empty promise. - * @remarks Use batchInsert method instead. + * Appends a set of leaf values to the tree. + * @param leaves - The set of leaves to be appended. */ - public appendLeaves(_leaves: Buffer[]): Promise { + appendLeaves(leaves: Buffer[]): Promise { throw new Error('Not implemented'); } @@ -115,11 +106,11 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @returns The value of the leaf at the given index or undefined if the leaf is empty. */ public getLeafValue(index: bigint, includeUncommitted: boolean): Promise { - const leaf = this.getLatestLeafDataCopy(Number(index), includeUncommitted); + const leaf = this.getLatestLeafPreimageCopy(Number(index), includeUncommitted); if (!leaf) { return Promise.resolve(undefined); } - return Promise.resolve(toBufferBE(leaf.value, 32)); + return Promise.resolve(leaf.toBuffer()); } /** @@ -128,7 +119,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - findIndexOfPreviousValue( + findIndexOfPreviousKey( newValue: bigint, includeUncommitted: boolean, ): { @@ -145,39 +136,38 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { const diff: bigint[] = []; for (let i = 0; i < numLeaves; i++) { - const storedLeaf = this.getLatestLeafDataCopy(i, includeUncommitted)!; + const storedPreimage = this.getLatestLeafPreimageCopy(i, includeUncommitted); // The stored leaf can be undefined if it addresses an empty leaf // If the leaf is empty we do the same as if the leaf was larger - if (storedLeaf === undefined) { + if (storedPreimage === undefined) { diff.push(newValue); - } else if (storedLeaf.value > newValue) { + } else if (storedPreimage.key > newValue) { diff.push(newValue); - } else if (storedLeaf.value === newValue) { + } else if (storedPreimage.key === newValue) { return { index: i, alreadyPresent: true }; } else { - diff.push(newValue - storedLeaf.value); + diff.push(newValue - storedPreimage.key); } } - const minIndex = this.findMinIndex(diff); + const minIndex = this.findMinKey(diff); return { index: minIndex, alreadyPresent: false }; } /** - * Gets the latest LeafData copy. - * @param index - Index of the leaf of which to obtain the LeafData copy. + * Gets the latest LeafPreimage copy. + * @param index - Index of the leaf of which to obtain the LeafPreimage copy. * @param includeUncommitted - If true, the uncommitted changes are included in the search. - * @returns A copy of the leaf data at the given index or undefined if the leaf was not found. + * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. */ - public getLatestLeafDataCopy(index: number, includeUncommitted: boolean): LeafData | undefined { - const leaf = !includeUncommitted ? this.leaves[index] : this.cachedLeaves[index] ?? this.leaves[index]; - return leaf - ? ({ - value: leaf.value, - nextIndex: leaf.nextIndex, - nextValue: leaf.nextValue, - } as LeafData) - : undefined; + public getLatestLeafPreimageCopy( + index: number, + includeUncommitted: boolean, + ): IndexedTreeLeafPreimage | undefined { + const preimage = !includeUncommitted + ? this.leafPreimages[index] + : this.cachedLeafPreimages[index] ?? this.leafPreimages[index]; + return preimage?.clone(); } /** @@ -185,7 +175,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param values - The collection of values to be searched. * @returns The index of the minimum value in the array. */ - private findMinIndex(values: bigint[]) { + private findMinKey(values: bigint[]) { if (!values.length) { return 0; } @@ -217,22 +207,19 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { throw new Error(`Prefilled size must be at least 1!`); } - const leaves: LeafData[] = []; + const leaves: IndexedTreeLeafPreimage[] = []; for (let i = 0n; i < prefilledSize; i++) { - const newLeaf = { - value: toBigIntBE(Buffer.from([Number(i)])), - nextIndex: i + 1n, - nextValue: i + 1n, - }; - leaves.push(newLeaf); + const newLeaf = this.leafFactory.buildDummy(toBigIntBE(Buffer.from([Number(i)]))); + const newLeafPreimage = this.leafPreimageFactory.fromLeaf(newLeaf, i + 1n, i + 1n); + leaves.push(newLeafPreimage); } - // Make the first leaf have 0 value - leaves[0].value = 0n; + // Make the first leaf have 0 key + leaves[0].key = 0n; // Make the last leaf point to the first leaf leaves[prefilledSize - 1].nextIndex = 0n; - leaves[prefilledSize - 1].nextValue = 0n; + leaves[prefilledSize - 1].nextKey = 0n; await this.encodeAndAppendLeaves(leaves, true); await this.commit(); @@ -243,16 +230,16 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { */ public async initFromDb(): Promise { const startingIndex = 0n; - const values: LeafData[] = []; + const preimages: IndexedTreeLeafPreimage[] = []; const promise = new Promise((resolve, reject) => { this.db .createReadStream({ - gte: indexToKeyLeaf(this.getName(), startingIndex), - lte: indexToKeyLeaf(this.getName(), 2n ** BigInt(this.getDepth())), + gte: leafKeyToDbKey(this.getName(), startingIndex), + lte: leafKeyToDbKey(this.getName(), 2n ** BigInt(this.getDepth())), }) - .on('data', function (data) { - const index = keyLeafToIndex(data.key.toString('utf-8')); - values[Number(index)] = decodeTreeValue(data.value); + .on('data', data => { + const leafKey = dbKeyToLeafKey(data.key.toString('utf-8')); + preimages[Number(leafKey)] = this.leafPreimageFactory.fromBuffer(data.value); }) .on('close', function () {}) .on('end', function () { @@ -264,7 +251,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { }); }); await promise; - this.leaves = values; + this.leafPreimages = preimages; } /** @@ -272,11 +259,11 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { */ private async commitLeaves(): Promise { const batch = this.db.batch(); - const keys = Object.getOwnPropertyNames(this.cachedLeaves); + const keys = Object.getOwnPropertyNames(this.cachedLeafPreimages); for (const key of keys) { const index = Number(key); - batch.put(indexToKeyLeaf(this.getName(), BigInt(index)), encodeTreeValue(this.cachedLeaves[index])); - this.leaves[index] = this.cachedLeaves[index]; + batch.put(leafKeyToDbKey(this.getName(), BigInt(index)), this.cachedLeafPreimages[index].toBuffer()); + this.leafPreimages[index] = this.cachedLeafPreimages[index]; } await batch.write(); this.clearCachedLeaves(); @@ -286,20 +273,20 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * Clears the cache. */ private clearCachedLeaves() { - this.cachedLeaves = {}; + this.cachedLeafPreimages = {}; } /** * Updates a leaf in the tree. - * @param leaf - New contents of the leaf. + * @param preimage - New contents of the leaf. * @param index - Index of the leaf to be updated. */ - protected async updateLeaf(leaf: LeafData, index: bigint) { + protected async updateLeaf(preimage: IndexedTreeLeafPreimage, index: bigint) { if (index > this.maxIndex) { throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); } - const encodedLeaf = this.encodeLeaf(leaf, true); + const encodedLeaf = this.encodeLeaf(preimage, true); await this.addLeafToCacheAndHashToRoot(encodedLeaf, index); const numLeaves = this.getNumLeaves(true); if (index >= numLeaves) { @@ -422,46 +409,46 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { >( leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise> { - const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight); + ): Promise> { + const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); // Accumulators - const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); - const pendingInsertionSubtree: LeafData[] = leaves.map(() => zeroLeaf); + const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); + const pendingInsertionSubtree: IndexedTreeLeafPreimage[] = leaves.map(() => this.leafPreimageFactory.empty()); // Start info const startInsertionIndex = this.getNumLeaves(true); - const leavesToInsert = leaves.map(leaf => toBigIntBE(leaf)); + const leavesToInsert = leaves.map(leaf => this.leafFactory.fromBuffer(leaf)); const sortedDescendingLeafTuples = leavesToInsert .map((leaf, index) => ({ leaf, index })) - .sort((a, b) => Number(b.leaf - a.leaf)); + .sort((a, b) => Number(b.leaf.getKey() - a.leaf.getKey())); const sortedDescendingLeaves = sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf); // Get insertion path for each leaf for (let i = 0; i < leavesToInsert.length; i++) { - const newValue = sortedDescendingLeaves[i]; - const originalIndex = leavesToInsert.indexOf(newValue); + const newLeaf = sortedDescendingLeaves[i]; + const originalIndex = leavesToInsert.indexOf(newLeaf); - if (newValue === 0n) { + if (newLeaf.isEmpty()) { continue; } - const indexOfPrevious = this.findIndexOfPreviousValue(newValue, true); + const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); // get the low leaf - const lowLeaf = this.getLatestLeafDataCopy(indexOfPrevious.index, true); - if (lowLeaf === undefined) { + const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true); + if (lowLeafPreimage === undefined) { return { lowLeavesWitnessData: undefined, - sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => new Fr(leafTuple.leaf).toBuffer()), + sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf.toBuffer()), sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), newSubtreeSiblingPath: await this.getSubtreeSiblingPath(subtreeHeight, true), }; } const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - const witness: LowLeafWitnessData = { - leafData: { ...lowLeaf }, + const witness: LowLeafWitnessData = { + leafData: lowLeafPreimage.clone(), index: BigInt(indexOfPrevious.index), siblingPath, }; @@ -469,20 +456,20 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { // Update the running paths lowLeavesWitnesses[i] = witness; - const currentPendingLeaf: LeafData = { - value: newValue, - nextValue: lowLeaf.nextValue, - nextIndex: lowLeaf.nextIndex, - }; + const currentPendingPreimageLeaf = this.leafPreimageFactory.fromLeaf( + newLeaf, + lowLeafPreimage.nextKey, + lowLeafPreimage.nextIndex, + ); - pendingInsertionSubtree[originalIndex] = currentPendingLeaf; + pendingInsertionSubtree[originalIndex] = currentPendingPreimageLeaf; - lowLeaf.nextValue = newValue; - lowLeaf.nextIndex = startInsertionIndex + BigInt(originalIndex); + lowLeafPreimage.nextKey = newLeaf.getKey(); + lowLeafPreimage.nextIndex = startInsertionIndex + BigInt(originalIndex); const lowLeafIndex = indexOfPrevious.index; - this.cachedLeaves[lowLeafIndex] = lowLeaf; - await this.updateLeaf(lowLeaf, BigInt(lowLeafIndex)); + this.cachedLeafPreimages[lowLeafIndex] = lowLeafPreimage; + await this.updateLeaf(lowLeafPreimage, BigInt(lowLeafIndex)); } const newSubtreeSiblingPath = await this.getSubtreeSiblingPath( @@ -497,7 +484,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { return { lowLeavesWitnessData: lowLeavesWitnesses, - sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => Buffer.from(new Fr(leafTuple.leaf).toBuffer())), + sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf.toBuffer()), sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), newSubtreeSiblingPath, }; @@ -516,19 +503,19 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { /** * Encodes leaves and appends them to a tree. - * @param leaves - Leaves to encode. + * @param preimages - Leaves to encode. * @param hash0Leaf - Indicates whether 0 value leaf should be hashed. See {@link encodeLeaf}. * @returns Empty promise */ - private async encodeAndAppendLeaves(leaves: LeafData[], hash0Leaf: boolean): Promise { + private async encodeAndAppendLeaves(preimages: IndexedTreeLeafPreimage[], hash0Leaf: boolean): Promise { const startInsertionIndex = Number(this.getNumLeaves(true)); - const serializedLeaves = leaves.map((leaf, i) => { - this.cachedLeaves[startInsertionIndex + i] = leaf; - return this.encodeLeaf(leaf, hash0Leaf); + const hashedLeaves = preimages.map((preimage, i) => { + this.cachedLeafPreimages[startInsertionIndex + i] = preimage; + return this.encodeLeaf(preimage, hash0Leaf); }); - await super.appendLeaves(serializedLeaves); + await super.appendLeaves(hashedLeaves); } /** @@ -539,14 +526,12 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * nullifier it is improbable that a valid nullifier would be 0. * @returns Leaf encoded in a buffer. */ - private encodeLeaf(leaf: LeafData, hash0Leaf: boolean): Buffer { + private encodeLeaf(leaf: IndexedTreeLeafPreimage, hash0Leaf: boolean): Buffer { let encodedLeaf; - if (!hash0Leaf && leaf.value == 0n) { + if (!hash0Leaf && leaf.key == 0n) { encodedLeaf = toBufferBE(0n, 32); } else { - encodedLeaf = this.hasher.hashInputs( - [leaf.value, leaf.nextIndex, leaf.nextValue].map(val => toBufferBE(val, 32)), - ); + encodedLeaf = this.hasher.hashInputs(leaf.toHashInputs()); } return encodedLeaf; } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index 2f6db0b1ac1..b9154eb7dc6 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -1,5 +1,5 @@ import { toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { Hasher, SiblingPath } from '@aztec/types'; +import { Hasher, NullifierLeaf, NullifierLeafPreimage, SiblingPath } from '@aztec/types'; import { default as levelup } from 'levelup'; @@ -9,15 +9,40 @@ import { createMemDown } from '../../test/utils/create_mem_down.js'; import { StandardIndexedTreeWithAppend } from './standard_indexed_tree_with_append.js'; const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { - return await newTree(StandardIndexedTreeWithAppend, levelUp, hasher, name, depth, prefilledSize); + return await newTree( + (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => { + return new StandardIndexedTreeWithAppend(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf); + }, + levelUp, + hasher, + name, + depth, + prefilledSize, + ); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(StandardIndexedTreeWithAppend, levelUp, hasher, name); + return await loadTree( + (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => { + return new StandardIndexedTreeWithAppend( + db, + hasher, + name, + depth, + size, + NullifierLeafPreimage, + NullifierLeaf, + root, + ); + }, + levelUp, + hasher, + name, + ); }; const createIndexedTreeLeaf = (value: number, nextIndex: number, nextValue: number) => { - return [toBufferBE(BigInt(value), 32), toBufferBE(BigInt(nextIndex), 32), toBufferBE(BigInt(nextValue), 32)]; + return new NullifierLeafPreimage(BigInt(value), BigInt(nextValue), BigInt(nextIndex)).toHashInputs(); }; const verifyCommittedState = async ( diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts index 49a90e611f1..992cc31f112 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts @@ -1,5 +1,4 @@ -import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; -import { LeafData } from '@aztec/types'; +import { IndexedTreeLeaf } from '@aztec/types'; import { StandardIndexedTree } from '../../index.js'; @@ -8,7 +7,7 @@ import { StandardIndexedTree } from '../../index.js'; * that was replaced by the more efficient batchInsert method. We keep the original implementation around as it useful * for testing that the more complex batchInsert method works correctly. */ -export class StandardIndexedTreeWithAppend extends StandardIndexedTree { +export class StandardIndexedTreeWithAppend extends StandardIndexedTree { /** * Appends the given leaves to the tree. * @param leaves - The leaves to append. @@ -27,10 +26,10 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { * @returns Empty promise. */ private async appendLeaf(leaf: Buffer): Promise { - const newValue = toBigIntBE(leaf); + const newLeaf = this.leafFactory.fromBuffer(leaf); // Special case when appending zero - if (newValue === 0n) { + if (newLeaf.getKey() === 0n) { const newSize = (this.cachedSize ?? this.size) + 1n; if (newSize - 1n > this.maxIndex) { throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); @@ -39,27 +38,28 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { return; } - const indexOfPrevious = this.findIndexOfPreviousValue(newValue, true); - const previousLeafCopy = this.getLatestLeafDataCopy(indexOfPrevious.index, true); + const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); + const previousLeafCopy = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true); if (previousLeafCopy === undefined) { throw new Error(`Previous leaf not found!`); } - const newLeaf = { - value: newValue, - nextIndex: previousLeafCopy.nextIndex, - nextValue: previousLeafCopy.nextValue, - } as LeafData; + const newLeafPreimage = this.leafPreimageFactory.fromLeaf( + newLeaf, + previousLeafCopy.nextKey, + previousLeafCopy.nextIndex, + ); + if (indexOfPrevious.alreadyPresent) { return; } // insert a new leaf at the highest index and update the values of our previous leaf copy const currentSize = this.getNumLeaves(true); previousLeafCopy.nextIndex = BigInt(currentSize); - previousLeafCopy.nextValue = newLeaf.value; - this.cachedLeaves[Number(currentSize)] = newLeaf; - this.cachedLeaves[Number(indexOfPrevious.index)] = previousLeafCopy; + previousLeafCopy.nextKey = newLeaf.getKey(); + this.cachedLeafPreimages[Number(currentSize)] = newLeafPreimage; + this.cachedLeafPreimages[Number(indexOfPrevious.index)] = previousLeafCopy; await this.updateLeaf(previousLeafCopy, BigInt(indexOfPrevious.index)); - await this.updateLeaf(newLeaf, this.getNumLeaves(true)); + await this.updateLeaf(newLeafPreimage, this.getNumLeaves(true)); } } diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index ee3191f42ff..db67453c4bc 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,7 +4,7 @@ import { Hasher } from '@aztec/types'; import { default as levelup } from 'levelup'; import { loadTree } from '../load_tree.js'; -import { newTree } from '../new_tree.js'; +import { builder, newTree } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { createMemDown } from '../test/utils/create_mem_down.js'; @@ -13,11 +13,11 @@ import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number) => { - return await newTree(StandardTree, levelUp, hasher, name, depth); + return await newTree(builder(StandardTree), levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(StandardTree, levelUp, hasher, name); + return await loadTree(builder(StandardTree), levelUp, hasher, name); }; treeTestSuite('StandardTree', createDb, createFromName); diff --git a/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr b/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr index 65cf52e96b5..1bb62e9f3f6 100644 --- a/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr +++ b/yarn-project/noir-contracts/src/contracts/test_contract/src/interface.nr @@ -1,5 +1,5 @@ /* Autogenerated file, do not edit! */ - + use dep::std; use dep::aztec::context::{ PrivateContext, PublicContext }; use dep::aztec::constants_gen::RETURN_VALUES_LENGTH; @@ -26,6 +26,7 @@ struct ManyNotesADeepStructTestCodeGenStruct { secret_hash: Field, } + // Interface for calling Test functions from a private context struct TestPrivateContextInterface { address: Field, @@ -241,6 +242,9 @@ impl TestPrivateContextInterface { } } + + + // Interface for calling Test functions from a public context struct TestPublicContextInterface { @@ -326,4 +330,5 @@ impl TestPublicContextInterface { } } - + + diff --git a/yarn-project/types/src/interfaces/index.ts b/yarn-project/types/src/interfaces/index.ts index f351287cab2..9832cc337ff 100644 --- a/yarn-project/types/src/interfaces/index.ts +++ b/yarn-project/types/src/interfaces/index.ts @@ -6,5 +6,5 @@ export * from './deployed-contract.js'; export * from './node-info.js'; export * from './sync-status.js'; export * from './configs.js'; -export * from './leaf_data.js'; -export * from './nullifier_witness.js'; +export * from './indexed_tree.js'; +export * from './nullifier_tree.js'; diff --git a/yarn-project/types/src/interfaces/indexed_tree.ts b/yarn-project/types/src/interfaces/indexed_tree.ts new file mode 100644 index 00000000000..d2a63e0dbd2 --- /dev/null +++ b/yarn-project/types/src/interfaces/indexed_tree.ts @@ -0,0 +1,36 @@ +/** + * A leaf of a tree. + */ +export interface LeafData { + /** + * A value of the leaf. + */ + value: bigint; + /** + * An index of the next leaf. + */ + nextIndex: bigint; + /** + * A value of the next leaf. + */ + nextValue: bigint; +} + +/* eslint-disable */ + +export interface IndexedTreeLeaf { + getKey(): bigint; + toBuffer(): Buffer; + isEmpty(): boolean; +} + +export interface IndexedTreeLeafPreimage { + key: bigint; + nextKey: bigint; + nextIndex: bigint; + + asLeaf(): Leaf; + toBuffer(): Buffer; + toHashInputs(): Buffer[]; + clone(): IndexedTreeLeafPreimage; +} diff --git a/yarn-project/types/src/interfaces/leaf_data.ts b/yarn-project/types/src/interfaces/leaf_data.ts deleted file mode 100644 index 2edc8e09818..00000000000 --- a/yarn-project/types/src/interfaces/leaf_data.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * A leaf of a tree. - */ -export interface LeafData { - /** - * A value of the leaf. - */ - value: bigint; - /** - * An index of the next leaf. - */ - nextIndex: bigint; - /** - * A value of the next leaf. - */ - nextValue: bigint; -} diff --git a/yarn-project/types/src/interfaces/nullifier_tree.ts b/yarn-project/types/src/interfaces/nullifier_tree.ts new file mode 100644 index 00000000000..180472a988e --- /dev/null +++ b/yarn-project/types/src/interfaces/nullifier_tree.ts @@ -0,0 +1,107 @@ +import { Fr, NULLIFIER_TREE_HEIGHT } from '@aztec/circuits.js'; +import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; + +import { SiblingPath } from '../sibling_path.js'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage, LeafData } from './indexed_tree.js'; + +/** + * Nullifier membership witness. + * @remarks When this represents membership witness of a low nullifier it can be used to perform a nullifier + * non-inclusion proof by leveraging the "linked list structure" of leaves and proving that a lower nullifier + * is pointing to a bigger next value than the nullifier we are trying to prove non-inclusion for. + */ +export class NullifierMembershipWitness { + constructor( + /** + * The index of the nullifier in a tree. + */ + public readonly index: bigint, + /** + * Preimage of the nullifier. + */ + public readonly leafData: LeafData, + /** + * Sibling path to prove membership of the nullifier. + */ + public readonly siblingPath: SiblingPath, + ) {} + + /** + * Returns a field array representation of a nullifier witness. + * @returns A field array representation of a nullifier witness. + */ + public toFieldArray(): Fr[] { + return [ + new Fr(this.index), + new Fr(this.leafData.value), + new Fr(this.leafData.nextIndex), + new Fr(this.leafData.nextValue), + ...this.siblingPath.toFieldArray(), + ]; + } +} + +/* eslint-disable */ + +export class NullifierLeafPreimage implements IndexedTreeLeafPreimage { + constructor(public key: bigint, public nextKey: bigint, public nextIndex: bigint) {} + + asLeaf(): NullifierLeaf { + return new NullifierLeaf(new Fr(this.key)); + } + + toBuffer(): Buffer { + return Buffer.concat(this.toHashInputs()); + } + + toHashInputs(): Buffer[] { + return [ + Buffer.from(toBufferBE(this.key, 32)), + Buffer.from(toBufferBE(this.nextIndex, 32)), + Buffer.from(toBufferBE(this.nextKey, 32)), + ]; + } + + clone(): NullifierLeafPreimage { + return new NullifierLeafPreimage(this.key, this.nextKey, this.nextIndex); + } + + static empty(): NullifierLeafPreimage { + return new NullifierLeafPreimage(0n, 0n, 0n); + } + + static fromBuffer(buf: Buffer): NullifierLeafPreimage { + const key = toBigIntBE(buf.subarray(0, 32)); + const nextKey = toBigIntBE(buf.subarray(32, 64)); + const nextIndex = toBigIntBE(buf.subarray(64, 96)); + return new NullifierLeafPreimage(key, nextKey, nextIndex); + } + + static fromLeaf(leaf: NullifierLeaf, nextKey: bigint, nextIndex: bigint): NullifierLeafPreimage { + return new NullifierLeafPreimage(leaf.nullifier.toBigInt(), nextKey, nextIndex); + } +} + +export class NullifierLeaf implements IndexedTreeLeaf { + constructor(public nullifier: Fr) {} + + getKey(): bigint { + return this.nullifier.toBigInt(); + } + + toBuffer(): Buffer { + return this.nullifier.toBuffer(); + } + + isEmpty(): boolean { + return this.nullifier.isZero(); + } + + static buildDummy(key: bigint): NullifierLeaf { + return new NullifierLeaf(new Fr(key)); + } + + static fromBuffer(buf: Buffer): NullifierLeaf { + return new NullifierLeaf(Fr.fromBuffer(buf)); + } +} diff --git a/yarn-project/types/src/interfaces/nullifier_witness.ts b/yarn-project/types/src/interfaces/nullifier_witness.ts deleted file mode 100644 index 90dc6d9a1c7..00000000000 --- a/yarn-project/types/src/interfaces/nullifier_witness.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Fr, NULLIFIER_TREE_HEIGHT } from '@aztec/circuits.js'; - -import { SiblingPath } from '../sibling_path.js'; -import { LeafData } from './leaf_data.js'; - -/** - * Nullifier membership witness. - * @remarks When this represents membership witness of a low nullifier it can be used to perform a nullifier - * non-inclusion proof by leveraging the "linked list structure" of leaves and proving that a lower nullifier - * is pointing to a bigger next value than the nullifier we are trying to prove non-inclusion for. - */ -export class NullifierMembershipWitness { - constructor( - /** - * The index of the nullifier in a tree. - */ - public readonly index: bigint, - /** - * Preimage of the nullifier. - */ - public readonly leafData: LeafData, - /** - * Sibling path to prove membership of the nullifier. - */ - public readonly siblingPath: SiblingPath, - ) {} - - /** - * Returns a field array representation of a nullifier witness. - * @returns A field array representation of a nullifier witness. - */ - public toFieldArray(): Fr[] { - return [ - new Fr(this.index), - new Fr(this.leafData.value), - new Fr(this.leafData.nextIndex), - new Fr(this.leafData.nextValue), - ...this.siblingPath.toFieldArray(), - ]; - } -} diff --git a/yarn-project/types/src/interfaces/state_provider.ts b/yarn-project/types/src/interfaces/state_provider.ts index 68d01812a7e..75690f6a564 100644 --- a/yarn-project/types/src/interfaces/state_provider.ts +++ b/yarn-project/types/src/interfaces/state_provider.ts @@ -12,7 +12,7 @@ import { L1ToL2MessageAndIndex } from '../l1_to_l2_message.js'; import { L2Block } from '../l2_block.js'; import { MerkleTreeId } from '../merkle_tree_id.js'; import { SiblingPath } from '../sibling_path.js'; -import { NullifierMembershipWitness } from './nullifier_witness.js'; +import { NullifierMembershipWitness } from './nullifier_tree.js'; /** * Interface providing methods for retrieving information about content of the state trees. diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 31240357748..5725ec141fe 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -1,6 +1,6 @@ import { Fr } from '@aztec/foundation/fields'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; import { CurrentTreeRoots, HandleL2BlockResult, MerkleTreeDb, MerkleTreeOperations, TreeInfo } from '../index.js'; @@ -89,8 +89,12 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param index - The index of the leaf to get. * @returns Leaf data. */ - getLeafData(treeId: MerkleTreeId.NULLIFIER_TREE, index: number): Promise { - return this.trees.getLeafData(treeId, index, this.includeUncommitted); + async getLeafPreimage( + treeId: MerkleTreeId.NULLIFIER_TREE, + index: number, + ): Promise | undefined> { + const preimage = await this.trees.getLeafPreimage(treeId, index, this.includeUncommitted); + return preimage as IndexedTreeLeafPreimage | undefined; } /** @@ -171,11 +175,11 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The data for the leaves to be updated when inserting the new ones. */ - public batchInsert( + public batchInsert( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise> { + ): Promise> { return this.trees.batchInsert(treeId, leaves, subtreeHeight); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 8f191517a82..40be71456aa 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -2,7 +2,7 @@ import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; /** * Type alias for the nullifier tree ID. @@ -146,7 +146,10 @@ export interface MerkleTreeOperations { * @param treeId - The tree for which leaf data should be returned. * @param index - The index of the leaf required. */ - getLeafData(treeId: IndexedTreeId, index: number): Promise; + getLeafPreimage( + treeId: IndexedTreeId, + index: number, + ): Promise | undefined>; /** * Update the leaf data at the given index. @@ -195,11 +198,11 @@ export interface MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The witness data for the leaves to be updated when inserting the new ones. */ - batchInsert( + batchInsert( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise>; + ): Promise>; /** * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree). diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index a27b1706612..577705dac81 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -25,7 +25,7 @@ import { loadTree, newTree, } from '@aztec/merkle-tree'; -import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; import { default as levelup } from 'levelup'; @@ -321,7 +321,7 @@ export class MerkleTrees implements MerkleTreeDb { alreadyPresent: boolean; }> { return await this.synchronize(() => - Promise.resolve(this._getIndexedTree(treeId).findIndexOfPreviousValue(value, includeUncommitted)), + Promise.resolve(this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)), ); } @@ -332,13 +332,13 @@ export class MerkleTrees implements MerkleTreeDb { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns Leaf data. */ - public async getLeafData( + public async getLeafPreimage( treeId: IndexedTreeId, index: number, includeUncommitted: boolean, - ): Promise { + ): Promise | undefined> { return await this.synchronize(() => - Promise.resolve(this._getIndexedTree(treeId).getLatestLeafDataCopy(index, includeUncommitted)), + Promise.resolve(this._getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted)), ); } @@ -397,12 +397,13 @@ export class MerkleTrees implements MerkleTreeDb { TreeHeight extends number, SubtreeHeight extends number, SubtreeSiblingPathHeight extends number, + Leaf extends IndexedTreeLeaf, >( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise> { - const tree = this.trees[treeId] as StandardIndexedTree; + ): Promise> { + const tree = this.trees[treeId] as StandardIndexedTree; if (!('batchInsert' in tree)) { throw new Error('Tree does not support `batchInsert` method'); } @@ -453,8 +454,8 @@ export class MerkleTrees implements MerkleTreeDb { * @param treeId - Id of the tree to get an instance of. * @returns The indexed tree for the specified tree id. */ - private _getIndexedTree(treeId: IndexedTreeId): IndexedTree { - return this.trees[treeId] as IndexedTree; + private _getIndexedTree(treeId: IndexedTreeId): IndexedTree { + return this.trees[treeId] as IndexedTree; } /** From ed471d49c25def39a6d835ebe9b4b2140df9b101 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 12:25:44 +0000 Subject: [PATCH 02/18] feat: experiment with generic indexed trees --- .../src/client/private_execution.test.ts | 4 +- .../aztec-node/src/aztec-node/server.ts | 12 +- .../src/structs/rollup/base_rollup.ts | 94 ++++++++++++-- .../circuits.js/src/tests/factories.ts | 2 +- .../src/e2e_blacklist_token_contract.test.ts | 10 +- .../end-to-end/src/e2e_slow_tree.test.ts | 10 +- yarn-project/foundation/package.json | 1 + yarn-project/foundation/src/index.ts | 1 + yarn-project/foundation/src/trees/index.ts | 17 +++ yarn-project/merkle-tree/src/index.ts | 2 +- .../src/interfaces/indexed_tree.ts | 19 ++- .../standard_indexed_tree.ts | 119 +++++++++--------- .../test/standard_indexed_tree.test.ts | 43 +++---- .../test/standard_indexed_tree_with_append.ts | 26 ++-- .../src/type_conversion.ts | 4 +- .../src/block_builder/solo_block_builder.ts | 28 ++--- .../types/src/interfaces/indexed_tree.ts | 19 --- .../types/src/interfaces/nullifier_tree.ts | 75 +---------- .../merkle_tree_operations_facade.ts | 18 ++- .../src/world-state-db/merkle_tree_db.ts | 16 ++- .../src/world-state-db/merkle_trees.ts | 41 +++--- 21 files changed, 315 insertions(+), 246 deletions(-) create mode 100644 yarn-project/foundation/src/trees/index.ts diff --git a/yarn-project/acir-simulator/src/client/private_execution.test.ts b/yarn-project/acir-simulator/src/client/private_execution.test.ts index b96d69dc338..0635036183f 100644 --- a/yarn-project/acir-simulator/src/client/private_execution.test.ts +++ b/yarn-project/acir-simulator/src/client/private_execution.test.ts @@ -27,7 +27,7 @@ import { pedersenHash } from '@aztec/foundation/crypto'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr, GrumpkinScalar } from '@aztec/foundation/fields'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { AppendOnlyTree, Pedersen, StandardTree, newTree } from '@aztec/merkle-tree'; +import { AppendOnlyTree, Pedersen, StandardTree, newTree, treeBuilder } from '@aztec/merkle-tree'; import { ChildContractArtifact, ImportTestContractArtifact, @@ -126,7 +126,7 @@ describe('Private Execution test suite', () => { if (!trees[name]) { const db = levelup(createMemDown()); const pedersen = new Pedersen(); - trees[name] = await newTree(StandardTree, db, pedersen, name, treeHeights[name]); + trees[name] = await newTree(treeBuilder(StandardTree), db, pedersen, name, treeHeights[name]); } await trees[name].appendLeaves(leaves.map(l => l.toBuffer())); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 817b1a6b206..a197bbfccd1 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -8,6 +8,8 @@ import { L1_TO_L2_MSG_TREE_HEIGHT, NOTE_HASH_TREE_HEIGHT, NULLIFIER_TREE_HEIGHT, + NullifierLeaf, + NullifierLeafPreimage, PUBLIC_DATA_TREE_HEIGHT, } from '@aztec/circuits.js'; import { computeGlobalsHash, computePublicDataTreeIndex } from '@aztec/circuits.js/abis'; @@ -401,7 +403,10 @@ export class AztecNodeService implements AztecNode { return undefined; } - const leafDataPromise = committedDb.getLeafData(MerkleTreeId.NULLIFIER_TREE, Number(index)); + const leafDataPromise = committedDb.getLeafPreimage( + MerkleTreeId.NULLIFIER_TREE, + Number(index), + ); const siblingPathPromise = committedDb.getSiblingPath( MerkleTreeId.NULLIFIER_TREE, BigInt(index), @@ -442,7 +447,10 @@ export class AztecNodeService implements AztecNode { if (alreadyPresent) { this.log.warn(`Nullifier ${nullifier.toBigInt()} already exists in the tree`); } - const leafData = await committedDb.getLeafData(MerkleTreeId.NULLIFIER_TREE, index); + const leafData = await committedDb.getLeafPreimage( + MerkleTreeId.NULLIFIER_TREE, + index, + ); if (!leafData) { return undefined; } diff --git a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts index 454532ec646..894f3ec0d93 100644 --- a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts +++ b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts @@ -1,5 +1,8 @@ +import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, Tuple } from '@aztec/foundation/serialize'; +import { IndexedTreeLeaf } from '@aztec/foundation/trees'; +import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, @@ -25,28 +28,103 @@ import { AppendOnlyTreeSnapshot } from './append_only_tree_snapshot.js'; * Class containing the data of a preimage of a single leaf in the nullifier tree. * Note: It's called preimage because this data gets hashed before being inserted as a node into the `IndexedTree`. */ -export class NullifierLeafPreimage { +export class NullifierLeafPreimage implements IndexedTreeLeafPreimage { constructor( /** * Leaf value inside the indexed tree's linked list. */ - public leafValue: Fr, + public nullifier: Fr, /** * Next value inside the indexed tree's linked list. */ - public nextValue: Fr, + public nextNullifier: Fr, /** * Index of the next leaf in the indexed tree's linked list. */ - public nextIndex: UInt32, + public nextIndex: bigint, ) {} - toBuffer() { - return serializeToBuffer(this.leafValue, this.nextValue, this.nextIndex); + getKey(): bigint { + return this.nullifier.toBigInt(); + } + + getNextKey(): bigint { + return this.nextNullifier.toBigInt(); + } + + getNextIndex(): bigint { + return this.nextIndex; + } + + asLeaf(): NullifierLeaf { + return new NullifierLeaf(this.nullifier); + } + + toBuffer(): Buffer { + return Buffer.concat(this.toHashInputs()); + } + + toHashInputs(): Buffer[] { + return [ + Buffer.from(this.nullifier.toBuffer()), + Buffer.from(toBufferBE(this.nextIndex, 32)), + Buffer.from(this.nextNullifier.toBuffer()), + ]; + } + + clone(): NullifierLeafPreimage { + return new NullifierLeafPreimage(this.nullifier, this.nextNullifier, this.nextIndex); + } + + static empty(): NullifierLeafPreimage { + return new NullifierLeafPreimage(Fr.ZERO, Fr.ZERO, 0n); + } + + static fromBuffer(buf: Buffer): NullifierLeafPreimage { + const key = Fr.fromBuffer(buf.subarray(0, 32)); + const nextKey = Fr.fromBuffer(buf.subarray(32, 64)); + const nextIndex = toBigIntBE(buf.subarray(64, 96)); + return new NullifierLeafPreimage(key, nextKey, nextIndex); + } + + static fromLeaf(leaf: NullifierLeaf, nextKey: bigint, nextIndex: bigint): NullifierLeafPreimage { + return new NullifierLeafPreimage(leaf.nullifier, new Fr(nextKey), nextIndex); + } + + static clone(preimage: NullifierLeafPreimage): NullifierLeafPreimage { + return new NullifierLeafPreimage(preimage.nullifier, preimage.nextNullifier, preimage.nextIndex); + } +} + +/** + * A nullifier to be inserted in the nullifier tree. + */ +export class NullifierLeaf implements IndexedTreeLeaf { + constructor( + /** + * Nullifier value. + */ + public nullifier: Fr, + ) {} + + getKey(): bigint { + return this.nullifier.toBigInt(); + } + + toBuffer(): Buffer { + return this.nullifier.toBuffer(); + } + + isEmpty(): boolean { + return this.nullifier.isZero(); + } + + static buildDummy(key: bigint): NullifierLeaf { + return new NullifierLeaf(new Fr(key)); } - static empty() { - return new NullifierLeafPreimage(Fr.ZERO, Fr.ZERO, 0); + static fromBuffer(buf: Buffer): NullifierLeaf { + return new NullifierLeaf(Fr.fromBuffer(buf)); } } diff --git a/yarn-project/circuits.js/src/tests/factories.ts b/yarn-project/circuits.js/src/tests/factories.ts index 6c03169795a..3d5acb065d4 100644 --- a/yarn-project/circuits.js/src/tests/factories.ts +++ b/yarn-project/circuits.js/src/tests/factories.ts @@ -906,7 +906,7 @@ export function makeBaseRollupInputs(seed = 0): BaseRollupInputs { const lowNullifierLeafPreimages = makeTuple( MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, - x => new NullifierLeafPreimage(fr(x), fr(x + 0x100), x + 0x200), + x => new NullifierLeafPreimage(fr(x), fr(x + 0x100), BigInt(x + 0x200)), seed + 0x1000, ); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts index 1c1825c7d83..6a2232eee47 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts @@ -14,7 +14,7 @@ import { computeAuthWitMessageHash, computeMessageSecretHash, } from '@aztec/aztec.js'; -import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; +import { Pedersen, SparseTree, newTree, treeBuilder } from '@aztec/merkle-tree'; import { SlowTreeContract, TokenBlacklistContract, TokenContract } from '@aztec/noir-contracts/types'; import { jest } from '@jest/globals'; @@ -107,7 +107,13 @@ describe('e2e_blacklist_token_contract', () => { slowTree = await SlowTreeContract.deploy(wallets[0]).send().deployed(); const depth = 254; - slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); + slowUpdateTreeSimulator = await newTree( + treeBuilder(SparseTree), + levelup(createMemDown()), + new Pedersen(), + 'test', + depth, + ); const deployTx = TokenBlacklistContract.deploy(wallets[0], accounts[0], slowTree.address).send({}); const receipt = await deployTx.wait(); diff --git a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts index 87c267e5e82..e0d58be36ee 100644 --- a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts +++ b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts @@ -1,6 +1,6 @@ /* eslint-disable camelcase */ import { CheatCodes, DebugLogger, Fr, Wallet } from '@aztec/aztec.js'; -import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; +import { Pedersen, SparseTree, newTree, treeBuilder } from '@aztec/merkle-tree'; import { SlowTreeContract } from '@aztec/noir-contracts/types'; import { default as levelup } from 'levelup'; @@ -27,7 +27,13 @@ describe('e2e_slow_tree', () => { it('Messing around with noir slow tree', async () => { const depth = 254; - const slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); + const slowUpdateTreeSimulator = await newTree( + treeBuilder(SparseTree), + levelup(createMemDown()), + new Pedersen(), + 'test', + depth, + ); const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, diff --git a/yarn-project/foundation/package.json b/yarn-project/foundation/package.json index 98a10914965..bb413f7e0d2 100644 --- a/yarn-project/foundation/package.json +++ b/yarn-project/foundation/package.json @@ -28,6 +28,7 @@ "./sleep": "./dest/sleep/index.js", "./timer": "./dest/timer/index.js", "./transport": "./dest/transport/index.js", + "./trees": "./dest/trees/index.js", "./wasm": "./dest/wasm/index.js", "./worker": "./dest/worker/index.js", "./bigint-buffer": "./dest/bigint-buffer/index.js", diff --git a/yarn-project/foundation/src/index.ts b/yarn-project/foundation/src/index.ts index b34a0bf474e..300b417c288 100644 --- a/yarn-project/foundation/src/index.ts +++ b/yarn-project/foundation/src/index.ts @@ -21,6 +21,7 @@ export * as serialize from './serialize/index.js'; export * as sleep from './sleep/index.js'; export * as timer from './timer/index.js'; export * as transport from './transport/index.js'; +export * as trees from './trees/index.js'; export * as types from './types/index.js'; export * as url from './url/index.js'; export * as wasm from './wasm/index.js'; diff --git a/yarn-project/foundation/src/trees/index.ts b/yarn-project/foundation/src/trees/index.ts new file mode 100644 index 00000000000..139b6383620 --- /dev/null +++ b/yarn-project/foundation/src/trees/index.ts @@ -0,0 +1,17 @@ +/* eslint-disable */ + +export interface IndexedTreeLeaf { + getKey(): bigint; + toBuffer(): Buffer; + isEmpty(): boolean; +} + +export interface IndexedTreeLeafPreimage { + getKey(): bigint; + getNextKey(): bigint; + getNextIndex(): bigint; + + asLeaf(): Leaf; + toBuffer(): Buffer; + toHashInputs(): Buffer[]; +} diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index de19e295bfd..ae9d417a40b 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -7,5 +7,5 @@ export * from './sparse_tree/sparse_tree.js'; export * from './standard_indexed_tree/standard_indexed_tree.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF } from './tree_base.js'; -export { newTree } from './new_tree.js'; +export { newTree, builder as treeBuilder } from './new_tree.js'; export { loadTree } from './load_tree.js'; diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 17c66a9100d..25ee825aadc 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -1,4 +1,5 @@ -import { IndexedTreeLeaf, IndexedTreeLeafPreimage, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { SiblingPath } from '@aztec/types'; import { AppendOnlyTree } from './append_only_tree.js'; @@ -7,11 +8,15 @@ import { AppendOnlyTree } from './append_only_tree.js'; /** * All of the data to be return during batch insertion. */ -export interface LowLeafWitnessData { +export interface LowLeafWitnessData< + N extends number, + Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, +> { /** * Preimage of the low nullifier that proves non membership. */ - leafData: IndexedTreeLeafPreimage; + leafData: Preimage; /** * Sibling path to prove membership of low nullifier. */ @@ -29,11 +34,12 @@ export interface BatchInsertionResult< TreeHeight extends number, SubtreeSiblingPathHeight extends number, Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, > { /** * Data for the leaves to be updated when inserting the new ones. */ - lowLeavesWitnessData?: LowLeafWitnessData[]; + lowLeavesWitnessData?: LowLeafWitnessData[]; /** * Sibling path "pointing to" where the new subtree should be inserted into the tree. */ @@ -51,7 +57,8 @@ export interface BatchInsertionResult< /** * Indexed merkle tree. */ -export interface IndexedTree extends AppendOnlyTree { +export interface IndexedTree> + extends AppendOnlyTree { /** * Finds the index of the largest leaf whose value is less than or equal to the provided value. * @param newValue - The new value to be inserted into the tree. @@ -90,5 +97,5 @@ export interface IndexedTree extends AppendOnlyTre leaves: Buffer[], subtreeHeight: SubtreeHeight, includeUncommitted: boolean, - ): Promise>; + ): Promise>; } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index b18e00275ae..32c8944bcad 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,6 +1,7 @@ import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; import { createDebugLogger } from '@aztec/foundation/log'; -import { Hasher, IndexedTreeLeaf, IndexedTreeLeafPreimage, SiblingPath } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { Hasher, SiblingPath } from '@aztec/types'; import { LevelUp } from 'levelup'; @@ -13,22 +14,6 @@ const log = createDebugLogger('aztec:standard-indexed-tree'); // TODO -/** - * Pre-compute empty witness. - * @param treeHeight - Height of tree for sibling path. - * @returns An empty witness. - */ -function getEmptyLowLeafWitness( - treeHeight: N, - preimageFactory: Empty>, -): LowLeafWitnessData { - return { - leafData: preimageFactory.empty(), - index: 0n, - siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), - }; -} - interface Empty { empty(): T; } @@ -41,8 +26,12 @@ interface FromBuffer { fromBuffer(buffer: Buffer): T; } -interface PreimageFactory { - fromLeaf(leaf: T, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage; +interface Clone { + clone(t: T): T; +} + +interface PreimageFactory> { + fromLeaf(leaf: Leaf, nextKey: bigint, nextIndex: bigint): Preimage; } const leafKeyToDbKey = (name: string, key: bigint) => { @@ -54,9 +43,12 @@ const dbKeyToLeafKey = (key: string): bigint => { return toBigIntBE(Buffer.from(index, 'hex')); }; -export class StandardIndexedTree extends TreeBase implements IndexedTree { - protected leafPreimages: IndexedTreeLeafPreimage[] = []; - protected cachedLeafPreimages: { [key: number]: IndexedTreeLeafPreimage } = {}; +export class StandardIndexedTree> + extends TreeBase + implements IndexedTree +{ + protected leafPreimages: Preimage[] = []; + protected cachedLeafPreimages: { [key: number]: Preimage } = {}; public constructor( db: LevelUp, @@ -64,9 +56,10 @@ export class StandardIndexedTree extends TreeBase name: string, depth: number, size: bigint = 0n, - protected leafPreimageFactory: FromBuffer> & - Empty> & - PreimageFactory, + protected leafPreimageFactory: FromBuffer & + Empty & + PreimageFactory & + Clone, protected leafFactory: FromBuffer & DummyBuilder, root?: Buffer, ) { @@ -142,12 +135,12 @@ export class StandardIndexedTree extends TreeBase // If the leaf is empty we do the same as if the leaf was larger if (storedPreimage === undefined) { diff.push(newValue); - } else if (storedPreimage.key > newValue) { + } else if (storedPreimage.getKey() > newValue) { diff.push(newValue); - } else if (storedPreimage.key === newValue) { + } else if (storedPreimage.getKey() === newValue) { return { index: i, alreadyPresent: true }; } else { - diff.push(newValue - storedPreimage.key); + diff.push(newValue - storedPreimage.getKey()); } } const minIndex = this.findMinKey(diff); @@ -160,14 +153,11 @@ export class StandardIndexedTree extends TreeBase * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. */ - public getLatestLeafPreimageCopy( - index: number, - includeUncommitted: boolean, - ): IndexedTreeLeafPreimage | undefined { + public getLatestLeafPreimageCopy(index: number, includeUncommitted: boolean): Preimage | undefined { const preimage = !includeUncommitted ? this.leafPreimages[index] : this.cachedLeafPreimages[index] ?? this.leafPreimages[index]; - return preimage?.clone(); + return this.leafPreimageFactory.clone(preimage); } /** @@ -207,19 +197,15 @@ export class StandardIndexedTree extends TreeBase throw new Error(`Prefilled size must be at least 1!`); } - const leaves: IndexedTreeLeafPreimage[] = []; + const leaves: Preimage[] = []; for (let i = 0n; i < prefilledSize; i++) { - const newLeaf = this.leafFactory.buildDummy(toBigIntBE(Buffer.from([Number(i)]))); + const newLeaf = this.leafFactory.buildDummy(i); const newLeafPreimage = this.leafPreimageFactory.fromLeaf(newLeaf, i + 1n, i + 1n); leaves.push(newLeafPreimage); } - // Make the first leaf have 0 key - leaves[0].key = 0n; - // Make the last leaf point to the first leaf - leaves[prefilledSize - 1].nextIndex = 0n; - leaves[prefilledSize - 1].nextKey = 0n; + leaves[prefilledSize - 1] = this.leafPreimageFactory.fromLeaf(leaves[prefilledSize - 1].asLeaf(), 0n, 0n); await this.encodeAndAppendLeaves(leaves, true); await this.commit(); @@ -230,7 +216,7 @@ export class StandardIndexedTree extends TreeBase */ public async initFromDb(): Promise { const startingIndex = 0n; - const preimages: IndexedTreeLeafPreimage[] = []; + const preimages: Preimage[] = []; const promise = new Promise((resolve, reject) => { this.db .createReadStream({ @@ -281,7 +267,7 @@ export class StandardIndexedTree extends TreeBase * @param preimage - New contents of the leaf. * @param index - Index of the leaf to be updated. */ - protected async updateLeaf(preimage: IndexedTreeLeafPreimage, index: bigint) { + protected async updateLeaf(preimage: Preimage, index: bigint) { if (index > this.maxIndex) { throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); } @@ -294,6 +280,22 @@ export class StandardIndexedTree extends TreeBase } } + /** + * Pre-compute empty witness. + * @param treeHeight - Height of tree for sibling path. + * @returns An empty witness. + */ + getEmptyLowLeafWitnes( + treeHeight: N, + preimageFactory: Empty, + ): LowLeafWitnessData { + return { + leafData: preimageFactory.empty(), + index: 0n, + siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), + }; + } + /* eslint-disable jsdoc/require-description-complete-sentence */ /* The following doc block messes up with complete-sentence, so we just disable it */ @@ -409,11 +411,11 @@ export class StandardIndexedTree extends TreeBase >( leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise> { - const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); + ): Promise> { + const emptyLowLeafWitness = this.getEmptyLowLeafWitnes(this.getDepth() as TreeHeight, this.leafPreimageFactory); // Accumulators - const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); - const pendingInsertionSubtree: IndexedTreeLeafPreimage[] = leaves.map(() => this.leafPreimageFactory.empty()); + const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); + const pendingInsertionSubtree: Preimage[] = leaves.map(() => this.leafPreimageFactory.empty()); // Start info const startInsertionIndex = this.getNumLeaves(true); @@ -447,8 +449,8 @@ export class StandardIndexedTree extends TreeBase } const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - const witness: LowLeafWitnessData = { - leafData: lowLeafPreimage.clone(), + const witness: LowLeafWitnessData = { + leafData: lowLeafPreimage, index: BigInt(indexOfPrevious.index), siblingPath, }; @@ -458,18 +460,21 @@ export class StandardIndexedTree extends TreeBase const currentPendingPreimageLeaf = this.leafPreimageFactory.fromLeaf( newLeaf, - lowLeafPreimage.nextKey, - lowLeafPreimage.nextIndex, + lowLeafPreimage.getNextKey(), + lowLeafPreimage.getNextIndex(), ); pendingInsertionSubtree[originalIndex] = currentPendingPreimageLeaf; - lowLeafPreimage.nextKey = newLeaf.getKey(); - lowLeafPreimage.nextIndex = startInsertionIndex + BigInt(originalIndex); + const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( + lowLeafPreimage.asLeaf(), + newLeaf.getKey(), + startInsertionIndex + BigInt(originalIndex), + ); const lowLeafIndex = indexOfPrevious.index; - this.cachedLeafPreimages[lowLeafIndex] = lowLeafPreimage; - await this.updateLeaf(lowLeafPreimage, BigInt(lowLeafIndex)); + this.cachedLeafPreimages[lowLeafIndex] = newLowLeafPreimage; + await this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex)); } const newSubtreeSiblingPath = await this.getSubtreeSiblingPath( @@ -507,7 +512,7 @@ export class StandardIndexedTree extends TreeBase * @param hash0Leaf - Indicates whether 0 value leaf should be hashed. See {@link encodeLeaf}. * @returns Empty promise */ - private async encodeAndAppendLeaves(preimages: IndexedTreeLeafPreimage[], hash0Leaf: boolean): Promise { + private async encodeAndAppendLeaves(preimages: Preimage[], hash0Leaf: boolean): Promise { const startInsertionIndex = Number(this.getNumLeaves(true)); const hashedLeaves = preimages.map((preimage, i) => { @@ -526,9 +531,9 @@ export class StandardIndexedTree extends TreeBase * nullifier it is improbable that a valid nullifier would be 0. * @returns Leaf encoded in a buffer. */ - private encodeLeaf(leaf: IndexedTreeLeafPreimage, hash0Leaf: boolean): Buffer { + private encodeLeaf(leaf: Preimage, hash0Leaf: boolean): Buffer { let encodedLeaf; - if (!hash0Leaf && leaf.key == 0n) { + if (!hash0Leaf && leaf.getKey() == 0n) { encodedLeaf = toBufferBE(0n, 32); } else { encodedLeaf = this.hasher.hashInputs(leaf.toHashInputs()); diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index b9154eb7dc6..015193954eb 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -1,5 +1,6 @@ +import { Fr, NullifierLeaf, NullifierLeafPreimage } from '@aztec/circuits.js'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { Hasher, NullifierLeaf, NullifierLeafPreimage, SiblingPath } from '@aztec/types'; +import { Hasher, SiblingPath } from '@aztec/types'; import { default as levelup } from 'levelup'; @@ -41,8 +42,8 @@ const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: st ); }; -const createIndexedTreeLeaf = (value: number, nextIndex: number, nextValue: number) => { - return new NullifierLeafPreimage(BigInt(value), BigInt(nextValue), BigInt(nextIndex)).toHashInputs(); +const createIndexedTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { + return new NullifierLeafPreimage(new Fr(value), new Fr(nextValue), BigInt(nextIndex)).toHashInputs(); }; const verifyCommittedState = async ( @@ -82,7 +83,7 @@ describe('StandardIndexedTreeSpecific', () => { * nextVal 0 0 0 0 0 0 0 0. */ - const initialLeafHash = pedersen.hashInputs(createIndexedTreeLeaf(0, 0, 0)); + const initialLeafHash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 0, 0)); const level1ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); const level2ZeroHash = pedersen.hash(level1ZeroHash, level1ZeroHash); @@ -116,8 +117,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 1 0 0 0 0 0 0 0 * nextVal 30 0 0 0 0 0 0 0. */ - index0Hash = pedersen.hashInputs(createIndexedTreeLeaf(0, 1, 30)); - let index1Hash = pedersen.hashInputs(createIndexedTreeLeaf(30, 0, 0)); + index0Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 1, 30)); + let index1Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(30, 0, 0)); e10 = pedersen.hash(index0Hash, index1Hash); e20 = pedersen.hash(e10, level1ZeroHash); root = pedersen.hash(e20, level2ZeroHash); @@ -143,8 +144,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 2 0 1 0 0 0 0 0 * nextVal 10 0 30 0 0 0 0 0. */ - index0Hash = pedersen.hashInputs(createIndexedTreeLeaf(0, 2, 10)); - let index2Hash = pedersen.hashInputs(createIndexedTreeLeaf(10, 1, 30)); + index0Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 2, 10)); + let index2Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(10, 1, 30)); e10 = pedersen.hash(index0Hash, index1Hash); let e11 = pedersen.hash(index2Hash, INITIAL_LEAF); e20 = pedersen.hash(e10, e11); @@ -176,8 +177,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextVal 10 0 20 30 0 0 0 0. */ e10 = pedersen.hash(index0Hash, index1Hash); - index2Hash = pedersen.hashInputs(createIndexedTreeLeaf(10, 3, 20)); - const index3Hash = pedersen.hashInputs(createIndexedTreeLeaf(20, 1, 30)); + index2Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(10, 3, 20)); + const index3Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(20, 1, 30)); e11 = pedersen.hash(index2Hash, index3Hash); e20 = pedersen.hash(e10, e11); root = pedersen.hash(e20, level2ZeroHash); @@ -207,8 +208,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 2 4 3 1 0 0 0 0 * nextVal 10 50 20 30 0 0 0 0. */ - index1Hash = pedersen.hashInputs(createIndexedTreeLeaf(30, 4, 50)); - const index4Hash = pedersen.hashInputs(createIndexedTreeLeaf(50, 0, 0)); + index1Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(30, 4, 50)); + const index4Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(50, 0, 0)); e10 = pedersen.hash(index0Hash, index1Hash); e20 = pedersen.hash(e10, e11); const e12 = pedersen.hash(index4Hash, INITIAL_LEAF); @@ -280,7 +281,7 @@ describe('StandardIndexedTreeSpecific', () => { */ const INITIAL_LEAF = toBufferBE(0n, 32); - const initialLeafHash = pedersen.hashInputs(createIndexedTreeLeaf(0, 0, 0)); + const initialLeafHash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 0, 0)); const level1ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); const level2ZeroHash = pedersen.hash(level1ZeroHash, level1ZeroHash); let index0Hash = initialLeafHash; @@ -314,8 +315,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 1 0 0 0 0 0 0 0 * nextVal 30 0 0 0 0 0 0 0. */ - index0Hash = pedersen.hashInputs(createIndexedTreeLeaf(0, 1, 30)); - let index1Hash = pedersen.hashInputs(createIndexedTreeLeaf(30, 0, 0)); + index0Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 1, 30)); + let index1Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(30, 0, 0)); e10 = pedersen.hash(index0Hash, index1Hash); e20 = pedersen.hash(e10, level1ZeroHash); root = pedersen.hash(e20, level2ZeroHash); @@ -340,8 +341,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 2 0 1 0 0 0 0 0 * nextVal 10 0 30 0 0 0 0 0. */ - index0Hash = pedersen.hashInputs(createIndexedTreeLeaf(0, 2, 10)); - let index2Hash = pedersen.hashInputs(createIndexedTreeLeaf(10, 1, 30)); + index0Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(0, 2, 10)); + let index2Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(10, 1, 30)); e10 = pedersen.hash(index0Hash, index1Hash); let e11 = pedersen.hash(index2Hash, INITIAL_LEAF); e20 = pedersen.hash(e10, e11); @@ -373,8 +374,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextVal 10 0 20 30 0 0 0 0. */ e10 = pedersen.hash(index0Hash, index1Hash); - index2Hash = pedersen.hashInputs(createIndexedTreeLeaf(10, 3, 20)); - const index3Hash = pedersen.hashInputs(createIndexedTreeLeaf(20, 1, 30)); + index2Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(10, 3, 20)); + const index3Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(20, 1, 30)); e11 = pedersen.hash(index2Hash, index3Hash); e20 = pedersen.hash(e10, e11); root = pedersen.hash(e20, level2ZeroHash); @@ -412,8 +413,8 @@ describe('StandardIndexedTreeSpecific', () => { * nextIdx 2 6 3 1 0 0 0 0 * nextVal 10 50 20 30 0 0 0 0. */ - index1Hash = pedersen.hashInputs(createIndexedTreeLeaf(30, 6, 50)); - const index6Hash = pedersen.hashInputs(createIndexedTreeLeaf(50, 0, 0)); + index1Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(30, 6, 50)); + const index6Hash = pedersen.hashInputs(createIndexedTreeLeafHashInputs(50, 0, 0)); e10 = pedersen.hash(index0Hash, index1Hash); e20 = pedersen.hash(e10, e11); const e13 = pedersen.hash(index6Hash, INITIAL_LEAF); diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts index 992cc31f112..bd0e5d549b3 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts @@ -1,4 +1,4 @@ -import { IndexedTreeLeaf } from '@aztec/types'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { StandardIndexedTree } from '../../index.js'; @@ -7,7 +7,10 @@ import { StandardIndexedTree } from '../../index.js'; * that was replaced by the more efficient batchInsert method. We keep the original implementation around as it useful * for testing that the more complex batchInsert method works correctly. */ -export class StandardIndexedTreeWithAppend extends StandardIndexedTree { +export class StandardIndexedTreeWithAppend< + Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, +> extends StandardIndexedTree { /** * Appends the given leaves to the tree. * @param leaves - The leaves to append. @@ -39,15 +42,15 @@ export class StandardIndexedTreeWithAppend extends } const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); - const previousLeafCopy = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true); + const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true); - if (previousLeafCopy === undefined) { + if (lowLeafPreimage === undefined) { throw new Error(`Previous leaf not found!`); } const newLeafPreimage = this.leafPreimageFactory.fromLeaf( newLeaf, - previousLeafCopy.nextKey, - previousLeafCopy.nextIndex, + lowLeafPreimage.getNextKey(), + lowLeafPreimage.getNextIndex(), ); if (indexOfPrevious.alreadyPresent) { @@ -55,11 +58,14 @@ export class StandardIndexedTreeWithAppend extends } // insert a new leaf at the highest index and update the values of our previous leaf copy const currentSize = this.getNumLeaves(true); - previousLeafCopy.nextIndex = BigInt(currentSize); - previousLeafCopy.nextKey = newLeaf.getKey(); + const newLowLeafPreimage = this.leafPreimageFactory.fromLeaf( + lowLeafPreimage.asLeaf(), + newLeaf.getKey(), + BigInt(currentSize), + ); this.cachedLeafPreimages[Number(currentSize)] = newLeafPreimage; - this.cachedLeafPreimages[Number(indexOfPrevious.index)] = previousLeafCopy; - await this.updateLeaf(previousLeafCopy, BigInt(indexOfPrevious.index)); + this.cachedLeafPreimages[Number(indexOfPrevious.index)] = newLowLeafPreimage; + await this.updateLeaf(newLowLeafPreimage, BigInt(indexOfPrevious.index)); await this.updateLeaf(newLeafPreimage, this.getNumLeaves(true)); } } diff --git a/yarn-project/noir-protocol-circuits/src/type_conversion.ts b/yarn-project/noir-protocol-circuits/src/type_conversion.ts index 8a539742be1..3dbcdaecb9d 100644 --- a/yarn-project/noir-protocol-circuits/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits/src/type_conversion.ts @@ -1360,8 +1360,8 @@ export function mapNullifierLeafPreimageToNoir( nullifierLeafPreimage: NullifierLeafPreimage, ): NullifierLeafPreimageNoir { return { - leaf_value: mapFieldToNoir(nullifierLeafPreimage.leafValue), - next_value: mapFieldToNoir(nullifierLeafPreimage.nextValue), + leaf_value: mapFieldToNoir(nullifierLeafPreimage.nullifier), + next_value: mapFieldToNoir(nullifierLeafPreimage.nextNullifier), next_index: mapFieldToNoir(new Fr(nullifierLeafPreimage.nextIndex)), }; } diff --git a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts index 464e766a69e..e22fe389a7c 100644 --- a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts @@ -22,6 +22,7 @@ import { NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, NULLIFIER_TREE_HEIGHT, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, + NullifierLeaf, NullifierLeafPreimage, PUBLIC_DATA_TREE_HEIGHT, PreviousKernelData, @@ -61,8 +62,6 @@ import { BlockBuilder } from './index.js'; import { AllowedTreeNames, OutputWithTreeSnapshot } from './types.js'; const frToBigInt = (fr: Fr) => toBigIntBE(fr.toBuffer()); -const bigintToFr = (num: bigint) => new Fr(num); -const bigintToNum = (num: bigint) => Number(num); // Denotes fields that are not used now, but will be in the future const FUTURE_FR = new Fr(0n); @@ -577,19 +576,15 @@ export class SoloBlockBuilder implements BlockBuilder { const tree = MerkleTreeId.NULLIFIER_TREE; const prevValueIndex = await this.db.getPreviousValueIndex(tree, frToBigInt(nullifier)); - const prevValueInfo = await this.db.getLeafData(tree, prevValueIndex.index); - if (!prevValueInfo) { + const prevValuePreimage = await this.db.getLeafPreimage(tree, prevValueIndex.index); + if (!prevValuePreimage) { throw new Error(`Nullifier tree should have one initial leaf`); } const prevValueSiblingPath = await this.db.getSiblingPath(tree, BigInt(prevValueIndex.index)); return { index: prevValueIndex, - leafPreimage: new NullifierLeafPreimage( - bigintToFr(prevValueInfo.value), - bigintToFr(prevValueInfo.nextValue), - bigintToNum(prevValueInfo.nextIndex), - ), + leafPreimage: prevValuePreimage, witness: new MembershipWitness( NULLIFIER_TREE_HEIGHT, BigInt(prevValueIndex.index), @@ -713,7 +708,12 @@ export class SoloBlockBuilder implements BlockBuilder { newSubtreeSiblingPath: newNullifiersSubtreeSiblingPath, sortedNewLeaves: sortedNewNullifiers, sortedNewLeavesIndexes: sortednewNullifiersIndexes, - } = await this.db.batchInsert( + } = await this.db.batchInsert< + typeof NULLIFIER_TREE_HEIGHT, + typeof NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, + NullifierLeaf, + NullifierLeafPreimage + >( MerkleTreeId.NULLIFIER_TREE, newNullifiers.map(fr => fr.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, @@ -747,13 +747,7 @@ export class SoloBlockBuilder implements BlockBuilder { newPublicDataUpdateRequestsSiblingPaths, newPublicDataReadsSiblingPaths, lowNullifierLeafPreimages: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => - i < nullifierWitnessLeaves.length - ? new NullifierLeafPreimage( - new Fr(nullifierWitnessLeaves[i].leafData.value), - new Fr(nullifierWitnessLeaves[i].leafData.nextValue), - Number(nullifierWitnessLeaves[i].leafData.nextIndex), - ) - : new NullifierLeafPreimage(Fr.ZERO, Fr.ZERO, 0), + i < nullifierWitnessLeaves.length ? nullifierWitnessLeaves[i].leafData : NullifierLeafPreimage.empty(), ), lowNullifierMembershipWitness: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => i < lowNullifierMembershipWitnesses.length diff --git a/yarn-project/types/src/interfaces/indexed_tree.ts b/yarn-project/types/src/interfaces/indexed_tree.ts index d2a63e0dbd2..2edc8e09818 100644 --- a/yarn-project/types/src/interfaces/indexed_tree.ts +++ b/yarn-project/types/src/interfaces/indexed_tree.ts @@ -15,22 +15,3 @@ export interface LeafData { */ nextValue: bigint; } - -/* eslint-disable */ - -export interface IndexedTreeLeaf { - getKey(): bigint; - toBuffer(): Buffer; - isEmpty(): boolean; -} - -export interface IndexedTreeLeafPreimage { - key: bigint; - nextKey: bigint; - nextIndex: bigint; - - asLeaf(): Leaf; - toBuffer(): Buffer; - toHashInputs(): Buffer[]; - clone(): IndexedTreeLeafPreimage; -} diff --git a/yarn-project/types/src/interfaces/nullifier_tree.ts b/yarn-project/types/src/interfaces/nullifier_tree.ts index 180472a988e..ab3053ef858 100644 --- a/yarn-project/types/src/interfaces/nullifier_tree.ts +++ b/yarn-project/types/src/interfaces/nullifier_tree.ts @@ -1,8 +1,6 @@ -import { Fr, NULLIFIER_TREE_HEIGHT } from '@aztec/circuits.js'; -import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; +import { Fr, NULLIFIER_TREE_HEIGHT, NullifierLeafPreimage } from '@aztec/circuits.js'; import { SiblingPath } from '../sibling_path.js'; -import { IndexedTreeLeaf, IndexedTreeLeafPreimage, LeafData } from './indexed_tree.js'; /** * Nullifier membership witness. @@ -19,7 +17,7 @@ export class NullifierMembershipWitness { /** * Preimage of the nullifier. */ - public readonly leafData: LeafData, + public readonly leafData: NullifierLeafPreimage, /** * Sibling path to prove membership of the nullifier. */ @@ -33,75 +31,10 @@ export class NullifierMembershipWitness { public toFieldArray(): Fr[] { return [ new Fr(this.index), - new Fr(this.leafData.value), + new Fr(this.leafData.nullifier), new Fr(this.leafData.nextIndex), - new Fr(this.leafData.nextValue), + new Fr(this.leafData.nextNullifier), ...this.siblingPath.toFieldArray(), ]; } } - -/* eslint-disable */ - -export class NullifierLeafPreimage implements IndexedTreeLeafPreimage { - constructor(public key: bigint, public nextKey: bigint, public nextIndex: bigint) {} - - asLeaf(): NullifierLeaf { - return new NullifierLeaf(new Fr(this.key)); - } - - toBuffer(): Buffer { - return Buffer.concat(this.toHashInputs()); - } - - toHashInputs(): Buffer[] { - return [ - Buffer.from(toBufferBE(this.key, 32)), - Buffer.from(toBufferBE(this.nextIndex, 32)), - Buffer.from(toBufferBE(this.nextKey, 32)), - ]; - } - - clone(): NullifierLeafPreimage { - return new NullifierLeafPreimage(this.key, this.nextKey, this.nextIndex); - } - - static empty(): NullifierLeafPreimage { - return new NullifierLeafPreimage(0n, 0n, 0n); - } - - static fromBuffer(buf: Buffer): NullifierLeafPreimage { - const key = toBigIntBE(buf.subarray(0, 32)); - const nextKey = toBigIntBE(buf.subarray(32, 64)); - const nextIndex = toBigIntBE(buf.subarray(64, 96)); - return new NullifierLeafPreimage(key, nextKey, nextIndex); - } - - static fromLeaf(leaf: NullifierLeaf, nextKey: bigint, nextIndex: bigint): NullifierLeafPreimage { - return new NullifierLeafPreimage(leaf.nullifier.toBigInt(), nextKey, nextIndex); - } -} - -export class NullifierLeaf implements IndexedTreeLeaf { - constructor(public nullifier: Fr) {} - - getKey(): bigint { - return this.nullifier.toBigInt(); - } - - toBuffer(): Buffer { - return this.nullifier.toBuffer(); - } - - isEmpty(): boolean { - return this.nullifier.isZero(); - } - - static buildDummy(key: bigint): NullifierLeaf { - return new NullifierLeaf(new Fr(key)); - } - - static fromBuffer(buf: Buffer): NullifierLeaf { - return new NullifierLeaf(Fr.fromBuffer(buf)); - } -} diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 5725ec141fe..35594003f6a 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -1,6 +1,7 @@ import { Fr } from '@aztec/foundation/fields'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; import { CurrentTreeRoots, HandleL2BlockResult, MerkleTreeDb, MerkleTreeOperations, TreeInfo } from '../index.js'; @@ -89,12 +90,12 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param index - The index of the leaf to get. * @returns Leaf data. */ - async getLeafPreimage( + async getLeafPreimage>( treeId: MerkleTreeId.NULLIFIER_TREE, index: number, - ): Promise | undefined> { + ): Promise { const preimage = await this.trees.getLeafPreimage(treeId, index, this.includeUncommitted); - return preimage as IndexedTreeLeafPreimage | undefined; + return preimage as Preimage | undefined; } /** @@ -175,11 +176,16 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The data for the leaves to be updated when inserting the new ones. */ - public batchInsert( + public batchInsert< + TreeHeight extends number, + SubtreeSiblingPathHeight extends number, + Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, + >( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise> { + ): Promise> { return this.trees.batchInsert(treeId, leaves, subtreeHeight); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 40be71456aa..25b80d9c3ec 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,8 +1,9 @@ import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; /** * Type alias for the nullifier tree ID. @@ -146,10 +147,10 @@ export interface MerkleTreeOperations { * @param treeId - The tree for which leaf data should be returned. * @param index - The index of the leaf required. */ - getLeafPreimage( + getLeafPreimage>( treeId: IndexedTreeId, index: number, - ): Promise | undefined>; + ): Promise; /** * Update the leaf data at the given index. @@ -198,11 +199,16 @@ export interface MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The witness data for the leaves to be updated when inserting the new ones. */ - batchInsert( + batchInsert< + TreeHeight extends number, + SubtreeSiblingPathHeight extends number, + Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, + >( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise>; + ): Promise>; /** * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree). diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 577705dac81..341404f3ae6 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -7,12 +7,15 @@ import { NOTE_HASH_TREE_HEIGHT, NULLIFIER_SUBTREE_HEIGHT, NULLIFIER_TREE_HEIGHT, + NullifierLeaf, + NullifierLeafPreimage, PUBLIC_DATA_TREE_HEIGHT, } from '@aztec/circuits.js'; import { computeBlockHash, computeGlobalsHash } from '@aztec/circuits.js/abis'; import { Committable } from '@aztec/foundation/committable'; import { SerialQueue } from '@aztec/foundation/fifo'; import { createDebugLogger } from '@aztec/foundation/log'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { AppendOnlyTree, BatchInsertionResult, @@ -24,8 +27,9 @@ import { UpdateOnlyTree, loadTree, newTree, + treeBuilder, } from '@aztec/merkle-tree'; -import { IndexedTreeLeaf, IndexedTreeLeafPreimage, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { Hasher, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; import { default as levelup } from 'levelup'; @@ -75,14 +79,16 @@ export class MerkleTrees implements MerkleTreeDb { const hasher = new Pedersen(); const contractTree: AppendOnlyTree = await initializeTree( - StandardTree, + treeBuilder(StandardTree), this.db, hasher, `${MerkleTreeId[MerkleTreeId.CONTRACT_TREE]}`, CONTRACT_TREE_HEIGHT, ); const nullifierTree = await initializeTree( - StandardIndexedTree, + (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => { + return new StandardIndexedTree(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf); + }, this.db, hasher, `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, @@ -90,28 +96,28 @@ export class MerkleTrees implements MerkleTreeDb { INITIAL_NULLIFIER_TREE_SIZE, ); const noteHashTree: AppendOnlyTree = await initializeTree( - StandardTree, + treeBuilder(StandardTree), this.db, hasher, `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, NOTE_HASH_TREE_HEIGHT, ); const publicDataTree: UpdateOnlyTree = await initializeTree( - SparseTree, + treeBuilder(SparseTree), this.db, hasher, `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, PUBLIC_DATA_TREE_HEIGHT, ); const l1Tol2MessagesTree: AppendOnlyTree = await initializeTree( - StandardTree, + treeBuilder(StandardTree), this.db, hasher, `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGES_TREE]}`, L1_TO_L2_MSG_TREE_HEIGHT, ); const historicBlocksTree: AppendOnlyTree = await initializeTree( - StandardTree, + treeBuilder(StandardTree), this.db, hasher, `${MerkleTreeId[MerkleTreeId.BLOCKS_TREE]}`, @@ -332,13 +338,15 @@ export class MerkleTrees implements MerkleTreeDb { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns Leaf data. */ - public async getLeafPreimage( + public async getLeafPreimage>( treeId: IndexedTreeId, index: number, includeUncommitted: boolean, ): Promise | undefined> { return await this.synchronize(() => - Promise.resolve(this._getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted)), + Promise.resolve( + this._getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted), + ), ); } @@ -398,12 +406,13 @@ export class MerkleTrees implements MerkleTreeDb { SubtreeHeight extends number, SubtreeSiblingPathHeight extends number, Leaf extends IndexedTreeLeaf, + Preimage extends IndexedTreeLeafPreimage, >( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise> { - const tree = this.trees[treeId] as StandardIndexedTree; + ): Promise> { + const tree = this.trees[treeId] as StandardIndexedTree; if (!('batchInsert' in tree)) { throw new Error('Tree does not support `batchInsert` method'); } @@ -454,8 +463,10 @@ export class MerkleTrees implements MerkleTreeDb { * @param treeId - Id of the tree to get an instance of. * @returns The indexed tree for the specified tree id. */ - private _getIndexedTree(treeId: IndexedTreeId): IndexedTree { - return this.trees[treeId] as IndexedTree; + private _getIndexedTree>( + treeId: IndexedTreeId, + ): IndexedTree { + return this.trees[treeId] as IndexedTree; } /** @@ -560,7 +571,9 @@ export class MerkleTrees implements MerkleTreeDb { } // Sync the indexed trees - await (this.trees[MerkleTreeId.NULLIFIER_TREE] as StandardIndexedTree).batchInsert( + await ( + this.trees[MerkleTreeId.NULLIFIER_TREE] as StandardIndexedTree + ).batchInsert( l2Block.newNullifiers.map(fr => fr.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, ); From 38fcf05080852985242e4a7456847a818bcf3f20 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 13:05:18 +0000 Subject: [PATCH 03/18] chore: add test dep to devDependencies --- yarn-project/merkle-tree/package.json | 1 + yarn-project/yarn.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index 6418d219963..4c7b53d8f42 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -40,6 +40,7 @@ "tslib": "^2.4.0" }, "devDependencies": { + "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/levelup": "^5.1.2", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 4c4ad8d3a13..ebd3927a90f 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -549,6 +549,7 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/merkle-tree@workspace:merkle-tree" dependencies: + "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 From bc4915b1cf05b8b08b6f395997e242a13ca793cd Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 13:11:42 +0000 Subject: [PATCH 04/18] chore: remove old LeafData struct --- yarn-project/end-to-end/tsconfig.json | 4 +--- .../src/interfaces/update_only_tree.ts | 5 +---- yarn-project/merkle-tree/tsconfig.json | 3 +++ yarn-project/types/src/interfaces/index.ts | 1 - .../types/src/interfaces/indexed_tree.ts | 17 ----------------- .../merkle_tree_operations_facade.ts | 5 +++-- .../src/world-state-db/merkle_tree_db.ts | 6 +++--- .../src/world-state-db/merkle_trees.ts | 10 +++------- 8 files changed, 14 insertions(+), 37 deletions(-) delete mode 100644 yarn-project/types/src/interfaces/indexed_tree.ts diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index 8792db4c3c5..c44790b98d8 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -55,7 +55,5 @@ "path": "../world-state" } ], - "include": [ - "src" - ] + "include": ["src"] } diff --git a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts b/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts index 59a82d0b118..d798c5bf435 100644 --- a/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/update_only_tree.ts @@ -1,5 +1,3 @@ -import { LeafData } from '@aztec/types'; - import { MerkleTree } from './merkle_tree.js'; /** @@ -11,6 +9,5 @@ export interface UpdateOnlyTree extends MerkleTree { * @param leaf - The leaf value to be updated. * @param index - The leaf to be updated. */ - // TODO: Make this strictly a Buffer - updateLeaf(leaf: Buffer | LeafData, index: bigint): Promise; + updateLeaf(leaf: Buffer, index: bigint): Promise; } diff --git a/yarn-project/merkle-tree/tsconfig.json b/yarn-project/merkle-tree/tsconfig.json index 831130c7c84..35f81f8b801 100644 --- a/yarn-project/merkle-tree/tsconfig.json +++ b/yarn-project/merkle-tree/tsconfig.json @@ -11,6 +11,9 @@ }, { "path": "../types" + }, + { + "path": "../circuits.js" } ], "include": ["src"] diff --git a/yarn-project/types/src/interfaces/index.ts b/yarn-project/types/src/interfaces/index.ts index 9832cc337ff..cdba82c6847 100644 --- a/yarn-project/types/src/interfaces/index.ts +++ b/yarn-project/types/src/interfaces/index.ts @@ -6,5 +6,4 @@ export * from './deployed-contract.js'; export * from './node-info.js'; export * from './sync-status.js'; export * from './configs.js'; -export * from './indexed_tree.js'; export * from './nullifier_tree.js'; diff --git a/yarn-project/types/src/interfaces/indexed_tree.ts b/yarn-project/types/src/interfaces/indexed_tree.ts deleted file mode 100644 index 2edc8e09818..00000000000 --- a/yarn-project/types/src/interfaces/indexed_tree.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * A leaf of a tree. - */ -export interface LeafData { - /** - * A value of the leaf. - */ - value: bigint; - /** - * An index of the next leaf. - */ - nextIndex: bigint; - /** - * A value of the next leaf. - */ - nextValue: bigint; -} diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 35594003f6a..d5ddc2025bf 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -1,7 +1,8 @@ +import { NullifierLeafPreimage } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types'; import { CurrentTreeRoots, HandleL2BlockResult, MerkleTreeDb, MerkleTreeOperations, TreeInfo } from '../index.js'; @@ -80,7 +81,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param index - The index to insert into. * @returns Empty promise. */ - updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: LeafData, index: bigint): Promise { + updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: NullifierLeafPreimage, index: bigint): Promise { return this.trees.updateLeaf(treeId, leaf, index); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 25b80d9c3ec..7c3aeba073e 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,9 +1,9 @@ -import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; +import { MAX_NEW_NULLIFIERS_PER_TX, NullifierLeafPreimage } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; -import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types'; /** * Type alias for the nullifier tree ID. @@ -158,7 +158,7 @@ export interface MerkleTreeOperations { * @param leaf - The updated leaf value. * @param index - The index of the leaf to be updated. */ - updateLeaf(treeId: IndexedTreeId | PublicTreeId, leaf: LeafData | Buffer, index: bigint): Promise; + updateLeaf(treeId: IndexedTreeId | PublicTreeId, leaf: NullifierLeafPreimage | Buffer, index: bigint): Promise; /** * Returns the index containing a leaf value. diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 341404f3ae6..b5361513d03 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -29,7 +29,7 @@ import { newTree, treeBuilder, } from '@aztec/merkle-tree'; -import { Hasher, L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; +import { Hasher, L2Block, MerkleTreeId, SiblingPath } from '@aztec/types'; import { default as levelup } from 'levelup'; @@ -381,7 +381,7 @@ export class MerkleTrees implements MerkleTreeDb { * @param index - The index to insert into. * @returns Empty promise. */ - public async updateLeaf(treeId: IndexedTreeId | PublicTreeId, leaf: LeafData | Buffer, index: bigint): Promise { + public async updateLeaf(treeId: IndexedTreeId | PublicTreeId, leaf: Buffer, index: bigint): Promise { return await this.synchronize(() => this._updateLeaf(treeId, leaf, index)); } @@ -498,11 +498,7 @@ export class MerkleTrees implements MerkleTreeDb { return await tree.appendLeaves(leaves); } - private async _updateLeaf( - treeId: IndexedTreeId | PublicTreeId, - leaf: LeafData | Buffer, - index: bigint, - ): Promise { + private async _updateLeaf(treeId: IndexedTreeId | PublicTreeId, leaf: Buffer, index: bigint): Promise { const tree = this.trees[treeId]; if (!('updateLeaf' in tree)) { throw new Error('Tree does not support `updateLeaf` method'); From 47a3aee14df7b7a06e402e49735c2ca34d0df460 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 13:34:35 +0000 Subject: [PATCH 05/18] fix: getLatestLeafPreimage can return undefined --- .../src/standard_indexed_tree/standard_indexed_tree.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index 32c8944bcad..5753a258a47 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -157,7 +157,7 @@ export class StandardIndexedTree Date: Fri, 1 Dec 2023 15:39:29 +0000 Subject: [PATCH 06/18] feat: remove local leaf cache in indexed tree --- .../aztec-node/src/aztec-node/server.ts | 2 +- .../src/structs/rollup/base_rollup.ts | 8 +- .../src/interfaces/indexed_tree.ts | 11 +- yarn-project/merkle-tree/src/load_tree.ts | 1 - .../standard_indexed_tree.ts | 192 +++++++++--------- .../test/standard_indexed_tree_with_append.ts | 10 +- yarn-project/merkle-tree/src/tree_base.ts | 7 - .../noir-protocol-circuits/src/index.ts | 1 + .../src/type_conversion.ts | 5 +- .../merkle_tree_operations_facade.ts | 4 +- .../src/world-state-db/merkle_tree_db.ts | 4 +- .../src/world-state-db/merkle_trees.ts | 8 +- 12 files changed, 124 insertions(+), 129 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index a197bbfccd1..53aa86cc19b 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -405,7 +405,7 @@ export class AztecNodeService implements AztecNode { const leafDataPromise = committedDb.getLeafPreimage( MerkleTreeId.NULLIFIER_TREE, - Number(index), + index, ); const siblingPathPromise = committedDb.getSiblingPath( MerkleTreeId.NULLIFIER_TREE, diff --git a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts index 894f3ec0d93..bbc2b03b4e4 100644 --- a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts +++ b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts @@ -81,10 +81,10 @@ export class NullifierLeafPreimage implements IndexedTreeLeafPreimage; /** * Gets the latest LeafPreimage copy. @@ -85,7 +85,10 @@ export interface IndexedTree | undefined; + getLatestLeafPreimageCopy( + index: bigint, + includeUncommitted: boolean, + ): Promise | undefined>; /** * Batch insert multiple leaves into the tree. diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts index 7bbb85634aa..bd852b930fd 100644 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -22,6 +22,5 @@ export async function loadTree( const { root, depth, size } = decodeMeta(meta); const tree = c(db, hasher, name, depth, size, root); - await tree.initFromDb(); return tree; } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index 5753a258a47..a1dcce18409 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -34,21 +34,19 @@ interface PreimageFactory { - return `${name}:leaf:${toBufferBE(key, 32).toString('hex')}`; +const leafIndexToDbKey = (name: string, index: bigint) => { + return `${name}:leaf_by_index:${toBufferBE(index, 32).toString('hex')}`; }; -const dbKeyToLeafKey = (key: string): bigint => { - const index = key.split(':')[2]; - return toBigIntBE(Buffer.from(index, 'hex')); +const leafKeyToDbKey = (name: string, key: bigint) => { + return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}`; }; export class StandardIndexedTree> extends TreeBase implements IndexedTree { - protected leafPreimages: Preimage[] = []; - protected cachedLeafPreimages: { [key: number]: Preimage } = {}; + protected cachedLeafPreimages: { [key: string]: Preimage } = {}; public constructor( db: LevelUp, @@ -98,12 +96,9 @@ export class StandardIndexedTree { - const leaf = this.getLatestLeafPreimageCopy(Number(index), includeUncommitted); - if (!leaf) { - return Promise.resolve(undefined); - } - return Promise.resolve(leaf.toBuffer()); + public async getLeafValue(index: bigint, includeUncommitted: boolean): Promise { + const leaf = await this.getLatestLeafPreimageCopy(index, includeUncommitted); + return leaf && leaf.toBuffer(); } /** @@ -112,39 +107,87 @@ export class StandardIndexedTree newValue) { - diff.push(newValue); - } else if (storedPreimage.getKey() === newValue) { - return { index: i, alreadyPresent: true }; - } else { - diff.push(newValue - storedPreimage.getKey()); + }> { + let lowLeafIndex = await this.getDbLowLeafIndex(newKey); + let lowLeafPreimage = lowLeafIndex !== undefined ? await this.getDbPreimage(lowLeafIndex) : undefined; + + if (includeUncommitted) { + const cachedLowLeafIndex = this.getCachedLowLeafIndex(newKey); + const cachedLowLeafPreimage = + cachedLowLeafIndex !== undefined ? this.getCachedPreimage(cachedLowLeafIndex) : undefined; + if (cachedLowLeafIndex && (!lowLeafIndex || cachedLowLeafPreimage!.getKey() > lowLeafPreimage!.getKey())) { + lowLeafIndex = cachedLowLeafIndex; + lowLeafPreimage = cachedLowLeafPreimage; } } - const minIndex = this.findMinKey(diff); - return { index: minIndex, alreadyPresent: false }; + + if (lowLeafIndex === undefined || !lowLeafPreimage) { + throw new Error('Low leaf not found'); + } + + return { + index: lowLeafIndex, + alreadyPresent: lowLeafPreimage.getKey() === newKey, + }; + } + + private getCachedLowLeafIndex(key: bigint): bigint | undefined { + const indexes = Object.getOwnPropertyNames(this.cachedLeafPreimages); + const lowLeafIndexes = indexes + .map(index => ({ + index: BigInt(index), + key: this.cachedLeafPreimages[index].getKey(), + })) + .filter(({ key: candidateKey }) => candidateKey <= key) + .sort((a, b) => Number(b.key - a.key)); + return lowLeafIndexes[0]?.index; + } + + private async getDbLowLeafIndex(key: bigint): Promise { + return await new Promise((resolve, reject) => { + let lowLeafIndex: bigint | undefined; + this.db + .createReadStream({ + lte: leafKeyToDbKey(this.getName(), key), + limit: 1, + reverse: true, + }) + .on('data', data => { + lowLeafIndex = toBigIntBE(data.value); + }) + .on('close', function () {}) + .on('end', function () { + resolve(lowLeafIndex); + }) + .on('error', function () { + log.error('stream error'); + reject(); + }); + }); + } + + private async getDbPreimage(index: bigint): Promise { + const dbPreimage = await this.db + .get(leafIndexToDbKey(this.getName(), index)) + .then(data => this.leafPreimageFactory.fromBuffer(data)) + .catch(() => undefined); + return dbPreimage; + } + + private getCachedPreimage(index: bigint): Preimage | undefined { + return this.cachedLeafPreimages[index.toString()]; } /** @@ -153,31 +196,13 @@ export class StandardIndexedTree { const preimage = !includeUncommitted - ? this.leafPreimages[index] - : this.cachedLeafPreimages[index] ?? this.leafPreimages[index]; + ? await this.getDbPreimage(index) + : this.getCachedPreimage(index) ?? (await this.getDbPreimage(index)); return preimage && this.leafPreimageFactory.clone(preimage); } - /** - * Finds the index of the minimum value in an array. - * @param values - The collection of values to be searched. - * @returns The index of the minimum value in the array. - */ - private findMinKey(values: bigint[]) { - if (!values.length) { - return 0; - } - let minIndex = 0; - for (let i = 1; i < values.length; i++) { - if (values[minIndex] > values[i]) { - minIndex = i; - } - } - return minIndex; - } - /** * Initializes the tree. * @param prefilledSize - A number of leaves that are prefilled with values. @@ -211,35 +236,6 @@ export class StandardIndexedTree { - const startingIndex = 0n; - const preimages: Preimage[] = []; - const promise = new Promise((resolve, reject) => { - this.db - .createReadStream({ - gte: leafKeyToDbKey(this.getName(), startingIndex), - lte: leafKeyToDbKey(this.getName(), 2n ** BigInt(this.getDepth())), - }) - .on('data', data => { - const leafKey = dbKeyToLeafKey(data.key.toString('utf-8')); - preimages[Number(leafKey)] = this.leafPreimageFactory.fromBuffer(data.value); - }) - .on('close', function () {}) - .on('end', function () { - resolve(); - }) - .on('error', function () { - log.error('stream error'); - reject(); - }); - }); - await promise; - this.leafPreimages = preimages; - } - /** * Commits all the leaves to the database and removes them from a cache. */ @@ -247,9 +243,10 @@ export class StandardIndexedTree( + getEmptyLowLeafWitness( treeHeight: N, preimageFactory: Empty, ): LowLeafWitnessData { @@ -412,7 +410,7 @@ export class StandardIndexedTree> { - const emptyLowLeafWitness = this.getEmptyLowLeafWitnes(this.getDepth() as TreeHeight, this.leafPreimageFactory); + const emptyLowLeafWitness = this.getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); // Accumulators const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); const pendingInsertionSubtree: Preimage[] = leaves.map(() => this.leafPreimageFactory.empty()); @@ -435,10 +433,10 @@ export class StandardIndexedTree( @@ -513,10 +511,10 @@ export class StandardIndexedTree { - const startInsertionIndex = Number(this.getNumLeaves(true)); + const startInsertionIndex = this.getNumLeaves(true); const hashedLeaves = preimages.map((preimage, i) => { - this.cachedLeafPreimages[startInsertionIndex + i] = preimage; + this.cachedLeafPreimages[(startInsertionIndex + BigInt(i)).toString()] = preimage; return this.encodeLeaf(preimage, hash0Leaf); }); diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts index bd0e5d549b3..047aea52e19 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts @@ -41,8 +41,8 @@ export class StandardIndexedTreeWithAppend< return; } - const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); - const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true); + const lowLeafIndex = await this.findIndexOfPreviousKey(newLeaf.getKey(), true); + const lowLeafPreimage = await this.getLatestLeafPreimageCopy(lowLeafIndex.index, true); if (lowLeafPreimage === undefined) { throw new Error(`Previous leaf not found!`); @@ -53,7 +53,7 @@ export class StandardIndexedTreeWithAppend< lowLeafPreimage.getNextIndex(), ); - if (indexOfPrevious.alreadyPresent) { + if (lowLeafIndex.alreadyPresent) { return; } // insert a new leaf at the highest index and update the values of our previous leaf copy @@ -64,8 +64,8 @@ export class StandardIndexedTreeWithAppend< BigInt(currentSize), ); this.cachedLeafPreimages[Number(currentSize)] = newLeafPreimage; - this.cachedLeafPreimages[Number(indexOfPrevious.index)] = newLowLeafPreimage; - await this.updateLeaf(newLowLeafPreimage, BigInt(indexOfPrevious.index)); + this.cachedLeafPreimages[Number(lowLeafIndex.index)] = newLowLeafPreimage; + await this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); await this.updateLeaf(newLeafPreimage, this.getNumLeaves(true)); } } diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index 6b715280380..f7e3fae9639 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -221,13 +221,6 @@ export abstract class TreeBase implements MerkleTree { await this.writeMeta(); } - /** - * Initializes the tree from the database. - */ - public async initFromDb(): Promise { - // Implemented only by Indexed Tree to populate the leaf cache. - } - /** * Writes meta data to the provided batch. * @param batch - The batch to which to write the meta data. diff --git a/yarn-project/noir-protocol-circuits/src/index.ts b/yarn-project/noir-protocol-circuits/src/index.ts index 97671183a7c..dc5baffd5fd 100644 --- a/yarn-project/noir-protocol-circuits/src/index.ts +++ b/yarn-project/noir-protocol-circuits/src/index.ts @@ -416,6 +416,7 @@ async function executeMergeRollupWithACVM(input: MergeRollupInputType): Promise< * Executes the base rollup with the given inputs using the acvm. */ async function executeBaseRollupWithACVM(input: BaseRollupInputType): Promise { + // console.log(JSON.stringify(input, null, 4)); const initialWitnessMap = abiEncode(BaseRollupJson.abi as Abi, input as any); // Execute the circuit on those initial witness values diff --git a/yarn-project/noir-protocol-circuits/src/type_conversion.ts b/yarn-project/noir-protocol-circuits/src/type_conversion.ts index 3dbcdaecb9d..afb3acee731 100644 --- a/yarn-project/noir-protocol-circuits/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits/src/type_conversion.ts @@ -160,6 +160,9 @@ export function mapNumberFromNoir(number: NoirField): number { * */ export function mapNumberToNoir(number: number): NoirField { + if (number > 2 ** 32) { + throw new Error('Number out of range'); + } return new Fr(BigInt(number)).toString(); } @@ -1362,7 +1365,7 @@ export function mapNullifierLeafPreimageToNoir( return { leaf_value: mapFieldToNoir(nullifierLeafPreimage.nullifier), next_value: mapFieldToNoir(nullifierLeafPreimage.nextNullifier), - next_index: mapFieldToNoir(new Fr(nullifierLeafPreimage.nextIndex)), + next_index: mapNumberToNoir(Number(nullifierLeafPreimage.nextIndex)), }; } diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index d5ddc2025bf..52b14fb637b 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -65,7 +65,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { /** * The index of the found leaf. */ - index: number; + index: bigint; /** * A flag indicating if the corresponding leaf's value is equal to `newValue`. */ @@ -93,7 +93,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { */ async getLeafPreimage>( treeId: MerkleTreeId.NULLIFIER_TREE, - index: number, + index: bigint, ): Promise { const preimage = await this.trees.getLeafPreimage(treeId, index, this.includeUncommitted); return preimage as Preimage | undefined; diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 7c3aeba073e..f94808728e7 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -135,7 +135,7 @@ export interface MerkleTreeOperations { /** * The index of the found leaf. */ - index: number; + index: bigint; /** * A flag indicating if the corresponding leaf's value is equal to `newValue`. */ @@ -149,7 +149,7 @@ export interface MerkleTreeOperations { */ getLeafPreimage>( treeId: IndexedTreeId, - index: number, + index: bigint, ): Promise; /** diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index b5361513d03..91d7640210a 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -320,15 +320,13 @@ export class MerkleTrees implements MerkleTreeDb { /** * The index of the found leaf. */ - index: number; + index: bigint; /** * A flag indicating if the corresponding leaf's value is equal to `newValue`. */ alreadyPresent: boolean; }> { - return await this.synchronize(() => - Promise.resolve(this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)), - ); + return await this.synchronize(() => this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)); } /** @@ -340,7 +338,7 @@ export class MerkleTrees implements MerkleTreeDb { */ public async getLeafPreimage>( treeId: IndexedTreeId, - index: number, + index: bigint, includeUncommitted: boolean, ): Promise | undefined> { return await this.synchronize(() => From d458756c1f65b8c42429656e25b98799cddb01d7 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Fri, 1 Dec 2023 15:51:14 +0000 Subject: [PATCH 07/18] feat: optimize findLeafIndex for indexed tree --- .../merkle-tree/src/interfaces/merkle_tree.ts | 8 +++++ .../standard_indexed_tree.ts | 31 +++++++++++++++++++ yarn-project/merkle-tree/src/tree_base.ts | 17 ++++++++++ .../src/world-state-db/merkle_trees.ts | 8 +---- 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index e4f65b326a2..ba3ffb4309b 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -49,4 +49,12 @@ export interface MerkleTree extends SiblingPathSource { * @param includeUncommitted - Set to true to include uncommitted updates in the data set. */ getLeafValue(index: bigint, includeUncommitted: boolean): Promise; + + /** + * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. + * @param leaf - The leaf value to look for. + * @param includeUncommitted - Indicates whether to include uncommitted data. + * @returns The index of the first leaf found with a given value (undefined if not found). + */ + findLeafIndex(leaf: Buffer, includeUncommitted: boolean): Promise; } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index a1dcce18409..9e262aa63d7 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -155,6 +155,16 @@ export class StandardIndexedTree { + return this.cachedLeafPreimages[index].getKey() === key; + }); + if (index) { + return BigInt(index); + } + return undefined; + } + private async getDbLowLeafIndex(key: bigint): Promise { return await new Promise((resolve, reject) => { let lowLeafIndex: bigint | undefined; @@ -203,6 +213,27 @@ export class StandardIndexedTree { + let leaf = this.leafFactory.fromBuffer(value); + let index = await this.db + .get(leafKeyToDbKey(this.getName(), leaf.getKey())) + .then(data => toBigIntBE(data)) + .catch(() => undefined); + + if (includeUncommitted && index === undefined) { + const cachedIndex = this.getCachedLeafIndex(leaf.getKey()); + index = cachedIndex; + } + return index; + } + /** * Initializes the tree. * @param prefilledSize - A number of leaves that are prefilled with values. diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index f7e3fae9639..ff2b8baf3f6 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -280,4 +280,21 @@ export abstract class TreeBase implements MerkleTree { } this.cachedSize = numLeaves + BigInt(leaves.length); } + + /** + * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. + * @param treeId - The ID of the tree. + * @param value - The leaf value to look for. + * @param includeUncommitted - Indicates whether to include uncommitted data. + * @returns The index of the first leaf found with a given value (undefined if not found). + */ + public async findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + for (let i = 0n; i < this.getNumLeaves(includeUncommitted); i++) { + const currentValue = await this.getLeafValue(i, includeUncommitted); + if (currentValue && currentValue.equals(value)) { + return i; + } + } + return undefined; + } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 91d7640210a..1e6a98e43a7 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -362,13 +362,7 @@ export class MerkleTrees implements MerkleTreeDb { ): Promise { return await this.synchronize(async () => { const tree = this.trees[treeId]; - for (let i = 0n; i < tree.getNumLeaves(includeUncommitted); i++) { - const currentValue = await tree.getLeafValue(i, includeUncommitted); - if (currentValue && currentValue.equals(value)) { - return i; - } - } - return undefined; + return await tree.findLeafIndex(value, includeUncommitted); }); } From ae6b6a0bb2c47457b80f58c5bfe31967210ad725 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 09:26:43 +0000 Subject: [PATCH 08/18] docs: added jsdoc --- yarn-project/foundation/src/trees/index.ts | 35 +++++++- .../src/interfaces/indexed_tree.ts | 2 - .../standard_indexed_tree.ts | 88 +++++++++++-------- 3 files changed, 85 insertions(+), 40 deletions(-) diff --git a/yarn-project/foundation/src/trees/index.ts b/yarn-project/foundation/src/trees/index.ts index 139b6383620..3c60bec3efe 100644 --- a/yarn-project/foundation/src/trees/index.ts +++ b/yarn-project/foundation/src/trees/index.ts @@ -1,17 +1,48 @@ -/* eslint-disable */ - +/** + * A leaf of an indexed merkle tree. + */ export interface IndexedTreeLeaf { + /** + * Returns key of the leaf. It's used for indexing. + */ getKey(): bigint; + /** + * Serializes the leaf into a buffer. + */ toBuffer(): Buffer; + /** + * Returns true if the leaf is empty. + */ isEmpty(): boolean; } +/** + * Preimage of an indexed merkle tree leaf. + */ export interface IndexedTreeLeafPreimage { + /** + * Returns key of the leaf corresponding to this preimage. + */ getKey(): bigint; + /** + * Returns the key of the next leaf. + */ getNextKey(): bigint; + /** + * Returns the index of the next leaf. + */ getNextIndex(): bigint; + /** + * Returns the preimage as a leaf. + */ asLeaf(): Leaf; + /** + * Serializes the preimage into a buffer. + */ toBuffer(): Buffer; + /** + * Serializes the preimage to an array of buffers for hashing. + */ toHashInputs(): Buffer[]; } diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 617aea4989d..ecf6f68d14e 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -3,8 +3,6 @@ import { SiblingPath } from '@aztec/types'; import { AppendOnlyTree } from './append_only_tree.js'; -/* eslint-disable */ - /** * All of the data to be return during batch insertion. */ diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index 9e262aa63d7..daa526ee368 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -10,28 +10,47 @@ import { TreeBase } from '../tree_base.js'; const log = createDebugLogger('aztec:standard-indexed-tree'); -/* eslint-disable */ - -// TODO - -interface Empty { - empty(): T; -} - -interface DummyBuilder { - buildDummy(key: bigint): T; -} - -interface FromBuffer { - fromBuffer(buffer: Buffer): T; -} - -interface Clone { - clone(t: T): T; +/** + * Factory for creating leaf preimages. + */ +export interface PreimageFactory> { + /** + * Creates a new preimage from a leaf. + * @param leaf - Leaf to create a preimage from. + * @param nextKey - Next key of the leaf. + * @param nextIndex - Next index of the leaf. + */ + fromLeaf(leaf: Leaf, nextKey: bigint, nextIndex: bigint): Preimage; + /** + * Creates a new preimage from a buffer. + * @param buffer - Buffer to create a preimage from. + */ + fromBuffer(buffer: Buffer): Preimage; + /** + * Creates an empty preimage. + */ + empty(): Preimage; + /** + * Creates a copy of a preimage. + * @param preimage - Preimage to be cloned. + */ + clone(preimage: Preimage): Preimage; } -interface PreimageFactory> { - fromLeaf(leaf: Leaf, nextKey: bigint, nextIndex: bigint): Preimage; +/** + * Factory for creating leaves. + */ +export interface LeafFactory { + /** + * Creates a new leaf from a buffer. + * @param key - Key of the leaf. + */ + buildDummy(key: bigint): Leaf; + /** + * Creates a new leaf from a buffer. + * @param buffer - Buffer to create a leaf from. + */ + fromBuffer(buffer: Buffer): Leaf; } const leafIndexToDbKey = (name: string, index: bigint) => { @@ -42,6 +61,9 @@ const leafKeyToDbKey = (name: string, key: bigint) => { return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}`; }; +/** + * Standard implementation of an indexed tree. + */ export class StandardIndexedTree> extends TreeBase implements IndexedTree @@ -54,11 +76,8 @@ export class StandardIndexedTree & - Empty & - PreimageFactory & - Clone, - protected leafFactory: FromBuffer & DummyBuilder, + protected leafPreimageFactory: PreimageFactory, + protected leafFactory: LeafFactory, root?: Buffer, ) { super(db, hasher, name, depth, size, root); @@ -66,9 +85,9 @@ export class StandardIndexedTree { + appendLeaves(_leaves: Buffer[]): Promise { throw new Error('Not implemented'); } @@ -103,7 +122,7 @@ export class StandardIndexedTree { + const index = Object.keys(this.cachedLeafPreimages).find(index => { return this.cachedLeafPreimages[index].getKey() === key; }); if (index) { @@ -221,7 +240,7 @@ export class StandardIndexedTree { - let leaf = this.leafFactory.fromBuffer(value); + const leaf = this.leafFactory.fromBuffer(value); let index = await this.db .get(leafKeyToDbKey(this.getName(), leaf.getKey())) .then(data => toBigIntBE(data)) @@ -314,12 +333,9 @@ export class StandardIndexedTree( - treeHeight: N, - preimageFactory: Empty, - ): LowLeafWitnessData { + getEmptyLowLeafWitness(treeHeight: N): LowLeafWitnessData { return { - leafData: preimageFactory.empty(), + leafData: this.leafPreimageFactory.empty(), index: 0n, siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), }; @@ -441,7 +457,7 @@ export class StandardIndexedTree> { - const emptyLowLeafWitness = this.getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); + const emptyLowLeafWitness = this.getEmptyLowLeafWitness(this.getDepth() as TreeHeight); // Accumulators const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); const pendingInsertionSubtree: Preimage[] = leaves.map(() => this.leafPreimageFactory.empty()); From 47bb7f010425efd2dc47d4a84ae256b6c3edc89d Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 09:37:02 +0000 Subject: [PATCH 09/18] refactor: low leaf getter returns option --- .../aztec-node/src/aztec-node/server.ts | 19 +++++------ .../src/interfaces/indexed_tree.ts | 23 +++++++------ .../standard_indexed_tree.ts | 33 ++++++++++--------- .../test/standard_indexed_tree_with_append.ts | 6 ++-- .../src/block_builder/solo_block_builder.ts | 5 +-- .../merkle_tree_operations_facade.ts | 23 +++++++------ .../src/world-state-db/merkle_tree_db.ts | 23 +++++++------ .../src/world-state-db/merkle_trees.ts | 23 +++++++------ 8 files changed, 85 insertions(+), 70 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 53aa86cc19b..d08034b553a 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -440,25 +440,24 @@ export class AztecNodeService implements AztecNode { nullifier: Fr, ): Promise { const committedDb = await this.#getWorldState(); - const { index, alreadyPresent } = await committedDb.getPreviousValueIndex( - MerkleTreeId.NULLIFIER_TREE, - nullifier.toBigInt(), - ); + const findResult = await committedDb.getPreviousValueIndex(MerkleTreeId.NULLIFIER_TREE, nullifier.toBigInt()); + if (!findResult) { + return undefined; + } + const { index, alreadyPresent } = findResult; if (alreadyPresent) { this.log.warn(`Nullifier ${nullifier.toBigInt()} already exists in the tree`); } - const leafData = await committedDb.getLeafPreimage( + const preimageData = (await committedDb.getLeafPreimage( MerkleTreeId.NULLIFIER_TREE, index, - ); - if (!leafData) { - return undefined; - } + ))!; + const siblingPath = await committedDb.getSiblingPath( MerkleTreeId.NULLIFIER_TREE, BigInt(index), ); - return new NullifierMembershipWitness(BigInt(index), leafData, siblingPath); + return new NullifierMembershipWitness(BigInt(index), preimageData, siblingPath); } /** diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index ecf6f68d14e..5dfd4d3c0f1 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -66,16 +66,19 @@ export interface IndexedTree; + ): Promise< + | { + /** + * The index of the found leaf. + */ + index: bigint; + /** + * A flag indicating if the corresponding leaf's value is equal to `newValue`. + */ + alreadyPresent: boolean; + } + | undefined + >; /** * Gets the latest LeafPreimage copy. diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index daa526ee368..6aa4a508da5 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -129,16 +129,19 @@ export class StandardIndexedTree { + ): Promise< + | { + /** + * The index of the found leaf. + */ + index: bigint; + /** + * A flag indicating if the corresponding leaf's value is equal to `newValue`. + */ + alreadyPresent: boolean; + } + | undefined + > { let lowLeafIndex = await this.getDbLowLeafIndex(newKey); let lowLeafPreimage = lowLeafIndex !== undefined ? await this.getDbPreimage(lowLeafIndex) : undefined; @@ -153,7 +156,7 @@ export class StandardIndexedTree leafTuple.leaf.toBuffer()), @@ -492,6 +492,9 @@ export class StandardIndexedTree(BigInt(indexOfPrevious.index), true); const witness: LowLeafWitnessData = { diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts index 047aea52e19..83f17c1aace 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts @@ -42,11 +42,11 @@ export class StandardIndexedTreeWithAppend< } const lowLeafIndex = await this.findIndexOfPreviousKey(newLeaf.getKey(), true); - const lowLeafPreimage = await this.getLatestLeafPreimageCopy(lowLeafIndex.index, true); - - if (lowLeafPreimage === undefined) { + if (lowLeafIndex === undefined) { throw new Error(`Previous leaf not found!`); } + const lowLeafPreimage = (await this.getLatestLeafPreimageCopy(lowLeafIndex.index, true))!; + const newLeafPreimage = this.leafPreimageFactory.fromLeaf( newLeaf, lowLeafPreimage.getNextKey(), diff --git a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts index e22fe389a7c..2325a49eb9b 100644 --- a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts @@ -576,10 +576,11 @@ export class SoloBlockBuilder implements BlockBuilder { const tree = MerkleTreeId.NULLIFIER_TREE; const prevValueIndex = await this.db.getPreviousValueIndex(tree, frToBigInt(nullifier)); - const prevValuePreimage = await this.db.getLeafPreimage(tree, prevValueIndex.index); - if (!prevValuePreimage) { + if (!prevValueIndex) { throw new Error(`Nullifier tree should have one initial leaf`); } + const prevValuePreimage = (await this.db.getLeafPreimage(tree, prevValueIndex.index))!; + const prevValueSiblingPath = await this.db.getSiblingPath(tree, BigInt(prevValueIndex.index)); return { diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 52b14fb637b..f00420898ab 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -61,16 +61,19 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { getPreviousValueIndex( treeId: MerkleTreeId.NULLIFIER_TREE, value: bigint, - ): Promise<{ - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - }> { + ): Promise< + | { + /** + * The index of the found leaf. + */ + index: bigint; + /** + * A flag indicating if the corresponding leaf's value is equal to `newValue`. + */ + alreadyPresent: boolean; + } + | undefined + > { return this.trees.getPreviousValueIndex(treeId, value, this.includeUncommitted); } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index f94808728e7..ec02c7cc22e 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -131,16 +131,19 @@ export interface MerkleTreeOperations { getPreviousValueIndex( treeId: IndexedTreeId, value: bigint, - ): Promise<{ - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - }>; + ): Promise< + | { + /** + * The index of the found leaf. + */ + index: bigint; + /** + * A flag indicating if the corresponding leaf's value is equal to `newValue`. + */ + alreadyPresent: boolean; + } + | undefined + >; /** * Returns the data at a specific leaf. diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 1e6a98e43a7..5e1ca83fd6f 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -316,16 +316,19 @@ export class MerkleTrees implements MerkleTreeDb { treeId: IndexedTreeId, value: bigint, includeUncommitted: boolean, - ): Promise<{ - /** - * The index of the found leaf. - */ - index: bigint; - /** - * A flag indicating if the corresponding leaf's value is equal to `newValue`. - */ - alreadyPresent: boolean; - }> { + ): Promise< + | { + /** + * The index of the found leaf. + */ + index: bigint; + /** + * A flag indicating if the corresponding leaf's value is equal to `newValue`. + */ + alreadyPresent: boolean; + } + | undefined + > { return await this.synchronize(() => this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)); } From 7250939625f40b7c960f9045207effd9fd69deb9 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 10:49:16 +0000 Subject: [PATCH 10/18] fix: make sure low leaf in range --- .../src/standard_indexed_tree/standard_indexed_tree.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index 9d51e6f416a..d165d3b26a3 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -196,6 +196,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { let lowLeafIndex: bigint | undefined; this.db .createReadStream({ + gte: buildDbKeyForLeafIndex(this.getName(), 0n), lte: buildDbKeyForLeafIndex(this.getName(), key), limit: 1, reverse: true, From 240e8b512407dcfaedbb9f6455562dec6ba1f0fc Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 11:55:12 +0000 Subject: [PATCH 11/18] fix: improve performance of getLeafIndex --- .../merkle-tree/src/snapshots/append_only_snapshot.ts | 11 +++++++++++ .../merkle-tree/src/snapshots/base_full_snapshot.ts | 11 +++++++++++ .../src/snapshots/indexed_tree_snapshot.test.ts | 2 +- .../src/snapshots/indexed_tree_snapshot.ts | 9 ++++++++- .../merkle-tree/src/snapshots/snapshot_builder.ts | 10 +++++++++- .../merkle_tree_snapshot_operations_facade.ts | 9 +-------- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts index b530e981b27..46361bd5913 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts @@ -229,4 +229,15 @@ class AppendOnlySnapshot implements TreeSnapshot { return undefined; } } + + async findLeafIndex(value: Buffer): Promise { + const numLeaves = this.getNumLeaves(); + for (let i = 0n; i < numLeaves; i++) { + const currentValue = await this.getLeafValue(i); + if (currentValue && currentValue.equals(value)) { + return i; + } + } + return undefined; + } } diff --git a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts index 438e77261ff..b1157cf9d44 100644 --- a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts @@ -218,4 +218,15 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { path.reverse(); return path; } + + async findLeafIndex(value: Buffer): Promise { + const numLeaves = this.getNumLeaves(); + for (let i = 0n; i < numLeaves; i++) { + const currentValue = await this.getLeafValue(i); + if (currentValue && currentValue.equals(value)) { + return i; + } + } + return undefined; + } } diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts index 3c17d8969ea..748ea507327 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts @@ -101,7 +101,7 @@ describe('IndexedTreeSnapshotBuilder', () => { await tree.appendLeaves([Buffer.from('c'), Buffer.from('b'), Buffer.from('e')]); await tree.commit(); - await expect(snapshot.findIndexOfPreviousValue(2n)).resolves.toEqual(historicalPrevValue); + await expect(snapshot.findIndexOfPreviousKey(2n)).resolves.toEqual(historicalPrevValue); }); }); }); diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts index 86e823cbaf3..006605df283 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts @@ -59,7 +59,7 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre } } - async findIndexOfPreviousValue(newValue: bigint): Promise<{ + async findIndexOfPreviousKey(newValue: bigint): Promise<{ /** * The index of the found leaf. */ @@ -98,4 +98,11 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre return { index: BigInt(minIndex), alreadyPresent: false }; } + + async findLeafIndex(value: Buffer): Promise { + const index = await this.tree.findLeafIndex(value, false); + if (index && index < this.getNumLeaves()) { + return index; + } + } } diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts index 730d83b261e..b1fd74f9bdc 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts @@ -48,6 +48,14 @@ export interface TreeSnapshot { * @param index - The index of the leaf for which a sibling path is required. */ getSiblingPath(index: bigint): Promise>; + + /** + * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. + * @param treeId - The ID of the tree. + * @param value - The leaf value to look for. + * @returns The index of the first leaf found with a given value (undefined if not found). + */ + findLeafIndex(value: Buffer): Promise; } /** A snapshot of an indexed tree */ @@ -63,7 +71,7 @@ export interface IndexedTreeSnapshot extends TreeSnapshot { * @param newValue - The new value to be inserted into the tree. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - findIndexOfPreviousValue(newValue: bigint): Promise<{ + findIndexOfPreviousKey(newValue: bigint): Promise<{ /** * The index of the found leaf. */ diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_snapshot_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_snapshot_operations_facade.ts index 83420c687a2..ef112c30e35 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_snapshot_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_snapshot_operations_facade.ts @@ -29,14 +29,7 @@ export class MerkleTreeSnapshotOperationsFacade implements MerkleTreeOperations async findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise { const tree = await this.#getTreeSnapshot(treeId); - const numLeaves = tree.getNumLeaves(); - for (let i = 0n; i < numLeaves; i++) { - const currentValue = await tree.getLeafValue(i); - if (currentValue && currentValue.equals(value)) { - return i; - } - } - return undefined; + return tree.findLeafIndex(value); } getLatestGlobalVariablesHash(): Promise { From bcc8ae0e5eddaa7bc38d0560bd20513129ee9dd8 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 12:37:49 +0000 Subject: [PATCH 12/18] chore: cleanups --- .../aztec-node/src/aztec-node/server.ts | 8 ++-- .../src/interfaces/indexed_tree.ts | 2 +- .../src/snapshots/indexed_tree_snapshot.ts | 10 ++--- .../snapshots/snapshot_builder_test_suite.ts | 20 +++++++++ .../standard_indexed_tree.ts | 11 ++--- .../merkle-tree/src/test/test_suite.ts | 14 ++++++ yarn-project/merkle-tree/src/tree_base.ts | 1 - yarn-project/noir-contracts/scripts/types.sh | 45 +++++++++---------- .../noir-protocol-circuits/src/index.ts | 1 - .../src/type_conversion.ts | 3 -- .../src/block_builder/solo_block_builder.ts | 4 +- .../types/src/interfaces/nullifier_tree.ts | 8 ++-- .../merkle_tree_operations_facade.ts | 2 +- .../src/world-state-db/merkle_trees.ts | 2 +- 14 files changed, 79 insertions(+), 52 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 46c218cbcf9..bc415839d7b 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -430,19 +430,19 @@ export class AztecNodeService implements AztecNode { return undefined; } - const leafDataPromise = db.getLeafPreimage(MerkleTreeId.NULLIFIER_TREE, index); + const leafPreimagePromise = db.getLeafPreimage(MerkleTreeId.NULLIFIER_TREE, index); const siblingPathPromise = db.getSiblingPath( MerkleTreeId.NULLIFIER_TREE, BigInt(index), ); - const [leafData, siblingPath] = await Promise.all([leafDataPromise, siblingPathPromise]); + const [leafPreimage, siblingPath] = await Promise.all([leafPreimagePromise, siblingPathPromise]); - if (!leafData) { + if (!leafPreimage) { return undefined; } - return new NullifierMembershipWitness(BigInt(index), leafData as NullifierLeafPreimage, siblingPath); + return new NullifierMembershipWitness(BigInt(index), leafPreimage as NullifierLeafPreimage, siblingPath); } /** diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 4dac599fd51..eee22a3ee2a 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -10,7 +10,7 @@ export interface LowLeafWitnessData { /** * Preimage of the low nullifier that proves non membership. */ - leafData: IndexedTreeLeafPreimage; + leafPreimage: IndexedTreeLeafPreimage; /** * Sibling path to prove membership of low nullifier. */ diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts index 006605df283..98e5373537c 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts @@ -25,9 +25,9 @@ export class IndexedTreeSnapshotBuilder } protected async handleLeaf(index: bigint, node: Buffer, batch: LevelUpChain) { - const leafData = await this.tree.getLatestLeafPreimageCopy(index, false); - if (leafData) { - batch.put(snapshotLeafValue(node, index), leafData.toBuffer()); + const leafPreimage = await this.tree.getLatestLeafPreimageCopy(index, false); + if (leafPreimage) { + batch.put(snapshotLeafValue(node, index), leafPreimage.toBuffer()); } } } @@ -45,8 +45,8 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre } async getLeafValue(index: bigint): Promise { - const leafData = await this.getLatestLeafPreimageCopy(index); - return leafData?.toBuffer(); + const leafPreimage = await this.getLatestLeafPreimageCopy(index); + return leafPreimage?.toBuffer(); } async getLatestLeafPreimageCopy(index: bigint): Promise { diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts index 227919645b1..f50ff1d69ae 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts @@ -194,5 +194,25 @@ export function describeSnapshotBuilderTestSuite { + it('returns the historical leaf index when the snapshot was taken', async () => { + await modifyTree(tree); + await tree.commit(); + const snapshot = await snapshotBuilder.snapshot(1); + + const initialLastLeafIndex = tree.getNumLeaves(false) - 1n; + let lastLeaf = await tree.getLeafValue(initialLastLeafIndex, false); + expect(await snapshot.findLeafIndex(lastLeaf!)).toBe(initialLastLeafIndex); + + await modifyTree(tree); + await tree.commit(); + + const newLastLeafIndex = tree.getNumLeaves(false) - 1n; + lastLeaf = await tree.getLeafValue(newLastLeafIndex, false); + + expect(await snapshot.findLeafIndex(lastLeaf!)).toBe(undefined); + }); + }); }); } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index d165d3b26a3..a935bcef75e 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -88,8 +88,10 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { } /** - * Appends a set of leaf values to the tree. - * @param _leaves - The set of leaves to be appended. + * Appends the given leaves to the tree. + * @param _leaves - The leaves to append. + * @returns Empty promise. + * @remarks Use batchInsert method instead. */ appendLeaves(_leaves: Buffer[]): Promise { throw new Error('Not implemented'); @@ -245,7 +247,6 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { /** * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param treeId - The ID of the tree. * @param value - The leaf value to look for. * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). @@ -346,7 +347,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { */ getEmptyLowLeafWitness(treeHeight: N): LowLeafWitnessData { return { - leafData: this.leafPreimageFactory.empty(), + leafPreimage: this.leafPreimageFactory.empty(), index: 0n, siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), }; @@ -506,7 +507,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); const witness: LowLeafWitnessData = { - leafData: lowLeafPreimage, + leafPreimage: lowLeafPreimage, index: BigInt(indexOfPrevious.index), siblingPath, }; diff --git a/yarn-project/merkle-tree/src/test/test_suite.ts b/yarn-project/merkle-tree/src/test/test_suite.ts index 1940f32ce91..51886a090b3 100644 --- a/yarn-project/merkle-tree/src/test/test_suite.ts +++ b/yarn-project/merkle-tree/src/test/test_suite.ts @@ -157,5 +157,19 @@ export const treeTestSuite = ( expect(deserialized2.elem).toEqual(siblingPath); expect(deserialized2.adv).toBe(4 + 10 * 32); }); + + it('should be able to find indexes of leaves', async () => { + const db = levelup(createMemDown()); + const tree = await createDb(db, pedersen, 'test', 10); + await appendLeaves(tree, values.slice(0, 1)); + + expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); + expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); + expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); + + await tree.commit(); + + expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); + }); }); }; diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index b3182abd5ba..f9f58068cbe 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -303,7 +303,6 @@ export abstract class TreeBase implements MerkleTree { /** * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. - * @param treeId - The ID of the tree. * @param value - The leaf value to look for. * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 2747a38ef9b..14fa63bfcd1 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -1,4 +1,3 @@ - #!/bin/bash # Example: @@ -8,7 +7,7 @@ # Enable strict mode: # Exit on error (set -e), treat unset variables as an error (set -u), -set -eu; +set -eu artifacts_dir="src/artifacts" types_dir="src/types" @@ -17,42 +16,40 @@ types_dir="src/types" mkdir -p $types_dir mkdir -p $artifacts_dir - ROOT=$(pwd) write_import() { - CONTRACT_NAME=$1 - NAME=$(echo $CONTRACT_NAME | perl -pe 's/(^|_)(\w)/\U$2/g') + CONTRACT_NAME=$1 + NAME=$(echo $CONTRACT_NAME | perl -pe 's/(^|_)(\w)/\U$2/g') - echo "import ${NAME}Json from './${CONTRACT_NAME}_contract.json' assert { type: 'json' };" >> "$artifacts_dir/index.ts"; + echo "import ${NAME}Json from './${CONTRACT_NAME}_contract.json' assert { type: 'json' };" >>"$artifacts_dir/index.ts" } write_export() { - CONTRACT_NAME=$1 - NAME=$(echo $CONTRACT_NAME | perl -pe 's/(^|_)(\w)/\U$2/g') + CONTRACT_NAME=$1 + NAME=$(echo $CONTRACT_NAME | perl -pe 's/(^|_)(\w)/\U$2/g') - # artifacts - echo "export const ${NAME}ContractArtifact = ${NAME}Json as ContractArtifact;" >> "$artifacts_dir/index.ts"; - echo "Written typescript for $NAME" + # artifacts + echo "export const ${NAME}ContractArtifact = ${NAME}Json as ContractArtifact;" >>"$artifacts_dir/index.ts" + echo "Written typescript for $NAME" - # types - echo "export * from './${CONTRACT_NAME}.js';" >> "$types_dir/index.ts"; + # types + echo "export * from './${CONTRACT_NAME}.js';" >>"$types_dir/index.ts" } - process() { CONTRACT=$1 cd $ROOT - NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_source.ts $CONTRACT_NAME + NODE_OPTIONS="--no-warnings --loader ts-node/esm" yarn ts-node --esm src/scripts/copy_source.ts $CONTRACT_NAME echo "Creating types for $CONTRACT" - NODE_OPTIONS=--no-warnings yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME + NODE_OPTIONS="--no-warnings --loader ts-node/esm" yarn ts-node --esm src/scripts/copy_output.ts $CONTRACT_NAME } -format(){ +format() { echo "Formatting contract folders" - yarn run -T prettier -w ../aztec.js/src/artifacts/*.json ./$types_dir/*.ts + yarn run -T prettier -w ../aztec.js/src/artifacts/*.json ./$types_dir/*.ts echo -e "Done\n" } @@ -69,15 +66,15 @@ wait rm -f $artifacts_dir/index.ts || true # Generate artifacts package index.ts -echo "// Auto generated module\n" > "$artifacts_dir/index.ts"; -echo "import { ContractArtifact } from '@aztec/foundation/abi';" >> "$artifacts_dir/index.ts"; +echo "// Auto generated module\n" >"$artifacts_dir/index.ts" +echo "import { ContractArtifact } from '@aztec/foundation/abi';" >>"$artifacts_dir/index.ts" # Generate types package index.ts -echo "// Auto generated module\n" > "$types_dir/index.ts"; +echo "// Auto generated module\n" >"$types_dir/index.ts" for CONTRACT_NAME in "$@"; do - write_import $CONTRACT_NAME - write_export $CONTRACT_NAME + write_import $CONTRACT_NAME + write_export $CONTRACT_NAME done # only run the rest when the full flag is set -format +format \ No newline at end of file diff --git a/yarn-project/noir-protocol-circuits/src/index.ts b/yarn-project/noir-protocol-circuits/src/index.ts index dc5baffd5fd..97671183a7c 100644 --- a/yarn-project/noir-protocol-circuits/src/index.ts +++ b/yarn-project/noir-protocol-circuits/src/index.ts @@ -416,7 +416,6 @@ async function executeMergeRollupWithACVM(input: MergeRollupInputType): Promise< * Executes the base rollup with the given inputs using the acvm. */ async function executeBaseRollupWithACVM(input: BaseRollupInputType): Promise { - // console.log(JSON.stringify(input, null, 4)); const initialWitnessMap = abiEncode(BaseRollupJson.abi as Abi, input as any); // Execute the circuit on those initial witness values diff --git a/yarn-project/noir-protocol-circuits/src/type_conversion.ts b/yarn-project/noir-protocol-circuits/src/type_conversion.ts index d0e49d71442..3ff307a6034 100644 --- a/yarn-project/noir-protocol-circuits/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits/src/type_conversion.ts @@ -163,9 +163,6 @@ export function mapNumberFromNoir(number: NoirField): number { * */ export function mapNumberToNoir(number: number): NoirField { - if (number > 2 ** 32) { - throw new Error('Number out of range'); - } return new Fr(BigInt(number)).toString(); } diff --git a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts index b1602f5d93c..f160b8dffe4 100644 --- a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts @@ -708,7 +708,7 @@ export class SoloBlockBuilder implements BlockBuilder { newSubtreeSiblingPath: newNullifiersSubtreeSiblingPath, sortedNewLeaves: sortedNewNullifiers, sortedNewLeavesIndexes: sortednewNullifiersIndexes, - } = await this.db.batchInsert( + } = await this.db.batchInsert( MerkleTreeId.NULLIFIER_TREE, newNullifiers.map(fr => fr.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, @@ -743,7 +743,7 @@ export class SoloBlockBuilder implements BlockBuilder { newPublicDataReadsSiblingPaths, lowNullifierLeafPreimages: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => i < nullifierWitnessLeaves.length - ? (nullifierWitnessLeaves[i].leafData as NullifierLeafPreimage) + ? (nullifierWitnessLeaves[i].leafPreimage as NullifierLeafPreimage) : NullifierLeafPreimage.empty(), ), lowNullifierMembershipWitness: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => diff --git a/yarn-project/types/src/interfaces/nullifier_tree.ts b/yarn-project/types/src/interfaces/nullifier_tree.ts index ab3053ef858..14fdf426b8d 100644 --- a/yarn-project/types/src/interfaces/nullifier_tree.ts +++ b/yarn-project/types/src/interfaces/nullifier_tree.ts @@ -17,7 +17,7 @@ export class NullifierMembershipWitness { /** * Preimage of the nullifier. */ - public readonly leafData: NullifierLeafPreimage, + public readonly leafPreimage: NullifierLeafPreimage, /** * Sibling path to prove membership of the nullifier. */ @@ -31,9 +31,9 @@ export class NullifierMembershipWitness { public toFieldArray(): Fr[] { return [ new Fr(this.index), - new Fr(this.leafData.nullifier), - new Fr(this.leafData.nextIndex), - new Fr(this.leafData.nextNullifier), + new Fr(this.leafPreimage.nullifier), + new Fr(this.leafPreimage.nextIndex), + new Fr(this.leafPreimage.nextNullifier), ...this.siblingPath.toFieldArray(), ]; } diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 05a25791246..3b7f4c13454 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -92,7 +92,7 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * Gets the leaf data at a given index and tree. * @param treeId - The ID of the tree get the leaf from. * @param index - The index of the leaf to get. - * @returns Leaf data. + * @returns Leaf preimage. */ async getLeafPreimage( treeId: MerkleTreeId.NULLIFIER_TREE, diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index bf8bf02d362..9a7f1ad629e 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -337,7 +337,7 @@ export class MerkleTrees implements MerkleTreeDb { * @param treeId - The ID of the tree get the leaf from. * @param index - The index of the leaf to get. * @param includeUncommitted - Indicates whether to include uncommitted data. - * @returns Leaf data. + * @returns Leaf preimage. */ public async getLeafPreimage( treeId: IndexedTreeId, From 1b7c9f88b6327fa8978201c7ee1b61770bd96a09 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 12:54:45 +0000 Subject: [PATCH 13/18] style: fix formatting --- yarn-project/circuits.js/src/structs/rollup/base_rollup.ts | 3 +-- .../src/merkle-tree/merkle_tree_operations_facade.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts index 78bf7c054e2..0ff77419842 100644 --- a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts +++ b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts @@ -1,8 +1,7 @@ import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; import { Fr } from '@aztec/foundation/fields'; import { BufferReader, Tuple } from '@aztec/foundation/serialize'; -import { IndexedTreeLeaf } from '@aztec/foundation/trees'; -import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BLOCKS_TREE_HEIGHT, diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 3b7f4c13454..494040e381e 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -1,6 +1,6 @@ import { NullifierLeafPreimage } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; -import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { BatchInsertionResult } from '@aztec/merkle-tree'; import { L2Block, MerkleTreeId, SiblingPath } from '@aztec/types'; From 08791c53d89df9d706cbe2090b4e20ef6c2ad031 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Mon, 4 Dec 2023 14:12:05 +0000 Subject: [PATCH 14/18] chore: reset file to master --- yarn-project/noir-contracts/scripts/types.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/noir-contracts/scripts/types.sh b/yarn-project/noir-contracts/scripts/types.sh index 14fa63bfcd1..c25603dda38 100755 --- a/yarn-project/noir-contracts/scripts/types.sh +++ b/yarn-project/noir-contracts/scripts/types.sh @@ -77,4 +77,4 @@ for CONTRACT_NAME in "$@"; do done # only run the rest when the full flag is set -format \ No newline at end of file +format From edb4548ba3b25fc83201c25d9436062c839a1c58 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Tue, 5 Dec 2023 09:38:11 +0000 Subject: [PATCH 15/18] fix: addressed some PR comments --- yarn-project/merkle-tree/src/index.ts | 2 +- yarn-project/merkle-tree/src/new_tree.ts | 2 +- .../src/snapshots/indexed_tree_snapshot.ts | 2 +- .../src/sparse_tree/sparse_tree.test.ts | 6 +-- .../standard_indexed_tree.ts | 42 ++++++++++--------- .../src/standard_tree/standard_tree.test.ts | 6 +-- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index c77fc1c9dec..10ac123395a 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -7,7 +7,7 @@ export * from './sparse_tree/sparse_tree.js'; export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF } from './tree_base.js'; -export { newTree, builder as treeBuilder } from './new_tree.js'; +export { newTree, treeBuilder } from './new_tree.js'; export { loadTree } from './load_tree.js'; export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts index 819c0409c6c..31bfdefd9d8 100644 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -9,7 +9,7 @@ import { TreeBase } from './tree_base.js'; * @param clazz - The class to be instantiated. * @returns A builder function. */ -export function builder( +export function treeBuilder( clazz: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => T, ) { return (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => { diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts index 98e5373537c..28aeefdc953 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts @@ -101,7 +101,7 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre async findLeafIndex(value: Buffer): Promise { const index = await this.tree.findLeafIndex(value, false); - if (index && index < this.getNumLeaves()) { + if (index !== undefined && index < this.getNumLeaves()) { return index; } } diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index 7f1df12cc0f..3e23772fb2e 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -7,7 +7,7 @@ import { default as levelup } from 'levelup'; import { INITIAL_LEAF, newTree } from '../index.js'; import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { loadTree } from '../load_tree.js'; -import { builder } from '../new_tree.js'; +import { treeBuilder } from '../new_tree.js'; import { Pedersen } from '../pedersen.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; @@ -22,11 +22,11 @@ const createDb = async ( name: string, depth: number, ): Promise => { - return await newTree(builder(SparseTree), levelUp, hasher, name, depth); + return await newTree(treeBuilder(SparseTree), levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string): Promise => { - return await loadTree(builder(SparseTree), levelUp, hasher, name); + return await loadTree(treeBuilder(SparseTree), levelUp, hasher, name); }; const TEST_TREE_DEPTH = 3; diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index a935bcef75e..325b438a0f1 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -67,6 +67,22 @@ export const buildDbKeyForLeafIndex = (name: string, key: bigint) => { return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}`; }; +/** + * Pre-compute empty witness. + * @param treeHeight - Height of tree for sibling path. + * @returns An empty witness. + */ +function getEmptyLowLeafWitness( + treeHeight: N, + leafPreimageFactory: PreimageFactory, +): LowLeafWitnessData { + return { + leafPreimage: leafPreimageFactory.empty(), + index: 0n, + siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), + }; +} + /** * Standard implementation of an indexed tree. */ @@ -153,11 +169,12 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { if (includeUncommitted) { const cachedLowLeafIndex = this.getCachedLowLeafIndex(newKey); - const cachedLowLeafPreimage = - cachedLowLeafIndex !== undefined ? this.getCachedPreimage(cachedLowLeafIndex) : undefined; - if (cachedLowLeafIndex && (!lowLeafIndex || cachedLowLeafPreimage!.getKey() > lowLeafPreimage!.getKey())) { - lowLeafIndex = cachedLowLeafIndex; - lowLeafPreimage = cachedLowLeafPreimage; + if (cachedLowLeafIndex !== undefined) { + const cachedLowLeafPreimage = this.getCachedPreimage(cachedLowLeafIndex)!; + if (!lowLeafPreimage || cachedLowLeafPreimage.getKey() > lowLeafPreimage.getKey()) { + lowLeafIndex = cachedLowLeafIndex; + lowLeafPreimage = cachedLowLeafPreimage; + } } } @@ -340,19 +357,6 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { } } - /** - * Pre-compute empty witness. - * @param treeHeight - Height of tree for sibling path. - * @returns An empty witness. - */ - getEmptyLowLeafWitness(treeHeight: N): LowLeafWitnessData { - return { - leafPreimage: this.leafPreimageFactory.empty(), - index: 0n, - siblingPath: new SiblingPath(treeHeight, Array(treeHeight).fill(toBufferBE(0n, 32))), - }; - } - /* eslint-disable jsdoc/require-description-complete-sentence */ /* The following doc block messes up with complete-sentence, so we just disable it */ @@ -469,7 +473,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { leaves: Buffer[], subtreeHeight: SubtreeHeight, ): Promise> { - const emptyLowLeafWitness = this.getEmptyLowLeafWitness(this.getDepth() as TreeHeight); + const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight, this.leafPreimageFactory); // Accumulators const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); const pendingInsertionSubtree: IndexedTreeLeafPreimage[] = leaves.map(() => this.leafPreimageFactory.empty()); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index db67453c4bc..5849c4e5806 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,7 +4,7 @@ import { Hasher } from '@aztec/types'; import { default as levelup } from 'levelup'; import { loadTree } from '../load_tree.js'; -import { builder, newTree } from '../new_tree.js'; +import { newTree, treeBuilder } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { createMemDown } from '../test/utils/create_mem_down.js'; @@ -13,11 +13,11 @@ import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number) => { - return await newTree(builder(StandardTree), levelUp, hasher, name, depth); + return await newTree(treeBuilder(StandardTree), levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(builder(StandardTree), levelUp, hasher, name); + return await loadTree(treeBuilder(StandardTree), levelUp, hasher, name); }; treeTestSuite('StandardTree', createDb, createFromName); From e731f97ae6749bd5bba32585dfdb67bcc034c746 Mon Sep 17 00:00:00 2001 From: sirasistant Date: Tue, 5 Dec 2023 10:10:04 +0000 Subject: [PATCH 16/18] fix: avoid findLeafIndex in sparse trees --- .../merkle-tree/src/sparse_tree/sparse_tree.ts | 4 ++++ .../test/standard_indexed_tree.test.ts | 16 ++++++++++++++++ .../src/standard_tree/standard_tree.test.ts | 16 ++++++++++++++++ .../src/standard_tree/standard_tree.ts | 10 ++++++++++ yarn-project/merkle-tree/src/test/test_suite.ts | 14 -------------- yarn-project/merkle-tree/src/tree_base.ts | 10 +--------- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts index 92cdc4152fc..4c85461987e 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts @@ -41,4 +41,8 @@ export class SparseTree extends TreeBase implements UpdateOnlyTree { public getSnapshot(block: number): Promise { return this.#snapshotBuilder.getSnapshot(block); } + + public findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + throw new Error('Finding leaf index is not supported for sparse trees'); + } } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index 015193954eb..9bcef10bf78 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -495,4 +495,20 @@ describe('StandardIndexedTreeSpecific', () => { const actualRoot = insertTree.getRoot(true); expect(actualRoot).toEqual(expectedRoot); }); + + it('should be able to find indexes of leaves', async () => { + const db = levelup(createMemDown()); + const tree = await createDb(db, pedersen, 'test', 3); + const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; + + await tree.appendLeaves([values[0]]); + + expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); + expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); + expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); + + await tree.commit(); + + expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); + }); }); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index 5849c4e5806..d2d2d0396c4 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -69,4 +69,20 @@ describe('StandardTree_batchAppend', () => { expect(tree.getRoot(true)).toEqual(root); }); + + it('should be able to find indexes of leaves', async () => { + const db = levelup(createMemDown()); + const tree = await createDb(db, pedersen, 'test', 3); + const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; + + await tree.appendLeaves([values[0]]); + + expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); + expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); + expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); + + await tree.commit(); + + expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); + }); }); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts index 0b92572a4b8..55b4f532469 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts @@ -24,4 +24,14 @@ export class StandardTree extends TreeBase implements AppendOnlyTree { public getSnapshot(block: number): Promise { return this.#snapshotBuilder.getSnapshot(block); } + + public async findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + for (let i = 0n; i < this.getNumLeaves(includeUncommitted); i++) { + const currentValue = await this.getLeafValue(i, includeUncommitted); + if (currentValue && currentValue.equals(value)) { + return i; + } + } + return undefined; + } } diff --git a/yarn-project/merkle-tree/src/test/test_suite.ts b/yarn-project/merkle-tree/src/test/test_suite.ts index 51886a090b3..1940f32ce91 100644 --- a/yarn-project/merkle-tree/src/test/test_suite.ts +++ b/yarn-project/merkle-tree/src/test/test_suite.ts @@ -157,19 +157,5 @@ export const treeTestSuite = ( expect(deserialized2.elem).toEqual(siblingPath); expect(deserialized2.adv).toBe(4 + 10 * 32); }); - - it('should be able to find indexes of leaves', async () => { - const db = levelup(createMemDown()); - const tree = await createDb(db, pedersen, 'test', 10); - await appendLeaves(tree, values.slice(0, 1)); - - expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); - expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); - expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); - - await tree.commit(); - - expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); - }); }); }; diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index f9f58068cbe..291ac258082 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -307,13 +307,5 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - public async findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { - for (let i = 0n; i < this.getNumLeaves(includeUncommitted); i++) { - const currentValue = await this.getLeafValue(i, includeUncommitted); - if (currentValue && currentValue.equals(value)) { - return i; - } - } - return undefined; - } + abstract findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise; } From 037a47961bfd2993758f03f3e5de2796cfa7ccbe Mon Sep 17 00:00:00 2001 From: sirasistant Date: Tue, 5 Dec 2023 10:37:17 +0000 Subject: [PATCH 17/18] refactor: avoid createTree helper --- .../src/client/private_execution.test.ts | 4 +-- .../src/e2e_blacklist_token_contract.test.ts | 10 ++---- .../end-to-end/src/e2e_slow_tree.test.ts | 10 ++---- yarn-project/merkle-tree/src/index.ts | 2 +- yarn-project/merkle-tree/src/load_tree.ts | 4 +-- yarn-project/merkle-tree/src/new_tree.ts | 17 ++------- .../snapshots/append_only_snapshot.test.ts | 4 +-- .../src/snapshots/full_snapshot.test.ts | 4 +-- .../snapshots/indexed_tree_snapshot.test.ts | 16 ++++----- .../src/sparse_tree/sparse_tree.test.ts | 5 ++- .../test/standard_indexed_tree.test.ts | 35 +++++-------------- .../src/standard_tree/standard_tree.test.ts | 6 ++-- .../src/world-state-db/merkle_trees.ts | 21 ++++++----- 13 files changed, 47 insertions(+), 91 deletions(-) diff --git a/yarn-project/acir-simulator/src/client/private_execution.test.ts b/yarn-project/acir-simulator/src/client/private_execution.test.ts index c35c9cf0ae0..6b0a7816a51 100644 --- a/yarn-project/acir-simulator/src/client/private_execution.test.ts +++ b/yarn-project/acir-simulator/src/client/private_execution.test.ts @@ -27,7 +27,7 @@ import { pedersenHash } from '@aztec/foundation/crypto'; import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr, GrumpkinScalar } from '@aztec/foundation/fields'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { AppendOnlyTree, Pedersen, StandardTree, newTree, treeBuilder } from '@aztec/merkle-tree'; +import { AppendOnlyTree, Pedersen, StandardTree, newTree } from '@aztec/merkle-tree'; import { ChildContractArtifact, ImportTestContractArtifact, @@ -126,7 +126,7 @@ describe('Private Execution test suite', () => { if (!trees[name]) { const db = levelup(createMemDown()); const pedersen = new Pedersen(); - trees[name] = await newTree(treeBuilder(StandardTree), db, pedersen, name, treeHeights[name]); + trees[name] = await newTree(StandardTree, db, pedersen, name, treeHeights[name]); } await trees[name].appendLeaves(leaves.map(l => l.toBuffer())); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts index 6a2232eee47..1c1825c7d83 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts @@ -14,7 +14,7 @@ import { computeAuthWitMessageHash, computeMessageSecretHash, } from '@aztec/aztec.js'; -import { Pedersen, SparseTree, newTree, treeBuilder } from '@aztec/merkle-tree'; +import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; import { SlowTreeContract, TokenBlacklistContract, TokenContract } from '@aztec/noir-contracts/types'; import { jest } from '@jest/globals'; @@ -107,13 +107,7 @@ describe('e2e_blacklist_token_contract', () => { slowTree = await SlowTreeContract.deploy(wallets[0]).send().deployed(); const depth = 254; - slowUpdateTreeSimulator = await newTree( - treeBuilder(SparseTree), - levelup(createMemDown()), - new Pedersen(), - 'test', - depth, - ); + slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); const deployTx = TokenBlacklistContract.deploy(wallets[0], accounts[0], slowTree.address).send({}); const receipt = await deployTx.wait(); diff --git a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts index e0d58be36ee..87c267e5e82 100644 --- a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts +++ b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts @@ -1,6 +1,6 @@ /* eslint-disable camelcase */ import { CheatCodes, DebugLogger, Fr, Wallet } from '@aztec/aztec.js'; -import { Pedersen, SparseTree, newTree, treeBuilder } from '@aztec/merkle-tree'; +import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; import { SlowTreeContract } from '@aztec/noir-contracts/types'; import { default as levelup } from 'levelup'; @@ -27,13 +27,7 @@ describe('e2e_slow_tree', () => { it('Messing around with noir slow tree', async () => { const depth = 254; - const slowUpdateTreeSimulator = await newTree( - treeBuilder(SparseTree), - levelup(createMemDown()), - new Pedersen(), - 'test', - depth, - ); + const slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, diff --git a/yarn-project/merkle-tree/src/index.ts b/yarn-project/merkle-tree/src/index.ts index 10ac123395a..68826f44e42 100644 --- a/yarn-project/merkle-tree/src/index.ts +++ b/yarn-project/merkle-tree/src/index.ts @@ -7,7 +7,7 @@ export * from './sparse_tree/sparse_tree.js'; export { StandardIndexedTree } from './standard_indexed_tree/standard_indexed_tree.js'; export * from './standard_tree/standard_tree.js'; export { INITIAL_LEAF } from './tree_base.js'; -export { newTree, treeBuilder } from './new_tree.js'; +export { newTree } from './new_tree.js'; export { loadTree } from './load_tree.js'; export * from './snapshots/snapshot_builder.js'; export * from './snapshots/full_snapshot.js'; diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts index bd852b930fd..9753a2d528d 100644 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -13,7 +13,7 @@ import { TreeBase, decodeMeta } from './tree_base.js'; * @returns The newly created tree. */ export async function loadTree( - c: (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, + c: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, db: LevelUp, hasher: Hasher, name: string, @@ -21,6 +21,6 @@ export async function loadTree( const meta: Buffer = await db.get(name); const { root, depth, size } = decodeMeta(meta); - const tree = c(db, hasher, name, depth, size, root); + const tree = new c(db, hasher, name, depth, size, root); return tree; } diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts index 31bfdefd9d8..1395d012d25 100644 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -4,19 +4,6 @@ import { LevelUp } from 'levelup'; import { TreeBase } from './tree_base.js'; -/** - * Wraps a constructor in a regular builder - * @param clazz - The class to be instantiated. - * @returns A builder function. - */ -export function treeBuilder( - clazz: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => T, -) { - return (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root?: Buffer) => { - return new clazz(db, hasher, name, depth, size, root); - }; -} - /** * Creates a new tree. * @param c - The class of the tree to be instantiated. @@ -28,14 +15,14 @@ export function treeBuilder( * @returns The newly created tree. */ export async function newTree( - c: (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => T, + c: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => T, db: LevelUp, hasher: Hasher, name: string, depth: number, prefilledSize = 1, ): Promise { - const tree = c(db, hasher, name, depth, 0n); + const tree = new c(db, hasher, name, depth, 0n); await tree.init(prefilledSize); return tree; } diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts index 23291de51da..b66eb2af22b 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts @@ -1,6 +1,6 @@ import levelup, { LevelUp } from 'levelup'; -import { Pedersen, StandardTree, newTree, treeBuilder } from '../index.js'; +import { Pedersen, StandardTree, newTree } from '../index.js'; import { createMemDown } from '../test/utils/create_mem_down.js'; import { AppendOnlySnapshotBuilder } from './append_only_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; @@ -13,7 +13,7 @@ describe('AppendOnlySnapshot', () => { beforeEach(async () => { db = levelup(createMemDown()); const hasher = new Pedersen(); - tree = await newTree(treeBuilder(StandardTree), db, hasher, 'test', 4); + tree = await newTree(StandardTree, db, hasher, 'test', 4); snapshotBuilder = new AppendOnlySnapshotBuilder(db, tree, hasher); }); diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts index b17c7b6185d..3f2cc2af791 100644 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts @@ -1,6 +1,6 @@ import levelup, { LevelUp } from 'levelup'; -import { Pedersen, StandardTree, newTree, treeBuilder } from '../index.js'; +import { Pedersen, StandardTree, newTree } from '../index.js'; import { createMemDown } from '../test/utils/create_mem_down.js'; import { FullTreeSnapshotBuilder } from './full_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; @@ -12,7 +12,7 @@ describe('FullSnapshotBuilder', () => { beforeEach(async () => { db = levelup(createMemDown()); - tree = await newTree(treeBuilder(StandardTree), db, new Pedersen(), 'test', 4); + tree = await newTree(StandardTree, db, new Pedersen(), 'test', 4); snapshotBuilder = new FullTreeSnapshotBuilder(db, tree); }); diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts index 748ea507327..3846bbcc21d 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts @@ -9,6 +9,12 @@ import { createMemDown } from '../test/utils/create_mem_down.js'; import { IndexedTreeSnapshotBuilder } from './indexed_tree_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; +class NullifierTree extends StandardIndexedTreeWithAppend { + constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + } +} + describe('IndexedTreeSnapshotBuilder', () => { let db: LevelUp; let tree: StandardIndexedTreeWithAppend; @@ -16,15 +22,7 @@ describe('IndexedTreeSnapshotBuilder', () => { beforeEach(async () => { db = levelup(createMemDown()); - tree = await newTree( - (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => { - return new StandardIndexedTreeWithAppend(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf); - }, - db, - new Pedersen(), - 'test', - 4, - ); + tree = await newTree(NullifierTree, db, new Pedersen(), 'test', 4); snapshotBuilder = new IndexedTreeSnapshotBuilder(db, tree, NullifierLeafPreimage); }); diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts index 3e23772fb2e..bd4f1e6bd0b 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.test.ts @@ -7,7 +7,6 @@ import { default as levelup } from 'levelup'; import { INITIAL_LEAF, newTree } from '../index.js'; import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { loadTree } from '../load_tree.js'; -import { treeBuilder } from '../new_tree.js'; import { Pedersen } from '../pedersen.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; @@ -22,11 +21,11 @@ const createDb = async ( name: string, depth: number, ): Promise => { - return await newTree(treeBuilder(SparseTree), levelUp, hasher, name, depth); + return await newTree(SparseTree, levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string): Promise => { - return await loadTree(treeBuilder(SparseTree), levelUp, hasher, name); + return await loadTree(SparseTree, levelUp, hasher, name); }; const TEST_TREE_DEPTH = 3; diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index 9bcef10bf78..9ebc8c30472 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -9,37 +9,18 @@ import { treeTestSuite } from '../../test/test_suite.js'; import { createMemDown } from '../../test/utils/create_mem_down.js'; import { StandardIndexedTreeWithAppend } from './standard_indexed_tree_with_append.js'; +class NullifierTree extends StandardIndexedTreeWithAppend { + constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + } +} + const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { - return await newTree( - (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => { - return new StandardIndexedTreeWithAppend(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf); - }, - levelUp, - hasher, - name, - depth, - prefilledSize, - ); + return await newTree(NullifierTree, levelUp, hasher, name, depth, prefilledSize); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree( - (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => { - return new StandardIndexedTreeWithAppend( - db, - hasher, - name, - depth, - size, - NullifierLeafPreimage, - NullifierLeaf, - root, - ); - }, - levelUp, - hasher, - name, - ); + return await loadTree(NullifierTree, levelUp, hasher, name); }; const createIndexedTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index d2d2d0396c4..b211017d851 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -4,7 +4,7 @@ import { Hasher } from '@aztec/types'; import { default as levelup } from 'levelup'; import { loadTree } from '../load_tree.js'; -import { newTree, treeBuilder } from '../new_tree.js'; +import { newTree } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; import { createMemDown } from '../test/utils/create_mem_down.js'; @@ -13,11 +13,11 @@ import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number) => { - return await newTree(treeBuilder(StandardTree), levelUp, hasher, name, depth); + return await newTree(StandardTree, levelUp, hasher, name, depth); }; const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(treeBuilder(StandardTree), levelUp, hasher, name); + return await loadTree(StandardTree, levelUp, hasher, name); }; treeTestSuite('StandardTree', createDb, createFromName); diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 9a7f1ad629e..ad196545558 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -27,7 +27,6 @@ import { UpdateOnlyTree, loadTree, newTree, - treeBuilder, } from '@aztec/merkle-tree'; import { Hasher, L2Block, MerkleTreeId, SiblingPath } from '@aztec/types'; @@ -57,6 +56,12 @@ interface FromDbOptions { const LAST_GLOBAL_VARS_HASH = 'lastGlobalVarsHash'; +class NullifierTree extends StandardIndexedTree { + constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + } +} + /** * A convenience class for managing multiple merkle trees. */ @@ -79,16 +84,14 @@ export class MerkleTrees implements MerkleTreeDb { const hasher = new Pedersen(); const contractTree: AppendOnlyTree = await initializeTree( - treeBuilder(StandardTree), + StandardTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.CONTRACT_TREE]}`, CONTRACT_TREE_HEIGHT, ); const nullifierTree = await initializeTree( - (db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => { - return new StandardIndexedTree(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf); - }, + NullifierTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, @@ -96,28 +99,28 @@ export class MerkleTrees implements MerkleTreeDb { INITIAL_NULLIFIER_TREE_SIZE, ); const noteHashTree: AppendOnlyTree = await initializeTree( - treeBuilder(StandardTree), + StandardTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, NOTE_HASH_TREE_HEIGHT, ); const publicDataTree: UpdateOnlyTree = await initializeTree( - treeBuilder(SparseTree), + SparseTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, PUBLIC_DATA_TREE_HEIGHT, ); const l1Tol2MessagesTree: AppendOnlyTree = await initializeTree( - treeBuilder(StandardTree), + StandardTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGES_TREE]}`, L1_TO_L2_MSG_TREE_HEIGHT, ); const blocksTree: AppendOnlyTree = await initializeTree( - treeBuilder(StandardTree), + StandardTree, this.db, hasher, `${MerkleTreeId[MerkleTreeId.BLOCKS_TREE]}`, From a217fe2d70f525d1cd0d3253c3fde2c52e02633a Mon Sep 17 00:00:00 2001 From: sirasistant Date: Tue, 5 Dec 2023 10:39:59 +0000 Subject: [PATCH 18/18] fix: formatting --- yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts | 2 +- yarn-project/world-state/src/world-state-db/merkle_trees.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts index 4c85461987e..138ca8f21e7 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts @@ -42,7 +42,7 @@ export class SparseTree extends TreeBase implements UpdateOnlyTree { return this.#snapshotBuilder.getSnapshot(block); } - public findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + public findLeafIndex(_value: Buffer, _includeUncommitted: boolean): Promise { throw new Error('Finding leaf index is not supported for sparse trees'); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index ad196545558..56e6ee96972 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -56,6 +56,9 @@ interface FromDbOptions { const LAST_GLOBAL_VARS_HASH = 'lastGlobalVarsHash'; +/** + * The nullifier tree is an indexed tree. + */ class NullifierTree extends StandardIndexedTree { constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root);