From 187d2f79d9390e43ec2e2ce6a0db0d6718cc1716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Rodr=C3=ADguez?= Date: Thu, 30 Nov 2023 11:49:14 +0100 Subject: [PATCH] feat: circuit optimized indexed tree batch insertion (#3367) Changes the batch insertion algorithm for indexed trees to a circuit-optimized one. --- .../src/structs/rollup/base_rollup.ts | 10 + .../circuits.js/src/tests/factories.ts | 11 +- .../src/interfaces/indexed_tree.ts | 27 +- .../standard_indexed_tree.ts | 218 ++++------ .../src/crates/rollup-base/src/main.nr | 3 +- .../src/abis/nullifier_leaf_preimage.nr | 2 +- .../rollup-lib/src/base/base_rollup_inputs.nr | 406 ++++++++---------- .../src/crates/rollup-lib/src/indexed_tree.nr | 121 ++++++ .../src/crates/rollup-lib/src/lib.nr | 4 +- .../src/type_conversion.ts | 12 + .../src/types/rollup_base_types.ts | 2 + .../src/block_builder/solo_block_builder.ts | 9 +- .../merkle_tree_operations_facade.ts | 6 +- .../src/world-state-db/merkle_tree_db.ts | 6 +- .../src/world-state-db/merkle_trees.ts | 7 +- 15 files changed, 472 insertions(+), 372 deletions(-) create mode 100644 yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/indexed_tree.nr diff --git a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts index 13e8edf46e1..454532ec646 100644 --- a/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts +++ b/yarn-project/circuits.js/src/structs/rollup/base_rollup.ts @@ -144,6 +144,14 @@ export class BaseRollupInputs { */ public startHistoricBlocksTreeSnapshot: AppendOnlyTreeSnapshot, + /** + * The nullifiers to be inserted in the tree, sorted high to low. + */ + public sortedNewNullifiers: Tuple, + /** + * The indexes of the sorted nullifiers to the original ones. + */ + public sortednewNullifiersIndexes: Tuple, /** * The nullifiers which need to be updated to perform the batch insertion of the new nullifiers. * See `StandardIndexedTree.batchInsert` function for more details. @@ -210,6 +218,8 @@ export class BaseRollupInputs { fields.startContractTreeSnapshot, fields.startPublicDataTreeRoot, fields.startHistoricBlocksTreeSnapshot, + fields.sortedNewNullifiers, + fields.sortednewNullifiersIndexes, fields.lowNullifierLeafPreimages, fields.lowNullifierMembershipWitness, fields.newCommitmentsSubtreeSiblingPath, diff --git a/yarn-project/circuits.js/src/tests/factories.ts b/yarn-project/circuits.js/src/tests/factories.ts index 3df6e72dccd..6c03169795a 100644 --- a/yarn-project/circuits.js/src/tests/factories.ts +++ b/yarn-project/circuits.js/src/tests/factories.ts @@ -920,20 +920,23 @@ export function makeBaseRollupInputs(seed = 0): BaseRollupInputs { const newNullifiersSubtreeSiblingPath = makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, fr, seed + 0x4000); const newContractsSubtreeSiblingPath = makeTuple(CONTRACT_SUBTREE_SIBLING_PATH_LENGTH, fr, seed + 0x5000); + const sortedNewNullifiers = makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, fr, seed + 0x6000); + const sortednewNullifiersIndexes = makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => i, seed + 0x7000); + const newPublicDataUpdateRequestsSiblingPaths = makeTuple( MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_BASE_ROLLUP, x => makeTuple(PUBLIC_DATA_TREE_HEIGHT, fr, x), - seed + 0x6000, + seed + 0x8000, ); const newPublicDataReadsSiblingPaths = makeTuple( MAX_PUBLIC_DATA_READS_PER_BASE_ROLLUP, x => makeTuple(PUBLIC_DATA_TREE_HEIGHT, fr, x), - seed + 0x6000, + seed + 0x8000, ); const historicBlocksTreeRootMembershipWitnesses = makeTuple(KERNELS_PER_BASE_ROLLUP, x => - makeMembershipWitness(HISTORIC_BLOCKS_TREE_HEIGHT, seed + x * 0x1000 + 0x7000), + makeMembershipWitness(HISTORIC_BLOCKS_TREE_HEIGHT, seed + x * 0x1000 + 0x9000), ); const constants = makeConstantBaseRollupData(0x100); @@ -946,6 +949,8 @@ export function makeBaseRollupInputs(seed = 0): BaseRollupInputs { startContractTreeSnapshot, startPublicDataTreeRoot, startHistoricBlocksTreeSnapshot, + sortedNewNullifiers, + sortednewNullifiersIndexes, lowNullifierLeafPreimages, newCommitmentsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath, diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 1b8bade092e..46c13f49bd9 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -3,6 +3,28 @@ import { LeafData, SiblingPath } from '@aztec/types'; import { LowLeafWitnessData } from '../index.js'; import { AppendOnlyTree } from './append_only_tree.js'; +/** + * The result of a batch insertion in an indexed merkle tree. + */ +export interface BatchInsertionResult { + /** + * Data for the leaves to be updated when inserting the new ones. + */ + lowLeavesWitnessData?: LowLeafWitnessData[]; + /** + * Sibling path "pointing to" where the new subtree should be inserted into the tree. + */ + newSubtreeSiblingPath: SiblingPath; + /** + * The new leaves being inserted in high to low order. This order corresponds with the order of the low leaves witness. + */ + sortedNewLeaves: Buffer[]; + /** + * The indexes of the sorted new leaves to the original ones. + */ + sortedNewLeavesIndexes: number[]; +} + /** * Indexed merkle tree. */ @@ -45,8 +67,5 @@ export interface IndexedTree extends AppendOnlyTree { leaves: Buffer[], subtreeHeight: SubtreeHeight, includeUncommitted: boolean, - ): Promise< - | [LowLeafWitnessData[], SiblingPath] - | [undefined, SiblingPath] - >; + ): Promise>; } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index c32ce3221e1..f9d44353fa9 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,8 +1,9 @@ import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; +import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; import { LeafData, SiblingPath } from '@aztec/types'; -import { IndexedTree } from '../interfaces/indexed_tree.js'; +import { BatchInsertionResult, IndexedTree } from '../interfaces/indexed_tree.js'; import { TreeBase } from '../tree_base.js'; const log = createDebugLogger('aztec:standard-indexed-tree'); @@ -317,8 +318,6 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * * This offers massive circuit performance savings over doing incremental insertions. * - * A description of the algorithm can be found here: https://colab.research.google.com/drive/1A0gizduSi4FIiIJZ8OylwIpO9-OTqV-R - * * WARNING: This function has side effects, it will insert values into the tree. * * Assumptions: @@ -338,81 +337,78 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * roots. * * This become tricky when two items that are being batch inserted need to update the same low nullifier, or need to use - * a value that is part of the same batch insertion as their low nullifier. In this case a zero low nullifier path is given - * to the circuit, and it must determine from the set of batch inserted values if the insertion is valid. + * a value that is part of the same batch insertion as their low nullifier. What we do to avoid this case is to + * update the existing leaves in the tree with the nullifiers in high to low order, ensuring that this case never occurs. + * The circuit has to sort the nullifiers (or take a hint of the sorted nullifiers and prove that it's a valid permutation). + * Then we just batch insert the new nullifiers in the original order. * * The following example will illustrate attempting to insert 2,3,20,19 into a tree already containing 0,5,10,15 * * The example will explore two cases. In each case the values low nullifier will exist within the batch insertion, * One where the low nullifier comes before the item in the set (2,3), and one where it comes after (20,19). * + * First, we sort the nullifiers high to low, that's 20,19,3,2 + * * The original tree: Pending insertion subtree * - * index 0 2 3 4 - - - - + * index 0 1 2 3 - - - - * ------------------------------------- ---------------------------- * val 0 5 10 15 - - - - * nextIdx 1 2 3 0 - - - - * nextVal 5 10 15 0 - - - - * * - * Inserting 2: (happy path) - * 1. Find the low nullifier (0) - provide inclusion proof + * Inserting 20: + * 1. Find the low nullifier (3) - provide inclusion proof * 2. Update its pointers - * 3. Insert 2 into the pending subtree + * 3. Insert 20 into the pending subtree * - * index 0 2 3 4 5 - - - + * index 0 1 2 3 - - 6 - * ------------------------------------- ---------------------------- - * val 0 5 10 15 2 - - - - * nextIdx 5 2 3 0 2 - - - - * nextVal 2 10 15 0 5 - - - + * val 0 5 10 15 - - 20 - + * nextIdx 1 2 3 6 - - 0 - + * nextVal 5 10 15 20 - - 0 - * - * Inserting 3: The low nullifier exists within the insertion current subtree - * 1. When looking for the low nullifier for 3, we will receive 0 again as we have not inserted 2 into the main tree - * This is problematic, as we cannot use either 0 or 2 as our inclusion proof. - * Why cant we? - * - Index 0 has a val 0 and nextVal of 2. This is NOT enough to prove non inclusion of 2. - * - Our existing tree is in a state where we cannot prove non inclusion of 3. - * We do not provide a non inclusion proof to out circuit, but prompt it to look within the insertion subtree. - * 2. Update pending insertion subtree - * 3. Insert 3 into pending subtree + * Inserting 19: + * 1. Find the low nullifier (3) - provide inclusion proof + * 2. Update its pointers + * 3. Insert 19 into the pending subtree * - * (no inclusion proof provided) - * index 0 2 3 4 5 6 - - + * index 0 1 2 3 - - 6 7 * ------------------------------------- ---------------------------- - * val 0 5 10 15 2 3 - - - * nextIdx 5 2 3 0 6 2 - - - * nextVal 2 10 15 0 3 5 - - + * val 0 5 10 15 - - 20 19 + * nextIdx 1 2 3 7 - - 0 6 + * nextVal 5 10 15 19 - - 0 20 * - * Inserting 20: (happy path) - * 1. Find the low nullifier (15) - provide inclusion proof + * Inserting 3: + * 1. Find the low nullifier (0) - provide inclusion proof * 2. Update its pointers - * 3. Insert 20 into the pending subtree + * 3. Insert 3 into the pending subtree * - * index 0 2 3 4 5 6 7 - + * index 0 1 2 3 - 5 6 7 * ------------------------------------- ---------------------------- - * val 0 5 10 15 2 3 20 - - * nextIdx 5 2 3 7 6 2 0 - - * nextVal 2 10 15 20 3 5 0 - + * val 0 5 10 15 - 3 20 19 + * nextIdx 5 2 3 7 - 1 0 6 + * nextVal 3 10 15 19 - 5 0 20 * - * Inserting 19: - * 1. In this case we can find a low nullifier, but we are updating a low nullifier that has already been updated - * We can provide an inclusion proof of this intermediate tree state. + * Inserting 2: + * 1. Find the low nullifier (0) - provide inclusion proof * 2. Update its pointers - * 3. Insert 19 into the pending subtree + * 3. Insert 2 into the pending subtree * - * index 0 2 3 4 5 6 7 8 + * index 0 1 2 3 4 5 6 7 * ------------------------------------- ---------------------------- - * val 0 5 10 15 2 3 20 19 - * nextIdx 5 2 3 8 6 2 0 7 - * nextVal 2 10 15 19 3 5 0 20 + * val 0 5 10 15 2 3 20 19 + * nextIdx 4 2 3 7 5 1 0 6 + * nextVal 2 10 15 19 3 5 0 20 * * Perform subtree insertion * - * index 0 2 3 4 5 6 7 8 + * index 0 1 2 3 4 5 6 7 * --------------------------------------------------------------------- - * val 0 5 10 15 2 3 20 19 - * nextIdx 5 2 3 8 6 2 0 7 - * nextVal 2 10 15 19 3 5 0 20 + * val 0 5 10 15 2 3 20 19 + * nextIdx 4 2 3 7 5 1 0 6 + * nextVal 2 10 15 19 3 5 0 20 * * TODO: this implementation will change once the zero value is changed from h(0,0,0). Changes incoming over the next sprint * @param leaves - Values to insert into the tree. @@ -426,107 +422,67 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { >( leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise< - | [LowLeafWitnessData[], SiblingPath] - | [undefined, SiblingPath] - > { - // Keep track of touched low leaves - const touched = new Map(); - + ): Promise> { const emptyLowLeafWitness = getEmptyLowLeafWitness(this.getDepth() as TreeHeight); // Accumulators - const lowLeavesWitnesses: LowLeafWitnessData[] = []; - const pendingInsertionSubtree: LeafData[] = []; + const lowLeavesWitnesses: LowLeafWitnessData[] = leaves.map(() => emptyLowLeafWitness); + const pendingInsertionSubtree: LeafData[] = leaves.map(() => zeroLeaf); // Start info const startInsertionIndex = this.getNumLeaves(true); + const leavesToInsert = leaves.map(leaf => toBigIntBE(leaf)); + const sortedDescendingLeafTuples = leavesToInsert + .map((leaf, index) => ({ leaf, index })) + .sort((a, b) => Number(b.leaf - a.leaf)); + const sortedDescendingLeaves = sortedDescendingLeafTuples.map(leafTuple => leafTuple.leaf); + // Get insertion path for each leaf - for (let i = 0; i < leaves.length; i++) { - const newValue = toBigIntBE(leaves[i]); + for (let i = 0; i < leavesToInsert.length; i++) { + const newValue = sortedDescendingLeaves[i]; + const originalIndex = leavesToInsert.indexOf(newValue); - // Keep space and just insert zero values if (newValue === 0n) { - pendingInsertionSubtree.push(zeroLeaf); - lowLeavesWitnesses.push(emptyLowLeafWitness); continue; } const indexOfPrevious = this.findIndexOfPreviousValue(newValue, true); - // If a touched node has a value that is less than the current value - const prevNodes = touched.get(indexOfPrevious.index); - if (prevNodes && prevNodes.some(v => v < newValue)) { - // check the pending low nullifiers for a low nullifier that works - // This is the case where the next value is less than the pending - for (let j = 0; j < pendingInsertionSubtree.length; j++) { - if (pendingInsertionSubtree[j].value === 0n) { - continue; - } - - if ( - pendingInsertionSubtree[j].value < newValue && - (pendingInsertionSubtree[j].nextValue > newValue || pendingInsertionSubtree[j].nextValue === 0n) - ) { - // add the new value to the pending low nullifiers - const currentLowLeaf: LeafData = { - value: newValue, - nextValue: pendingInsertionSubtree[j].nextValue, - nextIndex: pendingInsertionSubtree[j].nextIndex, - }; - - pendingInsertionSubtree.push(currentLowLeaf); - - // Update the pending low leaf to point at the new value - pendingInsertionSubtree[j].nextValue = newValue; - pendingInsertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); - - break; - } - } - - // Any node updated in this space will need to calculate its low nullifier from a previously inserted value - lowLeavesWitnesses.push(emptyLowLeafWitness); - } else { - // Update the touched mapping - if (prevNodes) { - prevNodes.push(newValue); - touched.set(indexOfPrevious.index, prevNodes); - } else { - touched.set(indexOfPrevious.index, [newValue]); - } - - // get the low leaf - const lowLeaf = this.getLatestLeafDataCopy(indexOfPrevious.index, true); - if (lowLeaf === undefined) { - return [undefined, await this.getSubtreeSiblingPath(subtreeHeight, true)]; - } - const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - - const witness: LowLeafWitnessData = { - leafData: { ...lowLeaf }, - index: BigInt(indexOfPrevious.index), - siblingPath, + // get the low leaf + const lowLeaf = this.getLatestLeafDataCopy(indexOfPrevious.index, true); + if (lowLeaf === undefined) { + return { + lowLeavesWitnessData: undefined, + sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => new Fr(leafTuple.leaf).toBuffer()), + sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), + newSubtreeSiblingPath: await this.getSubtreeSiblingPath(subtreeHeight, true), }; + } + const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); - // Update the running paths - lowLeavesWitnesses.push(witness); + const witness: LowLeafWitnessData = { + leafData: { ...lowLeaf }, + index: BigInt(indexOfPrevious.index), + siblingPath, + }; - const currentLowLeaf: LeafData = { - value: newValue, - nextValue: lowLeaf.nextValue, - nextIndex: lowLeaf.nextIndex, - }; + // Update the running paths + lowLeavesWitnesses[i] = witness; + + const currentPendingLeaf: LeafData = { + value: newValue, + nextValue: lowLeaf.nextValue, + nextIndex: lowLeaf.nextIndex, + }; - pendingInsertionSubtree.push(currentLowLeaf); + pendingInsertionSubtree[originalIndex] = currentPendingLeaf; - lowLeaf.nextValue = newValue; - lowLeaf.nextIndex = startInsertionIndex + BigInt(i); + lowLeaf.nextValue = newValue; + lowLeaf.nextIndex = startInsertionIndex + BigInt(originalIndex); - const lowLeafIndex = indexOfPrevious.index; - this.cachedLeaves[lowLeafIndex] = lowLeaf; - await this.updateLeaf(lowLeaf, BigInt(lowLeafIndex)); - } + const lowLeafIndex = indexOfPrevious.index; + this.cachedLeaves[lowLeafIndex] = lowLeaf; + await this.updateLeaf(lowLeaf, BigInt(lowLeafIndex)); } const newSubtreeSiblingPath = await this.getSubtreeSiblingPath( @@ -538,7 +494,13 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { // Note: In this case we set `hash0Leaf` param to false because batch insertion algorithm use forced null leaf // inclusion. See {@link encodeLeaf} for a more through param explanation. await this.encodeAndAppendLeaves(pendingInsertionSubtree, false); - return [lowLeavesWitnesses, newSubtreeSiblingPath]; + + return { + lowLeavesWitnessData: lowLeavesWitnesses, + sortedNewLeaves: sortedDescendingLeafTuples.map(leafTuple => Buffer.from(new Fr(leafTuple.leaf).toBuffer())), + sortedNewLeavesIndexes: sortedDescendingLeafTuples.map(leafTuple => leafTuple.index), + newSubtreeSiblingPath, + }; } async getSubtreeSiblingPath( diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-base/src/main.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-base/src/main.nr index edf487213f5..7405a961633 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-base/src/main.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-base/src/main.nr @@ -1,5 +1,6 @@ use dep::rollup_lib::base::{BaseRollupInputs,BaseOrMergeRollupPublicInputs}; -fn main(inputs : BaseRollupInputs) -> pub BaseOrMergeRollupPublicInputs { +//TODO add a circuit variant +unconstrained fn main(inputs : BaseRollupInputs) -> pub BaseOrMergeRollupPublicInputs { inputs.base_rollup_circuit() } \ No newline at end of file diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/nullifier_leaf_preimage.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/nullifier_leaf_preimage.nr index 5fce155fef6..b55f943f25c 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/nullifier_leaf_preimage.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/abis/nullifier_leaf_preimage.nr @@ -6,7 +6,7 @@ struct NullifierLeafPreimage { impl NullifierLeafPreimage { pub fn default() -> Self { - NullifierLeafPreimage { + Self { leaf_value : 0, next_value : 0, next_index : 0, diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr index 55da22511da..172701cfb64 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/base/base_rollup_inputs.nr @@ -29,9 +29,10 @@ use dep::aztec::constants_gen::{ MAX_NEW_L2_TO_L1_MSGS_PER_TX, NUM_UNENCRYPTED_LOGS_HASHES_PER_TX, NULLIFIER_SUBTREE_HEIGHT, + NULLIFIER_TREE_HEIGHT, }; use dep::types::abis::previous_kernel_data::PreviousKernelData; -use dep::types::abis::membership_witness::NullifierMembershipWitness; +use dep::types::abis::membership_witness::{NullifierMembershipWitness, MembershipWitness}; use dep::types::abis::membership_witness::HistoricBlocksTreeRootMembershipWitness; struct BaseRollupInputs { @@ -42,6 +43,8 @@ struct BaseRollupInputs { start_public_data_tree_root: Field, start_historic_blocks_tree_snapshot: AppendOnlyTreeSnapshot, + sorted_new_nullifiers: [Field; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], + sorted_new_nullifiers_indexes: [u32; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], low_nullifier_leaf_preimages: [NullifierLeafPreimage; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], low_nullifier_membership_witness: [NullifierMembershipWitness; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], @@ -180,171 +183,64 @@ impl BaseRollupInputs { calculate_subtree(commitment_tree_leaves) } - unconstrained fn find_leaf_index(self, leaves: [NullifierLeafPreimage; MAX_NEW_NULLIFIERS_PER_TX * 2], nullifier: Field, nullifier_index: u64) -> u64 { - let mut matched = false; - let mut index = 0; - for k in 0..nullifier_index { - if !matched { - if (!leaves[k].is_empty()) { - if (full_field_less_than(leaves[k].leaf_value, nullifier) & - (full_field_greater_than(leaves[k].next_value, nullifier) | - (leaves[k].next_value == 0))) { - matched = true; - index = k; - } - } - } - - - } - // if not matched, our subtree will misformed - we must reject - assert(matched, "Nullifier subtree is malformed"); - index - } + fn check_nullifier_tree_non_membership_and_insert_to_tree(self) -> AppendOnlyTreeSnapshot { + let mut new_nullifiers = [0; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP]; - // TODO this should be done in circuit. Ideally using the sorting strategy. - unconstrained fn check_nullifier_tree_non_membership_and_insert_to_tree(self) -> AppendOnlyTreeSnapshot { - // The below monologue is by Madiaa. fwiw, the plan was not simple. - // - // LADIES AND GENTLEMEN The P L A N ( is simple ) - // 1. Get the previous nullifier set setup - // 2. Check for the first added nullifier that it doesnt exist - // 3. Update the nullifier set - // 4. Calculate a new root with the sibling path - // 5. Use that for the next nullifier check. - // 6. Iterate for all of em - // 7. le bosh (profit) - - // BOYS AND GIRLS THE P L A N ( once the first plan is complete ) - // GENERATE OUR NEW NULLIFIER SUBTREE - // 1. We need to point the new nullifiers to point to the index that the previous nullifier replaced - // 2. If we receive the 0 nullifier leaf (where all values are 0, we skip insertion and leave a sparse subtree) - - // New nullifier subtree - let mut nullifier_insertion_subtree = [NullifierLeafPreimage::default(); MAX_NEW_NULLIFIERS_PER_TX * 2]; - - // This will update on each iteration - let mut current_nullifier_tree_root = self.start_nullifier_tree_snapshot.root; - - // This will increase with every insertion - let start_insertion_index = self.start_nullifier_tree_snapshot.next_available_leaf_index; - let mut new_index = start_insertion_index; - - // For each kernel circuit - for i in 0..KERNELS_PER_BASE_ROLLUP { - let new_nullifiers = self.kernel_data[i].public_inputs.end.new_nullifiers; - // For each of our nullifiers - for j in 0..MAX_NEW_NULLIFIERS_PER_TX { - // Witness containing index and path - let nullifier_index = i * MAX_NEW_NULLIFIERS_PER_TX + j; - - let witness = self.low_nullifier_membership_witness[nullifier_index]; - // Preimage of the lo-index required for a non-membership proof - let low_nullifier_preimage = self.low_nullifier_leaf_preimages[nullifier_index]; - // Newly created nullifier - let nullifier = new_nullifiers[j]; - - // TODO(maddiaa): reason about this more strongly, can this cause issues? - if (nullifier != 0) { - // Create the nullifier leaf of the new nullifier to be inserted - let mut new_nullifier_leaf = NullifierLeafPreimage { - leaf_value : nullifier, - next_value : low_nullifier_preimage.next_value, - next_index : low_nullifier_preimage.next_index, - }; - - // Assuming populated premier subtree - if (low_nullifier_preimage.is_empty()) { - // check previous nullifier leaves - let index = self.find_leaf_index(nullifier_insertion_subtree, nullifier, nullifier_index as u64); - let same_batch_nullifier = nullifier_insertion_subtree[index]; - assert(!same_batch_nullifier.is_empty(), "Same batch batch nullifier is empty"); - assert(full_field_less_than(same_batch_nullifier.leaf_value, nullifier), "Invalid hint"); - assert(full_field_greater_than(same_batch_nullifier.next_value, nullifier) | (same_batch_nullifier.next_value == 0), "Invalid hint"); - - new_nullifier_leaf.next_index = nullifier_insertion_subtree[index].next_index; - new_nullifier_leaf.next_value = nullifier_insertion_subtree[index].next_value; - - // Update child - nullifier_insertion_subtree[index].next_index = new_index; - nullifier_insertion_subtree[index].next_value = nullifier; - } else { - let is_less_than_nullifier = full_field_less_than(low_nullifier_preimage.leaf_value, nullifier); - let is_next_greater_than = full_field_greater_than(low_nullifier_preimage.next_value, nullifier); - - assert(is_less_than_nullifier, "invalid nullifier range"); - assert( - is_next_greater_than | - ((low_nullifier_preimage.next_value == 0) & (low_nullifier_preimage.next_index == 0)), - "invalid nullifier range" - ); - - // Recreate the original low nullifier from the preimage - let original_low_nullifier = NullifierLeafPreimage{ - leaf_value : low_nullifier_preimage.leaf_value, - next_value : low_nullifier_preimage.next_value, - next_index : low_nullifier_preimage.next_index, - }; - - // perform membership check for the low nullifier against the original root - components::assert_check_membership( - original_low_nullifier.hash(), - witness.leaf_index, - witness.sibling_path, - current_nullifier_tree_root, - ); - - // Calculate the new value of the low_nullifier_leaf - let updated_low_nullifier = NullifierLeafPreimage{ - leaf_value : low_nullifier_preimage.leaf_value, - next_value : nullifier, - next_index : new_index - }; - - // We need another set of witness values for this - current_nullifier_tree_root = components::root_from_sibling_path( - updated_low_nullifier.hash(), witness.leaf_index, witness.sibling_path); - } - nullifier_insertion_subtree[nullifier_index] = new_nullifier_leaf; - } - - // increment insertion index - new_index = new_index + 1; + for i in 0..2 { + for j in 0..MAX_NEW_NULLIFIERS_PER_TX { + new_nullifiers[i * MAX_NEW_NULLIFIERS_PER_TX + j] = self.kernel_data[i].public_inputs.end.new_nullifiers[j]; } - } + }; - // Check that the new subtree is to be inserted at the next location, and is empty currently - let empty_nullifier_subtree_root = calculate_empty_tree_root(NULLIFIER_SUBTREE_HEIGHT); - let leafIndexNullifierSubtreeDepth = self.start_nullifier_tree_snapshot.next_available_leaf_index >> (NULLIFIER_SUBTREE_HEIGHT as u32); - components::assert_check_membership( - empty_nullifier_subtree_root, - leafIndexNullifierSubtreeDepth as Field, + crate::indexed_tree::batch_insert( + self.start_nullifier_tree_snapshot, + new_nullifiers, + self.sorted_new_nullifiers, + self.sorted_new_nullifiers_indexes, self.new_nullifiers_subtree_sibling_path, - current_nullifier_tree_root, - ); - - // Create new nullifier subtree to insert into the whole nullifier tree - let nullifier_sibling_path = self.new_nullifiers_subtree_sibling_path; - let nullifier_subtree_root = self.create_nullifier_subtree(nullifier_insertion_subtree); - - // Calculate the new root - // We are inserting a subtree rather than a full tree here - let subtree_index = start_insertion_index >> (NULLIFIER_SUBTREE_HEIGHT as u32); - let new_root = components::root_from_sibling_path(nullifier_subtree_root, subtree_index as Field, nullifier_sibling_path); - - // Return the new state of the nullifier tree - AppendOnlyTreeSnapshot { - next_available_leaf_index: new_index, - root: new_root, - } + self.low_nullifier_leaf_preimages, + self.low_nullifier_membership_witness.map(|witness: NullifierMembershipWitness| { + MembershipWitness { + leaf_index: witness.leaf_index, + sibling_path: witness.sibling_path, + } + }), + |a: Field, b: Field| {a == b}, // Nullifier equals + |nullifier: Field| {nullifier == 0}, // Nullifier is zero + |leaf: NullifierLeafPreimage| {leaf.hash()}, // Hash leaf + |low_leaf: NullifierLeafPreimage, nullifier: Field| { // Is valid low leaf + let is_less_than_nullifier = full_field_less_than(low_leaf.leaf_value, nullifier); + let is_next_greater_than = full_field_less_than(nullifier, low_leaf.next_value); + + (!low_leaf.is_empty()) & is_less_than_nullifier & ( + is_next_greater_than | + ((low_leaf.next_index == 0) & (low_leaf.next_value == 0)) + ) + }, + |low_leaf: NullifierLeafPreimage, nullifier: Field, nullifier_index: u32| { // Update low leaf + NullifierLeafPreimage{ + leaf_value : low_leaf.leaf_value, + next_value : nullifier, + next_index : nullifier_index, + } + }, + |nullifier: Field, low_leaf: NullifierLeafPreimage| { // Build insertion leaf + NullifierLeafPreimage { + leaf_value : nullifier, + next_value : low_leaf.next_value, + next_index : low_leaf.next_index, + } + }, + [0; NULLIFIER_SUBTREE_HEIGHT], + [0; NULLIFIER_TREE_HEIGHT], + ) } fn create_nullifier_subtree(self, leaves: [NullifierLeafPreimage; N]) -> Field { calculate_subtree(leaves.map(|leaf:NullifierLeafPreimage| leaf.hash())) } - // TODO this should be changed to append only and done in-circuit - unconstrained fn validate_and_process_public_state(self) -> Field { + fn validate_and_process_public_state(self) -> Field { // TODO(#2521) - data read validation should happen against the current state of the tx and not the start state. // Blocks all interesting usecases that read and write to the same public state in the same tx. // https://aztecprotocol.slack.com/archives/C02M7VC7TN0/p1695809629015719?thread_ts=1695653252.007339&cid=C02M7VC7TN0 @@ -629,7 +525,8 @@ mod tests { CALL_DATA_HASH_LOG_FIELDS, NOTE_HASH_SUBTREE_WIDTH, NUM_CONTRACT_LEAVES, - BaseRollupInputs + BaseRollupInputs, + full_field_less_than, }, merkle_tree::{calculate_subtree, calculate_empty_tree_root}, abis::append_only_tree_snapshot::AppendOnlyTreeSnapshot, @@ -662,6 +559,7 @@ mod tests { abis::new_contract_data::NewContractData, abis::public_data_read::PublicDataRead, abis::public_data_update_request::PublicDataUpdateRequest, + abis::previous_kernel_data::PreviousKernelData, tests::previous_kernel_data_builder::PreviousKernelDataBuilder, address::{Address, EthAddress}, utils::bounded_vec::BoundedVec, @@ -670,10 +568,18 @@ mod tests { use dep::std::option::Option; struct NullifierInsertion { - existing_index: Option, + existing_index: u64, + value: Field, + } + + + struct SortedNullifierTuple { value: Field, + original_index: u32, } + global MAX_NEW_NULLIFIERS_PER_TEST = 4; + struct BaseRollupInputsBuilder { kernel_data: [PreviousKernelDataBuilder; KERNELS_PER_BASE_ROLLUP], pre_existing_notes: [Field; NOTE_HASH_SUBTREE_WIDTH], @@ -683,7 +589,7 @@ mod tests { pre_existing_blocks: [Field; KERNELS_PER_BASE_ROLLUP], public_data_reads: BoundedVec, public_data_writes: BoundedVec<(u64, Field), 2>, - new_nullifiers: BoundedVec, + new_nullifiers: BoundedVec, constants: ConstantRollupData, } @@ -718,6 +624,77 @@ mod tests { sibling_path } + fn update_nullifier_tree_with_new_leaves( + mut self, + nullifier_tree: &mut NonEmptyMerkleTree, + kernel_data: &mut [PreviousKernelData; KERNELS_PER_BASE_ROLLUP], + start_nullifier_tree_snapshot: AppendOnlyTreeSnapshot + ) -> ( + [NullifierLeafPreimage; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], + [NullifierMembershipWitness; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], + [Field; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], + [u32; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP], + ) { + let mut low_nullifier_leaf_preimages: [NullifierLeafPreimage; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP] = dep::std::unsafe::zeroed(); + let mut low_nullifier_membership_witness: [NullifierMembershipWitness; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP] = dep::std::unsafe::zeroed(); + + let mut sorted_new_nullifier_tuples = [SortedNullifierTuple { + value: 0, + original_index: 0, + }; MAX_NEW_NULLIFIERS_PER_TEST]; + + + for i in 0..MAX_NEW_NULLIFIERS_PER_TEST { + sorted_new_nullifier_tuples[i] = SortedNullifierTuple { + value: self.new_nullifiers.get_unchecked(i).value, + original_index: i as u32, + }; + } + sorted_new_nullifier_tuples = sorted_new_nullifier_tuples.sort_via(|a: SortedNullifierTuple, b: SortedNullifierTuple| {full_field_less_than(b.value, a.value)}); + + let mut sorted_new_nullifiers = [0; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP]; + let mut sorted_new_nullifiers_indexes = [0; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP]; + + for i in 0..MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP { + if (i as u32) < (MAX_NEW_NULLIFIERS_PER_TEST as u32) { + sorted_new_nullifiers[i] = sorted_new_nullifier_tuples[i].value; + sorted_new_nullifiers_indexes[i] = sorted_new_nullifier_tuples[i].original_index; + } else { + sorted_new_nullifiers[i] = 0; + sorted_new_nullifiers_indexes[i] = i as u32; + } + } + + let mut pre_existing_nullifiers = self.pre_existing_nullifiers; + + for i in 0..MAX_NEW_NULLIFIERS_PER_TEST { + if (i as u64) < (self.new_nullifiers.len() as u64) { + let sorted_tuple = sorted_new_nullifier_tuples[i]; + let new_nullifier = sorted_tuple.value; + let original_index = sorted_tuple.original_index; + + let low_index = self.new_nullifiers.get_unchecked(original_index as Field).existing_index; + + kernel_data[0].public_inputs.end.new_nullifiers[original_index] = new_nullifier; + + let mut low_preimage = pre_existing_nullifiers[low_index]; + low_nullifier_leaf_preimages[i] = low_preimage; + low_nullifier_membership_witness[i] = NullifierMembershipWitness { + leaf_index: low_index as Field, + sibling_path: nullifier_tree.get_sibling_path(low_index as Field) + }; + + low_preimage.next_value = new_nullifier; + low_preimage.next_index = start_nullifier_tree_snapshot.next_available_leaf_index + original_index; + pre_existing_nullifiers[low_index] = low_preimage; + + nullifier_tree.update_leaf(low_index, low_preimage.hash()); + } + } + + (low_nullifier_leaf_preimages, low_nullifier_membership_witness, sorted_new_nullifiers, sorted_new_nullifiers_indexes) + } + fn build_inputs(mut self) -> BaseRollupInputs { let mut kernel_data = self.kernel_data.map(|builder: PreviousKernelDataBuilder|{ builder.finish() @@ -736,12 +713,12 @@ mod tests { [0; NULLIFIER_TREE_HEIGHT - NULLIFIER_SUBTREE_HEIGHT], [0; NULLIFIER_SUBTREE_HEIGHT] ); - + let start_nullifier_tree_snapshot = AppendOnlyTreeSnapshot { root: start_nullifier_tree.get_root(), next_available_leaf_index: start_nullifier_tree.get_next_available_index() as u32, }; - + let start_contract_tree = NonEmptyMerkleTree::new(self.pre_existing_contracts, [0; CONTRACT_TREE_HEIGHT], [0; CONTRACT_TREE_HEIGHT - 1], [0; 1]); let start_contract_tree_snapshot = AppendOnlyTreeSnapshot { root: start_contract_tree.get_root(), @@ -761,7 +738,7 @@ mod tests { self.constants.start_historic_blocks_tree_roots_snapshot = start_historic_blocks_tree_snapshot; let mut new_public_data_reads_sibling_paths: [[Field; PUBLIC_DATA_TREE_HEIGHT]; MAX_PUBLIC_DATA_READS_PER_BASE_ROLLUP] = dep::std::unsafe::zeroed(); - + for i in 0..self.public_data_reads.max_len() { if (i as u64) < (self.public_data_reads.len() as u64) { let index = self.public_data_reads.get_unchecked(i); @@ -792,29 +769,13 @@ mod tests { } } - let mut low_nullifier_leaf_preimages: [NullifierLeafPreimage; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP] = dep::std::unsafe::zeroed(); - let mut low_nullifier_membership_witness: [NullifierMembershipWitness; MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP] = dep::std::unsafe::zeroed(); - - for i in 0..self.new_nullifiers.max_len() { - if (i as u64) < (self.new_nullifiers.len() as u64) { - let new_nullifier = self.new_nullifiers.get_unchecked(i); - kernel_data[0].public_inputs.end.new_nullifiers[i] = new_nullifier.value; - - if (new_nullifier.existing_index.is_some()) { - let low_index = new_nullifier.existing_index.unwrap_unchecked(); - let mut low_preimage = self.pre_existing_nullifiers[low_index]; - low_nullifier_leaf_preimages[i] = low_preimage; - low_nullifier_membership_witness[i] = NullifierMembershipWitness { - leaf_index: low_index as Field, - sibling_path: start_nullifier_tree.get_sibling_path(low_index as Field) - }; - - low_preimage.next_value = new_nullifier.value; - low_preimage.next_index = start_nullifier_tree_snapshot.next_available_leaf_index + (i as u32); - start_nullifier_tree.update_leaf(low_index, low_preimage.hash()); - } - } - } + let ( + low_nullifier_leaf_preimages, + low_nullifier_membership_witness, + sorted_new_nullifiers, + sorted_new_nullifiers_indexes + ) = self.update_nullifier_tree_with_new_leaves(&mut start_nullifier_tree, &mut kernel_data, start_nullifier_tree_snapshot); + let new_nullifiers_subtree_sibling_path = BaseRollupInputsBuilder::extract_subtree_sibling_path(start_nullifier_tree.get_sibling_path(self.pre_existing_nullifiers.len()), [0; NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH]); BaseRollupInputs { @@ -825,6 +786,9 @@ mod tests { start_public_data_tree_root, start_historic_blocks_tree_snapshot, + sorted_new_nullifiers, + sorted_new_nullifiers_indexes, + low_nullifier_leaf_preimages, low_nullifier_membership_witness, @@ -833,7 +797,7 @@ mod tests { new_contracts_subtree_sibling_path, new_public_data_update_requests_sibling_paths, new_public_data_reads_sibling_paths, - + historic_blocks_tree_root_membership_witnesses: [ HistoricBlocksTreeRootMembershipWitness { leaf_index: 0, @@ -844,7 +808,7 @@ mod tests { sibling_path: start_historic_blocks_tree.get_sibling_path(1) }, ], - + constants: self.constants, } } @@ -863,7 +827,7 @@ mod tests { } #[test] - fn no_new_contract_leaves() { + unconstrained fn no_new_contract_leaves() { let outputs = BaseRollupInputsBuilder::new().execute(); let expected_start_contract_tree_snapshot = AppendOnlyTreeSnapshot { root: test_compute_empty_root([0; CONTRACT_TREE_HEIGHT]), next_available_leaf_index: 2 }; let expected_end_contract_tree_snapshot = AppendOnlyTreeSnapshot { root: test_compute_empty_root([0; CONTRACT_TREE_HEIGHT]), next_available_leaf_index: 4 }; @@ -872,7 +836,7 @@ mod tests { } #[test] - fn contract_leaf_inserted() { + unconstrained fn contract_leaf_inserted() { let new_contract = NewContractData { contract_address: Address::from_field(1), portal_contract_address: EthAddress::from_field(2), @@ -903,7 +867,7 @@ mod tests { } #[test] - fn contract_leaf_inserted_in_non_empty_snapshot_tree() { + unconstrained fn contract_leaf_inserted_in_non_empty_snapshot_tree() { let new_contract = NewContractData { contract_address: Address::from_field(1), portal_contract_address: EthAddress::from_field(2), @@ -936,7 +900,7 @@ mod tests { } #[test] - fn new_commitments_tree() { + unconstrained fn new_commitments_tree() { let mut builder = BaseRollupInputsBuilder::new(); let new_commitments = [27, 28, 29, 30, 31, 32]; @@ -972,7 +936,7 @@ mod tests { } #[test] - fn new_nullifier_tree_empty() { + unconstrained fn new_nullifier_tree_empty() { /** * DESCRIPTION */ @@ -999,7 +963,7 @@ mod tests { } #[test] - fn nullifier_insertion_test() { + unconstrained fn nullifier_insertion_test() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_nullifiers[0] = NullifierLeafPreimage { @@ -1014,7 +978,7 @@ mod tests { }; builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::some(0), + existing_index: 0, value: 1, }); @@ -1047,7 +1011,7 @@ mod tests { } #[test] - fn new_nullifier_tree_all_larger() { + unconstrained fn new_nullifier_tree_all_larger() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_nullifiers[0] = NullifierLeafPreimage { @@ -1062,16 +1026,18 @@ mod tests { }; builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::some(1), + existing_index: 1, value: 8, }); for i in 1..builder.new_nullifiers.max_len() { builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::none(), + existing_index: 1, value: (8 + i) as Field, }); } + let output = builder.execute(); + let mut tree_nullifiers = [NullifierLeafPreimage::default(); MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP * 2]; tree_nullifiers[0] = builder.pre_existing_nullifiers[0]; @@ -1103,18 +1069,14 @@ mod tests { [0; NULLIFIER_SUBTREE_HEIGHT + 1] ); - let output = builder.execute(); - assert(output.end_nullifier_tree_snapshot.eq(AppendOnlyTreeSnapshot { root: end_nullifier_tree.get_root(), next_available_leaf_index: 2 * MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP as u32, })); } - // TODO(Alvaro) some nullifier tree tests. We are updating the nullifier tree insertion algorithm. - - #[test(should_fail_with = "membership check failed")] - fn new_nullifier_tree_double_spend() { + #[test(should_fail_with = "Invalid low leaf")] + unconstrained fn new_nullifier_tree_double_spend() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_nullifiers[0] = NullifierLeafPreimage { @@ -1129,20 +1091,19 @@ mod tests { }; builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::some(1), + existing_index: 1, value: 8, }); builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::some(1), + existing_index: 1, value: 8, }); builder.fails(); } - - #[test(should_fail_with = "Nullifier subtree is malformed")] - fn new_nullifier_tree_double_spend_same_batch() { + #[test(should_fail_with = "Invalid low leaf")] + unconstrained fn new_nullifier_tree_double_spend_same_batch() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_nullifiers[0] = NullifierLeafPreimage { @@ -1157,30 +1118,31 @@ mod tests { }; builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::some(1), + existing_index: 1, value: 8, }); builder.new_nullifiers.push(NullifierInsertion { - existing_index: Option::none(), + existing_index: 1, value: 8, }); builder.fails(); } - #[test] - fn empty_block_calldata_hash() { + unconstrained fn empty_block_calldata_hash() { let outputs = BaseRollupInputsBuilder::new().execute(); let hash_input_flattened = [0; CALL_DATA_HASH_FULL_FIELDS * 32 + CALL_DATA_HASH_LOG_FIELDS * 16]; let sha_digest = dep::std::hash::sha256(hash_input_flattened); let expected_calldata_hash = U256::from_bytes32(sha_digest).to_u128_limbs(); - assert_eq(outputs.calldata_hash, expected_calldata_hash); + for i in 0..NUM_FIELDS_PER_SHA256 { + assert_eq(outputs.calldata_hash[i], expected_calldata_hash[i]); + } } #[test(should_fail_with = "membership check failed")] - fn compute_membership_historic_blocks_tree_negative() { + unconstrained fn compute_membership_historic_blocks_tree_negative() { let mut inputs = BaseRollupInputsBuilder::new().build_inputs(); inputs.historic_blocks_tree_root_membership_witnesses[0].sibling_path[0] = 27; @@ -1189,7 +1151,7 @@ mod tests { } #[test] - fn constants_dont_change() { + unconstrained fn constants_dont_change() { let inputs = BaseRollupInputsBuilder::new().build_inputs(); let outputs = inputs.base_rollup_circuit(); @@ -1197,28 +1159,28 @@ mod tests { } #[test(should_fail_with = "kernel chain_id does not match the rollup chain_id")] - fn constants_dont_match_kernels_chain_id() { + unconstrained fn constants_dont_match_kernels_chain_id() { let mut builder = BaseRollupInputsBuilder::new(); builder.constants.global_variables.chain_id = 3; builder.fails(); } #[test(should_fail_with = "kernel version does not match the rollup version")] - fn constants_dont_match_kernels_version() { + unconstrained fn constants_dont_match_kernels_version() { let mut builder = BaseRollupInputsBuilder::new(); builder.constants.global_variables.version = 3; builder.fails(); } #[test] - fn subtree_height_is_0() { + unconstrained fn subtree_height_is_0() { let outputs = BaseRollupInputsBuilder::new().execute(); assert_eq(outputs.rollup_subtree_height, 0); } #[test] - fn single_public_state_read() { + unconstrained fn single_public_state_read() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_public_data[0] = 27; @@ -1228,7 +1190,7 @@ mod tests { } #[test] - fn single_public_state_write() { + unconstrained fn single_public_state_write() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_public_data[0] = 27; @@ -1247,7 +1209,7 @@ mod tests { } #[test] - fn multiple_public_state_read_writes() { + unconstrained fn multiple_public_state_read_writes() { let mut builder = BaseRollupInputsBuilder::new(); builder.pre_existing_public_data[0] = 27; @@ -1271,4 +1233,4 @@ mod tests { assert_eq(outputs.end_public_data_tree_root, expected_public_data_tree.get_root()); } -} +} \ No newline at end of file diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/indexed_tree.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/indexed_tree.nr new file mode 100644 index 00000000000..1f871270b2d --- /dev/null +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/indexed_tree.nr @@ -0,0 +1,121 @@ +use crate::abis::append_only_tree_snapshot::AppendOnlyTreeSnapshot; +use crate::merkle_tree::{calculate_subtree, calculate_empty_tree_root}; + +use dep::types::abis::membership_witness::MembershipWitness; + +fn check_permutation(original_array: [T; N], sorted_array: [T; N], indexes: [u32; N], is_equal: fn (T, T) -> bool) { + let mut seen_value = [false; N]; + for i in 0..N { + let index = indexes[i]; + let sorted_value = sorted_array[i]; + let original_value = original_array[index]; + assert(is_equal(sorted_value, original_value), "Invalid index"); + assert(!seen_value[index], "Duplicated index"); + seen_value[index] = true; + } +} + +#[test] +fn check_permutation_basic_test(){ + let original_array = [1, 2, 3]; + let sorted_array = [3, 1, 2]; + let indexes = [2, 0, 1]; + let is_equal = |a: Field, b: Field| a == b; + check_permutation(original_array, sorted_array, indexes, is_equal); +} + +#[test(should_fail_with = "Duplicated index")] +fn check_permutation_duplicated_index(){ + let original_array = [0, 1, 0]; + let sorted_array = [1, 0, 0]; + let indexes = [1, 0, 0]; + let is_equal = |a: Field, b: Field| a == b; + check_permutation(original_array, sorted_array, indexes, is_equal); +} + +#[test(should_fail_with = "Invalid index")] +fn check_permutation_invalid_index(){ + let original_array = [0, 1, 2]; + let sorted_array = [1, 0, 0]; + let indexes = [1, 0, 2]; + let is_equal = |a: Field, b: Field| a == b; + check_permutation(original_array, sorted_array, indexes, is_equal); +} + +pub fn batch_insert( + start_snapshot: AppendOnlyTreeSnapshot, + values_to_insert: [Value; SubtreeWidth], + sorted_values: [Value; SubtreeWidth], + sorted_values_indexes: [u32; SubtreeWidth], + new_subtree_sibling_path: [Field; SiblingPathLength], + low_leaf_preimages: [Leaf; SubtreeWidth], + low_leaf_membership_witnesses: [MembershipWitness; SubtreeWidth], + is_equal: fn (Value, Value) -> bool, + is_empty_value: fn (Value) -> bool, + hash_leaf: fn (Leaf) -> Field, + is_valid_low_leaf: fn(Leaf, Value) -> bool, + update_low_leaf: fn(Leaf, Value, u32) -> Leaf, + build_insertion_leaf: fn (Value, Leaf) -> Leaf, + _subtree_height: [Field; SubtreeHeight], + _tree_height: [Field; TreeHeight], +) -> AppendOnlyTreeSnapshot { + // A permutation to the values is provided to make the insertion use only one insertion strategy + check_permutation(values_to_insert, sorted_values, sorted_values_indexes, is_equal); + + // Now, update the existing leaves with the new leaves + let mut current_tree_root = start_snapshot.root; + let mut insertion_subtree: [Leaf; SubtreeWidth] = dep::std::unsafe::zeroed(); + let start_insertion_index = start_snapshot.next_available_leaf_index; + + for i in 0..sorted_values.len() { + let value = sorted_values[i]; + if !is_empty_value(value) { + let low_leaf_preimage = low_leaf_preimages[i]; + let witness = low_leaf_membership_witnesses[i]; + + assert(is_valid_low_leaf(low_leaf_preimage, value), "Invalid low leaf"); + + // perform membership check for the low leaf against the original root + crate::components::assert_check_membership( + hash_leaf(low_leaf_preimage), + witness.leaf_index, + witness.sibling_path, + current_tree_root, + ); + + let value_index = sorted_values_indexes[i]; + + // Calculate the new value of the low_leaf + let updated_low_leaf= update_low_leaf(low_leaf_preimage, value, start_insertion_index+value_index); + + current_tree_root = crate::components::root_from_sibling_path( + hash_leaf(updated_low_leaf), witness.leaf_index, witness.sibling_path); + + insertion_subtree[value_index] = build_insertion_leaf(value, low_leaf_preimage); + } + } + + let empty_subtree_root = calculate_empty_tree_root(SubtreeHeight); + let leaf_index_subtree_depth = start_insertion_index >> (SubtreeHeight as u32); + + crate::components::assert_check_membership( + empty_subtree_root, + leaf_index_subtree_depth as Field, + new_subtree_sibling_path, + current_tree_root, + ); + + // Create new subtree to insert into the whole indexed tree + let subtree_root = calculate_subtree(insertion_subtree.map(hash_leaf)); + + // Calculate the new root + // We are inserting a subtree rather than a full tree here + let subtree_index = start_insertion_index >> (SubtreeHeight as u32); + let new_root = crate::components::root_from_sibling_path(subtree_root, subtree_index as Field, new_subtree_sibling_path); + + AppendOnlyTreeSnapshot { + root: new_root, + next_available_leaf_index: start_insertion_index + (values_to_insert.len() as u32), + } +} + diff --git a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/lib.nr b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/lib.nr index 3a225975d81..1d0df93cc9d 100644 --- a/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/lib.nr +++ b/yarn-project/noir-protocol-circuits/src/crates/rollup-lib/src/lib.nr @@ -15,4 +15,6 @@ mod hash; mod merkle_tree; -mod tests; \ No newline at end of file +mod tests; + +mod indexed_tree; diff --git a/yarn-project/noir-protocol-circuits/src/type_conversion.ts b/yarn-project/noir-protocol-circuits/src/type_conversion.ts index 0ae1b87bc33..8a539742be1 100644 --- a/yarn-project/noir-protocol-circuits/src/type_conversion.ts +++ b/yarn-project/noir-protocol-circuits/src/type_conversion.ts @@ -156,6 +156,13 @@ export function mapNumberFromNoir(number: NoirField): number { return Number(Fr.fromString(number).toBigInt()); } +/** + * + */ +export function mapNumberToNoir(number: number): NoirField { + return new Fr(BigInt(number)).toString(); +} + /** * Maps a point to a noir point. * @param point - The point. @@ -1406,6 +1413,11 @@ export function mapBaseRollupInputsToNoir(inputs: BaseRollupInputs): BaseRollupI start_contract_tree_snapshot: mapAppendOnlyTreeSnapshotToNoir(inputs.startContractTreeSnapshot), start_public_data_tree_root: mapFieldToNoir(inputs.startPublicDataTreeRoot), start_historic_blocks_tree_snapshot: mapAppendOnlyTreeSnapshotToNoir(inputs.startHistoricBlocksTreeSnapshot), + sorted_new_nullifiers: inputs.sortedNewNullifiers.map(mapFieldToNoir) as FixedLengthArray, + sorted_new_nullifiers_indexes: inputs.sortednewNullifiersIndexes.map(mapNumberToNoir) as FixedLengthArray< + NoirField, + 128 + >, low_nullifier_leaf_preimages: inputs.lowNullifierLeafPreimages.map( mapNullifierLeafPreimageToNoir, ) as FixedLengthArray, diff --git a/yarn-project/noir-protocol-circuits/src/types/rollup_base_types.ts b/yarn-project/noir-protocol-circuits/src/types/rollup_base_types.ts index a4ac38b38eb..424535bad73 100644 --- a/yarn-project/noir-protocol-circuits/src/types/rollup_base_types.ts +++ b/yarn-project/noir-protocol-circuits/src/types/rollup_base_types.ts @@ -191,6 +191,8 @@ export interface BaseRollupInputs { start_contract_tree_snapshot: AppendOnlyTreeSnapshot; start_public_data_tree_root: Field; start_historic_blocks_tree_snapshot: AppendOnlyTreeSnapshot; + sorted_new_nullifiers: FixedLengthArray; + sorted_new_nullifiers_indexes: FixedLengthArray; low_nullifier_leaf_preimages: FixedLengthArray; low_nullifier_membership_witness: FixedLengthArray; new_commitments_subtree_sibling_path: FixedLengthArray; diff --git a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts index da6b621773c..464e766a69e 100644 --- a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.ts @@ -708,7 +708,12 @@ export class SoloBlockBuilder implements BlockBuilder { // Update the nullifier tree, capturing the low nullifier info for each individual operation const newNullifiers = [...left.data.end.newNullifiers, ...right.data.end.newNullifiers]; - const [nullifierWitnessLeaves, newNullifiersSubtreeSiblingPath] = await this.db.batchInsert( + const { + lowLeavesWitnessData: nullifierWitnessLeaves, + newSubtreeSiblingPath: newNullifiersSubtreeSiblingPath, + sortedNewLeaves: sortedNewNullifiers, + sortedNewLeavesIndexes: sortednewNullifiersIndexes, + } = await this.db.batchInsert( MerkleTreeId.NULLIFIER_TREE, newNullifiers.map(fr => fr.toBuffer()), NULLIFIER_SUBTREE_HEIGHT, @@ -732,6 +737,8 @@ export class SoloBlockBuilder implements BlockBuilder { startNoteHashTreeSnapshot, startPublicDataTreeRoot: startPublicDataTreeSnapshot.root, startHistoricBlocksTreeSnapshot, + sortedNewNullifiers: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => Fr.fromBuffer(sortedNewNullifiers[i])), + sortednewNullifiersIndexes: makeTuple(MAX_NEW_NULLIFIERS_PER_BASE_ROLLUP, i => sortednewNullifiersIndexes[i]), newCommitmentsSubtreeSiblingPath, newContractsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath: makeTuple(NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH, i => diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 5917a863694..31240357748 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -1,5 +1,5 @@ import { Fr } from '@aztec/foundation/fields'; -import { LowLeafWitnessData } from '@aztec/merkle-tree'; +import { BatchInsertionResult } from '@aztec/merkle-tree'; import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; import { CurrentTreeRoots, HandleL2BlockResult, MerkleTreeDb, MerkleTreeOperations, TreeInfo } from '../index.js'; @@ -171,11 +171,11 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The data for the leaves to be updated when inserting the new ones. */ - public batchInsert( + public batchInsert( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise<[LowLeafWitnessData[], SiblingPath] | [undefined, SiblingPath]> { + ): Promise> { return this.trees.batchInsert(treeId, leaves, subtreeHeight); } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts index 04f4d749fee..8f191517a82 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_tree_db.ts @@ -1,7 +1,7 @@ import { MAX_NEW_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; -import { LowLeafWitnessData } from '@aztec/merkle-tree'; +import { BatchInsertionResult } from '@aztec/merkle-tree'; import { L2Block, LeafData, MerkleTreeId, SiblingPath } from '@aztec/types'; /** @@ -195,11 +195,11 @@ export interface MerkleTreeOperations { * @param subtreeHeight - Height of the subtree. * @returns The witness data for the leaves to be updated when inserting the new ones. */ - batchInsert( + batchInsert( treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: number, - ): Promise<[LowLeafWitnessData[], SiblingPath] | [undefined, SiblingPath]>; + ): Promise>; /** * Handles a single L2 block (i.e. Inserts the new commitments into the merkle tree). diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index cdf1aa5c9bd..a27b1706612 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -15,8 +15,8 @@ import { SerialQueue } from '@aztec/foundation/fifo'; import { createDebugLogger } from '@aztec/foundation/log'; import { AppendOnlyTree, + BatchInsertionResult, IndexedTree, - LowLeafWitnessData, Pedersen, SparseTree, StandardIndexedTree, @@ -401,10 +401,7 @@ export class MerkleTrees implements MerkleTreeDb { treeId: MerkleTreeId, leaves: Buffer[], subtreeHeight: SubtreeHeight, - ): Promise< - | [LowLeafWitnessData[], SiblingPath] - | [undefined, SiblingPath] - > { + ): Promise> { const tree = this.trees[treeId] as StandardIndexedTree; if (!('batchInsert' in tree)) { throw new Error('Tree does not support `batchInsert` method');