feat: process note logs in aztec-nr (#10651)

Closes #9576

This PR offloads the later stages of note log processing from PXE into
aztec-nr. The plan is to take over at the decrypted log payload stage,
and later on expand scope to also include decryption and (eventually)
tagging indices management.

Update: this now works, with some caveats. I'll leave some comments here
re. current status in case anyone wants to take a look, and for me to
resume work once I'm back.

Contracts are now expected to have a `process_logs` function that will
be called during note syncing. This function is not yet being
autogenerated, but I did manually add it to SchnorrAccount and
TokenContract for local testing - it should be fairly easy to
autogenerate it.

PXE still performs tagging index synchronization, fetches all relevant
logs, decrypts them, and merges the public and private components when
they are partial note logs. This will continue to be this way for a
while: we'll tackle these problems in #10723 and #10724. However, past
this point we delegate work to the contract.[^1] The contract performs
nonce discovery and computes note hash and nullifier, and then calls a
new PXE oracle called `deliverNote`. PXE validates that the note hash
exists in the tree, and adds the note to its database. **Edit:** I now
updated this section to remove all of the old relevant TS code. We no
longer do nonce discovery in PXE.

With this first step, PXE no longer needs to know about note type ids,
which become exclusively an aztec-nr concept. It will continue to know
about storage slots, but only because we index by them. More importantly
however, this makes us quite ready to continue building on top of this
work in order to fully move the other parts of the stack (notably
decryption and partial notes) into the contract as well.

[^1]: As of right now we're still doing all of the work in PXE,
including payload destructuring and nonce discovery, but we discard the
results and re-compute them in the contract. Changing this involves
deleting a bunch of files and re-structuring some dataflows, and I
haven't gotten round to it yet. We should do this in this PR.

---------

Co-authored-by: Jan Beneš <[email protected]>

noir-projects/aztec-nr/aztec/src/macros/mod.nr

@@ -25,15 +25,19 @@ pub comptime fn aztec(m: Module) -> Quoted {
     for f in unconstrained_functions {
         transform_unconstrained(f);
     }
+
     let compute_note_hash_and_optionally_a_nullifier =
         generate_compute_note_hash_and_optionally_a_nullifier();
+    let process_logs = generate_process_log();
     let note_exports = generate_note_exports();
     let public_dispatch = generate_public_dispatch(m);
     let sync_notes = generate_sync_notes();
+
     quote {
         $note_exports
         $interface
         $compute_note_hash_and_optionally_a_nullifier
+        $process_logs
         $public_dispatch
         $sync_notes
     }
@@ -169,6 +173,124 @@ comptime fn generate_compute_note_hash_and_optionally_a_nullifier() -> Quoted {
     }
 }
 
+comptime fn generate_process_log() -> Quoted {
+    // This mandatory function processes a log emitted by the contract. This is currently used to recover note contents
+    // and deliver the note to PXE.
+    // The bulk of the work of this function is done by aztec::note::discovery::do_process_log, so all we need to do
+    // is call that function. However, one of its parameters is a lambda function that computes note hash and nullifier
+    // given note contents and metadata (e.g. note type id), since this behavior is contract-specific (as it
+    // depends on the note types implemented by each contract).
+    // The job of this macro is therefore to implement this lambda function and then call `do_process_log` with it.
+
+    // A typical implementation of the lambda looks something like this:
+    // ```
+    // |serialized_note_content: BoundedVec<Field, MAX_NOTE_SERIALIZED_LEN>, note_header: NoteHeader, note_type_id: Field| {
+    //     let hashes = if note_type_id == MyNoteType::get_note_type_id() {
+    //         assert(serialized_note_content.len() == MY_NOTE_TYPE_SERIALIZATION_LENGTH);
+    //         dep::aztec::note::utils::compute_note_hash_and_optionally_a_nullifier(
+    //             MyNoteType::deserialize_content,
+    //             note_header,
+    //             true,
+    //             serialized_note_content.storage(),
+    //         )
+    //     } else {
+    //         panic(f"Unknown note type id {note_type_id}")
+    //     };
+    //
+    //     Option::some(dep::aztec::note::discovery::NoteHashesAndNullifier {
+    //         note_hash: hashes[0],
+    //         unique_note_hash: hashes[1],
+    //         inner_nullifier: hashes[3],
+    //     })
+    // }
+    // ```
+    //
+    // We create this implementation by iterating over the different note types, creating an `if` or `else if` clause
+    // for each of them and calling `compute_note_hash_and_optionally_a_nullifier` with the note's deserialization
+    // function, and finally produce the required `NoteHashesAndNullifier` object.
+
+    let notes = NOTES.entries();
+
+    let mut if_note_type_id_match_statements_list = &[];
+    for i in 0..notes.len() {
+        let (typ, (_, serialized_note_length, _, _)) = notes[i];
+
+        let if_or_else_if = if i == 0 {
+            quote { if }
+        } else {
+            quote { else if }
+        };
+
+        if_note_type_id_match_statements_list = if_note_type_id_match_statements_list.push_back(
+            quote {
+                $if_or_else_if note_type_id == $typ::get_note_type_id() {
+                    // As an extra safety check we make sure that the serialized_note_content bounded vec has the
+                    // expected length, to avoid scenarios in which compute_note_hash_and_optionally_a_nullifier
+                    // silently trims the end if the log were to be longer.
+                    let expected_len = $serialized_note_length;
+                    let actual_len = serialized_note_content.len();
+                    assert(
+                        actual_len == expected_len,
+                        f"Expected note content of length {expected_len} but got {actual_len} for note type id {note_type_id}"
+                    );
+
+                    aztec::note::utils::compute_note_hash_and_optionally_a_nullifier($typ::deserialize_content, note_header, true, serialized_note_content.storage())
+                }
+            },
+        );
+    }
+
+    let if_note_type_id_match_statements = if_note_type_id_match_statements_list.join(quote {});
+
+    let body = if notes.len() > 0 {
+        quote {
+            // Because this unconstrained function is injected after the contract is processed by the macros, it'll not
+            // be modified by the macros that alter unconstrained functions. As such, we need to manually inject the
+            // unconstrained execution context since it will not be available otherwise.
+            let context = dep::aztec::context::unconstrained_context::UnconstrainedContext::new();
+
+            dep::aztec::note::discovery::do_process_log(
+                context,
+                log_plaintext,
+                tx_hash,
+                unique_note_hashes_in_tx,
+                first_nullifier_in_tx,
+                recipient,
+                |serialized_note_content: BoundedVec<Field, _>, note_header, note_type_id| {
+                    let hashes = $if_note_type_id_match_statements
+                    else {
+                        panic(f"Unknown note type id {note_type_id}")
+                    };
+
+                    Option::some(
+                        dep::aztec::note::discovery::NoteHashesAndNullifier {
+                            note_hash: hashes[0],
+                            unique_note_hash: hashes[1],
+                            inner_nullifier: hashes[3],
+                        },
+                    )
+                }
+            );
+        }
+    } else {
+        quote {
+            panic(f"No notes defined")
+        }
+    };
+
+    quote {
+        unconstrained fn process_log(
+            log_plaintext: BoundedVec<Field, dep::aztec::protocol_types::constants::PRIVATE_LOG_SIZE_IN_FIELDS>,
+            tx_hash: Field,
+            unique_note_hashes_in_tx: BoundedVec<Field, dep::aztec::protocol_types::constants::MAX_NOTE_HASHES_PER_TX>,
+            first_nullifier_in_tx: Field,
+            recipient: aztec::protocol_types::address::AztecAddress,
+        ) {
+            $body
+        }
+    }
+}
+
 comptime fn generate_note_exports() -> Quoted {
     let notes = NOTES.values();
     // Second value in each tuple is `note_serialized_len` and that is ignored here because it's only used when

noir-projects/aztec-nr/aztec/src/note/discovery/mod.nr

@@ -0,0 +1,134 @@
+use std::static_assert;
+
+use crate::{
+    context::unconstrained_context::UnconstrainedContext, note::note_header::NoteHeader,
+    oracle::note_discovery::deliver_note, utils::array,
+};
+
+use dep::protocol_types::{
+    address::AztecAddress,
+    constants::{MAX_NOTE_HASHES_PER_TX, PRIVATE_LOG_SIZE_IN_FIELDS},
+    hash::compute_note_hash_nonce,
+};
+
+// We reserve two fields in the note log that are not part of the note content: one for the storage slot, and one for
+// the note type id.
+global NOTE_LOG_RESERVED_FIELDS: u32 = 2;
+pub global MAX_NOTE_SERIALIZED_LEN: u32 = PRIVATE_LOG_SIZE_IN_FIELDS - NOTE_LOG_RESERVED_FIELDS;
+
+pub struct NoteHashesAndNullifier {
+    pub note_hash: Field,
+    pub unique_note_hash: Field,
+    pub inner_nullifier: Field,
+}
+
+/// Processes a log given its plaintext by trying to find notes encoded in it. This process involves the discovery of
+/// the nonce of any such notes, which requires knowledge of the transaction hash in which the notes would've been
+/// created, along with the list of unique note hashes in said transaction.
+///
+/// Additionally, this requires a `compute_note_hash_and_nullifier` lambda that is able to compute these values for any
+/// note in the contract given their contents. A typical implementation of such a function would look like this:
+///
+/// ```
+/// |serialized_note_content, note_header, note_type_id| {
+///     let hashes = if note_type_id == MyNoteType::get_note_type_id() {
+///         assert(serialized_note_content.len() == MY_NOTE_TYPE_SERIALIZATION_LENGTH);
+///         dep::aztec::note::utils::compute_note_hash_and_optionally_a_nullifier(
+///             MyNoteType::deserialize_content,
+///             note_header,
+///             true,
+///             serialized_note_content.storage(),
+///         )
+///     } else {
+///         panic(f"Unknown note type id {note_type_id}")
+///     };
+///
+///     Option::some(dep::aztec::oracle::management::NoteHashesAndNullifier {
+///         note_hash: hashes[0],
+///         unique_note_hash: hashes[1],
+///         inner_nullifier: hashes[3],
+///     })
+/// }
+/// ```
+pub unconstrained fn do_process_log<Env>(
+    context: UnconstrainedContext,
+    log_plaintext: BoundedVec<Field, PRIVATE_LOG_SIZE_IN_FIELDS>,
+    tx_hash: Field,
+    unique_note_hashes_in_tx: BoundedVec<Field, MAX_NOTE_HASHES_PER_TX>,
+    first_nullifier_in_tx: Field,
+    recipient: AztecAddress,
+    compute_note_hash_and_nullifier: fn[Env](BoundedVec<Field, MAX_NOTE_SERIALIZED_LEN>, NoteHeader, Field) -> Option<NoteHashesAndNullifier>,
+) {
+    let (storage_slot, note_type_id, serialized_note_content) =
+        destructure_log_plaintext(log_plaintext);
+
+    // We need to find the note's nonce, which is the one that results in one of the unique note hashes from tx_hash
+    for_each_in_bounded_vec(
+        unique_note_hashes_in_tx,
+        |expected_unique_note_hash, i| {
+            let candidate_nonce = compute_note_hash_nonce(first_nullifier_in_tx, i);
+
+            let header = NoteHeader::new(context.this_address(), candidate_nonce, storage_slot);
+
+            // TODO(#11157): handle failed note_hash_and_nullifier computation
+            let hashes = compute_note_hash_and_nullifier(
+                serialized_note_content,
+                header,
+                note_type_id,
+            )
+                .unwrap();
+
+            if hashes.unique_note_hash == expected_unique_note_hash {
+                // TODO(#10726): push these into a vec to deliver all at once instead of having one oracle call per note
+
+                assert(
+                    deliver_note(
+                        context.this_address(), // TODO(#10727): allow other contracts to deliver notes
+                        storage_slot,
+                        candidate_nonce,
+                        serialized_note_content,
+                        hashes.note_hash,
+                        hashes.inner_nullifier,
+                        tx_hash,
+                        recipient,
+                    ),
+                    "Failed to deliver note",
+                );
+
+                // We don't exit the loop - it is possible (though rare) for the exact same note content to be present
+                // multiple times in the same transaction with different nonces. This typically doesn't happen due to
+                // notes containing random values in order to hide their contents.
+            }
+        },
+    );
+}
+
+unconstrained fn destructure_log_plaintext(
+    log_plaintext: BoundedVec<Field, PRIVATE_LOG_SIZE_IN_FIELDS>,
+) -> (Field, Field, BoundedVec<Field, MAX_NOTE_SERIALIZED_LEN>) {
+    assert(log_plaintext.len() >= NOTE_LOG_RESERVED_FIELDS);
+
+    // If NOTE_LOG_RESERVED_FIELDS is changed, causing the assertion below to fail, then the declarations for
+    // `storage_slot` and `note_type_id` must be updated as well.
+    static_assert(
+        NOTE_LOG_RESERVED_FIELDS == 2,
+        "unepxected value for NOTE_LOG_RESERVED_FIELDS",
+    );
+    let storage_slot = log_plaintext.get(0);
+    let note_type_id = log_plaintext.get(1);
+
+    let serialized_note_content = array::subbvec(log_plaintext, NOTE_LOG_RESERVED_FIELDS);
+
+    (storage_slot, note_type_id, serialized_note_content)
+}
+
+fn for_each_in_bounded_vec<T, let MaxLen: u32, Env>(
+    vec: BoundedVec<T, MaxLen>,
+    f: fn[Env](T, u32) -> (),
+) {
+    for i in 0..MaxLen {
+        if i < vec.len() {
+            f(vec.get_unchecked(i), i);
+        }
+    }
+}

noir-projects/aztec-nr/aztec/src/note/mod.nr

@@ -1,4 +1,5 @@
 pub mod constants;
+pub mod discovery;
 pub mod lifecycle;
 pub mod note_getter;
 pub mod note_getter_options;

noir-projects/aztec-nr/aztec/src/note/note_interface.nr

@@ -17,41 +17,44 @@ where
 }
 
 pub trait NullifiableNote {
-    // This function MUST be called with the correct note hash for consumption! It will otherwise silently fail and
-    // compute an incorrect value.
-    // The reason why we receive this as an argument instead of computing it ourselves directly is because the
-    // caller will typically already have computed this note hash, and we can reuse that value to reduce the total
-    // gate count of the circuit.
+    /// Returns the non-siloed nullifier, which will be later siloed by contract address by the kernels before being
+    /// committed to the state tree.
+    ///
+    /// This function MUST be called with the correct note hash for consumption! It will otherwise silently fail and
+    /// compute an incorrect value. The reason why we receive this as an argument instead of computing it ourselves
+    /// directly is because the caller will typically already have computed this note hash, and we can reuse that value
+    /// to reduce the total gate count of the circuit.
+    ///
+    /// This function receives the context since nullifier computation typically involves proving nullifying keys, and
+    /// we require the kernel's assistance to do this in order to prevent having to reveal private keys to application
+    /// circuits.
     fn compute_nullifier(self, context: &mut PrivateContext, note_hash_for_nullify: Field) -> Field;
 
-    // Unlike compute_nullifier, this function does not take a note hash since it'll only be invoked in unconstrained
-    // contexts, where there is no gate count.
+    /// Same as compute_nullifier, but unconstrained. This version does not take a note hash because it'll only be
+    /// invoked in unconstrained contexts, where there is no gate count.
     unconstrained fn compute_nullifier_without_context(self) -> Field;
 }
 
 // docs:start:note_interface
 // Autogenerated by the #[note] macro
-
 pub trait NoteInterface<let N: u32> {
-    // Autogenerated by the #[note] macro
     fn serialize_content(self) -> [Field; N];
 
-    // Autogenerated by the #[note] macro
     fn deserialize_content(fields: [Field; N]) -> Self;
 
-    // Autogenerated by the #[note] macro
     fn get_header(self) -> NoteHeader;
 
-    // Autogenerated by the #[note] macro
     fn set_header(&mut self, header: NoteHeader) -> ();
 
-    // Autogenerated by the #[note] macro
     fn get_note_type_id() -> Field;
 
-    // Autogenerated by the #[note] macro
     fn to_be_bytes(self, storage_slot: Field) -> [u8; N * 32 + 64];
 
-    // Autogenerated by the #[note] macro
+    /// Returns the non-siloed note hash, i.e. the inner hash computed by the contract during private execution. Note
+    /// hashes are later siloed by contract address and nonce by the kernels before being committed to the state tree.
+    ///
+    /// This should be a commitment to the note contents, including the storage slot (for indexing) and some random
+    /// value (to prevent brute force trial-hashing attacks).
     fn compute_note_hash(self) -> Field;
 }
 // docs:end:note_interface

noir-projects/aztec-nr/aztec/src/oracle/mod.nr

@@ -11,6 +11,7 @@ pub mod get_public_data_witness;
 pub mod get_membership_witness;
 pub mod keys;
 pub mod key_validation_request;
+pub mod note_discovery;
 pub mod random;
 pub mod enqueue_public_function_call;
 pub mod block_header;